diff --git "a/checkpoint-5070/trainer_state.json" "b/checkpoint-5070/trainer_state.json"
new file mode 100644--- /dev/null
+++ "b/checkpoint-5070/trainer_state.json"
@@ -0,0 +1,37742 @@
+{
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 0.10068713508360805,
+  "global_step": 5070,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.0,
+      "learning_rate": 9.92063492063492e-07,
+      "loss": 10.8568,
+      "theoretical_loss": 20.812814784551147,
+      "tokens_seen": 65536
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 1.984126984126984e-06,
+      "loss": 10.8727,
+      "theoretical_loss": 17.56619808733264,
+      "tokens_seen": 131072
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 2.9761904761904763e-06,
+      "loss": 10.7356,
+      "theoretical_loss": 15.939474075840563,
+      "tokens_seen": 196608
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 3.968253968253968e-06,
+      "loss": 10.7113,
+      "theoretical_loss": 14.892313738992565,
+      "tokens_seen": 262144
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 4.96031746031746e-06,
+      "loss": 10.3498,
+      "theoretical_loss": 14.13621392076697,
+      "tokens_seen": 327680
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 5.9523809523809525e-06,
+      "loss": 10.1826,
+      "theoretical_loss": 13.552558455554218,
+      "tokens_seen": 393216
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 6.944444444444444e-06,
+      "loss": 10.0176,
+      "theoretical_loss": 13.081805984405184,
+      "tokens_seen": 458752
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 7.936507936507936e-06,
+      "loss": 9.6515,
+      "theoretical_loss": 12.690126608487319,
+      "tokens_seen": 524288
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 8.928571428571428e-06,
+      "loss": 9.2598,
+      "theoretical_loss": 12.356589446877619,
+      "tokens_seen": 589824
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 9.92063492063492e-06,
+      "loss": 9.1219,
+      "theoretical_loss": 12.067409590039073,
+      "tokens_seen": 655360
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 1.0912698412698412e-05,
+      "loss": 9.2934,
+      "theoretical_loss": 11.813063214105672,
+      "tokens_seen": 720896
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 1.1904761904761905e-05,
+      "loss": 9.0973,
+      "theoretical_loss": 11.586716191710725,
+      "tokens_seen": 786432
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 1.2896825396825396e-05,
+      "loss": 9.4804,
+      "theoretical_loss": 11.383311123190783,
+      "tokens_seen": 851968
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 1.3888888888888888e-05,
+      "loss": 9.3208,
+      "theoretical_loss": 11.199008685115867,
+      "tokens_seen": 917504
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 1.4880952380952381e-05,
+      "loss": 9.1864,
+      "theoretical_loss": 11.030830900981908,
+      "tokens_seen": 983040
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 1.5873015873015872e-05,
+      "loss": 9.398,
+      "theoretical_loss": 10.876425069460945,
+      "tokens_seen": 1048576
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 1.6865079365079364e-05,
+      "loss": 8.9394,
+      "theoretical_loss": 10.73390272306672,
+      "tokens_seen": 1114112
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 1.7857142857142855e-05,
+      "loss": 9.0636,
+      "theoretical_loss": 10.601726859234274,
+      "tokens_seen": 1179648
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 1.884920634920635e-05,
+      "loss": 9.1583,
+      "theoretical_loss": 10.478631155360636,
+      "tokens_seen": 1245184
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 1.984126984126984e-05,
+      "loss": 9.1124,
+      "theoretical_loss": 10.363560926767326,
+      "tokens_seen": 1310720
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 2.0833333333333333e-05,
+      "loss": 9.0087,
+      "theoretical_loss": 10.25562920390074,
+      "tokens_seen": 1376256
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 2.1825396825396824e-05,
+      "loss": 9.0788,
+      "theoretical_loss": 10.154083536274014,
+      "tokens_seen": 1441792
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 2.2817460317460315e-05,
+      "loss": 8.4406,
+      "theoretical_loss": 10.058280544736593,
+      "tokens_seen": 1507328
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 2.380952380952381e-05,
+      "loss": 8.9754,
+      "theoretical_loss": 9.967666161844274,
+      "tokens_seen": 1572864
+    },
+    {
+      "epoch": 0.0,
+      "objective/train/docs_used": 12267,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 8.854658126831055,
+      "objective/train/theoretical_loss": 9.881760109397105,
+      "objective/train/tokens_used": 22098400,
+      "theoretical_loss": 9.881760109397105,
+      "tokens_seen": 1638400
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 2.48015873015873e-05,
+      "loss": 8.9164,
+      "theoretical_loss": 9.881760109397105,
+      "tokens_seen": 1638400
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 2.5793650793650793e-05,
+      "loss": 7.9452,
+      "theoretical_loss": 9.800143574544556,
+      "tokens_seen": 1703936
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 2.6785714285714284e-05,
+      "loss": 8.81,
+      "theoretical_loss": 9.722449329911441,
+      "tokens_seen": 1769472
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 2.7777777777777776e-05,
+      "loss": 9.0741,
+      "theoretical_loss": 9.648353742085542,
+      "tokens_seen": 1835008
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 2.876984126984127e-05,
+      "loss": 8.8256,
+      "theoretical_loss": 9.577570254149634,
+      "tokens_seen": 1900544
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 2.9761904761904762e-05,
+      "loss": 8.3274,
+      "theoretical_loss": 9.509844029768848,
+      "tokens_seen": 1966080
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 3.075396825396825e-05,
+      "loss": 8.55,
+      "theoretical_loss": 9.44494752063593,
+      "tokens_seen": 2031616
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 3.1746031746031745e-05,
+      "loss": 8.0979,
+      "theoretical_loss": 9.382676773914453,
+      "tokens_seen": 2097152
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 3.273809523809524e-05,
+      "loss": 8.3374,
+      "theoretical_loss": 9.322848337237977,
+      "tokens_seen": 2162688
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 3.373015873015873e-05,
+      "loss": 8.5177,
+      "theoretical_loss": 9.26529664966427,
+      "tokens_seen": 2228224
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 3.472222222222222e-05,
+      "loss": 8.6686,
+      "theoretical_loss": 9.209871830448751,
+      "tokens_seen": 2293760
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 3.571428571428571e-05,
+      "loss": 8.675,
+      "theoretical_loss": 9.156437795512288,
+      "tokens_seen": 2359296
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 3.670634920634921e-05,
+      "loss": 8.4694,
+      "theoretical_loss": 9.104870645417346,
+      "tokens_seen": 2424832
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 3.76984126984127e-05,
+      "loss": 8.2407,
+      "theoretical_loss": 9.05505727953773,
+      "tokens_seen": 2490368
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 3.8690476190476195e-05,
+      "loss": 8.5934,
+      "theoretical_loss": 9.006894199647824,
+      "tokens_seen": 2555904
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 3.968253968253968e-05,
+      "loss": 7.8491,
+      "theoretical_loss": 8.960286472913351,
+      "tokens_seen": 2621440
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 4.067460317460318e-05,
+      "loss": 7.8615,
+      "theoretical_loss": 8.915146829644607,
+      "tokens_seen": 2686976
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 4.1666666666666665e-05,
+      "loss": 7.7589,
+      "theoretical_loss": 8.871394875482219,
+      "tokens_seen": 2752512
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 4.265873015873016e-05,
+      "loss": 8.0014,
+      "theoretical_loss": 8.828956401157495,
+      "tokens_seen": 2818048
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 4.365079365079365e-05,
+      "loss": 7.8309,
+      "theoretical_loss": 8.787762775782408,
+      "tokens_seen": 2883584
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 4.464285714285714e-05,
+      "loss": 8.259,
+      "theoretical_loss": 8.747750411915451,
+      "tokens_seen": 2949120
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 4.563492063492063e-05,
+      "loss": 8.1107,
+      "theoretical_loss": 8.708860292524829,
+      "tokens_seen": 3014656
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 4.6626984126984126e-05,
+      "loss": 8.2947,
+      "theoretical_loss": 8.671037551512843,
+      "tokens_seen": 3080192
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 4.761904761904762e-05,
+      "loss": 7.764,
+      "theoretical_loss": 8.63423110073947,
+      "tokens_seen": 3145728
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 4.8611111111111115e-05,
+      "loss": 8.1678,
+      "theoretical_loss": 8.59839329754032,
+      "tokens_seen": 3211264
+    },
+    {
+      "epoch": 0.0,
+      "objective/train/docs_used": 13483,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 8.108365058898926,
+      "objective/train/theoretical_loss": 8.563479647615063,
+      "objective/train/tokens_used": 23736800,
+      "theoretical_loss": 8.563479647615063,
+      "tokens_seen": 3276800
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 4.96031746031746e-05,
+      "loss": 7.5801,
+      "theoretical_loss": 8.563479647615063,
+      "tokens_seen": 3276800
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 5.05952380952381e-05,
+      "loss": 7.8434,
+      "theoretical_loss": 8.52944853889911,
+      "tokens_seen": 3342336
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 5.1587301587301586e-05,
+      "loss": 7.8492,
+      "theoretical_loss": 8.496261002649998,
+      "tokens_seen": 3407872
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 5.257936507936508e-05,
+      "loss": 7.8312,
+      "theoretical_loss": 8.463880498501183,
+      "tokens_seen": 3473408
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 5.357142857142857e-05,
+      "loss": 7.3015,
+      "theoretical_loss": 8.432272720676774,
+      "tokens_seen": 3538944
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 5.4563492063492063e-05,
+      "loss": 7.9474,
+      "theoretical_loss": 8.401405422934712,
+      "tokens_seen": 3604480
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 5.555555555555555e-05,
+      "loss": 7.6148,
+      "theoretical_loss": 8.371248260124204,
+      "tokens_seen": 3670016
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 5.6547619047619046e-05,
+      "loss": 7.5108,
+      "theoretical_loss": 8.341772644515071,
+      "tokens_seen": 3735552
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 5.753968253968254e-05,
+      "loss": 7.6168,
+      "theoretical_loss": 8.312951615289325,
+      "tokens_seen": 3801088
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 5.8531746031746036e-05,
+      "loss": 7.8436,
+      "theoretical_loss": 8.284759719785177,
+      "tokens_seen": 3866624
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 5.9523809523809524e-05,
+      "loss": 7.0651,
+      "theoretical_loss": 8.257172905255858,
+      "tokens_seen": 3932160
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 6.051587301587302e-05,
+      "loss": 7.4859,
+      "theoretical_loss": 8.23016842005411,
+      "tokens_seen": 3997696
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 6.15079365079365e-05,
+      "loss": 7.531,
+      "theoretical_loss": 8.203724723281965,
+      "tokens_seen": 4063232
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 6.25e-05,
+      "loss": 7.5815,
+      "theoretical_loss": 8.177821402057042,
+      "tokens_seen": 4128768
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 6.349206349206349e-05,
+      "loss": 7.2471,
+      "theoretical_loss": 8.152439095643611,
+      "tokens_seen": 4194304
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 6.448412698412699e-05,
+      "loss": 7.4488,
+      "theoretical_loss": 8.127559425781495,
+      "tokens_seen": 4259840
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 6.547619047619048e-05,
+      "loss": 7.4517,
+      "theoretical_loss": 8.103164932619705,
+      "tokens_seen": 4325376
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 6.646825396825397e-05,
+      "loss": 7.1039,
+      "theoretical_loss": 8.079239015726635,
+      "tokens_seen": 4390912
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 6.746031746031745e-05,
+      "loss": 7.1279,
+      "theoretical_loss": 8.055765879705412,
+      "tokens_seen": 4456448
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 6.845238095238096e-05,
+      "loss": 7.3354,
+      "theoretical_loss": 8.032730483993003,
+      "tokens_seen": 4521984
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 6.944444444444444e-05,
+      "loss": 7.1485,
+      "theoretical_loss": 8.01011849646583,
+      "tokens_seen": 4587520
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 7.043650793650793e-05,
+      "loss": 6.9851,
+      "theoretical_loss": 7.987916250513374,
+      "tokens_seen": 4653056
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 7.142857142857142e-05,
+      "loss": 6.9432,
+      "theoretical_loss": 7.966110705275796,
+      "tokens_seen": 4718592
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 7.242063492063492e-05,
+      "loss": 6.9933,
+      "theoretical_loss": 7.944689408771984,
+      "tokens_seen": 4784128
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 7.341269841269842e-05,
+      "loss": 7.0325,
+      "theoretical_loss": 7.923640463671514,
+      "tokens_seen": 4849664
+    },
+    {
+      "epoch": 0.0,
+      "objective/train/docs_used": 14033,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 6.057826519012451,
+      "objective/train/theoretical_loss": 7.902952495488062,
+      "objective/train/tokens_used": 25375200,
+      "theoretical_loss": 7.902952495488062,
+      "tokens_seen": 4915200
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 7.440476190476191e-05,
+      "loss": 6.7791,
+      "theoretical_loss": 7.902952495488062,
+      "tokens_seen": 4915200
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 7.53968253968254e-05,
+      "loss": 6.7978,
+      "theoretical_loss": 7.882614622993199,
+      "tokens_seen": 4980736
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 7.63888888888889e-05,
+      "loss": 6.8558,
+      "theoretical_loss": 7.8626164306686235,
+      "tokens_seen": 5046272
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 7.738095238095239e-05,
+      "loss": 6.9141,
+      "theoretical_loss": 7.842947943031932,
+      "tokens_seen": 5111808
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 7.837301587301588e-05,
+      "loss": 6.9255,
+      "theoretical_loss": 7.8235996006863076,
+      "tokens_seen": 5177344
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 7.936507936507937e-05,
+      "loss": 6.7895,
+      "theoretical_loss": 7.8045622379581605,
+      "tokens_seen": 5242880
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 8.035714285714287e-05,
+      "loss": 6.7059,
+      "theoretical_loss": 7.785827061999068,
+      "tokens_seen": 5308416
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 8.134920634920635e-05,
+      "loss": 6.815,
+      "theoretical_loss": 7.767385633239359,
+      "tokens_seen": 5373952
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 8.234126984126984e-05,
+      "loss": 6.8445,
+      "theoretical_loss": 7.749229847090614,
+      "tokens_seen": 5439488
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 8.333333333333333e-05,
+      "loss": 6.7849,
+      "theoretical_loss": 7.731351916803314,
+      "tokens_seen": 5505024
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 8.432539682539683e-05,
+      "loss": 6.5349,
+      "theoretical_loss": 7.7137443573939155,
+      "tokens_seen": 5570560
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 8.531746031746032e-05,
+      "loss": 6.5103,
+      "theoretical_loss": 7.696399970562929,
+      "tokens_seen": 5636096
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 8.630952380952381e-05,
+      "loss": 6.9307,
+      "theoretical_loss": 7.679311830532177,
+      "tokens_seen": 5701632
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 8.73015873015873e-05,
+      "loss": 6.8115,
+      "theoretical_loss": 7.662473270735324,
+      "tokens_seen": 5767168
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 8.82936507936508e-05,
+      "loss": 6.6601,
+      "theoretical_loss": 7.645877871301275,
+      "tokens_seen": 5832704
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 8.928571428571429e-05,
+      "loss": 6.2238,
+      "theoretical_loss": 7.6295194472748555,
+      "tokens_seen": 5898240
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 9.027777777777777e-05,
+      "loss": 6.6176,
+      "theoretical_loss": 7.613392037523692,
+      "tokens_seen": 5963776
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 9.126984126984126e-05,
+      "loss": 6.5932,
+      "theoretical_loss": 7.597489894284275,
+      "tokens_seen": 6029312
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 9.226190476190476e-05,
+      "loss": 6.51,
+      "theoretical_loss": 7.581807473303883,
+      "tokens_seen": 6094848
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 9.325396825396825e-05,
+      "loss": 6.3768,
+      "theoretical_loss": 7.566339424538366,
+      "tokens_seen": 6160384
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 9.424603174603175e-05,
+      "loss": 6.5465,
+      "theoretical_loss": 7.551080583368945,
+      "tokens_seen": 6225920
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 9.523809523809524e-05,
+      "loss": 6.8931,
+      "theoretical_loss": 7.536025962303915,
+      "tokens_seen": 6291456
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 9.623015873015874e-05,
+      "loss": 6.5266,
+      "theoretical_loss": 7.521170743133757,
+      "tokens_seen": 6356992
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 9.722222222222223e-05,
+      "loss": 6.4856,
+      "theoretical_loss": 7.506510269510493,
+      "tokens_seen": 6422528
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 9.821428571428572e-05,
+      "loss": 6.4763,
+      "theoretical_loss": 7.492040039924245,
+      "tokens_seen": 6488064
+    },
+    {
+      "epoch": 0.0,
+      "objective/train/docs_used": 15357,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 6.699707984924316,
+      "objective/train/theoretical_loss": 7.4777557010520255,
+      "objective/train/tokens_used": 27013600,
+      "theoretical_loss": 7.4777557010520255,
+      "tokens_seen": 6553600
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 9.92063492063492e-05,
+      "loss": 6.6532,
+      "theoretical_loss": 7.4777557010520255,
+      "tokens_seen": 6553600
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00010019841269841271,
+      "loss": 6.4,
+      "theoretical_loss": 7.463653041455457,
+      "tokens_seen": 6619136
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.0001011904761904762,
+      "loss": 6.2462,
+      "theoretical_loss": 7.449727985605911,
+      "tokens_seen": 6684672
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00010218253968253968,
+      "loss": 6.6399,
+      "theoretical_loss": 7.435976588217015,
+      "tokens_seen": 6750208
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00010317460317460317,
+      "loss": 6.1887,
+      "theoretical_loss": 7.4223950288659,
+      "tokens_seen": 6815744
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00010416666666666667,
+      "loss": 6.4129,
+      "theoretical_loss": 7.40897960688587,
+      "tokens_seen": 6881280
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00010515873015873016,
+      "loss": 6.2296,
+      "theoretical_loss": 7.395726736514341,
+      "tokens_seen": 6946816
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00010615079365079365,
+      "loss": 6.3505,
+      "theoretical_loss": 7.3826329422810275,
+      "tokens_seen": 7012352
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00010714285714285714,
+      "loss": 6.5043,
+      "theoretical_loss": 7.369694854622368,
+      "tokens_seen": 7077888
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00010813492063492064,
+      "loss": 6.3689,
+      "theoretical_loss": 7.3569092057090835,
+      "tokens_seen": 7143424
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00010912698412698413,
+      "loss": 6.5583,
+      "theoretical_loss": 7.3442728254746825,
+      "tokens_seen": 7208960
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00011011904761904761,
+      "loss": 6.3757,
+      "theoretical_loss": 7.3317826378335145,
+      "tokens_seen": 7274496
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.0001111111111111111,
+      "loss": 6.3164,
+      "theoretical_loss": 7.319435657077673,
+      "tokens_seen": 7340032
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.0001121031746031746,
+      "loss": 6.205,
+      "theoretical_loss": 7.307228984442819,
+      "tokens_seen": 7405568
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00011309523809523809,
+      "loss": 6.4975,
+      "theoretical_loss": 7.29515980483356,
+      "tokens_seen": 7471104
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00011408730158730158,
+      "loss": 6.087,
+      "theoretical_loss": 7.283225383699648,
+      "tokens_seen": 7536640
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00011507936507936508,
+      "loss": 6.0785,
+      "theoretical_loss": 7.271423064054828,
+      "tokens_seen": 7602176
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00011607142857142858,
+      "loss": 6.4949,
+      "theoretical_loss": 7.259750263630618,
+      "tokens_seen": 7667712
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00011706349206349207,
+      "loss": 6.2155,
+      "theoretical_loss": 7.248204472157866,
+      "tokens_seen": 7733248
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00011805555555555556,
+      "loss": 6.0229,
+      "theoretical_loss": 7.236783248769257,
+      "tokens_seen": 7798784
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00011904761904761905,
+      "loss": 6.6244,
+      "theoretical_loss": 7.225484219516492,
+      "tokens_seen": 7864320
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00012003968253968255,
+      "loss": 5.9387,
+      "theoretical_loss": 7.2143050749961155,
+      "tokens_seen": 7929856
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00012103174603174604,
+      "loss": 6.2842,
+      "theoretical_loss": 7.2032435680783955,
+      "tokens_seen": 7995392
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00012202380952380953,
+      "loss": 6.2047,
+      "theoretical_loss": 7.19229751173401,
+      "tokens_seen": 8060928
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.000123015873015873,
+      "loss": 6.2571,
+      "theoretical_loss": 7.181464776953511,
+      "tokens_seen": 8126464
+    },
+    {
+      "epoch": 0.0,
+      "objective/train/docs_used": 16711,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 6.2001729011535645,
+      "objective/train/theoretical_loss": 7.1707432907549595,
+      "objective/train/tokens_used": 28652000,
+      "theoretical_loss": 7.1707432907549595,
+      "tokens_seen": 8192000
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.0001240079365079365,
+      "loss": 6.0533,
+      "theoretical_loss": 7.1707432907549595,
+      "tokens_seen": 8192000
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.000125,
+      "loss": 6.0526,
+      "theoretical_loss": 7.160131034275267,
+      "tokens_seen": 8257536
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.0001259920634920635,
+      "loss": 6.0532,
+      "theoretical_loss": 7.149626040941133,
+      "tokens_seen": 8323072
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00012698412698412698,
+      "loss": 5.9719,
+      "theoretical_loss": 7.139226394715633,
+      "tokens_seen": 8388608
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00012797619047619048,
+      "loss": 6.0403,
+      "theoretical_loss": 7.12893022841679,
+      "tokens_seen": 8454144
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00012896825396825398,
+      "loss": 5.7847,
+      "theoretical_loss": 7.118735722104612,
+      "tokens_seen": 8519680
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00012996031746031748,
+      "loss": 5.9094,
+      "theoretical_loss": 7.10864110153334,
+      "tokens_seen": 8585216
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00013095238095238096,
+      "loss": 6.0872,
+      "theoretical_loss": 7.098644636665766,
+      "tokens_seen": 8650752
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00013194444444444446,
+      "loss": 6.0185,
+      "theoretical_loss": 7.088744640246688,
+      "tokens_seen": 8716288
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00013293650793650793,
+      "loss": 6.0768,
+      "theoretical_loss": 7.078939466432745,
+      "tokens_seen": 8781824
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00013392857142857144,
+      "loss": 6.3022,
+      "theoretical_loss": 7.069227509475962,
+      "tokens_seen": 8847360
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.0001349206349206349,
+      "loss": 6.0881,
+      "theoretical_loss": 7.059607202458563,
+      "tokens_seen": 8912896
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.0001359126984126984,
+      "loss": 5.7381,
+      "theoretical_loss": 7.0500770160766635,
+      "tokens_seen": 8978432
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.0001369047619047619,
+      "loss": 5.9916,
+      "theoretical_loss": 7.04063545747062,
+      "tokens_seen": 9043968
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00013789682539682541,
+      "loss": 6.0983,
+      "theoretical_loss": 7.031281069099929,
+      "tokens_seen": 9109504
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.0001388888888888889,
+      "loss": 6.0662,
+      "theoretical_loss": 7.022012427660673,
+      "tokens_seen": 9175040
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.0001398809523809524,
+      "loss": 5.7078,
+      "theoretical_loss": 7.012828143043604,
+      "tokens_seen": 9240576
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00014087301587301586,
+      "loss": 6.2455,
+      "theoretical_loss": 7.003726857331067,
+      "tokens_seen": 9306112
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00014186507936507937,
+      "loss": 5.9609,
+      "theoretical_loss": 6.994707243831053,
+      "tokens_seen": 9371648
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00014285714285714284,
+      "loss": 5.9423,
+      "theoretical_loss": 6.985768006146776,
+      "tokens_seen": 9437184
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00014384920634920634,
+      "loss": 6.0329,
+      "theoretical_loss": 6.976907877280185,
+      "tokens_seen": 9502720
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00014484126984126984,
+      "loss": 5.7799,
+      "theoretical_loss": 6.96812561876801,
+      "tokens_seen": 9568256
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00014583333333333335,
+      "loss": 5.9063,
+      "theoretical_loss": 6.95942001984889,
+      "tokens_seen": 9633792
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00014682539682539685,
+      "loss": 5.9317,
+      "theoretical_loss": 6.950789896660304,
+      "tokens_seen": 9699328
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00014781746031746032,
+      "loss": 5.8925,
+      "theoretical_loss": 6.942234091464025,
+      "tokens_seen": 9764864
+    },
+    {
+      "epoch": 0.0,
+      "objective/train/docs_used": 17166,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 6.479184627532959,
+      "objective/train/theoretical_loss": 6.933751471898896,
+      "objective/train/tokens_used": 30290400,
+      "theoretical_loss": 6.933751471898896,
+      "tokens_seen": 9830400
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00014880952380952382,
+      "loss": 6.2539,
+      "theoretical_loss": 6.933751471898896,
+      "tokens_seen": 9830400
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.0001498015873015873,
+      "loss": 5.996,
+      "theoretical_loss": 6.925340930259813,
+      "tokens_seen": 9895936
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.0001507936507936508,
+      "loss": 5.8197,
+      "theoretical_loss": 6.917001382801793,
+      "tokens_seen": 9961472
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00015178571428571427,
+      "loss": 6.0534,
+      "theoretical_loss": 6.908731769068142,
+      "tokens_seen": 10027008
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.0001527777777777778,
+      "loss": 5.7838,
+      "theoretical_loss": 6.900531051241684,
+      "tokens_seen": 10092544
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00015376984126984128,
+      "loss": 5.7799,
+      "theoretical_loss": 6.892398213518156,
+      "tokens_seen": 10158080
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00015476190476190478,
+      "loss": 6.1056,
+      "theoretical_loss": 6.884332261500866,
+      "tokens_seen": 10223616
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00015575396825396825,
+      "loss": 5.5432,
+      "theoretical_loss": 6.876332221615746,
+      "tokens_seen": 10289152
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00015674603174603175,
+      "loss": 5.8572,
+      "theoretical_loss": 6.8683971405459925,
+      "tokens_seen": 10354688
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00015773809523809523,
+      "loss": 6.226,
+      "theoretical_loss": 6.860526084685546,
+      "tokens_seen": 10420224
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00015873015873015873,
+      "loss": 5.7117,
+      "theoretical_loss": 6.852718139610625,
+      "tokens_seen": 10485760
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.0001597222222222222,
+      "loss": 5.6684,
+      "theoretical_loss": 6.844972409568637,
+      "tokens_seen": 10551296
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00016071428571428573,
+      "loss": 5.8166,
+      "theoretical_loss": 6.8372880169837895,
+      "tokens_seen": 10616832
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.0001617063492063492,
+      "loss": 5.971,
+      "theoretical_loss": 6.829664101978745,
+      "tokens_seen": 10682368
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.0001626984126984127,
+      "loss": 5.7343,
+      "theoretical_loss": 6.8220998219117135,
+      "tokens_seen": 10747904
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00016369047619047618,
+      "loss": 5.8061,
+      "theoretical_loss": 6.81459435092839,
+      "tokens_seen": 10813440
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00016468253968253969,
+      "loss": 6.0029,
+      "theoretical_loss": 6.807146879528176,
+      "tokens_seen": 10878976
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00016567460317460316,
+      "loss": 5.5367,
+      "theoretical_loss": 6.79975661414414,
+      "tokens_seen": 10944512
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00016666666666666666,
+      "loss": 5.6256,
+      "theoretical_loss": 6.79242277673622,
+      "tokens_seen": 11010048
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00016765873015873016,
+      "loss": 5.7868,
+      "theoretical_loss": 6.785144604397143,
+      "tokens_seen": 11075584
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00016865079365079366,
+      "loss": 5.6046,
+      "theoretical_loss": 6.777921348970633,
+      "tokens_seen": 11141120
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00016964285714285717,
+      "loss": 5.9753,
+      "theoretical_loss": 6.770752276681419,
+      "tokens_seen": 11206656
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00017063492063492064,
+      "loss": 5.9022,
+      "theoretical_loss": 6.76363666777662,
+      "tokens_seen": 11272192
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00017162698412698414,
+      "loss": 5.8921,
+      "theoretical_loss": 6.756573816178118,
+      "tokens_seen": 11337728
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00017261904761904762,
+      "loss": 5.9722,
+      "theoretical_loss": 6.749563029145481,
+      "tokens_seen": 11403264
+    },
+    {
+      "epoch": 0.0,
+      "objective/train/docs_used": 18369,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 5.672413349151611,
+      "objective/train/theoretical_loss": 6.742603626949085,
+      "objective/train/tokens_used": 31928800,
+      "theoretical_loss": 6.742603626949085,
+      "tokens_seen": 11468800
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00017361111111111112,
+      "loss": 5.4506,
+      "theoretical_loss": 6.742603626949085,
+      "tokens_seen": 11468800
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.0001746031746031746,
+      "loss": 6.0078,
+      "theoretical_loss": 6.735694942553071,
+      "tokens_seen": 11534336
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.0001755952380952381,
+      "loss": 5.8091,
+      "theoretical_loss": 6.728836321307757,
+      "tokens_seen": 11599872
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.0001765873015873016,
+      "loss": 5.6113,
+      "theoretical_loss": 6.722027120651221,
+      "tokens_seen": 11665408
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.0001775793650793651,
+      "loss": 5.6758,
+      "theoretical_loss": 6.715266709819684,
+      "tokens_seen": 11730944
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00017857142857142857,
+      "loss": 5.6378,
+      "theoretical_loss": 6.708554469566408,
+      "tokens_seen": 11796480
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00017956349206349207,
+      "loss": 5.7411,
+      "theoretical_loss": 6.701889791888819,
+      "tokens_seen": 11862016
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00018055555555555555,
+      "loss": 5.8311,
+      "theoretical_loss": 6.695272079763555,
+      "tokens_seen": 11927552
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00018154761904761905,
+      "loss": 5.2846,
+      "theoretical_loss": 6.688700746889175,
+      "tokens_seen": 11993088
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00018253968253968252,
+      "loss": 5.7687,
+      "theoretical_loss": 6.682175217436269,
+      "tokens_seen": 12058624
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00018353174603174602,
+      "loss": 5.7687,
+      "theoretical_loss": 6.675694925804711,
+      "tokens_seen": 12124160
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00018452380952380953,
+      "loss": 5.6209,
+      "theoretical_loss": 6.66925931638781,
+      "tokens_seen": 12189696
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00018551587301587303,
+      "loss": 5.9193,
+      "theoretical_loss": 6.662867843343154,
+      "tokens_seen": 12255232
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.0001865079365079365,
+      "loss": 5.6754,
+      "theoretical_loss": 6.6565199703698745,
+      "tokens_seen": 12320768
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.0001875,
+      "loss": 5.8597,
+      "theoretical_loss": 6.6502151704921655,
+      "tokens_seen": 12386304
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.0001884920634920635,
+      "loss": 5.709,
+      "theoretical_loss": 6.643952925848826,
+      "tokens_seen": 12451840
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00018948412698412698,
+      "loss": 5.7054,
+      "theoretical_loss": 6.637732727488622,
+      "tokens_seen": 12517376
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00019047619047619048,
+      "loss": 5.5972,
+      "theoretical_loss": 6.6315540751713,
+      "tokens_seen": 12582912
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00019146825396825398,
+      "loss": 5.5887,
+      "theoretical_loss": 6.625416477174044,
+      "tokens_seen": 12648448
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00019246031746031748,
+      "loss": 5.7082,
+      "theoretical_loss": 6.619319450103218,
+      "tokens_seen": 12713984
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00019345238095238096,
+      "loss": 5.5405,
+      "theoretical_loss": 6.613262518711206,
+      "tokens_seen": 12779520
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00019444444444444446,
+      "loss": 5.5806,
+      "theoretical_loss": 6.607245215718208,
+      "tokens_seen": 12845056
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00019543650793650793,
+      "loss": 5.4612,
+      "theoretical_loss": 6.601267081638806,
+      "tokens_seen": 12910592
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00019642857142857144,
+      "loss": 5.6405,
+      "theoretical_loss": 6.595327664613176,
+      "tokens_seen": 12976128
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.0001974206349206349,
+      "loss": 5.3968,
+      "theoretical_loss": 6.58942652024278,
+      "tokens_seen": 13041664
+    },
+    {
+      "epoch": 0.0,
+      "objective/train/docs_used": 19017,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 5.398904323577881,
+      "objective/train/theoretical_loss": 6.583563211430409,
+      "objective/train/tokens_used": 33567200,
+      "theoretical_loss": 6.583563211430409,
+      "tokens_seen": 13107200
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.0001984126984126984,
+      "loss": 5.5392,
+      "theoretical_loss": 6.583563211430409,
+      "tokens_seen": 13107200
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00019940476190476191,
+      "loss": 5.3995,
+      "theoretical_loss": 6.577737308224426,
+      "tokens_seen": 13172736
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00020039682539682542,
+      "loss": 5.5665,
+      "theoretical_loss": 6.571948387667094,
+      "tokens_seen": 13238272
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.0002013888888888889,
+      "loss": 5.5745,
+      "theoretical_loss": 6.566196033646859,
+      "tokens_seen": 13303808
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.0002023809523809524,
+      "loss": 5.4756,
+      "theoretical_loss": 6.560479836754459,
+      "tokens_seen": 13369344
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00020337301587301587,
+      "loss": 5.7895,
+      "theoretical_loss": 6.554799394142741,
+      "tokens_seen": 13434880
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00020436507936507937,
+      "loss": 5.3965,
+      "theoretical_loss": 6.549154309390087,
+      "tokens_seen": 13500416
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00020535714285714284,
+      "loss": 5.494,
+      "theoretical_loss": 6.5435441923673245,
+      "tokens_seen": 13565952
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00020634920634920634,
+      "loss": 5.5559,
+      "theoretical_loss": 6.537968659108021,
+      "tokens_seen": 13631488
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00020734126984126985,
+      "loss": 5.9125,
+      "theoretical_loss": 6.532427331682063,
+      "tokens_seen": 13697024
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00020833333333333335,
+      "loss": 5.5235,
+      "theoretical_loss": 6.5269198380724145,
+      "tokens_seen": 13762560
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00020932539682539685,
+      "loss": 5.6307,
+      "theoretical_loss": 6.5214458120549725,
+      "tokens_seen": 13828096
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00021031746031746032,
+      "loss": 5.3661,
+      "theoretical_loss": 6.516004893081412,
+      "tokens_seen": 13893632
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00021130952380952382,
+      "loss": 5.2823,
+      "theoretical_loss": 6.510596726164945,
+      "tokens_seen": 13959168
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.0002123015873015873,
+      "loss": 5.4241,
+      "theoretical_loss": 6.505220961768906,
+      "tokens_seen": 14024704
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.0002132936507936508,
+      "loss": 5.3679,
+      "theoretical_loss": 6.499877255698063,
+      "tokens_seen": 14090240
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00021428571428571427,
+      "loss": 5.2005,
+      "theoretical_loss": 6.494565268992613,
+      "tokens_seen": 14155776
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.0002152777777777778,
+      "loss": 5.6155,
+      "theoretical_loss": 6.489284667824741,
+      "tokens_seen": 14221312
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00021626984126984128,
+      "loss": 5.6027,
+      "theoretical_loss": 6.484035123397694,
+      "tokens_seen": 14286848
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00021726190476190478,
+      "loss": 5.5959,
+      "theoretical_loss": 6.478816311847305,
+      "tokens_seen": 14352384
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00021825396825396825,
+      "loss": 5.7842,
+      "theoretical_loss": 6.473627914145865,
+      "tokens_seen": 14417920
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00021924603174603176,
+      "loss": 5.4174,
+      "theoretical_loss": 6.468469616008303,
+      "tokens_seen": 14483456
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00022023809523809523,
+      "loss": 5.4423,
+      "theoretical_loss": 6.463341107800611,
+      "tokens_seen": 14548992
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00022123015873015873,
+      "loss": 5.2413,
+      "theoretical_loss": 6.458242084450426,
+      "tokens_seen": 14614528
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.0002222222222222222,
+      "loss": 5.4841,
+      "theoretical_loss": 6.453172245359726,
+      "tokens_seen": 14680064
+    },
+    {
+      "epoch": 0.0,
+      "objective/train/docs_used": 20187,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 5.4210638999938965,
+      "objective/train/theoretical_loss": 6.448131294319589,
+      "objective/train/tokens_used": 35205600,
+      "theoretical_loss": 6.448131294319589,
+      "tokens_seen": 14745600
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00022321428571428573,
+      "loss": 5.318,
+      "theoretical_loss": 6.448131294319589,
+      "tokens_seen": 14745600
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.0002242063492063492,
+      "loss": 5.5298,
+      "theoretical_loss": 6.443118939426935,
+      "tokens_seen": 14811136
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.0002251984126984127,
+      "loss": 5.5104,
+      "theoretical_loss": 6.438134893003209,
+      "tokens_seen": 14876672
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00022619047619047618,
+      "loss": 5.4089,
+      "theoretical_loss": 6.43317887151496,
+      "tokens_seen": 14942208
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00022718253968253969,
+      "loss": 5.5783,
+      "theoretical_loss": 6.428250595496234,
+      "tokens_seen": 15007744
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00022817460317460316,
+      "loss": 5.4029,
+      "theoretical_loss": 6.4233497894727805,
+      "tokens_seen": 15073280
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00022916666666666666,
+      "loss": 5.6043,
+      "theoretical_loss": 6.418476181887964,
+      "tokens_seen": 15138816
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00023015873015873016,
+      "loss": 5.5123,
+      "theoretical_loss": 6.413629505030386,
+      "tokens_seen": 15204352
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00023115079365079367,
+      "loss": 5.5084,
+      "theoretical_loss": 6.408809494963135,
+      "tokens_seen": 15269888
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00023214285714285717,
+      "loss": 5.269,
+      "theoretical_loss": 6.404015891454651,
+      "tokens_seen": 15335424
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00023313492063492064,
+      "loss": 5.4379,
+      "theoretical_loss": 6.399248437911128,
+      "tokens_seen": 15400960
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00023412698412698414,
+      "loss": 5.4346,
+      "theoretical_loss": 6.394506881310447,
+      "tokens_seen": 15466496
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00023511904761904762,
+      "loss": 5.7191,
+      "theoretical_loss": 6.38979097213757,
+      "tokens_seen": 15532032
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00023611111111111112,
+      "loss": 5.5484,
+      "theoretical_loss": 6.385100464321383,
+      "tokens_seen": 15597568
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.0002371031746031746,
+      "loss": 5.4379,
+      "theoretical_loss": 6.380435115172919,
+      "tokens_seen": 15663104
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.0002380952380952381,
+      "loss": 5.4785,
+      "theoretical_loss": 6.375794685324962,
+      "tokens_seen": 15728640
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.0002390873015873016,
+      "loss": 5.2088,
+      "theoretical_loss": 6.371178938672962,
+      "tokens_seen": 15794176
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.0002400793650793651,
+      "loss": 5.5076,
+      "theoretical_loss": 6.366587642317243,
+      "tokens_seen": 15859712
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00024107142857142857,
+      "loss": 5.6351,
+      "theoretical_loss": 6.362020566506475,
+      "tokens_seen": 15925248
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00024206349206349207,
+      "loss": 5.6029,
+      "theoretical_loss": 6.3574774845823665,
+      "tokens_seen": 15990784
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00024305555555555555,
+      "loss": 5.1969,
+      "theoretical_loss": 6.352958172925549,
+      "tokens_seen": 16056320
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00024404761904761905,
+      "loss": 5.285,
+      "theoretical_loss": 6.348462410902625,
+      "tokens_seen": 16121856
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00024503968253968255,
+      "loss": 5.6805,
+      "theoretical_loss": 6.343989980814362,
+      "tokens_seen": 16187392
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.000246031746031746,
+      "loss": 5.5855,
+      "theoretical_loss": 6.339540667844965,
+      "tokens_seen": 16252928
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00024702380952380955,
+      "loss": 5.3559,
+      "theoretical_loss": 6.335114260012455,
+      "tokens_seen": 16318464
+    },
+    {
+      "epoch": 0.0,
+      "objective/train/docs_used": 20737,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 5.463630199432373,
+      "objective/train/theoretical_loss": 6.330710548120079,
+      "objective/train/tokens_used": 36844000,
+      "theoretical_loss": 6.330710548120079,
+      "tokens_seen": 16384000
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.000248015873015873,
+      "loss": 5.515,
+      "theoretical_loss": 6.330710548120079,
+      "tokens_seen": 16384000
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.0002490079365079365,
+      "loss": 5.3596,
+      "theoretical_loss": 6.326329325708746,
+      "tokens_seen": 16449536
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00025,
+      "loss": 5.249,
+      "theoretical_loss": 6.321970389010465,
+      "tokens_seen": 16515072
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0002509920634920635,
+      "loss": 5.3675,
+      "theoretical_loss": 6.317633536902759,
+      "tokens_seen": 16580608
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.000251984126984127,
+      "loss": 5.0919,
+      "theoretical_loss": 6.313318570864016,
+      "tokens_seen": 16646144
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00025297619047619046,
+      "loss": 5.1228,
+      "theoretical_loss": 6.30902529492978,
+      "tokens_seen": 16711680
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00025396825396825396,
+      "loss": 5.4573,
+      "theoretical_loss": 6.304753515649935,
+      "tokens_seen": 16777216
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00025496031746031746,
+      "loss": 5.1499,
+      "theoretical_loss": 6.30050304204677,
+      "tokens_seen": 16842752
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00025595238095238096,
+      "loss": 5.6432,
+      "theoretical_loss": 6.296273685573913,
+      "tokens_seen": 16908288
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0002569444444444444,
+      "loss": 5.439,
+      "theoretical_loss": 6.292065260076094,
+      "tokens_seen": 16973824
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00025793650793650796,
+      "loss": 5.2959,
+      "theoretical_loss": 6.287877581749726,
+      "tokens_seen": 17039360
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00025892857142857146,
+      "loss": 5.3502,
+      "theoretical_loss": 6.2837104691042915,
+      "tokens_seen": 17104896
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00025992063492063497,
+      "loss": 5.5134,
+      "theoretical_loss": 6.279563742924502,
+      "tokens_seen": 17170432
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0002609126984126984,
+      "loss": 5.4871,
+      "theoretical_loss": 6.275437226233224,
+      "tokens_seen": 17235968
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0002619047619047619,
+      "loss": 5.3132,
+      "theoretical_loss": 6.271330744255137,
+      "tokens_seen": 17301504
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0002628968253968254,
+      "loss": 5.1796,
+      "theoretical_loss": 6.267244124381133,
+      "tokens_seen": 17367040
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0002638888888888889,
+      "loss": 5.2656,
+      "theoretical_loss": 6.2631771961334035,
+      "tokens_seen": 17432576
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00026488095238095237,
+      "loss": 5.5896,
+      "theoretical_loss": 6.259129791131242,
+      "tokens_seen": 17498112
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00026587301587301587,
+      "loss": 5.14,
+      "theoretical_loss": 6.255101743057493,
+      "tokens_seen": 17563648
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00026686507936507937,
+      "loss": 5.3606,
+      "theoretical_loss": 6.251092887625685,
+      "tokens_seen": 17629184
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00026785714285714287,
+      "loss": 5.487,
+      "theoretical_loss": 6.247103062547796,
+      "tokens_seen": 17694720
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0002688492063492063,
+      "loss": 5.2887,
+      "theoretical_loss": 6.243132107502647,
+      "tokens_seen": 17760256
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0002698412698412698,
+      "loss": 5.4254,
+      "theoretical_loss": 6.239179864104911,
+      "tokens_seen": 17825792
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0002708333333333333,
+      "loss": 5.4668,
+      "theoretical_loss": 6.235246175874727,
+      "tokens_seen": 17891328
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0002718253968253968,
+      "loss": 5.3584,
+      "theoretical_loss": 6.231330888207894,
+      "tokens_seen": 17956864
+    },
+    {
+      "epoch": 0.01,
+      "objective/train/docs_used": 21782,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 5.119491100311279,
+      "objective/train/theoretical_loss": 6.227433848346639,
+      "objective/train/tokens_used": 38482400,
+      "theoretical_loss": 6.227433848346639,
+      "tokens_seen": 18022400
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0002728174603174603,
+      "loss": 5.1069,
+      "theoretical_loss": 6.227433848346639,
+      "tokens_seen": 18022400
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0002738095238095238,
+      "loss": 5.1568,
+      "theoretical_loss": 6.22355490535095,
+      "tokens_seen": 18087936
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0002748015873015873,
+      "loss": 5.4409,
+      "theoretical_loss": 6.219693910070452,
+      "tokens_seen": 18153472
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00027579365079365083,
+      "loss": 5.1939,
+      "theoretical_loss": 6.215850715116817,
+      "tokens_seen": 18219008
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00027678571428571433,
+      "loss": 5.2274,
+      "theoretical_loss": 6.212025174836697,
+      "tokens_seen": 18284544
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0002777777777777778,
+      "loss": 5.5299,
+      "theoretical_loss": 6.208217145285173,
+      "tokens_seen": 18350080
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0002787698412698413,
+      "loss": 5.4441,
+      "theoretical_loss": 6.204426484199696,
+      "tokens_seen": 18415616
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0002797619047619048,
+      "loss": 5.3135,
+      "theoretical_loss": 6.200653050974525,
+      "tokens_seen": 18481152
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0002807539682539683,
+      "loss": 5.189,
+      "theoretical_loss": 6.196896706635635,
+      "tokens_seen": 18546688
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00028174603174603173,
+      "loss": 5.0786,
+      "theoretical_loss": 6.1931573138160925,
+      "tokens_seen": 18612224
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00028273809523809523,
+      "loss": 5.0434,
+      "theoretical_loss": 6.189434736731897,
+      "tokens_seen": 18677760
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00028373015873015873,
+      "loss": 5.0132,
+      "theoretical_loss": 6.185728841158257,
+      "tokens_seen": 18743296
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00028472222222222223,
+      "loss": 5.1159,
+      "theoretical_loss": 6.182039494406309,
+      "tokens_seen": 18808832
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0002857142857142857,
+      "loss": 5.1786,
+      "theoretical_loss": 6.178366565300266,
+      "tokens_seen": 18874368
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0002867063492063492,
+      "loss": 5.1947,
+      "theoretical_loss": 6.174709924154978,
+      "tokens_seen": 18939904
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0002876984126984127,
+      "loss": 5.2396,
+      "theoretical_loss": 6.171069442753909,
+      "tokens_seen": 19005440
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0002886904761904762,
+      "loss": 5.3947,
+      "theoretical_loss": 6.1674449943275045,
+      "tokens_seen": 19070976
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0002896825396825397,
+      "loss": 4.8732,
+      "theoretical_loss": 6.16383645353196,
+      "tokens_seen": 19136512
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0002906746031746032,
+      "loss": 5.3178,
+      "theoretical_loss": 6.160243696428367,
+      "tokens_seen": 19202048
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0002916666666666667,
+      "loss": 5.1441,
+      "theoretical_loss": 6.156666600462238,
+      "tokens_seen": 19267584
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0002926587301587302,
+      "loss": 5.2695,
+      "theoretical_loss": 6.153105044443393,
+      "tokens_seen": 19333120
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0002936507936507937,
+      "loss": 5.3181,
+      "theoretical_loss": 6.149558908526206,
+      "tokens_seen": 19398656
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00029464285714285714,
+      "loss": 5.2188,
+      "theoretical_loss": 6.146028074190217,
+      "tokens_seen": 19464192
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00029563492063492064,
+      "loss": 5.1652,
+      "theoretical_loss": 6.14251242422106,
+      "tokens_seen": 19529728
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00029662698412698414,
+      "loss": 5.3475,
+      "theoretical_loss": 6.139011842691756,
+      "tokens_seen": 19595264
+    },
+    {
+      "epoch": 0.01,
+      "objective/train/docs_used": 22379,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 5.661308765411377,
+      "objective/train/theoretical_loss": 6.135526214944321,
+      "objective/train/tokens_used": 40120800,
+      "theoretical_loss": 6.135526214944321,
+      "tokens_seen": 19660800
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00029761904761904765,
+      "loss": 5.6218,
+      "theoretical_loss": 6.135526214944321,
+      "tokens_seen": 19660800
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0002986111111111111,
+      "loss": 5.4528,
+      "theoretical_loss": 6.1320554275717,
+      "tokens_seen": 19726336
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0002996031746031746,
+      "loss": 5.1971,
+      "theoretical_loss": 6.128599368400017,
+      "tokens_seen": 19791872
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0003005952380952381,
+      "loss": 5.2823,
+      "theoretical_loss": 6.125157926471134,
+      "tokens_seen": 19857408
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0003015873015873016,
+      "loss": 5.1154,
+      "theoretical_loss": 6.121730992025516,
+      "tokens_seen": 19922944
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00030257936507936505,
+      "loss": 4.8293,
+      "theoretical_loss": 6.118318456485394,
+      "tokens_seen": 19988480
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00030357142857142855,
+      "loss": 5.0591,
+      "theoretical_loss": 6.114920212438209,
+      "tokens_seen": 20054016
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00030456349206349205,
+      "loss": 5.2452,
+      "theoretical_loss": 6.111536153620355,
+      "tokens_seen": 20119552
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0003055555555555556,
+      "loss": 5.1158,
+      "theoretical_loss": 6.108166174901191,
+      "tokens_seen": 20185088
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00030654761904761905,
+      "loss": 5.0927,
+      "theoretical_loss": 6.104810172267331,
+      "tokens_seen": 20250624
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00030753968253968255,
+      "loss": 5.1097,
+      "theoretical_loss": 6.101468042807199,
+      "tokens_seen": 20316160
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00030853174603174605,
+      "loss": 5.0109,
+      "theoretical_loss": 6.098139684695851,
+      "tokens_seen": 20381696
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00030952380952380956,
+      "loss": 5.2,
+      "theoretical_loss": 6.094824997180048,
+      "tokens_seen": 20447232
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.000310515873015873,
+      "loss": 4.919,
+      "theoretical_loss": 6.091523880563589,
+      "tokens_seen": 20512768
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0003115079365079365,
+      "loss": 4.7726,
+      "theoretical_loss": 6.088236236192881,
+      "tokens_seen": 20578304
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0003125,
+      "loss": 5.1579,
+      "theoretical_loss": 6.0849619664427586,
+      "tokens_seen": 20643840
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0003134920634920635,
+      "loss": 5.2649,
+      "theoretical_loss": 6.0817009747025415,
+      "tokens_seen": 20709376
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.000314484126984127,
+      "loss": 4.9338,
+      "theoretical_loss": 6.07845316536232,
+      "tokens_seen": 20774912
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00031547619047619046,
+      "loss": 5.2449,
+      "theoretical_loss": 6.075218443799468,
+      "tokens_seen": 20840448
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00031646825396825396,
+      "loss": 5.2222,
+      "theoretical_loss": 6.071996716365382,
+      "tokens_seen": 20905984
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00031746031746031746,
+      "loss": 4.8122,
+      "theoretical_loss": 6.068787890372443,
+      "tokens_seen": 20971520
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00031845238095238096,
+      "loss": 4.983,
+      "theoretical_loss": 6.0655918740811865,
+      "tokens_seen": 21037056
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0003194444444444444,
+      "loss": 4.9428,
+      "theoretical_loss": 6.062408576687682,
+      "tokens_seen": 21102592
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00032043650793650796,
+      "loss": 5.0493,
+      "theoretical_loss": 6.059237908311129,
+      "tokens_seen": 21168128
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00032142857142857147,
+      "loss": 4.8675,
+      "theoretical_loss": 6.056079779981644,
+      "tokens_seen": 21233664
+    },
+    {
+      "epoch": 0.01,
+      "objective/train/docs_used": 23601,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 5.199455738067627,
+      "objective/train/theoretical_loss": 6.052934103628253,
+      "objective/train/tokens_used": 41759200,
+      "theoretical_loss": 6.052934103628253,
+      "tokens_seen": 21299200
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00032242063492063497,
+      "loss": 5.2637,
+      "theoretical_loss": 6.052934103628253,
+      "tokens_seen": 21299200
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0003234126984126984,
+      "loss": 5.4325,
+      "theoretical_loss": 6.049800792067078,
+      "tokens_seen": 21364736
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0003244047619047619,
+      "loss": 4.8274,
+      "theoretical_loss": 6.046679758989709,
+      "tokens_seen": 21430272
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0003253968253968254,
+      "loss": 5.2104,
+      "theoretical_loss": 6.043570918951775,
+      "tokens_seen": 21495808
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0003263888888888889,
+      "loss": 5.1036,
+      "theoretical_loss": 6.040474187361681,
+      "tokens_seen": 21561344
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00032738095238095237,
+      "loss": 4.8811,
+      "theoretical_loss": 6.037389480469548,
+      "tokens_seen": 21626880
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00032837301587301587,
+      "loss": 5.0504,
+      "theoretical_loss": 6.034316715356304,
+      "tokens_seen": 21692416
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00032936507936507937,
+      "loss": 5.0898,
+      "theoretical_loss": 6.031255809922974,
+      "tokens_seen": 21757952
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00033035714285714287,
+      "loss": 5.0121,
+      "theoretical_loss": 6.0282066828801195,
+      "tokens_seen": 21823488
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0003313492063492063,
+      "loss": 5.1363,
+      "theoretical_loss": 6.0251692537374595,
+      "tokens_seen": 21889024
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0003323412698412698,
+      "loss": 5.1503,
+      "theoretical_loss": 6.022143442793643,
+      "tokens_seen": 21954560
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0003333333333333333,
+      "loss": 5.1066,
+      "theoretical_loss": 6.0191291711261945,
+      "tokens_seen": 22020096
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0003343253968253968,
+      "loss": 5.0889,
+      "theoretical_loss": 6.016126360581609,
+      "tokens_seen": 22085632
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0003353174603174603,
+      "loss": 4.9077,
+      "theoretical_loss": 6.013134933765605,
+      "tokens_seen": 22151168
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0003363095238095238,
+      "loss": 5.2813,
+      "theoretical_loss": 6.010154814033528,
+      "tokens_seen": 22216704
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00033730158730158733,
+      "loss": 4.6999,
+      "theoretical_loss": 6.007185925480902,
+      "tokens_seen": 22282240
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00033829365079365083,
+      "loss": 5.0728,
+      "theoretical_loss": 6.0042281929341375,
+      "tokens_seen": 22347776
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00033928571428571433,
+      "loss": 5.1319,
+      "theoretical_loss": 6.001281541941363,
+      "tokens_seen": 22413312
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0003402777777777778,
+      "loss": 5.0915,
+      "theoretical_loss": 5.998345898763421,
+      "tokens_seen": 22478848
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0003412698412698413,
+      "loss": 4.8912,
+      "theoretical_loss": 5.995421190364983,
+      "tokens_seen": 22544384
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0003422619047619048,
+      "loss": 5.0317,
+      "theoretical_loss": 5.992507344405814,
+      "tokens_seen": 22609920
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0003432539682539683,
+      "loss": 5.1603,
+      "theoretical_loss": 5.9896042892321635,
+      "tokens_seen": 22675456
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00034424603174603173,
+      "loss": 4.9822,
+      "theoretical_loss": 5.986711953868287,
+      "tokens_seen": 22740992
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00034523809523809523,
+      "loss": 5.3948,
+      "theoretical_loss": 5.983830268008107,
+      "tokens_seen": 22806528
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00034623015873015873,
+      "loss": 5.222,
+      "theoretical_loss": 5.980959162006979,
+      "tokens_seen": 22872064
+    },
+    {
+      "epoch": 0.01,
+      "objective/train/docs_used": 24859,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 4.439995288848877,
+      "objective/train/theoretical_loss": 5.978098566873603,
+      "objective/train/tokens_used": 43397600,
+      "theoretical_loss": 5.978098566873603,
+      "tokens_seen": 22937600
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00034722222222222224,
+      "loss": 4.9666,
+      "theoretical_loss": 5.978098566873603,
+      "tokens_seen": 22937600
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0003482142857142857,
+      "loss": 4.7873,
+      "theoretical_loss": 5.975248414262053,
+      "tokens_seen": 23003136
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0003492063492063492,
+      "loss": 5.1429,
+      "theoretical_loss": 5.972408636463909,
+      "tokens_seen": 23068672
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0003501984126984127,
+      "loss": 4.8387,
+      "theoretical_loss": 5.969579166400528,
+      "tokens_seen": 23134208
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0003511904761904762,
+      "loss": 5.2741,
+      "theoretical_loss": 5.966759937615427,
+      "tokens_seen": 23199744
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0003521825396825397,
+      "loss": 4.9993,
+      "theoretical_loss": 5.96395088426676,
+      "tokens_seen": 23265280
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0003531746031746032,
+      "loss": 4.838,
+      "theoretical_loss": 5.961151941119932,
+      "tokens_seen": 23330816
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0003541666666666667,
+      "loss": 4.8937,
+      "theoretical_loss": 5.95836304354031,
+      "tokens_seen": 23396352
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0003551587301587302,
+      "loss": 5.1755,
+      "theoretical_loss": 5.9555841274860395,
+      "tokens_seen": 23461888
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0003561507936507937,
+      "loss": 5.0192,
+      "theoretical_loss": 5.952815129500973,
+      "tokens_seen": 23527424
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00035714285714285714,
+      "loss": 5.1528,
+      "theoretical_loss": 5.950055986707699,
+      "tokens_seen": 23592960
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00035813492063492064,
+      "loss": 5.2473,
+      "theoretical_loss": 5.9473066368006755,
+      "tokens_seen": 23658496
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00035912698412698415,
+      "loss": 5.1431,
+      "theoretical_loss": 5.944567018039454,
+      "tokens_seen": 23724032
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00036011904761904765,
+      "loss": 5.2078,
+      "theoretical_loss": 5.941837069242023,
+      "tokens_seen": 23789568
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0003611111111111111,
+      "loss": 5.2074,
+      "theoretical_loss": 5.939116729778224,
+      "tokens_seen": 23855104
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0003621031746031746,
+      "loss": 5.045,
+      "theoretical_loss": 5.936405939563279,
+      "tokens_seen": 23920640
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0003630952380952381,
+      "loss": 5.0272,
+      "theoretical_loss": 5.93370463905141,
+      "tokens_seen": 23986176
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0003640873015873016,
+      "loss": 5.0628,
+      "theoretical_loss": 5.931012769229536,
+      "tokens_seen": 24051712
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00036507936507936505,
+      "loss": 4.6127,
+      "theoretical_loss": 5.928330271611081,
+      "tokens_seen": 24117248
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00036607142857142855,
+      "loss": 4.8408,
+      "theoretical_loss": 5.925657088229862,
+      "tokens_seen": 24182784
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00036706349206349205,
+      "loss": 4.9603,
+      "theoretical_loss": 5.9229931616340545,
+      "tokens_seen": 24248320
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0003680555555555556,
+      "loss": 5.0127,
+      "theoretical_loss": 5.920338434880263,
+      "tokens_seen": 24313856
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00036904761904761905,
+      "loss": 5.3699,
+      "theoretical_loss": 5.9176928515276535,
+      "tokens_seen": 24379392
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00037003968253968255,
+      "loss": 4.9974,
+      "theoretical_loss": 5.915056355632197,
+      "tokens_seen": 24444928
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00037103174603174606,
+      "loss": 4.9348,
+      "theoretical_loss": 5.912428891740967,
+      "tokens_seen": 24510464
+    },
+    {
+      "epoch": 0.01,
+      "objective/train/docs_used": 25411,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 4.414747714996338,
+      "objective/train/theoretical_loss": 5.90981040488653,
+      "objective/train/tokens_used": 45036000,
+      "theoretical_loss": 5.90981040488653,
+      "tokens_seen": 24576000
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00037202380952380956,
+      "loss": 5.0376,
+      "theoretical_loss": 5.90981040488653,
+      "tokens_seen": 24576000
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.000373015873015873,
+      "loss": 5.0724,
+      "theoretical_loss": 5.907200840581417,
+      "tokens_seen": 24641536
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0003740079365079365,
+      "loss": 5.2247,
+      "theoretical_loss": 5.904600144812672,
+      "tokens_seen": 24707072
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.000375,
+      "loss": 5.4811,
+      "theoretical_loss": 5.902008264036468,
+      "tokens_seen": 24772608
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0003759920634920635,
+      "loss": 4.9107,
+      "theoretical_loss": 5.899425145172803,
+      "tokens_seen": 24838144
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.000376984126984127,
+      "loss": 5.0933,
+      "theoretical_loss": 5.896850735600281,
+      "tokens_seen": 24903680
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00037797619047619046,
+      "loss": 5.1802,
+      "theoretical_loss": 5.8942849831509445,
+      "tokens_seen": 24969216
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00037896825396825396,
+      "loss": 5.2521,
+      "theoretical_loss": 5.891727836105194,
+      "tokens_seen": 25034752
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00037996031746031746,
+      "loss": 4.8607,
+      "theoretical_loss": 5.889179243186776,
+      "tokens_seen": 25100288
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00038095238095238096,
+      "loss": 5.0973,
+      "theoretical_loss": 5.886639153557828,
+      "tokens_seen": 25165824
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0003819444444444444,
+      "loss": 5.3397,
+      "theoretical_loss": 5.8841075168140105,
+      "tokens_seen": 25231360
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00038293650793650797,
+      "loss": 4.7777,
+      "theoretical_loss": 5.88158428297969,
+      "tokens_seen": 25296896
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00038392857142857147,
+      "loss": 4.7219,
+      "theoretical_loss": 5.879069402503189,
+      "tokens_seen": 25362432
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00038492063492063497,
+      "loss": 5.0299,
+      "theoretical_loss": 5.876562826252119,
+      "tokens_seen": 25427968
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0003859126984126984,
+      "loss": 4.8369,
+      "theoretical_loss": 5.874064505508748,
+      "tokens_seen": 25493504
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0003869047619047619,
+      "loss": 5.0017,
+      "theoretical_loss": 5.871574391965453,
+      "tokens_seen": 25559040
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0003878968253968254,
+      "loss": 4.9705,
+      "theoretical_loss": 5.8690924377202265,
+      "tokens_seen": 25624576
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0003888888888888889,
+      "loss": 5.034,
+      "theoretical_loss": 5.866618595272241,
+      "tokens_seen": 25690112
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00038988095238095237,
+      "loss": 4.7181,
+      "theoretical_loss": 5.864152817517482,
+      "tokens_seen": 25755648
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00039087301587301587,
+      "loss": 4.9562,
+      "theoretical_loss": 5.8616950577444245,
+      "tokens_seen": 25821184
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00039186507936507937,
+      "loss": 4.9405,
+      "theoretical_loss": 5.859245269629783,
+      "tokens_seen": 25886720
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0003928571428571429,
+      "loss": 5.099,
+      "theoretical_loss": 5.85680340723431,
+      "tokens_seen": 25952256
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0003938492063492063,
+      "loss": 4.6503,
+      "theoretical_loss": 5.85436942499865,
+      "tokens_seen": 26017792
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0003948412698412698,
+      "loss": 4.8672,
+      "theoretical_loss": 5.851943277739253,
+      "tokens_seen": 26083328
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0003958333333333333,
+      "loss": 5.0201,
+      "theoretical_loss": 5.84952492064434,
+      "tokens_seen": 26148864
+    },
+    {
+      "epoch": 0.01,
+      "objective/train/docs_used": 26668,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 5.337137222290039,
+      "objective/train/theoretical_loss": 5.847114309269919,
+      "objective/train/tokens_used": 46674400,
+      "theoretical_loss": 5.847114309269919,
+      "tokens_seen": 26214400
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0003968253968253968,
+      "loss": 5.1138,
+      "theoretical_loss": 5.847114309269919,
+      "tokens_seen": 26214400
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0003978174603174603,
+      "loss": 4.8239,
+      "theoretical_loss": 5.844711399535855,
+      "tokens_seen": 26279936
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00039880952380952383,
+      "loss": 4.7304,
+      "theoretical_loss": 5.842316147722,
+      "tokens_seen": 26345472
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00039980158730158733,
+      "loss": 5.0092,
+      "theoretical_loss": 5.839928510464356,
+      "tokens_seen": 26411008
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00040079365079365083,
+      "loss": 4.4343,
+      "theoretical_loss": 5.837548444751306,
+      "tokens_seen": 26476544
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00040178571428571433,
+      "loss": 4.9309,
+      "theoretical_loss": 5.835175907919885,
+      "tokens_seen": 26542080
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004027777777777778,
+      "loss": 5.0148,
+      "theoretical_loss": 5.832810857652097,
+      "tokens_seen": 26607616
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004037698412698413,
+      "loss": 4.9709,
+      "theoretical_loss": 5.830453251971296,
+      "tokens_seen": 26673152
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004047619047619048,
+      "loss": 5.1225,
+      "theoretical_loss": 5.8281030492385835,
+      "tokens_seen": 26738688
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004057539682539683,
+      "loss": 4.9439,
+      "theoretical_loss": 5.82576020814929,
+      "tokens_seen": 26804224
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00040674603174603173,
+      "loss": 5.154,
+      "theoretical_loss": 5.823424687729469,
+      "tokens_seen": 26869760
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00040773809523809523,
+      "loss": 4.6484,
+      "theoretical_loss": 5.821096447332456,
+      "tokens_seen": 26935296
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00040873015873015874,
+      "loss": 4.6262,
+      "theoretical_loss": 5.818775446635469,
+      "tokens_seen": 27000832
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00040972222222222224,
+      "loss": 5.0236,
+      "theoretical_loss": 5.816461645636238,
+      "tokens_seen": 27066368
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004107142857142857,
+      "loss": 5.1234,
+      "theoretical_loss": 5.814155004649702,
+      "tokens_seen": 27131904
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004117063492063492,
+      "loss": 4.6509,
+      "theoretical_loss": 5.811855484304724,
+      "tokens_seen": 27197440
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004126984126984127,
+      "loss": 4.9814,
+      "theoretical_loss": 5.809563045540864,
+      "tokens_seen": 27262976
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004136904761904762,
+      "loss": 5.0187,
+      "theoretical_loss": 5.8072776496051866,
+      "tokens_seen": 27328512
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004146825396825397,
+      "loss": 4.9255,
+      "theoretical_loss": 5.804999258049106,
+      "tokens_seen": 27394048
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004156746031746032,
+      "loss": 5.0265,
+      "theoretical_loss": 5.802727832725283,
+      "tokens_seen": 27459584
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004166666666666667,
+      "loss": 4.8827,
+      "theoretical_loss": 5.800463335784541,
+      "tokens_seen": 27525120
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004176587301587302,
+      "loss": 5.0962,
+      "theoretical_loss": 5.798205729672842,
+      "tokens_seen": 27590656
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004186507936507937,
+      "loss": 5.0409,
+      "theoretical_loss": 5.795954977128286,
+      "tokens_seen": 27656192
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00041964285714285714,
+      "loss": 4.8332,
+      "theoretical_loss": 5.7937110411781525,
+      "tokens_seen": 27721728
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00042063492063492065,
+      "loss": 4.813,
+      "theoretical_loss": 5.79147388513598,
+      "tokens_seen": 27787264
+    },
+    {
+      "epoch": 0.01,
+      "objective/train/docs_used": 27295,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 4.60736083984375,
+      "objective/train/theoretical_loss": 5.789243472598683,
+      "objective/train/tokens_used": 48312800,
+      "theoretical_loss": 5.789243472598683,
+      "tokens_seen": 27852800
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00042162698412698415,
+      "loss": 4.735,
+      "theoretical_loss": 5.789243472598683,
+      "tokens_seen": 27852800
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00042261904761904765,
+      "loss": 5.0103,
+      "theoretical_loss": 5.787019767443696,
+      "tokens_seen": 27918336
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004236111111111111,
+      "loss": 4.9487,
+      "theoretical_loss": 5.784802733826166,
+      "tokens_seen": 27983872
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004246031746031746,
+      "loss": 5.0393,
+      "theoretical_loss": 5.782592336176171,
+      "tokens_seen": 28049408
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004255952380952381,
+      "loss": 4.8239,
+      "theoretical_loss": 5.780388539195972,
+      "tokens_seen": 28114944
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004265873015873016,
+      "loss": 4.8301,
+      "theoretical_loss": 5.778191307857307,
+      "tokens_seen": 28180480
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00042757936507936505,
+      "loss": 4.8818,
+      "theoretical_loss": 5.776000607398707,
+      "tokens_seen": 28246016
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00042857142857142855,
+      "loss": 5.0365,
+      "theoretical_loss": 5.773816403322854,
+      "tokens_seen": 28311552
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00042956349206349205,
+      "loss": 4.8806,
+      "theoretical_loss": 5.7716386613939665,
+      "tokens_seen": 28377088
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004305555555555556,
+      "loss": 4.8212,
+      "theoretical_loss": 5.769467347635221,
+      "tokens_seen": 28442624
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00043154761904761905,
+      "loss": 4.6856,
+      "theoretical_loss": 5.767302428326196,
+      "tokens_seen": 28508160
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00043253968253968256,
+      "loss": 5.1099,
+      "theoretical_loss": 5.765143870000358,
+      "tokens_seen": 28573696
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00043353174603174606,
+      "loss": 4.9942,
+      "theoretical_loss": 5.762991639442574,
+      "tokens_seen": 28639232
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00043452380952380956,
+      "loss": 4.7269,
+      "theoretical_loss": 5.7608457036866465,
+      "tokens_seen": 28704768
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.000435515873015873,
+      "loss": 4.7079,
+      "theoretical_loss": 5.758706030012889,
+      "tokens_seen": 28770304
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004365079365079365,
+      "loss": 5.0535,
+      "theoretical_loss": 5.756572585945728,
+      "tokens_seen": 28835840
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004375,
+      "loss": 4.9953,
+      "theoretical_loss": 5.754445339251326,
+      "tokens_seen": 28901376
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004384920634920635,
+      "loss": 4.8239,
+      "theoretical_loss": 5.752324257935244,
+      "tokens_seen": 28966912
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.000439484126984127,
+      "loss": 4.8738,
+      "theoretical_loss": 5.750209310240125,
+      "tokens_seen": 29032448
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00044047619047619046,
+      "loss": 4.5774,
+      "theoretical_loss": 5.7481004646434055,
+      "tokens_seen": 29097984
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00044146825396825396,
+      "loss": 4.7821,
+      "theoretical_loss": 5.745997689855058,
+      "tokens_seen": 29163520
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00044246031746031746,
+      "loss": 4.9197,
+      "theoretical_loss": 5.743900954815356,
+      "tokens_seen": 29229056
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00044345238095238096,
+      "loss": 4.9178,
+      "theoretical_loss": 5.741810228692663,
+      "tokens_seen": 29294592
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004444444444444444,
+      "loss": 4.9811,
+      "theoretical_loss": 5.739725480881262,
+      "tokens_seen": 29360128
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00044543650793650797,
+      "loss": 4.595,
+      "theoretical_loss": 5.737646680999193,
+      "tokens_seen": 29425664
+    },
+    {
+      "epoch": 0.01,
+      "objective/train/docs_used": 28553,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 4.193212032318115,
+      "objective/train/theoretical_loss": 5.73557379888612,
+      "objective/train/tokens_used": 49951200,
+      "theoretical_loss": 5.73557379888612,
+      "tokens_seen": 29491200
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00044642857142857147,
+      "loss": 4.6289,
+      "theoretical_loss": 5.73557379888612,
+      "tokens_seen": 29491200
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00044742063492063497,
+      "loss": 4.5175,
+      "theoretical_loss": 5.733506804601236,
+      "tokens_seen": 29556736
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004484126984126984,
+      "loss": 4.97,
+      "theoretical_loss": 5.7314456684211725,
+      "tokens_seen": 29622272
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004494047619047619,
+      "loss": 4.9007,
+      "theoretical_loss": 5.729390360837952,
+      "tokens_seen": 29687808
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004503968253968254,
+      "loss": 5.0718,
+      "theoretical_loss": 5.7273408525569485,
+      "tokens_seen": 29753344
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004513888888888889,
+      "loss": 4.8076,
+      "theoretical_loss": 5.725297114494884,
+      "tokens_seen": 29818880
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00045238095238095237,
+      "loss": 4.9151,
+      "theoretical_loss": 5.7232591177778405,
+      "tokens_seen": 29884416
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00045337301587301587,
+      "loss": 4.7611,
+      "theoretical_loss": 5.7212268337393,
+      "tokens_seen": 29949952
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00045436507936507937,
+      "loss": 4.6533,
+      "theoretical_loss": 5.719200233918203,
+      "tokens_seen": 30015488
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004553571428571429,
+      "loss": 4.7689,
+      "theoretical_loss": 5.717179290057032,
+      "tokens_seen": 30081024
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004563492063492063,
+      "loss": 4.753,
+      "theoretical_loss": 5.715163974099917,
+      "tokens_seen": 30146560
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004573412698412698,
+      "loss": 4.9783,
+      "theoretical_loss": 5.713154258190757,
+      "tokens_seen": 30212096
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004583333333333333,
+      "loss": 4.8919,
+      "theoretical_loss": 5.711150114671375,
+      "tokens_seen": 30277632
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004593253968253968,
+      "loss": 4.6246,
+      "theoretical_loss": 5.709151516079683,
+      "tokens_seen": 30343168
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00046031746031746033,
+      "loss": 4.8047,
+      "theoretical_loss": 5.707158435147875,
+      "tokens_seen": 30408704
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00046130952380952383,
+      "loss": 4.7619,
+      "theoretical_loss": 5.705170844800628,
+      "tokens_seen": 30474240
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00046230158730158733,
+      "loss": 5.0389,
+      "theoretical_loss": 5.703188718153347,
+      "tokens_seen": 30539776
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00046329365079365083,
+      "loss": 4.7978,
+      "theoretical_loss": 5.7012120285104055,
+      "tokens_seen": 30605312
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00046428571428571433,
+      "loss": 4.8182,
+      "theoretical_loss": 5.699240749363417,
+      "tokens_seen": 30670848
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004652777777777778,
+      "loss": 5.1046,
+      "theoretical_loss": 5.69727485438953,
+      "tokens_seen": 30736384
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004662698412698413,
+      "loss": 4.9301,
+      "theoretical_loss": 5.695314317449732,
+      "tokens_seen": 30801920
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004672619047619048,
+      "loss": 4.9513,
+      "theoretical_loss": 5.69335911258718,
+      "tokens_seen": 30867456
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004682539682539683,
+      "loss": 4.7506,
+      "theoretical_loss": 5.691409214025544,
+      "tokens_seen": 30932992
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00046924603174603173,
+      "loss": 4.9203,
+      "theoretical_loss": 5.689464596167383,
+      "tokens_seen": 30998528
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00047023809523809523,
+      "loss": 4.6588,
+      "theoretical_loss": 5.687525233592513,
+      "tokens_seen": 31064064
+    },
+    {
+      "epoch": 0.01,
+      "objective/train/docs_used": 29210,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 4.980744361877441,
+      "objective/train/theoretical_loss": 5.685591101056422,
+      "objective/train/tokens_used": 51589600,
+      "theoretical_loss": 5.685591101056422,
+      "tokens_seen": 31129600
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00047123015873015874,
+      "loss": 4.8131,
+      "theoretical_loss": 5.685591101056422,
+      "tokens_seen": 31129600
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00047222222222222224,
+      "loss": 4.9729,
+      "theoretical_loss": 5.683662173488678,
+      "tokens_seen": 31195136
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004732142857142857,
+      "loss": 4.9988,
+      "theoretical_loss": 5.681738425991377,
+      "tokens_seen": 31260672
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004742063492063492,
+      "loss": 4.9296,
+      "theoretical_loss": 5.679819833837586,
+      "tokens_seen": 31326208
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004751984126984127,
+      "loss": 4.8866,
+      "theoretical_loss": 5.677906372469826,
+      "tokens_seen": 31391744
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004761904761904762,
+      "loss": 4.8619,
+      "theoretical_loss": 5.675998017498549,
+      "tokens_seen": 31457280
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004771825396825397,
+      "loss": 4.4881,
+      "theoretical_loss": 5.674094744700648,
+      "tokens_seen": 31522816
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004781746031746032,
+      "loss": 4.7385,
+      "theoretical_loss": 5.672196530017979,
+      "tokens_seen": 31588352
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004791666666666667,
+      "loss": 4.9446,
+      "theoretical_loss": 5.670303349555893,
+      "tokens_seen": 31653888
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004801587301587302,
+      "loss": 4.5805,
+      "theoretical_loss": 5.668415179581795,
+      "tokens_seen": 31719424
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004811507936507937,
+      "loss": 4.943,
+      "theoretical_loss": 5.666531996523711,
+      "tokens_seen": 31784960
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00048214285714285715,
+      "loss": 4.9994,
+      "theoretical_loss": 5.664653776968867,
+      "tokens_seen": 31850496
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00048313492063492065,
+      "loss": 4.9432,
+      "theoretical_loss": 5.6627804976622995,
+      "tokens_seen": 31916032
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00048412698412698415,
+      "loss": 4.4612,
+      "theoretical_loss": 5.660912135505461,
+      "tokens_seen": 31981568
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00048511904761904765,
+      "loss": 5.0296,
+      "theoretical_loss": 5.659048667554853,
+      "tokens_seen": 32047104
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004861111111111111,
+      "loss": 4.7342,
+      "theoretical_loss": 5.657190071020672,
+      "tokens_seen": 32112640
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004871031746031746,
+      "loss": 5.0271,
+      "theoretical_loss": 5.65533632326547,
+      "tokens_seen": 32178176
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004880952380952381,
+      "loss": 4.5208,
+      "theoretical_loss": 5.6534874018028205,
+      "tokens_seen": 32243712
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004890873015873016,
+      "loss": 4.8883,
+      "theoretical_loss": 5.651643284296018,
+      "tokens_seen": 32309248
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004900793650793651,
+      "loss": 4.5918,
+      "theoretical_loss": 5.6498039485567695,
+      "tokens_seen": 32374784
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004910714285714286,
+      "loss": 4.8362,
+      "theoretical_loss": 5.6479693725439155,
+      "tokens_seen": 32440320
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.000492063492063492,
+      "loss": 4.4816,
+      "theoretical_loss": 5.646139534362161,
+      "tokens_seen": 32505856
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004930555555555556,
+      "loss": 4.7304,
+      "theoretical_loss": 5.644314412260815,
+      "tokens_seen": 32571392
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004940476190476191,
+      "loss": 4.7639,
+      "theoretical_loss": 5.642493984632544,
+      "tokens_seen": 32636928
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004950396825396826,
+      "loss": 4.486,
+      "theoretical_loss": 5.640678230012151,
+      "tokens_seen": 32702464
+    },
+    {
+      "epoch": 0.01,
+      "objective/train/docs_used": 30192,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 4.205135822296143,
+      "objective/train/theoretical_loss": 5.638867127075349,
+      "objective/train/tokens_used": 53228000,
+      "theoretical_loss": 5.638867127075349,
+      "tokens_seen": 32768000
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.000496031746031746,
+      "loss": 4.6383,
+      "theoretical_loss": 5.638867127075349,
+      "tokens_seen": 32768000
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004970238095238095,
+      "loss": 4.7665,
+      "theoretical_loss": 5.63706065463756,
+      "tokens_seen": 32833536
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.000498015873015873,
+      "loss": 4.6651,
+      "theoretical_loss": 5.635258791652723,
+      "tokens_seen": 32899072
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004990079365079365,
+      "loss": 4.6032,
+      "theoretical_loss": 5.633461517212115,
+      "tokens_seen": 32964608
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0005,
+      "loss": 4.8611,
+      "theoretical_loss": 5.631668810543181,
+      "tokens_seen": 33030144
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004999899699097292,
+      "loss": 4.4403,
+      "theoretical_loss": 5.629880651008384,
+      "tokens_seen": 33095680
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004999799398194584,
+      "loss": 4.6342,
+      "theoretical_loss": 5.628097018104057,
+      "tokens_seen": 33161216
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004999699097291876,
+      "loss": 4.789,
+      "theoretical_loss": 5.626317891459275,
+      "tokens_seen": 33226752
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004999598796389167,
+      "loss": 4.4669,
+      "theoretical_loss": 5.6245432508347335,
+      "tokens_seen": 33292288
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.000499949849548646,
+      "loss": 4.5265,
+      "theoretical_loss": 5.622773076121648,
+      "tokens_seen": 33357824
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004999398194583751,
+      "loss": 4.8668,
+      "theoretical_loss": 5.621007347340646,
+      "tokens_seen": 33423360
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004999297893681044,
+      "loss": 4.663,
+      "theoretical_loss": 5.619246044640693,
+      "tokens_seen": 33488896
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004999197592778335,
+      "loss": 4.51,
+      "theoretical_loss": 5.617489148298015,
+      "tokens_seen": 33554432
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004999097291875627,
+      "loss": 4.4827,
+      "theoretical_loss": 5.615736638715033,
+      "tokens_seen": 33619968
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004998996990972919,
+      "loss": 4.6552,
+      "theoretical_loss": 5.613988496419315,
+      "tokens_seen": 33685504
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004998896690070211,
+      "loss": 4.5444,
+      "theoretical_loss": 5.6122447020625374,
+      "tokens_seen": 33751040
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004998796389167503,
+      "loss": 4.7462,
+      "theoretical_loss": 5.610505236419448,
+      "tokens_seen": 33816576
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004998696088264795,
+      "loss": 4.6343,
+      "theoretical_loss": 5.608770080386854,
+      "tokens_seen": 33882112
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004998595787362087,
+      "loss": 4.3194,
+      "theoretical_loss": 5.6070392149826045,
+      "tokens_seen": 33947648
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004998495486459378,
+      "loss": 4.7376,
+      "theoretical_loss": 5.605312621344602,
+      "tokens_seen": 34013184
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.000499839518555667,
+      "loss": 4.6319,
+      "theoretical_loss": 5.603590280729803,
+      "tokens_seen": 34078720
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004998294884653962,
+      "loss": 4.3936,
+      "theoretical_loss": 5.601872174513245,
+      "tokens_seen": 34144256
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004998194583751254,
+      "loss": 4.8045,
+      "theoretical_loss": 5.60015828418708,
+      "tokens_seen": 34209792
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004998094282848546,
+      "loss": 4.8346,
+      "theoretical_loss": 5.59844859135961,
+      "tokens_seen": 34275328
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004997993981945837,
+      "loss": 4.4444,
+      "theoretical_loss": 5.596743077754337,
+      "tokens_seen": 34340864
+    },
+    {
+      "epoch": 0.01,
+      "objective/train/docs_used": 30758,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 4.619051933288574,
+      "objective/train/theoretical_loss": 5.5950417252090325,
+      "objective/train/tokens_used": 54866400,
+      "theoretical_loss": 5.5950417252090325,
+      "tokens_seen": 34406400
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004997893681043129,
+      "loss": 4.5684,
+      "theoretical_loss": 5.5950417252090325,
+      "tokens_seen": 34406400
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004997793380140421,
+      "loss": 4.7403,
+      "theoretical_loss": 5.593344515674795,
+      "tokens_seen": 34471936
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004997693079237714,
+      "loss": 4.7682,
+      "theoretical_loss": 5.591651431215139,
+      "tokens_seen": 34537472
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004997592778335005,
+      "loss": 4.9194,
+      "theoretical_loss": 5.589962454005072,
+      "tokens_seen": 34603008
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004997492477432298,
+      "loss": 4.7926,
+      "theoretical_loss": 5.588277566330207,
+      "tokens_seen": 34668544
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004997392176529588,
+      "loss": 4.5919,
+      "theoretical_loss": 5.586596750585855,
+      "tokens_seen": 34734080
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004997291875626881,
+      "loss": 4.8471,
+      "theoretical_loss": 5.584919989276146,
+      "tokens_seen": 34799616
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004997191574724173,
+      "loss": 4.8823,
+      "theoretical_loss": 5.583247265013155,
+      "tokens_seen": 34865152
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004997091273821465,
+      "loss": 4.7952,
+      "theoretical_loss": 5.581578560516027,
+      "tokens_seen": 34930688
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004996990972918757,
+      "loss": 4.8327,
+      "theoretical_loss": 5.57991385861013,
+      "tokens_seen": 34996224
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004996890672016048,
+      "loss": 4.3745,
+      "theoretical_loss": 5.578253142226191,
+      "tokens_seen": 35061760
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.000499679037111334,
+      "loss": 4.6921,
+      "theoretical_loss": 5.576596394399467,
+      "tokens_seen": 35127296
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004996690070210632,
+      "loss": 4.6698,
+      "theoretical_loss": 5.574943598268902,
+      "tokens_seen": 35192832
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004996589769307924,
+      "loss": 4.9374,
+      "theoretical_loss": 5.573294737076306,
+      "tokens_seen": 35258368
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004996489468405216,
+      "loss": 4.5814,
+      "theoretical_loss": 5.571649794165537,
+      "tokens_seen": 35323904
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004996389167502507,
+      "loss": 4.6064,
+      "theoretical_loss": 5.570008752981688,
+      "tokens_seen": 35389440
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004996288866599799,
+      "loss": 4.7137,
+      "theoretical_loss": 5.5683715970702945,
+      "tokens_seen": 35454976
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004996188565697091,
+      "loss": 4.7919,
+      "theoretical_loss": 5.566738310076531,
+      "tokens_seen": 35520512
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004996088264794383,
+      "loss": 4.5989,
+      "theoretical_loss": 5.5651088757444285,
+      "tokens_seen": 35586048
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004995987963891675,
+      "loss": 4.4273,
+      "theoretical_loss": 5.5634832779161,
+      "tokens_seen": 35651584
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004995887662988968,
+      "loss": 4.6147,
+      "theoretical_loss": 5.561861500530965,
+      "tokens_seen": 35717120
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004995787362086258,
+      "loss": 4.5803,
+      "theoretical_loss": 5.560243527624989,
+      "tokens_seen": 35782656
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004995687061183551,
+      "loss": 4.762,
+      "theoretical_loss": 5.558629343329924,
+      "tokens_seen": 35848192
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004995586760280842,
+      "loss": 4.7117,
+      "theoretical_loss": 5.557018931872566,
+      "tokens_seen": 35913728
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004995486459378135,
+      "loss": 4.7915,
+      "theoretical_loss": 5.555412277574007,
+      "tokens_seen": 35979264
+    },
+    {
+      "epoch": 0.01,
+      "objective/train/docs_used": 31707,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 4.547565460205078,
+      "objective/train/theoretical_loss": 5.553809364848902,
+      "objective/train/tokens_used": 56504800,
+      "theoretical_loss": 5.553809364848902,
+      "tokens_seen": 36044800
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004995386158475427,
+      "loss": 4.6425,
+      "theoretical_loss": 5.553809364848902,
+      "tokens_seen": 36044800
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004995285857572718,
+      "loss": 4.5963,
+      "theoretical_loss": 5.552210178204751,
+      "tokens_seen": 36110336
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.000499518555667001,
+      "loss": 4.6793,
+      "theoretical_loss": 5.550614702241162,
+      "tokens_seen": 36175872
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004995085255767302,
+      "loss": 4.7561,
+      "theoretical_loss": 5.549022921649151,
+      "tokens_seen": 36241408
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004994984954864594,
+      "loss": 4.6607,
+      "theoretical_loss": 5.54743482121043,
+      "tokens_seen": 36306944
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004994884653961886,
+      "loss": 4.6252,
+      "theoretical_loss": 5.545850385796712,
+      "tokens_seen": 36372480
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004994784353059178,
+      "loss": 4.4331,
+      "theoretical_loss": 5.544269600369009,
+      "tokens_seen": 36438016
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004994684052156469,
+      "loss": 4.643,
+      "theoretical_loss": 5.5426924499769505,
+      "tokens_seen": 36503552
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004994583751253761,
+      "loss": 4.7548,
+      "theoretical_loss": 5.541118919758107,
+      "tokens_seen": 36569088
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004994483450351053,
+      "loss": 4.7337,
+      "theoretical_loss": 5.539548994937308,
+      "tokens_seen": 36634624
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004994383149448345,
+      "loss": 4.3641,
+      "theoretical_loss": 5.537982660825981,
+      "tokens_seen": 36700160
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004994282848545637,
+      "loss": 4.5599,
+      "theoretical_loss": 5.536419902821491,
+      "tokens_seen": 36765696
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004994182547642928,
+      "loss": 4.6875,
+      "theoretical_loss": 5.534860706406477,
+      "tokens_seen": 36831232
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004994082246740221,
+      "loss": 4.7352,
+      "theoretical_loss": 5.533305057148215,
+      "tokens_seen": 36896768
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004993981945837512,
+      "loss": 4.6149,
+      "theoretical_loss": 5.531752940697968,
+      "tokens_seen": 36962304
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004993881644934805,
+      "loss": 4.5067,
+      "theoretical_loss": 5.5302043427903484,
+      "tokens_seen": 37027840
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004993781344032096,
+      "loss": 4.9474,
+      "theoretical_loss": 5.528659249242693,
+      "tokens_seen": 37093376
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004993681043129389,
+      "loss": 4.4017,
+      "theoretical_loss": 5.527117645954435,
+      "tokens_seen": 37158912
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.000499358074222668,
+      "loss": 4.5323,
+      "theoretical_loss": 5.525579518906484,
+      "tokens_seen": 37224448
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004993480441323972,
+      "loss": 4.4802,
+      "theoretical_loss": 5.524044854160613,
+      "tokens_seen": 37289984
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004993380140421264,
+      "loss": 4.4531,
+      "theoretical_loss": 5.522513637858855,
+      "tokens_seen": 37355520
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004993279839518556,
+      "loss": 4.6216,
+      "theoretical_loss": 5.520985856222893,
+      "tokens_seen": 37421056
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004993179538615848,
+      "loss": 4.5407,
+      "theoretical_loss": 5.519461495553473,
+      "tokens_seen": 37486592
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004993079237713139,
+      "loss": 4.4097,
+      "theoretical_loss": 5.5179405422298045,
+      "tokens_seen": 37552128
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004992978936810431,
+      "loss": 4.406,
+      "theoretical_loss": 5.516422982708983,
+      "tokens_seen": 37617664
+    },
+    {
+      "epoch": 0.01,
+      "objective/train/docs_used": 32203,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 4.049692630767822,
+      "objective/train/theoretical_loss": 5.514908803525402,
+      "objective/train/tokens_used": 58143200,
+      "theoretical_loss": 5.514908803525402,
+      "tokens_seen": 37683200
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004992878635907723,
+      "loss": 4.3638,
+      "theoretical_loss": 5.514908803525402,
+      "tokens_seen": 37683200
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004992778335005015,
+      "loss": 4.6329,
+      "theoretical_loss": 5.513397991290185,
+      "tokens_seen": 37748736
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004992678034102307,
+      "loss": 4.6538,
+      "theoretical_loss": 5.511890532690611,
+      "tokens_seen": 37814272
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004992577733199598,
+      "loss": 4.7513,
+      "theoretical_loss": 5.510386414489556,
+      "tokens_seen": 37879808
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.000499247743229689,
+      "loss": 4.6591,
+      "theoretical_loss": 5.5088856235249235,
+      "tokens_seen": 37945344
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004992377131394183,
+      "loss": 4.4618,
+      "theoretical_loss": 5.507388146709105,
+      "tokens_seen": 38010880
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004992276830491475,
+      "loss": 4.4231,
+      "theoretical_loss": 5.505893971028418,
+      "tokens_seen": 38076416
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004992176529588767,
+      "loss": 4.2713,
+      "theoretical_loss": 5.504403083542568,
+      "tokens_seen": 38141952
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004992076228686059,
+      "loss": 4.4092,
+      "theoretical_loss": 5.502915471384111,
+      "tokens_seen": 38207488
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.000499197592778335,
+      "loss": 4.7235,
+      "theoretical_loss": 5.501431121757914,
+      "tokens_seen": 38273024
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004991875626880642,
+      "loss": 4.3762,
+      "theoretical_loss": 5.49995002194063,
+      "tokens_seen": 38338560
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004991775325977934,
+      "loss": 4.6923,
+      "theoretical_loss": 5.498472159280171,
+      "tokens_seen": 38404096
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004991675025075226,
+      "loss": 4.8392,
+      "theoretical_loss": 5.4969975211951905,
+      "tokens_seen": 38469632
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004991574724172518,
+      "loss": 4.5345,
+      "theoretical_loss": 5.495526095174563,
+      "tokens_seen": 38535168
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.000499147442326981,
+      "loss": 4.6407,
+      "theoretical_loss": 5.494057868776883,
+      "tokens_seen": 38600704
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004991374122367101,
+      "loss": 4.5119,
+      "theoretical_loss": 5.492592829629946,
+      "tokens_seen": 38666240
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004991273821464393,
+      "loss": 4.6456,
+      "theoretical_loss": 5.491130965430262,
+      "tokens_seen": 38731776
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004991173520561685,
+      "loss": 4.2981,
+      "theoretical_loss": 5.489672263942543,
+      "tokens_seen": 38797312
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004991073219658977,
+      "loss": 4.7488,
+      "theoretical_loss": 5.488216712999222,
+      "tokens_seen": 38862848
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.000499097291875627,
+      "loss": 4.5654,
+      "theoretical_loss": 5.486764300499962,
+      "tokens_seen": 38928384
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.000499087261785356,
+      "loss": 4.3462,
+      "theoretical_loss": 5.4853150144111655,
+      "tokens_seen": 38993920
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004990772316950853,
+      "loss": 4.6389,
+      "theoretical_loss": 5.4838688427655065,
+      "tokens_seen": 39059456
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004990672016048144,
+      "loss": 4.5166,
+      "theoretical_loss": 5.482425773661445,
+      "tokens_seen": 39124992
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004990571715145437,
+      "loss": 4.5858,
+      "theoretical_loss": 5.480985795262759,
+      "tokens_seen": 39190528
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004990471414242729,
+      "loss": 4.6098,
+      "theoretical_loss": 5.479548895798082,
+      "tokens_seen": 39256064
+    },
+    {
+      "epoch": 0.01,
+      "objective/train/docs_used": 33134,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 4.761120319366455,
+      "objective/train/theoretical_loss": 5.478115063560433,
+      "objective/train/tokens_used": 59781600,
+      "theoretical_loss": 5.478115063560433,
+      "tokens_seen": 39321600
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.000499037111334002,
+      "loss": 4.4957,
+      "theoretical_loss": 5.478115063560433,
+      "tokens_seen": 39321600
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004990270812437312,
+      "loss": 4.7046,
+      "theoretical_loss": 5.476684286906764,
+      "tokens_seen": 39387136
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004990170511534604,
+      "loss": 4.2948,
+      "theoretical_loss": 5.475256554257497,
+      "tokens_seen": 39452672
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004990070210631896,
+      "loss": 4.5969,
+      "theoretical_loss": 5.473831854096085,
+      "tokens_seen": 39518208
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004989969909729188,
+      "loss": 4.5511,
+      "theoretical_loss": 5.472410174968553,
+      "tokens_seen": 39583744
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.000498986960882648,
+      "loss": 4.4106,
+      "theoretical_loss": 5.470991505483065,
+      "tokens_seen": 39649280
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004989769307923771,
+      "loss": 4.623,
+      "theoretical_loss": 5.469575834309477,
+      "tokens_seen": 39714816
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004989669007021063,
+      "loss": 4.5077,
+      "theoretical_loss": 5.468163150178907,
+      "tokens_seen": 39780352
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004989568706118355,
+      "loss": 4.475,
+      "theoretical_loss": 5.466753441883302,
+      "tokens_seen": 39845888
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004989468405215647,
+      "loss": 4.6451,
+      "theoretical_loss": 5.465346698275008,
+      "tokens_seen": 39911424
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004989368104312939,
+      "loss": 4.3366,
+      "theoretical_loss": 5.46394290826635,
+      "tokens_seen": 39976960
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.000498926780341023,
+      "loss": 4.4816,
+      "theoretical_loss": 5.462542060829209,
+      "tokens_seen": 40042496
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004989167502507523,
+      "loss": 4.5005,
+      "theoretical_loss": 5.461144144994606,
+      "tokens_seen": 40108032
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004989067201604814,
+      "loss": 4.5075,
+      "theoretical_loss": 5.459749149852287,
+      "tokens_seen": 40173568
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004988966900702107,
+      "loss": 4.555,
+      "theoretical_loss": 5.4583570645503166,
+      "tokens_seen": 40239104
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004988866599799398,
+      "loss": 4.411,
+      "theoretical_loss": 5.45696787829467,
+      "tokens_seen": 40304640
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004988766298896691,
+      "loss": 4.479,
+      "theoretical_loss": 5.455581580348831,
+      "tokens_seen": 40370176
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004988665997993982,
+      "loss": 4.424,
+      "theoretical_loss": 5.454198160033391,
+      "tokens_seen": 40435712
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004988565697091274,
+      "loss": 4.6216,
+      "theoretical_loss": 5.452817606725657,
+      "tokens_seen": 40501248
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004988465396188566,
+      "loss": 4.5915,
+      "theoretical_loss": 5.451439909859255,
+      "tokens_seen": 40566784
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004988365095285858,
+      "loss": 4.4225,
+      "theoretical_loss": 5.450065058923747,
+      "tokens_seen": 40632320
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.000498826479438315,
+      "loss": 4.4659,
+      "theoretical_loss": 5.448693043464235,
+      "tokens_seen": 40697856
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004988164493480441,
+      "loss": 4.7231,
+      "theoretical_loss": 5.447323853080991,
+      "tokens_seen": 40763392
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004988064192577733,
+      "loss": 4.561,
+      "theoretical_loss": 5.445957477429067,
+      "tokens_seen": 40828928
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004987963891675025,
+      "loss": 4.4559,
+      "theoretical_loss": 5.444593906217927,
+      "tokens_seen": 40894464
+    },
+    {
+      "epoch": 0.01,
+      "objective/train/docs_used": 33820,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 4.709114074707031,
+      "objective/train/theoretical_loss": 5.443233129211069,
+      "objective/train/tokens_used": 61420000,
+      "theoretical_loss": 5.443233129211069,
+      "tokens_seen": 40960000
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004987863590772317,
+      "loss": 4.5353,
+      "theoretical_loss": 5.443233129211069,
+      "tokens_seen": 40960000
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004987763289869609,
+      "loss": 4.5888,
+      "theoretical_loss": 5.441875136225658,
+      "tokens_seen": 41025536
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00049876629889669,
+      "loss": 4.6609,
+      "theoretical_loss": 5.440519917132159,
+      "tokens_seen": 41091072
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004987562688064192,
+      "loss": 4.6782,
+      "theoretical_loss": 5.439167461853971,
+      "tokens_seen": 41156608
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004987462387161484,
+      "loss": 4.3218,
+      "theoretical_loss": 5.437817760367073,
+      "tokens_seen": 41222144
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004987362086258777,
+      "loss": 4.6102,
+      "theoretical_loss": 5.436470802699661,
+      "tokens_seen": 41287680
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004987261785356068,
+      "loss": 4.4044,
+      "theoretical_loss": 5.43512657893179,
+      "tokens_seen": 41353216
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004987161484453361,
+      "loss": 4.5902,
+      "theoretical_loss": 5.433785079195035,
+      "tokens_seen": 41418752
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004987061183550651,
+      "loss": 4.4395,
+      "theoretical_loss": 5.432446293672129,
+      "tokens_seen": 41484288
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004986960882647944,
+      "loss": 4.6268,
+      "theoretical_loss": 5.43111021259663,
+      "tokens_seen": 41549824
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004986860581745236,
+      "loss": 4.7247,
+      "theoretical_loss": 5.429776826252565,
+      "tokens_seen": 41615360
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004986760280842528,
+      "loss": 4.7844,
+      "theoretical_loss": 5.428446124974103,
+      "tokens_seen": 41680896
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.000498665997993982,
+      "loss": 4.748,
+      "theoretical_loss": 5.427118099145207,
+      "tokens_seen": 41746432
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004986559679037111,
+      "loss": 4.4521,
+      "theoretical_loss": 5.425792739199306,
+      "tokens_seen": 41811968
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004986459378134403,
+      "loss": 4.5162,
+      "theoretical_loss": 5.424470035618963,
+      "tokens_seen": 41877504
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004986359077231695,
+      "loss": 4.5787,
+      "theoretical_loss": 5.423149978935547,
+      "tokens_seen": 41943040
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004986258776328987,
+      "loss": 4.4447,
+      "theoretical_loss": 5.421832559728902,
+      "tokens_seen": 42008576
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004986158475426279,
+      "loss": 4.5436,
+      "theoretical_loss": 5.420517768627027,
+      "tokens_seen": 42074112
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.000498605817452357,
+      "loss": 4.5508,
+      "theoretical_loss": 5.419205596305758,
+      "tokens_seen": 42139648
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004985957873620862,
+      "loss": 4.3851,
+      "theoretical_loss": 5.417896033488446,
+      "tokens_seen": 42205184
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004985857572718154,
+      "loss": 4.4232,
+      "theoretical_loss": 5.416589070945641,
+      "tokens_seen": 42270720
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004985757271815446,
+      "loss": 4.4672,
+      "theoretical_loss": 5.415284699494782,
+      "tokens_seen": 42336256
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004985656970912738,
+      "loss": 4.3494,
+      "theoretical_loss": 5.413982909999887,
+      "tokens_seen": 42401792
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004985556670010031,
+      "loss": 4.246,
+      "theoretical_loss": 5.412683693371241,
+      "tokens_seen": 42467328
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004985456369107321,
+      "loss": 4.5265,
+      "theoretical_loss": 5.411387040565092,
+      "tokens_seen": 42532864
+    },
+    {
+      "epoch": 0.01,
+      "objective/train/docs_used": 34938,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 3.999885082244873,
+      "objective/train/theoretical_loss": 5.410092942583357,
+      "objective/train/tokens_used": 63058400,
+      "theoretical_loss": 5.410092942583357,
+      "tokens_seen": 42598400
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004985356068204614,
+      "loss": 4.5919,
+      "theoretical_loss": 5.410092942583357,
+      "tokens_seen": 42598400
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004985255767301905,
+      "loss": 4.603,
+      "theoretical_loss": 5.408801390473304,
+      "tokens_seen": 42663936
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004985155466399198,
+      "loss": 4.546,
+      "theoretical_loss": 5.407512375327272,
+      "tokens_seen": 42729472
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.000498505516549649,
+      "loss": 4.5326,
+      "theoretical_loss": 5.406225888282363,
+      "tokens_seen": 42795008
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004984954864593782,
+      "loss": 4.3889,
+      "theoretical_loss": 5.404941920520156,
+      "tokens_seen": 42860544
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004984854563691073,
+      "loss": 4.1785,
+      "theoretical_loss": 5.403660463266413,
+      "tokens_seen": 42926080
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004984754262788365,
+      "loss": 4.5382,
+      "theoretical_loss": 5.402381507790792,
+      "tokens_seen": 42991616
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004984653961885657,
+      "loss": 4.5693,
+      "theoretical_loss": 5.401105045406557,
+      "tokens_seen": 43057152
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004984553660982949,
+      "loss": 4.4403,
+      "theoretical_loss": 5.399831067470302,
+      "tokens_seen": 43122688
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004984453360080241,
+      "loss": 4.7214,
+      "theoretical_loss": 5.398559565381662,
+      "tokens_seen": 43188224
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004984353059177532,
+      "loss": 4.4442,
+      "theoretical_loss": 5.397290530583037,
+      "tokens_seen": 43253760
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004984252758274825,
+      "loss": 4.309,
+      "theoretical_loss": 5.396023954559315,
+      "tokens_seen": 43319296
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004984152457372116,
+      "loss": 4.3474,
+      "theoretical_loss": 5.394759828837596,
+      "tokens_seen": 43384832
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004984052156469409,
+      "loss": 4.5364,
+      "theoretical_loss": 5.393498144986921,
+      "tokens_seen": 43450368
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00049839518555667,
+      "loss": 4.647,
+      "theoretical_loss": 5.392238894618,
+      "tokens_seen": 43515904
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004983851554663993,
+      "loss": 4.4688,
+      "theoretical_loss": 5.390982069382944,
+      "tokens_seen": 43581440
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004983751253761284,
+      "loss": 4.555,
+      "theoretical_loss": 5.389727660974998,
+      "tokens_seen": 43646976
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004983650952858576,
+      "loss": 4.2673,
+      "theoretical_loss": 5.38847566112828,
+      "tokens_seen": 43712512
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004983550651955868,
+      "loss": 4.223,
+      "theoretical_loss": 5.387226061617516,
+      "tokens_seen": 43778048
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.000498345035105316,
+      "loss": 4.5944,
+      "theoretical_loss": 5.38597885425778,
+      "tokens_seen": 43843584
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004983350050150452,
+      "loss": 4.2395,
+      "theoretical_loss": 5.384734030904238,
+      "tokens_seen": 43909120
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004983249749247743,
+      "loss": 4.2047,
+      "theoretical_loss": 5.383491583451892,
+      "tokens_seen": 43974656
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004983149448345035,
+      "loss": 4.6784,
+      "theoretical_loss": 5.382251503835324,
+      "tokens_seen": 44040192
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004983049147442327,
+      "loss": 4.5094,
+      "theoretical_loss": 5.381013784028445,
+      "tokens_seen": 44105728
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004982948846539619,
+      "loss": 4.2719,
+      "theoretical_loss": 5.379778416044247,
+      "tokens_seen": 44171264
+    },
+    {
+      "epoch": 0.01,
+      "objective/train/docs_used": 35534,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 4.2067718505859375,
+      "objective/train/theoretical_loss": 5.378545391934553,
+      "objective/train/tokens_used": 64696800,
+      "theoretical_loss": 5.378545391934553,
+      "tokens_seen": 44236800
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004982848545636911,
+      "loss": 4.3011,
+      "theoretical_loss": 5.378545391934553,
+      "tokens_seen": 44236800
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004982748244734202,
+      "loss": 4.3593,
+      "theoretical_loss": 5.377314703789773,
+      "tokens_seen": 44302336
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004982647943831494,
+      "loss": 4.5546,
+      "theoretical_loss": 5.376086343738656,
+      "tokens_seen": 44367872
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004982547642928786,
+      "loss": 4.2113,
+      "theoretical_loss": 5.3748603039480525,
+      "tokens_seen": 44433408
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004982447342026079,
+      "loss": 4.6548,
+      "theoretical_loss": 5.373636576622671,
+      "tokens_seen": 44498944
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.000498234704112337,
+      "loss": 4.6067,
+      "theoretical_loss": 5.372415154004842,
+      "tokens_seen": 44564480
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004982246740220663,
+      "loss": 4.5434,
+      "theoretical_loss": 5.371196028374278,
+      "tokens_seen": 44630016
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004982146439317953,
+      "loss": 4.219,
+      "theoretical_loss": 5.369979192047846,
+      "tokens_seen": 44695552
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004982046138415246,
+      "loss": 4.4616,
+      "theoretical_loss": 5.368764637379322,
+      "tokens_seen": 44761088
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004981945837512538,
+      "loss": 4.3927,
+      "theoretical_loss": 5.367552356759175,
+      "tokens_seen": 44826624
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.000498184553660983,
+      "loss": 4.3436,
+      "theoretical_loss": 5.366342342614322,
+      "tokens_seen": 44892160
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004981745235707122,
+      "loss": 4.3641,
+      "theoretical_loss": 5.365134587407918,
+      "tokens_seen": 44957696
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004981644934804413,
+      "loss": 4.3878,
+      "theoretical_loss": 5.363929083639112,
+      "tokens_seen": 45023232
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004981544633901705,
+      "loss": 4.252,
+      "theoretical_loss": 5.362725823842838,
+      "tokens_seen": 45088768
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004981444332998997,
+      "loss": 4.1657,
+      "theoretical_loss": 5.361524800589581,
+      "tokens_seen": 45154304
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004981344032096289,
+      "loss": 4.0417,
+      "theoretical_loss": 5.360326006485165,
+      "tokens_seen": 45219840
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004981243731193581,
+      "loss": 4.6023,
+      "theoretical_loss": 5.359129434170529,
+      "tokens_seen": 45285376
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004981143430290873,
+      "loss": 4.3602,
+      "theoretical_loss": 5.357935076321514,
+      "tokens_seen": 45350912
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004981043129388164,
+      "loss": 4.4446,
+      "theoretical_loss": 5.35674292564864,
+      "tokens_seen": 45416448
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004980942828485456,
+      "loss": 4.4122,
+      "theoretical_loss": 5.3555529748969,
+      "tokens_seen": 45481984
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004980842527582748,
+      "loss": 4.465,
+      "theoretical_loss": 5.354365216845546,
+      "tokens_seen": 45547520
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.000498074222668004,
+      "loss": 4.1501,
+      "theoretical_loss": 5.353179644307868,
+      "tokens_seen": 45613056
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004980641925777333,
+      "loss": 4.6183,
+      "theoretical_loss": 5.351996250131005,
+      "tokens_seen": 45678592
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004980541624874623,
+      "loss": 4.4826,
+      "theoretical_loss": 5.350815027195717,
+      "tokens_seen": 45744128
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004980441323971916,
+      "loss": 4.0969,
+      "theoretical_loss": 5.349635968416189,
+      "tokens_seen": 45809664
+    },
+    {
+      "epoch": 0.01,
+      "objective/train/docs_used": 36791,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 3.983624219894409,
+      "objective/train/theoretical_loss": 5.34845906673983,
+      "objective/train/tokens_used": 66335200,
+      "theoretical_loss": 5.34845906673983,
+      "tokens_seen": 45875200
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004980341023069207,
+      "loss": 4.1114,
+      "theoretical_loss": 5.34845906673983,
+      "tokens_seen": 45875200
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00049802407221665,
+      "loss": 4.3748,
+      "theoretical_loss": 5.347284315147059,
+      "tokens_seen": 45940736
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004980140421263792,
+      "loss": 4.3086,
+      "theoretical_loss": 5.346111706651114,
+      "tokens_seen": 46006272
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004980040120361084,
+      "loss": 4.5521,
+      "theoretical_loss": 5.344941234297847,
+      "tokens_seen": 46071808
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004979939819458375,
+      "loss": 4.4492,
+      "theoretical_loss": 5.343772891165527,
+      "tokens_seen": 46137344
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004979839518555667,
+      "loss": 4.4528,
+      "theoretical_loss": 5.34260667036464,
+      "tokens_seen": 46202880
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004979739217652959,
+      "loss": 4.6275,
+      "theoretical_loss": 5.3414425650377,
+      "tokens_seen": 46268416
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004979638916750251,
+      "loss": 4.5955,
+      "theoretical_loss": 5.3402805683590495,
+      "tokens_seen": 46333952
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004979538615847543,
+      "loss": 4.5274,
+      "theoretical_loss": 5.339120673534668,
+      "tokens_seen": 46399488
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004979438314944834,
+      "loss": 4.1984,
+      "theoretical_loss": 5.337962873801984,
+      "tokens_seen": 46465024
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004979338014042126,
+      "loss": 4.4472,
+      "theoretical_loss": 5.336807162429681,
+      "tokens_seen": 46530560
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004979237713139418,
+      "loss": 4.2688,
+      "theoretical_loss": 5.335653532717512,
+      "tokens_seen": 46596096
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.000497913741223671,
+      "loss": 4.5101,
+      "theoretical_loss": 5.33450197799611,
+      "tokens_seen": 46661632
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004979037111334002,
+      "loss": 4.4218,
+      "theoretical_loss": 5.33335249162681,
+      "tokens_seen": 46727168
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004978936810431293,
+      "loss": 4.365,
+      "theoretical_loss": 5.332205067001454,
+      "tokens_seen": 46792704
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004978836509528586,
+      "loss": 4.4012,
+      "theoretical_loss": 5.331059697542216,
+      "tokens_seen": 46858240
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004978736208625877,
+      "loss": 4.178,
+      "theoretical_loss": 5.329916376701417,
+      "tokens_seen": 46923776
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.000497863590772317,
+      "loss": 4.404,
+      "theoretical_loss": 5.328775097961346,
+      "tokens_seen": 46989312
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004978535606820461,
+      "loss": 4.3063,
+      "theoretical_loss": 5.327635854834084,
+      "tokens_seen": 47054848
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004978435305917754,
+      "loss": 4.3593,
+      "theoretical_loss": 5.326498640861321,
+      "tokens_seen": 47120384
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004978335005015045,
+      "loss": 4.458,
+      "theoretical_loss": 5.325363449614185,
+      "tokens_seen": 47185920
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004978234704112337,
+      "loss": 4.4505,
+      "theoretical_loss": 5.324230274693065,
+      "tokens_seen": 47251456
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004978134403209629,
+      "loss": 4.2784,
+      "theoretical_loss": 5.323099109727434,
+      "tokens_seen": 47316992
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004978034102306921,
+      "loss": 4.3645,
+      "theoretical_loss": 5.321969948375687,
+      "tokens_seen": 47382528
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004977933801404213,
+      "loss": 4.3349,
+      "theoretical_loss": 5.320842784324954,
+      "tokens_seen": 47448064
+    },
+    {
+      "epoch": 0.01,
+      "objective/train/docs_used": 38014,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 4.285874366760254,
+      "objective/train/theoretical_loss": 5.319717611290951,
+      "objective/train/tokens_used": 67973600,
+      "theoretical_loss": 5.319717611290951,
+      "tokens_seen": 47513600
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004977833500501504,
+      "loss": 4.2487,
+      "theoretical_loss": 5.319717611290951,
+      "tokens_seen": 47513600
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004977733199598796,
+      "loss": 4.227,
+      "theoretical_loss": 5.318594423017791,
+      "tokens_seen": 47579136
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004977632898696088,
+      "loss": 4.5068,
+      "theoretical_loss": 5.317473213277827,
+      "tokens_seen": 47644672
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.000497753259779338,
+      "loss": 4.4264,
+      "theoretical_loss": 5.316353975871486,
+      "tokens_seen": 47710208
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004977432296890672,
+      "loss": 4.4848,
+      "theoretical_loss": 5.315236704627106,
+      "tokens_seen": 47775744
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004977331995987965,
+      "loss": 4.4288,
+      "theoretical_loss": 5.314121393400763,
+      "tokens_seen": 47841280
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004977231695085255,
+      "loss": 4.0077,
+      "theoretical_loss": 5.313008036076115,
+      "tokens_seen": 47906816
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004977131394182548,
+      "loss": 4.3322,
+      "theoretical_loss": 5.311896626564246,
+      "tokens_seen": 47972352
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.000497703109327984,
+      "loss": 4.1105,
+      "theoretical_loss": 5.3107871588034925,
+      "tokens_seen": 48037888
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004976930792377132,
+      "loss": 4.3437,
+      "theoretical_loss": 5.309679626759296,
+      "tokens_seen": 48103424
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004976830491474424,
+      "loss": 4.292,
+      "theoretical_loss": 5.308574024424041,
+      "tokens_seen": 48168960
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004976730190571715,
+      "loss": 4.3868,
+      "theoretical_loss": 5.3074703458168955,
+      "tokens_seen": 48234496
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004976629889669007,
+      "loss": 4.436,
+      "theoretical_loss": 5.3063685849836615,
+      "tokens_seen": 48300032
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004976529588766299,
+      "loss": 4.368,
+      "theoretical_loss": 5.305268735996615,
+      "tokens_seen": 48365568
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004976429287863591,
+      "loss": 4.2524,
+      "theoretical_loss": 5.304170792954354,
+      "tokens_seen": 48431104
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004976328986960883,
+      "loss": 4.5041,
+      "theoretical_loss": 5.303074749981648,
+      "tokens_seen": 48496640
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004976228686058175,
+      "loss": 4.412,
+      "theoretical_loss": 5.301980601229285,
+      "tokens_seen": 48562176
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004976128385155466,
+      "loss": 4.3484,
+      "theoretical_loss": 5.300888340873924,
+      "tokens_seen": 48627712
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004976028084252758,
+      "loss": 4.4722,
+      "theoretical_loss": 5.299797963117941,
+      "tokens_seen": 48693248
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.000497592778335005,
+      "loss": 4.1945,
+      "theoretical_loss": 5.298709462189283,
+      "tokens_seen": 48758784
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004975827482447342,
+      "loss": 4.2003,
+      "theoretical_loss": 5.297622832341327,
+      "tokens_seen": 48824320
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004975727181544635,
+      "loss": 4.0957,
+      "theoretical_loss": 5.2965380678527225,
+      "tokens_seen": 48889856
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004975626880641925,
+      "loss": 4.2418,
+      "theoretical_loss": 5.295455163027258,
+      "tokens_seen": 48955392
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004975526579739218,
+      "loss": 4.3033,
+      "theoretical_loss": 5.29437411219371,
+      "tokens_seen": 49020928
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004975426278836509,
+      "loss": 4.2827,
+      "theoretical_loss": 5.293294909705702,
+      "tokens_seen": 49086464
+    },
+    {
+      "epoch": 0.01,
+      "objective/train/docs_used": 38689,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 4.5455546379089355,
+      "objective/train/theoretical_loss": 5.292217549941562,
+      "objective/train/tokens_used": 69612000,
+      "theoretical_loss": 5.292217549941562,
+      "tokens_seen": 49152000
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004975325977933802,
+      "loss": 4.2752,
+      "theoretical_loss": 5.292217549941562,
+      "tokens_seen": 49152000
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004975225677031094,
+      "loss": 4.1889,
+      "theoretical_loss": 5.291142027304185,
+      "tokens_seen": 49217536
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004975125376128386,
+      "loss": 3.9788,
+      "theoretical_loss": 5.29006833622089,
+      "tokens_seen": 49283072
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004975025075225677,
+      "loss": 4.4666,
+      "theoretical_loss": 5.28899647114328,
+      "tokens_seen": 49348608
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004974924774322969,
+      "loss": 4.3282,
+      "theoretical_loss": 5.287926426547106,
+      "tokens_seen": 49414144
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004974824473420261,
+      "loss": 4.3775,
+      "theoretical_loss": 5.2868581969321315,
+      "tokens_seen": 49479680
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004974724172517553,
+      "loss": 4.3022,
+      "theoretical_loss": 5.285791776821995,
+      "tokens_seen": 49545216
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004974623871614845,
+      "loss": 4.559,
+      "theoretical_loss": 5.2847271607640725,
+      "tokens_seen": 49610752
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004974523570712136,
+      "loss": 4.5333,
+      "theoretical_loss": 5.283664343329346,
+      "tokens_seen": 49676288
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004974423269809428,
+      "loss": 4.2267,
+      "theoretical_loss": 5.282603319112273,
+      "tokens_seen": 49741824
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.000497432296890672,
+      "loss": 4.3567,
+      "theoretical_loss": 5.281544082730649,
+      "tokens_seen": 49807360
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004974222668004012,
+      "loss": 4.2938,
+      "theoretical_loss": 5.280486628825479,
+      "tokens_seen": 49872896
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004974122367101304,
+      "loss": 4.3754,
+      "theoretical_loss": 5.279430952060843,
+      "tokens_seen": 49938432
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004974022066198595,
+      "loss": 4.3361,
+      "theoretical_loss": 5.278377047123777,
+      "tokens_seen": 50003968
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004973921765295888,
+      "loss": 4.2186,
+      "theoretical_loss": 5.277324908724132,
+      "tokens_seen": 50069504
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004973821464393179,
+      "loss": 4.2978,
+      "theoretical_loss": 5.276274531594453,
+      "tokens_seen": 50135040
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004973721163490472,
+      "loss": 4.3052,
+      "theoretical_loss": 5.27522591048985,
+      "tokens_seen": 50200576
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004973620862587763,
+      "loss": 4.352,
+      "theoretical_loss": 5.274179040187876,
+      "tokens_seen": 50266112
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004973520561685056,
+      "loss": 4.0427,
+      "theoretical_loss": 5.273133915488394,
+      "tokens_seen": 50331648
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004973420260782347,
+      "loss": 4.3078,
+      "theoretical_loss": 5.272090531213462,
+      "tokens_seen": 50397184
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004973319959879639,
+      "loss": 4.0645,
+      "theoretical_loss": 5.271048882207202,
+      "tokens_seen": 50462720
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004973219658976931,
+      "loss": 4.6409,
+      "theoretical_loss": 5.27000896333568,
+      "tokens_seen": 50528256
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004973119358074223,
+      "loss": 4.0496,
+      "theoretical_loss": 5.268970769486789,
+      "tokens_seen": 50593792
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004973019057171515,
+      "loss": 4.4574,
+      "theoretical_loss": 5.267934295570118,
+      "tokens_seen": 50659328
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004972918756268806,
+      "loss": 4.2343,
+      "theoretical_loss": 5.266899536516842,
+      "tokens_seen": 50724864
+    },
+    {
+      "epoch": 0.02,
+      "objective/train/docs_used": 39885,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 4.198443412780762,
+      "objective/train/theoretical_loss": 5.265866487279597,
+      "objective/train/tokens_used": 71250400,
+      "theoretical_loss": 5.265866487279597,
+      "tokens_seen": 50790400
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004972818455366098,
+      "loss": 4.2608,
+      "theoretical_loss": 5.265866487279597,
+      "tokens_seen": 50790400
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.000497271815446339,
+      "loss": 4.2763,
+      "theoretical_loss": 5.264835142832364,
+      "tokens_seen": 50855936
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004972617853560682,
+      "loss": 4.1906,
+      "theoretical_loss": 5.263805498170351,
+      "tokens_seen": 50921472
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004972517552657974,
+      "loss": 4.1761,
+      "theoretical_loss": 5.26277754830987,
+      "tokens_seen": 50987008
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004972417251755266,
+      "loss": 4.2526,
+      "theoretical_loss": 5.261751288288236,
+      "tokens_seen": 51052544
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004972316950852557,
+      "loss": 4.275,
+      "theoretical_loss": 5.260726713163637,
+      "tokens_seen": 51118080
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004972216649949849,
+      "loss": 4.3822,
+      "theoretical_loss": 5.259703818015023,
+      "tokens_seen": 51183616
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004972116349047142,
+      "loss": 4.235,
+      "theoretical_loss": 5.258682597941998,
+      "tokens_seen": 51249152
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004972016048144433,
+      "loss": 4.1933,
+      "theoretical_loss": 5.257663048064704,
+      "tokens_seen": 51314688
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004971915747241726,
+      "loss": 4.0819,
+      "theoretical_loss": 5.256645163523704,
+      "tokens_seen": 51380224
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004971815446339017,
+      "loss": 4.2302,
+      "theoretical_loss": 5.255628939479877,
+      "tokens_seen": 51445760
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004971715145436309,
+      "loss": 4.1741,
+      "theoretical_loss": 5.254614371114306,
+      "tokens_seen": 51511296
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004971614844533601,
+      "loss": 4.0105,
+      "theoretical_loss": 5.253601453628165,
+      "tokens_seen": 51576832
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004971514543630893,
+      "loss": 4.1921,
+      "theoretical_loss": 5.2525901822426135,
+      "tokens_seen": 51642368
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004971414242728185,
+      "loss": 4.1033,
+      "theoretical_loss": 5.251580552198687,
+      "tokens_seen": 51707904
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004971313941825477,
+      "loss": 4.5223,
+      "theoretical_loss": 5.250572558757186,
+      "tokens_seen": 51773440
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004971213640922768,
+      "loss": 4.2685,
+      "theoretical_loss": 5.2495661971985745,
+      "tokens_seen": 51838976
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.000497111334002006,
+      "loss": 4.4627,
+      "theoretical_loss": 5.248561462822871,
+      "tokens_seen": 51904512
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004971013039117352,
+      "loss": 4.2511,
+      "theoretical_loss": 5.247558350949539,
+      "tokens_seen": 51970048
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004970912738214644,
+      "loss": 4.0554,
+      "theoretical_loss": 5.246556856917392,
+      "tokens_seen": 52035584
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004970812437311936,
+      "loss": 4.1646,
+      "theoretical_loss": 5.24555697608448,
+      "tokens_seen": 52101120
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004970712136409227,
+      "loss": 4.2295,
+      "theoretical_loss": 5.244558703827986,
+      "tokens_seen": 52166656
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004970611835506519,
+      "loss": 4.1183,
+      "theoretical_loss": 5.243562035544132,
+      "tokens_seen": 52232192
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004970511534603811,
+      "loss": 4.4799,
+      "theoretical_loss": 5.24256696664807,
+      "tokens_seen": 52297728
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004970411233701103,
+      "loss": 4.1907,
+      "theoretical_loss": 5.24157349257378,
+      "tokens_seen": 52363264
+    },
+    {
+      "epoch": 0.02,
+      "objective/train/docs_used": 40418,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 3.979517936706543,
+      "objective/train/theoretical_loss": 5.240581608773973,
+      "objective/train/tokens_used": 72888800,
+      "theoretical_loss": 5.240581608773973,
+      "tokens_seen": 52428800
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004970310932798396,
+      "loss": 4.0593,
+      "theoretical_loss": 5.240581608773973,
+      "tokens_seen": 52428800
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004970210631895686,
+      "loss": 4.2029,
+      "theoretical_loss": 5.239591310719987,
+      "tokens_seen": 52494336
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004970110330992979,
+      "loss": 4.0965,
+      "theoretical_loss": 5.238602593901694,
+      "tokens_seen": 52559872
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004970010030090271,
+      "loss": 4.0066,
+      "theoretical_loss": 5.23761545382739,
+      "tokens_seen": 52625408
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004969909729187563,
+      "loss": 4.1287,
+      "theoretical_loss": 5.236629886023708,
+      "tokens_seen": 52690944
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004969809428284855,
+      "loss": 4.1441,
+      "theoretical_loss": 5.235645886035517,
+      "tokens_seen": 52756480
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004969709127382147,
+      "loss": 4.2684,
+      "theoretical_loss": 5.2346634494258195,
+      "tokens_seen": 52822016
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004969608826479438,
+      "loss": 4.1129,
+      "theoretical_loss": 5.233682571775664,
+      "tokens_seen": 52887552
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.000496950852557673,
+      "loss": 4.44,
+      "theoretical_loss": 5.232703248684045,
+      "tokens_seen": 52953088
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004969408224674022,
+      "loss": 4.3805,
+      "theoretical_loss": 5.231725475767806,
+      "tokens_seen": 53018624
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004969307923771314,
+      "loss": 4.012,
+      "theoretical_loss": 5.2307492486615494,
+      "tokens_seen": 53084160
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004969207622868606,
+      "loss": 4.3914,
+      "theoretical_loss": 5.229774563017541,
+      "tokens_seen": 53149696
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004969107321965897,
+      "loss": 4.2825,
+      "theoretical_loss": 5.2288014145056145,
+      "tokens_seen": 53215232
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.000496900702106319,
+      "loss": 3.9345,
+      "theoretical_loss": 5.227829798813083,
+      "tokens_seen": 53280768
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004968906720160481,
+      "loss": 4.1139,
+      "theoretical_loss": 5.2268597116446465,
+      "tokens_seen": 53346304
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004968806419257774,
+      "loss": 4.2213,
+      "theoretical_loss": 5.225891148722296,
+      "tokens_seen": 53411840
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004968706118355065,
+      "loss": 4.1851,
+      "theoretical_loss": 5.224924105785227,
+      "tokens_seen": 53477376
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004968605817452358,
+      "loss": 4.1925,
+      "theoretical_loss": 5.22395857858975,
+      "tokens_seen": 53542912
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004968505516549649,
+      "loss": 3.8839,
+      "theoretical_loss": 5.222994562909199,
+      "tokens_seen": 53608448
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004968405215646941,
+      "loss": 4.1607,
+      "theoretical_loss": 5.222032054533841,
+      "tokens_seen": 53673984
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004968304914744233,
+      "loss": 4.2475,
+      "theoretical_loss": 5.2210710492707895,
+      "tokens_seen": 53739520
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004968204613841525,
+      "loss": 3.881,
+      "theoretical_loss": 5.220111542943918,
+      "tokens_seen": 53805056
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004968104312938817,
+      "loss": 4.0171,
+      "theoretical_loss": 5.21915353139377,
+      "tokens_seen": 53870592
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004968004012036108,
+      "loss": 4.0926,
+      "theoretical_loss": 5.218197010477477,
+      "tokens_seen": 53936128
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.00049679037111334,
+      "loss": 4.3292,
+      "theoretical_loss": 5.21724197606866,
+      "tokens_seen": 54001664
+    },
+    {
+      "epoch": 0.02,
+      "objective/train/docs_used": 41741,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 4.073785305023193,
+      "objective/train/theoretical_loss": 5.216288424057362,
+      "objective/train/tokens_used": 74527200,
+      "theoretical_loss": 5.216288424057362,
+      "tokens_seen": 54067200
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004967803410230692,
+      "loss": 4.2491,
+      "theoretical_loss": 5.216288424057362,
+      "tokens_seen": 54067200
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004967703109327984,
+      "loss": 4.2446,
+      "theoretical_loss": 5.21533635034995,
+      "tokens_seen": 54132736
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004967602808425276,
+      "loss": 4.2944,
+      "theoretical_loss": 5.214385750869031,
+      "tokens_seen": 54198272
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004967502507522568,
+      "loss": 4.1296,
+      "theoretical_loss": 5.213436621553377,
+      "tokens_seen": 54263808
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004967402206619859,
+      "loss": 4.17,
+      "theoretical_loss": 5.212488958357831,
+      "tokens_seen": 54329344
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004967301905717151,
+      "loss": 4.0717,
+      "theoretical_loss": 5.211542757253229,
+      "tokens_seen": 54394880
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004967201604814444,
+      "loss": 4.4436,
+      "theoretical_loss": 5.210598014226319,
+      "tokens_seen": 54460416
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004967101303911735,
+      "loss": 4.1394,
+      "theoretical_loss": 5.209654725279678,
+      "tokens_seen": 54525952
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004967001003009028,
+      "loss": 4.4479,
+      "theoretical_loss": 5.208712886431626,
+      "tokens_seen": 54591488
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.000496690070210632,
+      "loss": 3.9774,
+      "theoretical_loss": 5.207772493716154,
+      "tokens_seen": 54657024
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004966800401203611,
+      "loss": 4.0348,
+      "theoretical_loss": 5.206833543182833,
+      "tokens_seen": 54722560
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004966700100300903,
+      "loss": 4.1275,
+      "theoretical_loss": 5.205896030896748,
+      "tokens_seen": 54788096
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004966599799398195,
+      "loss": 4.034,
+      "theoretical_loss": 5.2049599529384025,
+      "tokens_seen": 54853632
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004966499498495487,
+      "loss": 4.2901,
+      "theoretical_loss": 5.204025305403653,
+      "tokens_seen": 54919168
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004966399197592779,
+      "loss": 4.0859,
+      "theoretical_loss": 5.2030920844036235,
+      "tokens_seen": 54984704
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.000496629889669007,
+      "loss": 4.2186,
+      "theoretical_loss": 5.202160286064628,
+      "tokens_seen": 55050240
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004966198595787362,
+      "loss": 4.1753,
+      "theoretical_loss": 5.2012299065281,
+      "tokens_seen": 55115776
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004966098294884654,
+      "loss": 4.2037,
+      "theoretical_loss": 5.2003009419505055,
+      "tokens_seen": 55181312
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004965997993981946,
+      "loss": 4.3058,
+      "theoretical_loss": 5.199373388503272,
+      "tokens_seen": 55246848
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004965897693079238,
+      "loss": 4.3641,
+      "theoretical_loss": 5.198447242372716,
+      "tokens_seen": 55312384
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004965797392176529,
+      "loss": 4.1662,
+      "theoretical_loss": 5.197522499759961,
+      "tokens_seen": 55377920
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004965697091273821,
+      "loss": 4.1908,
+      "theoretical_loss": 5.196599156880863,
+      "tokens_seen": 55443456
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004965596790371113,
+      "loss": 3.9863,
+      "theoretical_loss": 5.195677209965942,
+      "tokens_seen": 55508992
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004965496489468405,
+      "loss": 4.5222,
+      "theoretical_loss": 5.194756655260305,
+      "tokens_seen": 55574528
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004965396188565698,
+      "loss": 3.9698,
+      "theoretical_loss": 5.193837489023565,
+      "tokens_seen": 55640064
+    },
+    {
+      "epoch": 0.02,
+      "objective/train/docs_used": 42409,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 4.305491924285889,
+      "objective/train/theoretical_loss": 5.192919707529784,
+      "objective/train/tokens_used": 76165600,
+      "theoretical_loss": 5.192919707529784,
+      "tokens_seen": 55705600
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004965295887662988,
+      "loss": 4.121,
+      "theoretical_loss": 5.192919707529784,
+      "tokens_seen": 55705600
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004965195586760281,
+      "loss": 4.108,
+      "theoretical_loss": 5.1920033070673846,
+      "tokens_seen": 55771136
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004965095285857573,
+      "loss": 3.9647,
+      "theoretical_loss": 5.191088283939086,
+      "tokens_seen": 55836672
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004964994984954865,
+      "loss": 3.9453,
+      "theoretical_loss": 5.190174634461828,
+      "tokens_seen": 55902208
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004964894684052157,
+      "loss": 3.9193,
+      "theoretical_loss": 5.189262354966708,
+      "tokens_seen": 55967744
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004964794383149449,
+      "loss": 4.0925,
+      "theoretical_loss": 5.188351441798897,
+      "tokens_seen": 56033280
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.000496469408224674,
+      "loss": 4.1099,
+      "theoretical_loss": 5.187441891317581,
+      "tokens_seen": 56098816
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004964593781344032,
+      "loss": 4.0055,
+      "theoretical_loss": 5.186533699895888,
+      "tokens_seen": 56164352
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004964493480441324,
+      "loss": 4.1547,
+      "theoretical_loss": 5.18562686392081,
+      "tokens_seen": 56229888
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004964393179538616,
+      "loss": 4.1385,
+      "theoretical_loss": 5.184721379793145,
+      "tokens_seen": 56295424
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004964292878635908,
+      "loss": 4.1137,
+      "theoretical_loss": 5.183817243927425,
+      "tokens_seen": 56360960
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.00049641925777332,
+      "loss": 3.988,
+      "theoretical_loss": 5.182914452751846,
+      "tokens_seen": 56426496
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004964092276830491,
+      "loss": 3.9087,
+      "theoretical_loss": 5.1820130027081985,
+      "tokens_seen": 56492032
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004963991975927783,
+      "loss": 4.1078,
+      "theoretical_loss": 5.181112890251804,
+      "tokens_seen": 56557568
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004963891675025075,
+      "loss": 3.9657,
+      "theoretical_loss": 5.180214111851447,
+      "tokens_seen": 56623104
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004963791374122367,
+      "loss": 3.8969,
+      "theoretical_loss": 5.179316663989305,
+      "tokens_seen": 56688640
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004963691073219659,
+      "loss": 4.0938,
+      "theoretical_loss": 5.178420543160889,
+      "tokens_seen": 56754176
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004963590772316951,
+      "loss": 4.0717,
+      "theoretical_loss": 5.177525745874968,
+      "tokens_seen": 56819712
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004963490471414242,
+      "loss": 4.0293,
+      "theoretical_loss": 5.176632268653516,
+      "tokens_seen": 56885248
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004963390170511535,
+      "loss": 4.0914,
+      "theoretical_loss": 5.175740108031633,
+      "tokens_seen": 56950784
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004963289869608827,
+      "loss": 4.3386,
+      "theoretical_loss": 5.174849260557494,
+      "tokens_seen": 57016320
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004963189568706119,
+      "loss": 4.1438,
+      "theoretical_loss": 5.173959722792272,
+      "tokens_seen": 57081856
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.000496308926780341,
+      "loss": 4.1502,
+      "theoretical_loss": 5.173071491310086,
+      "tokens_seen": 57147392
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004962988966900702,
+      "loss": 3.9246,
+      "theoretical_loss": 5.172184562697928,
+      "tokens_seen": 57212928
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004962888665997994,
+      "loss": 4.0249,
+      "theoretical_loss": 5.171298933555606,
+      "tokens_seen": 57278464
+    },
+    {
+      "epoch": 0.02,
+      "objective/train/docs_used": 43695,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 3.649152994155884,
+      "objective/train/theoretical_loss": 5.170414600495677,
+      "objective/train/tokens_used": 77804000,
+      "theoretical_loss": 5.170414600495677,
+      "tokens_seen": 57344000
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004962788365095286,
+      "loss": 4.0093,
+      "theoretical_loss": 5.170414600495677,
+      "tokens_seen": 57344000
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004962688064192578,
+      "loss": 4.1786,
+      "theoretical_loss": 5.16953156014339,
+      "tokens_seen": 57409536
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.000496258776328987,
+      "loss": 4.0607,
+      "theoretical_loss": 5.1686498091366175,
+      "tokens_seen": 57475072
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004962487462387161,
+      "loss": 3.6266,
+      "theoretical_loss": 5.1677693441258,
+      "tokens_seen": 57540608
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004962387161484453,
+      "loss": 3.8942,
+      "theoretical_loss": 5.16689016177388,
+      "tokens_seen": 57606144
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004962286860581746,
+      "loss": 3.9167,
+      "theoretical_loss": 5.166012258756243,
+      "tokens_seen": 57671680
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004962186559679037,
+      "loss": 4.2399,
+      "theoretical_loss": 5.16513563176066,
+      "tokens_seen": 57737216
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.000496208625877633,
+      "loss": 3.7377,
+      "theoretical_loss": 5.164260277487221,
+      "tokens_seen": 57802752
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004961985957873621,
+      "loss": 4.0589,
+      "theoretical_loss": 5.163386192648282,
+      "tokens_seen": 57868288
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004961885656970913,
+      "loss": 4.2436,
+      "theoretical_loss": 5.162513373968403,
+      "tokens_seen": 57933824
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004961785356068205,
+      "loss": 4.2688,
+      "theoretical_loss": 5.1616418181842825,
+      "tokens_seen": 57999360
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004961685055165497,
+      "loss": 3.9842,
+      "theoretical_loss": 5.160771522044712,
+      "tokens_seen": 58064896
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004961584754262789,
+      "loss": 4.202,
+      "theoretical_loss": 5.159902482310506,
+      "tokens_seen": 58130432
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004961484453360081,
+      "loss": 4.0252,
+      "theoretical_loss": 5.15903469575445,
+      "tokens_seen": 58195968
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004961384152457372,
+      "loss": 4.0653,
+      "theoretical_loss": 5.1581681591612405,
+      "tokens_seen": 58261504
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004961283851554664,
+      "loss": 4.1853,
+      "theoretical_loss": 5.15730286932743,
+      "tokens_seen": 58327040
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004961183550651956,
+      "loss": 4.0287,
+      "theoretical_loss": 5.156438823061366,
+      "tokens_seen": 58392576
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004961083249749248,
+      "loss": 3.8706,
+      "theoretical_loss": 5.15557601718314,
+      "tokens_seen": 58458112
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.000496098294884654,
+      "loss": 3.4818,
+      "theoretical_loss": 5.154714448524525,
+      "tokens_seen": 58523648
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004960882647943831,
+      "loss": 4.1847,
+      "theoretical_loss": 5.153854113928924,
+      "tokens_seen": 58589184
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004960782347041123,
+      "loss": 4.0123,
+      "theoretical_loss": 5.152995010251313,
+      "tokens_seen": 58654720
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004960682046138415,
+      "loss": 3.8546,
+      "theoretical_loss": 5.1521371343581865,
+      "tokens_seen": 58720256
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004960581745235707,
+      "loss": 3.9651,
+      "theoretical_loss": 5.1512804831275,
+      "tokens_seen": 58785792
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004960481444333,
+      "loss": 3.9367,
+      "theoretical_loss": 5.1504250534486165,
+      "tokens_seen": 58851328
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.000496038114343029,
+      "loss": 4.3063,
+      "theoretical_loss": 5.149570842222257,
+      "tokens_seen": 58916864
+    },
+    {
+      "epoch": 0.02,
+      "objective/train/docs_used": 44395,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 4.063347339630127,
+      "objective/train/theoretical_loss": 5.1487178463604355,
+      "objective/train/tokens_used": 79442400,
+      "theoretical_loss": 5.1487178463604355,
+      "tokens_seen": 58982400
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004960280842527583,
+      "loss": 3.7506,
+      "theoretical_loss": 5.1487178463604355,
+      "tokens_seen": 58982400
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004960180541624875,
+      "loss": 3.9872,
+      "theoretical_loss": 5.147866062786418,
+      "tokens_seen": 59047936
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004960080240722167,
+      "loss": 3.9911,
+      "theoretical_loss": 5.147015488434661,
+      "tokens_seen": 59113472
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004959979939819459,
+      "loss": 3.8278,
+      "theoretical_loss": 5.14616612025076,
+      "tokens_seen": 59179008
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004959879638916751,
+      "loss": 3.9528,
+      "theoretical_loss": 5.145317955191397,
+      "tokens_seen": 59244544
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004959779338014042,
+      "loss": 3.7919,
+      "theoretical_loss": 5.144470990224288,
+      "tokens_seen": 59310080
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004959679037111334,
+      "loss": 3.9634,
+      "theoretical_loss": 5.143625222328134,
+      "tokens_seen": 59375616
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004959578736208626,
+      "loss": 4.096,
+      "theoretical_loss": 5.142780648492563,
+      "tokens_seen": 59441152
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004959478435305918,
+      "loss": 3.9749,
+      "theoretical_loss": 5.141937265718084,
+      "tokens_seen": 59506688
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.000495937813440321,
+      "loss": 4.0214,
+      "theoretical_loss": 5.141095071016032,
+      "tokens_seen": 59572224
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004959277833500501,
+      "loss": 3.9143,
+      "theoretical_loss": 5.14025406140852,
+      "tokens_seen": 59637760
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004959177532597793,
+      "loss": 4.0642,
+      "theoretical_loss": 5.139414233928385,
+      "tokens_seen": 59703296
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004959077231695085,
+      "loss": 3.9695,
+      "theoretical_loss": 5.138575585619142,
+      "tokens_seen": 59768832
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004958976930792377,
+      "loss": 4.1946,
+      "theoretical_loss": 5.13773811353493,
+      "tokens_seen": 59834368
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004958876629889669,
+      "loss": 4.0784,
+      "theoretical_loss": 5.1369018147404635,
+      "tokens_seen": 59899904
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.000495877632898696,
+      "loss": 4.0826,
+      "theoretical_loss": 5.136066686310986,
+      "tokens_seen": 59965440
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004958676028084253,
+      "loss": 4.0869,
+      "theoretical_loss": 5.135232725332212,
+      "tokens_seen": 60030976
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004958575727181544,
+      "loss": 3.9758,
+      "theoretical_loss": 5.134399928900292,
+      "tokens_seen": 60096512
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004958475426278837,
+      "loss": 4.0519,
+      "theoretical_loss": 5.13356829412175,
+      "tokens_seen": 60162048
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004958375125376129,
+      "loss": 4.1545,
+      "theoretical_loss": 5.132737818113444,
+      "tokens_seen": 60227584
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004958274824473421,
+      "loss": 4.2355,
+      "theoretical_loss": 5.131908498002513,
+      "tokens_seen": 60293120
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004958174523570712,
+      "loss": 3.8189,
+      "theoretical_loss": 5.131080330926334,
+      "tokens_seen": 60358656
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004958074222668004,
+      "loss": 4.1869,
+      "theoretical_loss": 5.130253314032469,
+      "tokens_seen": 60424192
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004957973921765296,
+      "loss": 3.728,
+      "theoretical_loss": 5.1294274444786225,
+      "tokens_seen": 60489728
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004957873620862588,
+      "loss": 4.1881,
+      "theoretical_loss": 5.128602719432592,
+      "tokens_seen": 60555264
+    },
+    {
+      "epoch": 0.02,
+      "objective/train/docs_used": 45571,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 4.731624603271484,
+      "objective/train/theoretical_loss": 5.12777913607222,
+      "objective/train/tokens_used": 81080800,
+      "theoretical_loss": 5.12777913607222,
+      "tokens_seen": 60620800
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.000495777331995988,
+      "loss": 4.3579,
+      "theoretical_loss": 5.12777913607222,
+      "tokens_seen": 60620800
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004957673019057172,
+      "loss": 3.8709,
+      "theoretical_loss": 5.126956691585351,
+      "tokens_seen": 60686336
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004957572718154463,
+      "loss": 3.9854,
+      "theoretical_loss": 5.1261353831697845,
+      "tokens_seen": 60751872
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004957472417251755,
+      "loss": 4.2274,
+      "theoretical_loss": 5.125315208033226,
+      "tokens_seen": 60817408
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004957372116349047,
+      "loss": 3.8993,
+      "theoretical_loss": 5.124496163393244,
+      "tokens_seen": 60882944
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004957271815446339,
+      "loss": 3.7806,
+      "theoretical_loss": 5.123678246477226,
+      "tokens_seen": 60948480
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004957171514543631,
+      "loss": 3.977,
+      "theoretical_loss": 5.122861454522329,
+      "tokens_seen": 61014016
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004957071213640923,
+      "loss": 3.9739,
+      "theoretical_loss": 5.122045784775439,
+      "tokens_seen": 61079552
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004956970912738214,
+      "loss": 3.8256,
+      "theoretical_loss": 5.1212312344931235,
+      "tokens_seen": 61145088
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004956870611835507,
+      "loss": 4.0096,
+      "theoretical_loss": 5.120417800941587,
+      "tokens_seen": 61210624
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004956770310932798,
+      "loss": 4.0399,
+      "theoretical_loss": 5.119605481396629,
+      "tokens_seen": 61276160
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004956670010030091,
+      "loss": 3.9718,
+      "theoretical_loss": 5.1187942731436,
+      "tokens_seen": 61341696
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004956569709127383,
+      "loss": 3.9982,
+      "theoretical_loss": 5.117984173477356,
+      "tokens_seen": 61407232
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004956469408224674,
+      "loss": 4.0777,
+      "theoretical_loss": 5.117175179702215,
+      "tokens_seen": 61472768
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004956369107321966,
+      "loss": 3.9865,
+      "theoretical_loss": 5.116367289131917,
+      "tokens_seen": 61538304
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004956268806419258,
+      "loss": 4.1915,
+      "theoretical_loss": 5.115560499089575,
+      "tokens_seen": 61603840
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.000495616850551655,
+      "loss": 3.8813,
+      "theoretical_loss": 5.1147548069076425,
+      "tokens_seen": 61669376
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004956068204613842,
+      "loss": 3.97,
+      "theoretical_loss": 5.113950209927859,
+      "tokens_seen": 61734912
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004955967903711133,
+      "loss": 3.5137,
+      "theoretical_loss": 5.113146705501217,
+      "tokens_seen": 61800448
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004955867602808425,
+      "loss": 3.7782,
+      "theoretical_loss": 5.112344290987915,
+      "tokens_seen": 61865984
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004955767301905717,
+      "loss": 4.1055,
+      "theoretical_loss": 5.1115429637573175,
+      "tokens_seen": 61931520
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004955667001003009,
+      "loss": 3.9588,
+      "theoretical_loss": 5.110742721187915,
+      "tokens_seen": 61997056
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004955566700100301,
+      "loss": 3.8558,
+      "theoretical_loss": 5.109943560667279,
+      "tokens_seen": 62062592
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004955466399197592,
+      "loss": 3.8969,
+      "theoretical_loss": 5.109145479592028,
+      "tokens_seen": 62128128
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004955366098294884,
+      "loss": 4.1078,
+      "theoretical_loss": 5.108348475367774,
+      "tokens_seen": 62193664
+    },
+    {
+      "epoch": 0.02,
+      "objective/train/docs_used": 46598,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 4.215669631958008,
+      "objective/train/theoretical_loss": 5.107552545409097,
+      "objective/train/tokens_used": 82719200,
+      "theoretical_loss": 5.107552545409097,
+      "tokens_seen": 62259200
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004955265797392177,
+      "loss": 4.056,
+      "theoretical_loss": 5.107552545409097,
+      "tokens_seen": 62259200
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004955165496489468,
+      "loss": 3.9227,
+      "theoretical_loss": 5.106757687139494,
+      "tokens_seen": 62324736
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004955065195586761,
+      "loss": 4.2246,
+      "theoretical_loss": 5.1059638979913435,
+      "tokens_seen": 62390272
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004954964894684052,
+      "loss": 4.0438,
+      "theoretical_loss": 5.105171175405864,
+      "tokens_seen": 62455808
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004954864593781344,
+      "loss": 4.0275,
+      "theoretical_loss": 5.104379516833075,
+      "tokens_seen": 62521344
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004954764292878636,
+      "loss": 4.005,
+      "theoretical_loss": 5.103588919731757,
+      "tokens_seen": 62586880
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004954663991975928,
+      "loss": 3.937,
+      "theoretical_loss": 5.1027993815694135,
+      "tokens_seen": 62652416
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.000495456369107322,
+      "loss": 4.1536,
+      "theoretical_loss": 5.10201089982223,
+      "tokens_seen": 62717952
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004954463390170512,
+      "loss": 3.971,
+      "theoretical_loss": 5.101223471975038,
+      "tokens_seen": 62783488
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004954363089267803,
+      "loss": 3.9062,
+      "theoretical_loss": 5.100437095521272,
+      "tokens_seen": 62849024
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004954262788365095,
+      "loss": 3.8866,
+      "theoretical_loss": 5.099651767962936,
+      "tokens_seen": 62914560
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004954162487462387,
+      "loss": 3.8535,
+      "theoretical_loss": 5.098867486810562,
+      "tokens_seen": 62980096
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004954062186559679,
+      "loss": 3.7859,
+      "theoretical_loss": 5.098084249583175,
+      "tokens_seen": 63045632
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004953961885656971,
+      "loss": 3.9093,
+      "theoretical_loss": 5.097302053808251,
+      "tokens_seen": 63111168
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004953861584754263,
+      "loss": 3.5672,
+      "theoretical_loss": 5.0965208970216835,
+      "tokens_seen": 63176704
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004953761283851555,
+      "loss": 3.8846,
+      "theoretical_loss": 5.095740776767743,
+      "tokens_seen": 63242240
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004953660982948846,
+      "loss": 3.9469,
+      "theoretical_loss": 5.0949616905990425,
+      "tokens_seen": 63307776
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004953560682046139,
+      "loss": 4.2765,
+      "theoretical_loss": 5.094183636076501,
+      "tokens_seen": 63373312
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004953460381143431,
+      "loss": 4.015,
+      "theoretical_loss": 5.093406610769302,
+      "tokens_seen": 63438848
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004953360080240723,
+      "loss": 4.2246,
+      "theoretical_loss": 5.092630612254862,
+      "tokens_seen": 63504384
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004953259779338014,
+      "loss": 3.9173,
+      "theoretical_loss": 5.091855638118792,
+      "tokens_seen": 63569920
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004953159478435306,
+      "loss": 4.0249,
+      "theoretical_loss": 5.091081685954863,
+      "tokens_seen": 63635456
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004953059177532598,
+      "loss": 3.7978,
+      "theoretical_loss": 5.090308753364967,
+      "tokens_seen": 63700992
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.000495295887662989,
+      "loss": 3.8605,
+      "theoretical_loss": 5.089536837959084,
+      "tokens_seen": 63766528
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004952858575727182,
+      "loss": 3.9657,
+      "theoretical_loss": 5.0887659373552445,
+      "tokens_seen": 63832064
+    },
+    {
+      "epoch": 0.02,
+      "objective/train/docs_used": 47189,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 3.3733479976654053,
+      "objective/train/theoretical_loss": 5.087996049179497,
+      "objective/train/tokens_used": 84357600,
+      "theoretical_loss": 5.087996049179497,
+      "tokens_seen": 63897600
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004952758274824474,
+      "loss": 3.6824,
+      "theoretical_loss": 5.087996049179497,
+      "tokens_seen": 63897600
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004952657973921765,
+      "loss": 3.7677,
+      "theoretical_loss": 5.087227171065869,
+      "tokens_seen": 63963136
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004952557673019057,
+      "loss": 3.9197,
+      "theoretical_loss": 5.086459300656337,
+      "tokens_seen": 64028672
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004952457372116349,
+      "loss": 3.927,
+      "theoretical_loss": 5.085692435600783,
+      "tokens_seen": 64094208
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004952357071213641,
+      "loss": 3.8821,
+      "theoretical_loss": 5.084926573556972,
+      "tokens_seen": 64159744
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004952256770310933,
+      "loss": 3.7931,
+      "theoretical_loss": 5.084161712190507,
+      "tokens_seen": 64225280
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004952156469408225,
+      "loss": 3.7647,
+      "theoretical_loss": 5.083397849174801,
+      "tokens_seen": 64290816
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004952056168505516,
+      "loss": 3.9431,
+      "theoretical_loss": 5.082634982191042,
+      "tokens_seen": 64356352
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004951955867602809,
+      "loss": 3.9173,
+      "theoretical_loss": 5.081873108928154,
+      "tokens_seen": 64421888
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.00049518555667001,
+      "loss": 4.0204,
+      "theoretical_loss": 5.081112227082771,
+      "tokens_seen": 64487424
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004951755265797393,
+      "loss": 3.816,
+      "theoretical_loss": 5.0803523343591985,
+      "tokens_seen": 64552960
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004951654964894685,
+      "loss": 3.8761,
+      "theoretical_loss": 5.079593428469382,
+      "tokens_seen": 64618496
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004951554663991976,
+      "loss": 4.0756,
+      "theoretical_loss": 5.078835507132873,
+      "tokens_seen": 64684032
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004951454363089268,
+      "loss": 3.8136,
+      "theoretical_loss": 5.078078568076798,
+      "tokens_seen": 64749568
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.000495135406218656,
+      "loss": 4.0581,
+      "theoretical_loss": 5.077322609035821,
+      "tokens_seen": 64815104
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004951253761283852,
+      "loss": 3.8215,
+      "theoretical_loss": 5.076567627752118,
+      "tokens_seen": 64880640
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004951153460381144,
+      "loss": 3.8355,
+      "theoretical_loss": 5.075813621975337,
+      "tokens_seen": 64946176
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004951053159478435,
+      "loss": 3.9719,
+      "theoretical_loss": 5.075060589462572,
+      "tokens_seen": 65011712
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004950952858575727,
+      "loss": 3.7383,
+      "theoretical_loss": 5.074308527978326,
+      "tokens_seen": 65077248
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004950852557673019,
+      "loss": 3.8755,
+      "theoretical_loss": 5.073557435294486,
+      "tokens_seen": 65142784
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004950752256770311,
+      "loss": 3.6252,
+      "theoretical_loss": 5.07280730919028,
+      "tokens_seen": 65208320
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004950651955867603,
+      "loss": 3.7529,
+      "theoretical_loss": 5.072058147452257,
+      "tokens_seen": 65273856
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004950551654964894,
+      "loss": 3.679,
+      "theoretical_loss": 5.071309947874248,
+      "tokens_seen": 65339392
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004950451354062186,
+      "loss": 3.5894,
+      "theoretical_loss": 5.070562708257339,
+      "tokens_seen": 65404928
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004950351053159479,
+      "loss": 3.9951,
+      "theoretical_loss": 5.069816426409837,
+      "tokens_seen": 65470464
+    },
+    {
+      "epoch": 0.02,
+      "objective/train/docs_used": 48433,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 3.8971009254455566,
+      "objective/train/theoretical_loss": 5.069071100147241,
+      "objective/train/tokens_used": 85996000,
+      "theoretical_loss": 5.069071100147241,
+      "tokens_seen": 65536000
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.000495025075225677,
+      "loss": 3.9133,
+      "theoretical_loss": 5.069071100147241,
+      "tokens_seen": 65536000
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004950150451354063,
+      "loss": 4.2034,
+      "theoretical_loss": 5.068326727292211,
+      "tokens_seen": 65601536
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004950050150451354,
+      "loss": 3.7703,
+      "theoretical_loss": 5.067583305674537,
+      "tokens_seen": 65667072
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004949949849548646,
+      "loss": 3.8678,
+      "theoretical_loss": 5.066840833131106,
+      "tokens_seen": 65732608
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004949849548645938,
+      "loss": 3.8256,
+      "theoretical_loss": 5.066099307505878,
+      "tokens_seen": 65798144
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.000494974924774323,
+      "loss": 3.6242,
+      "theoretical_loss": 5.06535872664985,
+      "tokens_seen": 65863680
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004949648946840522,
+      "loss": 3.9033,
+      "theoretical_loss": 5.064619088421029,
+      "tokens_seen": 65929216
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004949548645937814,
+      "loss": 3.5217,
+      "theoretical_loss": 5.0638803906844,
+      "tokens_seen": 65994752
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004949448345035105,
+      "loss": 3.8469,
+      "theoretical_loss": 5.0631426313119,
+      "tokens_seen": 66060288
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004949348044132397,
+      "loss": 3.7472,
+      "theoretical_loss": 5.062405808182383,
+      "tokens_seen": 66125824
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004949247743229689,
+      "loss": 3.4883,
+      "theoretical_loss": 5.061669919181599,
+      "tokens_seen": 66191360
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004949147442326981,
+      "loss": 3.8441,
+      "theoretical_loss": 5.060934962202155,
+      "tokens_seen": 66256896
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004949047141424273,
+      "loss": 3.9786,
+      "theoretical_loss": 5.060200935143492,
+      "tokens_seen": 66322432
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004948946840521565,
+      "loss": 4.0608,
+      "theoretical_loss": 5.059467835911857,
+      "tokens_seen": 66387968
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004948846539618856,
+      "loss": 3.7521,
+      "theoretical_loss": 5.058735662420268,
+      "tokens_seen": 66453504
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004948746238716148,
+      "loss": 3.6167,
+      "theoretical_loss": 5.0580044125884935,
+      "tokens_seen": 66519040
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.000494864593781344,
+      "loss": 3.76,
+      "theoretical_loss": 5.057274084343018,
+      "tokens_seen": 66584576
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004948545636910733,
+      "loss": 3.8083,
+      "theoretical_loss": 5.056544675617017,
+      "tokens_seen": 66650112
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004948445336008024,
+      "loss": 3.7601,
+      "theoretical_loss": 5.055816184350326,
+      "tokens_seen": 66715648
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004948345035105316,
+      "loss": 3.8069,
+      "theoretical_loss": 5.055088608489417,
+      "tokens_seen": 66781184
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004948244734202607,
+      "loss": 3.3711,
+      "theoretical_loss": 5.054361945987363,
+      "tokens_seen": 66846720
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.00049481444332999,
+      "loss": 3.3429,
+      "theoretical_loss": 5.05363619480382,
+      "tokens_seen": 66912256
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004948044132397192,
+      "loss": 4.0226,
+      "theoretical_loss": 5.052911352904992,
+      "tokens_seen": 66977792
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004947943831494484,
+      "loss": 3.7436,
+      "theoretical_loss": 5.052187418263609,
+      "tokens_seen": 67043328
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004947843530591776,
+      "loss": 3.8333,
+      "theoretical_loss": 5.051464388858893,
+      "tokens_seen": 67108864
+    },
+    {
+      "epoch": 0.02,
+      "objective/train/docs_used": 48877,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 3.7897393703460693,
+      "objective/train/theoretical_loss": 5.050742262676536,
+      "objective/train/tokens_used": 87634400,
+      "theoretical_loss": 5.050742262676536,
+      "tokens_seen": 67174400
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004947743229689067,
+      "loss": 3.7583,
+      "theoretical_loss": 5.050742262676536,
+      "tokens_seen": 67174400
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004947642928786359,
+      "loss": 4.0895,
+      "theoretical_loss": 5.050021037708673,
+      "tokens_seen": 67239936
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004947542627883651,
+      "loss": 3.642,
+      "theoretical_loss": 5.049300711953854,
+      "tokens_seen": 67305472
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004947442326980943,
+      "loss": 3.7434,
+      "theoretical_loss": 5.048581283417015,
+      "tokens_seen": 67371008
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004947342026078235,
+      "loss": 3.7182,
+      "theoretical_loss": 5.047862750109456,
+      "tokens_seen": 67436544
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004947241725175527,
+      "loss": 3.9326,
+      "theoretical_loss": 5.047145110048811,
+      "tokens_seen": 67502080
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004947141424272818,
+      "loss": 3.638,
+      "theoretical_loss": 5.046428361259022,
+      "tokens_seen": 67567616
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004947041123370111,
+      "loss": 3.7471,
+      "theoretical_loss": 5.045712501770317,
+      "tokens_seen": 67633152
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004946940822467402,
+      "loss": 3.699,
+      "theoretical_loss": 5.0449975296191765,
+      "tokens_seen": 67698688
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004946840521564695,
+      "loss": 3.8026,
+      "theoretical_loss": 5.0442834428483145,
+      "tokens_seen": 67764224
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004946740220661987,
+      "loss": 3.7185,
+      "theoretical_loss": 5.043570239506648,
+      "tokens_seen": 67829760
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004946639919759278,
+      "loss": 3.8962,
+      "theoretical_loss": 5.042857917649275,
+      "tokens_seen": 67895296
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.000494653961885657,
+      "loss": 3.7692,
+      "theoretical_loss": 5.042146475337448,
+      "tokens_seen": 67960832
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004946439317953862,
+      "loss": 3.5141,
+      "theoretical_loss": 5.041435910638544,
+      "tokens_seen": 68026368
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004946339017051154,
+      "loss": 3.8396,
+      "theoretical_loss": 5.040726221626048,
+      "tokens_seen": 68091904
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004946238716148446,
+      "loss": 3.8633,
+      "theoretical_loss": 5.040017406379521,
+      "tokens_seen": 68157440
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004946138415245737,
+      "loss": 3.6866,
+      "theoretical_loss": 5.0393094629845745,
+      "tokens_seen": 68222976
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004946038114343029,
+      "loss": 3.5706,
+      "theoretical_loss": 5.038602389532853,
+      "tokens_seen": 68288512
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004945937813440321,
+      "loss": 3.4773,
+      "theoretical_loss": 5.037896184122,
+      "tokens_seen": 68354048
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004945837512537613,
+      "loss": 3.7606,
+      "theoretical_loss": 5.037190844855641,
+      "tokens_seen": 68419584
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004945737211634905,
+      "loss": 3.8062,
+      "theoretical_loss": 5.036486369843355,
+      "tokens_seen": 68485120
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004945636910732196,
+      "loss": 3.6164,
+      "theoretical_loss": 5.03578275720065,
+      "tokens_seen": 68550656
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004945536609829488,
+      "loss": 3.7046,
+      "theoretical_loss": 5.03508000504894,
+      "tokens_seen": 68616192
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004945436308926781,
+      "loss": 3.7478,
+      "theoretical_loss": 5.034378111515521,
+      "tokens_seen": 68681728
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004945336008024072,
+      "loss": 3.663,
+      "theoretical_loss": 5.0336770747335455,
+      "tokens_seen": 68747264
+    },
+    {
+      "epoch": 0.02,
+      "objective/train/docs_used": 49922,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 2.9727299213409424,
+      "objective/train/theoretical_loss": 5.032976892842002,
+      "objective/train/tokens_used": 89272800,
+      "theoretical_loss": 5.032976892842002,
+      "tokens_seen": 68812800
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004945235707121365,
+      "loss": 3.6731,
+      "theoretical_loss": 5.032976892842002,
+      "tokens_seen": 68812800
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004945135406218656,
+      "loss": 3.8125,
+      "theoretical_loss": 5.032277563985687,
+      "tokens_seen": 68878336
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004945035105315948,
+      "loss": 3.8008,
+      "theoretical_loss": 5.031579086315183,
+      "tokens_seen": 68943872
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.000494493480441324,
+      "loss": 3.9551,
+      "theoretical_loss": 5.030881457986837,
+      "tokens_seen": 69009408
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004944834503510532,
+      "loss": 3.843,
+      "theoretical_loss": 5.030184677162734,
+      "tokens_seen": 69074944
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004944734202607824,
+      "loss": 3.5448,
+      "theoretical_loss": 5.029488742010676,
+      "tokens_seen": 69140480
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004944633901705116,
+      "loss": 3.8376,
+      "theoretical_loss": 5.028793650704154,
+      "tokens_seen": 69206016
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004944533600802407,
+      "loss": 3.9125,
+      "theoretical_loss": 5.028099401422336,
+      "tokens_seen": 69271552
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004944433299899699,
+      "loss": 3.6831,
+      "theoretical_loss": 5.0274059923500305,
+      "tokens_seen": 69337088
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004944332998996991,
+      "loss": 3.7007,
+      "theoretical_loss": 5.026713421677673,
+      "tokens_seen": 69402624
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004944232698094283,
+      "loss": 3.6241,
+      "theoretical_loss": 5.0260216876013,
+      "tokens_seen": 69468160
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004944132397191575,
+      "loss": 3.7006,
+      "theoretical_loss": 5.025330788322526,
+      "tokens_seen": 69533696
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004944032096288867,
+      "loss": 3.9117,
+      "theoretical_loss": 5.024640722048522,
+      "tokens_seen": 69599232
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004943931795386158,
+      "loss": 3.4086,
+      "theoretical_loss": 5.023951486991994,
+      "tokens_seen": 69664768
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.000494383149448345,
+      "loss": 3.9368,
+      "theoretical_loss": 5.023263081371157,
+      "tokens_seen": 69730304
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004943731193580742,
+      "loss": 3.8784,
+      "theoretical_loss": 5.022575503409717,
+      "tokens_seen": 69795840
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004943630892678035,
+      "loss": 3.941,
+      "theoretical_loss": 5.021888751336848,
+      "tokens_seen": 69861376
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004943530591775326,
+      "loss": 3.8625,
+      "theoretical_loss": 5.02120282338717,
+      "tokens_seen": 69926912
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004943430290872618,
+      "loss": 3.6762,
+      "theoretical_loss": 5.020517717800723,
+      "tokens_seen": 69992448
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004943329989969909,
+      "loss": 3.4889,
+      "theoretical_loss": 5.019833432822953,
+      "tokens_seen": 70057984
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004943229689067202,
+      "loss": 3.5122,
+      "theoretical_loss": 5.019149966704683,
+      "tokens_seen": 70123520
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004943129388164494,
+      "loss": 3.6293,
+      "theoretical_loss": 5.018467317702097,
+      "tokens_seen": 70189056
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004943029087261786,
+      "loss": 3.5085,
+      "theoretical_loss": 5.017785484076715,
+      "tokens_seen": 70254592
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004942928786359078,
+      "loss": 3.6348,
+      "theoretical_loss": 5.017104464095375,
+      "tokens_seen": 70320128
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004942828485456369,
+      "loss": 3.6907,
+      "theoretical_loss": 5.0164242560302075,
+      "tokens_seen": 70385664
+    },
+    {
+      "epoch": 0.02,
+      "objective/train/docs_used": 51260,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 3.9014105796813965,
+      "objective/train/theoretical_loss": 5.015744858158618,
+      "objective/train/tokens_used": 90911200,
+      "theoretical_loss": 5.015744858158618,
+      "tokens_seen": 70451200
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004942728184553661,
+      "loss": 3.8765,
+      "theoretical_loss": 5.015744858158618,
+      "tokens_seen": 70451200
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004942627883650953,
+      "loss": 3.8516,
+      "theoretical_loss": 5.015066268763264,
+      "tokens_seen": 70516736
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004942527582748245,
+      "loss": 3.6519,
+      "theoretical_loss": 5.014388486132036,
+      "tokens_seen": 70582272
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004942427281845537,
+      "loss": 3.9132,
+      "theoretical_loss": 5.0137115085580355,
+      "tokens_seen": 70647808
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004942326980942828,
+      "loss": 3.8834,
+      "theoretical_loss": 5.013035334339554,
+      "tokens_seen": 70713344
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.000494222668004012,
+      "loss": 3.8794,
+      "theoretical_loss": 5.012359961780053,
+      "tokens_seen": 70778880
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004942126379137412,
+      "loss": 3.8483,
+      "theoretical_loss": 5.011685389188142,
+      "tokens_seen": 70844416
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004942026078234704,
+      "loss": 3.4989,
+      "theoretical_loss": 5.011011614877562,
+      "tokens_seen": 70909952
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004941925777331996,
+      "loss": 3.7672,
+      "theoretical_loss": 5.010338637167162,
+      "tokens_seen": 70975488
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004941825476429289,
+      "loss": 3.6359,
+      "theoretical_loss": 5.009666454380877,
+      "tokens_seen": 71041024
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004941725175526579,
+      "loss": 3.8619,
+      "theoretical_loss": 5.008995064847716,
+      "tokens_seen": 71106560
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004941624874623872,
+      "loss": 3.8008,
+      "theoretical_loss": 5.0083244669017315,
+      "tokens_seen": 71172096
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004941524573721163,
+      "loss": 3.9147,
+      "theoretical_loss": 5.0076546588820054,
+      "tokens_seen": 71237632
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004941424272818456,
+      "loss": 3.3797,
+      "theoretical_loss": 5.00698563913263,
+      "tokens_seen": 71303168
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004941323971915748,
+      "loss": 3.617,
+      "theoretical_loss": 5.006317406002687,
+      "tokens_seen": 71368704
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004941223671013039,
+      "loss": 3.5556,
+      "theoretical_loss": 5.0056499578462255,
+      "tokens_seen": 71434240
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004941123370110331,
+      "loss": 3.4481,
+      "theoretical_loss": 5.0049832930222475,
+      "tokens_seen": 71499776
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004941023069207623,
+      "loss": 3.826,
+      "theoretical_loss": 5.004317409894682,
+      "tokens_seen": 71565312
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004940922768304915,
+      "loss": 3.6718,
+      "theoretical_loss": 5.003652306832372,
+      "tokens_seen": 71630848
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004940822467402207,
+      "loss": 3.8578,
+      "theoretical_loss": 5.002987982209052,
+      "tokens_seen": 71696384
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004940722166499498,
+      "loss": 3.758,
+      "theoretical_loss": 5.00232443440333,
+      "tokens_seen": 71761920
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.000494062186559679,
+      "loss": 3.4768,
+      "theoretical_loss": 5.001661661798666,
+      "tokens_seen": 71827456
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004940521564694082,
+      "loss": 3.6442,
+      "theoretical_loss": 5.000999662783359,
+      "tokens_seen": 71892992
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004940421263791374,
+      "loss": 3.7435,
+      "theoretical_loss": 5.0003384357505185,
+      "tokens_seen": 71958528
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004940320962888666,
+      "loss": 3.8745,
+      "theoretical_loss": 4.999677979098056,
+      "tokens_seen": 72024064
+    },
+    {
+      "epoch": 0.02,
+      "objective/train/docs_used": 51878,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 3.4065515995025635,
+      "objective/train/theoretical_loss": 4.999018291228659,
+      "objective/train/tokens_used": 92549600,
+      "theoretical_loss": 4.999018291228659,
+      "tokens_seen": 72089600
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004940220661985958,
+      "loss": 3.5808,
+      "theoretical_loss": 4.999018291228659,
+      "tokens_seen": 72089600
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004940120361083249,
+      "loss": 3.8043,
+      "theoretical_loss": 4.998359370549778,
+      "tokens_seen": 72155136
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004940020060180542,
+      "loss": 3.649,
+      "theoretical_loss": 4.997701215473601,
+      "tokens_seen": 72220672
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004939919759277834,
+      "loss": 3.6107,
+      "theoretical_loss": 4.997043824417044,
+      "tokens_seen": 72286208
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004939819458375126,
+      "loss": 3.6793,
+      "theoretical_loss": 4.996387195801723,
+      "tokens_seen": 72351744
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004939719157472418,
+      "loss": 3.6529,
+      "theoretical_loss": 4.995731328053944,
+      "tokens_seen": 72417280
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.000493961885656971,
+      "loss": 3.7394,
+      "theoretical_loss": 4.995076219604682,
+      "tokens_seen": 72482816
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004939518555667001,
+      "loss": 3.6256,
+      "theoretical_loss": 4.994421868889559,
+      "tokens_seen": 72548352
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004939418254764293,
+      "loss": 3.6209,
+      "theoretical_loss": 4.9937682743488345,
+      "tokens_seen": 72613888
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004939317953861585,
+      "loss": 3.5004,
+      "theoretical_loss": 4.993115434427376,
+      "tokens_seen": 72679424
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004939217652958877,
+      "loss": 3.5103,
+      "theoretical_loss": 4.992463347574654,
+      "tokens_seen": 72744960
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004939117352056169,
+      "loss": 3.7514,
+      "theoretical_loss": 4.991812012244717,
+      "tokens_seen": 72810496
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.000493901705115346,
+      "loss": 3.5319,
+      "theoretical_loss": 4.99116142689617,
+      "tokens_seen": 72876032
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004938916750250752,
+      "loss": 3.8628,
+      "theoretical_loss": 4.990511589992169,
+      "tokens_seen": 72941568
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004938816449348044,
+      "loss": 4.0163,
+      "theoretical_loss": 4.989862500000392,
+      "tokens_seen": 73007104
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004938716148445337,
+      "loss": 3.7264,
+      "theoretical_loss": 4.989214155393025,
+      "tokens_seen": 73072640
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004938615847542628,
+      "loss": 3.9026,
+      "theoretical_loss": 4.988566554646752,
+      "tokens_seen": 73138176
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.000493851554663992,
+      "loss": 3.4925,
+      "theoretical_loss": 4.9879196962427255,
+      "tokens_seen": 73203712
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004938415245737211,
+      "loss": 3.5623,
+      "theoretical_loss": 4.9872735786665565,
+      "tokens_seen": 73269248
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004938314944834504,
+      "loss": 3.415,
+      "theoretical_loss": 4.986628200408299,
+      "tokens_seen": 73334784
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004938214643931796,
+      "loss": 3.5253,
+      "theoretical_loss": 4.985983559962429,
+      "tokens_seen": 73400320
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004938114343029088,
+      "loss": 3.5016,
+      "theoretical_loss": 4.985339655827827,
+      "tokens_seen": 73465856
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.000493801404212638,
+      "loss": 3.8156,
+      "theoretical_loss": 4.984696486507767,
+      "tokens_seen": 73531392
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004937913741223671,
+      "loss": 3.7367,
+      "theoretical_loss": 4.984054050509893,
+      "tokens_seen": 73596928
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004937813440320963,
+      "loss": 3.6885,
+      "theoretical_loss": 4.983412346346206,
+      "tokens_seen": 73662464
+    },
+    {
+      "epoch": 0.02,
+      "objective/train/docs_used": 53017,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 3.889392137527466,
+      "objective/train/theoretical_loss": 4.982771372533048,
+      "objective/train/tokens_used": 94188000,
+      "theoretical_loss": 4.982771372533048,
+      "tokens_seen": 73728000
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004937713139418255,
+      "loss": 3.6721,
+      "theoretical_loss": 4.982771372533048,
+      "tokens_seen": 73728000
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004937612838515547,
+      "loss": 3.9426,
+      "theoretical_loss": 4.982131127591083,
+      "tokens_seen": 73793536
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004937512537612839,
+      "loss": 3.6135,
+      "theoretical_loss": 4.981491610045282,
+      "tokens_seen": 73859072
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.000493741223671013,
+      "loss": 3.6257,
+      "theoretical_loss": 4.980852818424905,
+      "tokens_seen": 73924608
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004937311935807422,
+      "loss": 3.6675,
+      "theoretical_loss": 4.980214751263491,
+      "tokens_seen": 73990144
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004937211634904714,
+      "loss": 3.6453,
+      "theoretical_loss": 4.979577407098831,
+      "tokens_seen": 74055680
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004937111334002006,
+      "loss": 3.7323,
+      "theoretical_loss": 4.978940784472962,
+      "tokens_seen": 74121216
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004937011033099298,
+      "loss": 3.4667,
+      "theoretical_loss": 4.978304881932145,
+      "tokens_seen": 74186752
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004936910732196591,
+      "loss": 3.9063,
+      "theoretical_loss": 4.97766969802685,
+      "tokens_seen": 74252288
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004936810431293881,
+      "loss": 3.3322,
+      "theoretical_loss": 4.977035231311745,
+      "tokens_seen": 74317824
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004936710130391174,
+      "loss": 3.6439,
+      "theoretical_loss": 4.976401480345672,
+      "tokens_seen": 74383360
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004936609829488465,
+      "loss": 3.5552,
+      "theoretical_loss": 4.975768443691636,
+      "tokens_seen": 74448896
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004936509528585758,
+      "loss": 3.5095,
+      "theoretical_loss": 4.97513611991679,
+      "tokens_seen": 74514432
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.000493640922768305,
+      "loss": 3.7738,
+      "theoretical_loss": 4.974504507592419,
+      "tokens_seen": 74579968
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004936308926780341,
+      "loss": 3.8278,
+      "theoretical_loss": 4.973873605293922,
+      "tokens_seen": 74645504
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004936208625877633,
+      "loss": 3.7248,
+      "theoretical_loss": 4.973243411600797,
+      "tokens_seen": 74711040
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004936108324974925,
+      "loss": 3.8762,
+      "theoretical_loss": 4.9726139250966295,
+      "tokens_seen": 74776576
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004936008024072217,
+      "loss": 3.6798,
+      "theoretical_loss": 4.971985144369073,
+      "tokens_seen": 74842112
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004935907723169509,
+      "loss": 3.6282,
+      "theoretical_loss": 4.9713570680098345,
+      "tokens_seen": 74907648
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.00049358074222668,
+      "loss": 3.7215,
+      "theoretical_loss": 4.970729694614662,
+      "tokens_seen": 74973184
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004935707121364092,
+      "loss": 3.5638,
+      "theoretical_loss": 4.970103022783325,
+      "tokens_seen": 75038720
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004935606820461384,
+      "loss": 3.5533,
+      "theoretical_loss": 4.969477051119606,
+      "tokens_seen": 75104256
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004935506519558676,
+      "loss": 3.496,
+      "theoretical_loss": 4.968851778231276,
+      "tokens_seen": 75169792
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004935406218655968,
+      "loss": 3.6459,
+      "theoretical_loss": 4.9682272027300876,
+      "tokens_seen": 75235328
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.000493530591775326,
+      "loss": 3.5601,
+      "theoretical_loss": 4.96760332323176,
+      "tokens_seen": 75300864
+    },
+    {
+      "epoch": 0.02,
+      "objective/train/docs_used": 53591,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 3.8066093921661377,
+      "objective/train/theoretical_loss": 4.966980138355957,
+      "objective/train/tokens_used": 95826400,
+      "theoretical_loss": 4.966980138355957,
+      "tokens_seen": 75366400
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004935205616850551,
+      "loss": 3.7377,
+      "theoretical_loss": 4.966980138355957,
+      "tokens_seen": 75366400
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004935105315947844,
+      "loss": 3.6051,
+      "theoretical_loss": 4.966357646726282,
+      "tokens_seen": 75431936
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004935005015045135,
+      "loss": 3.5847,
+      "theoretical_loss": 4.965735846970255,
+      "tokens_seen": 75497472
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004934904714142428,
+      "loss": 3.7027,
+      "theoretical_loss": 4.965114737719302,
+      "tokens_seen": 75563008
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004934804413239719,
+      "loss": 3.648,
+      "theoretical_loss": 4.964494317608743,
+      "tokens_seen": 75628544
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004934704112337011,
+      "loss": 3.4989,
+      "theoretical_loss": 4.963874585277772,
+      "tokens_seen": 75694080
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004934603811434303,
+      "loss": 3.8889,
+      "theoretical_loss": 4.963255539369445,
+      "tokens_seen": 75759616
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004934503510531595,
+      "loss": 3.5961,
+      "theoretical_loss": 4.9626371785306675,
+      "tokens_seen": 75825152
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004934403209628887,
+      "loss": 3.4261,
+      "theoretical_loss": 4.962019501412178,
+      "tokens_seen": 75890688
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004934302908726179,
+      "loss": 3.7473,
+      "theoretical_loss": 4.961402506668535,
+      "tokens_seen": 75956224
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.000493420260782347,
+      "loss": 3.4055,
+      "theoretical_loss": 4.960786192958103,
+      "tokens_seen": 76021760
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004934102306920762,
+      "loss": 3.6034,
+      "theoretical_loss": 4.960170558943037,
+      "tokens_seen": 76087296
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004934002006018054,
+      "loss": 3.4187,
+      "theoretical_loss": 4.959555603289269,
+      "tokens_seen": 76152832
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004933901705115346,
+      "loss": 3.6193,
+      "theoretical_loss": 4.9589413246664975,
+      "tokens_seen": 76218368
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004933801404212638,
+      "loss": 3.794,
+      "theoretical_loss": 4.958327721748168,
+      "tokens_seen": 76283904
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.000493370110330993,
+      "loss": 3.3982,
+      "theoretical_loss": 4.957714793211462,
+      "tokens_seen": 76349440
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004933600802407221,
+      "loss": 3.3334,
+      "theoretical_loss": 4.957102537737284,
+      "tokens_seen": 76414976
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004933500501504513,
+      "loss": 3.6421,
+      "theoretical_loss": 4.956490954010249,
+      "tokens_seen": 76480512
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004933400200601805,
+      "loss": 3.5119,
+      "theoretical_loss": 4.9558800407186645,
+      "tokens_seen": 76546048
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004933299899699098,
+      "loss": 3.4933,
+      "theoretical_loss": 4.95526979655452,
+      "tokens_seen": 76611584
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004933199598796389,
+      "loss": 3.7028,
+      "theoretical_loss": 4.954660220213473,
+      "tokens_seen": 76677120
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004933099297893682,
+      "loss": 3.7165,
+      "theoretical_loss": 4.954051310394836,
+      "tokens_seen": 76742656
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004932998996990972,
+      "loss": 3.5507,
+      "theoretical_loss": 4.953443065801563,
+      "tokens_seen": 76808192
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004932898696088265,
+      "loss": 3.4901,
+      "theoretical_loss": 4.952835485140236,
+      "tokens_seen": 76873728
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004932798395185557,
+      "loss": 3.2798,
+      "theoretical_loss": 4.952228567121052,
+      "tokens_seen": 76939264
+    },
+    {
+      "epoch": 0.02,
+      "objective/train/docs_used": 54447,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 3.6758010387420654,
+      "objective/train/theoretical_loss": 4.951622310457807,
+      "objective/train/tokens_used": 97464800,
+      "theoretical_loss": 4.951622310457807,
+      "tokens_seen": 77004800
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004932698094282849,
+      "loss": 3.4817,
+      "theoretical_loss": 4.951622310457807,
+      "tokens_seen": 77004800
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004932597793380141,
+      "loss": 3.5561,
+      "theoretical_loss": 4.95101671386789,
+      "tokens_seen": 77070336
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004932497492477432,
+      "loss": 3.6278,
+      "theoretical_loss": 4.950411776072261,
+      "tokens_seen": 77135872
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004932397191574724,
+      "loss": 3.7609,
+      "theoretical_loss": 4.949807495795445,
+      "tokens_seen": 77201408
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004932296890672016,
+      "loss": 3.6186,
+      "theoretical_loss": 4.949203871765516,
+      "tokens_seen": 77266944
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004932196589769308,
+      "loss": 3.6535,
+      "theoretical_loss": 4.948600902714083,
+      "tokens_seen": 77332480
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.00049320962888666,
+      "loss": 3.4356,
+      "theoretical_loss": 4.947998587376282,
+      "tokens_seen": 77398016
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004931995987963893,
+      "loss": 3.7607,
+      "theoretical_loss": 4.947396924490757,
+      "tokens_seen": 77463552
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004931895687061183,
+      "loss": 3.9334,
+      "theoretical_loss": 4.946795912799653,
+      "tokens_seen": 77529088
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004931795386158476,
+      "loss": 3.4391,
+      "theoretical_loss": 4.946195551048597,
+      "tokens_seen": 77594624
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004931695085255767,
+      "loss": 3.4097,
+      "theoretical_loss": 4.945595837986693,
+      "tokens_seen": 77660160
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.000493159478435306,
+      "loss": 3.4903,
+      "theoretical_loss": 4.944996772366503,
+      "tokens_seen": 77725696
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004931494483450352,
+      "loss": 3.5634,
+      "theoretical_loss": 4.944398352944038,
+      "tokens_seen": 77791232
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004931394182547643,
+      "loss": 3.5448,
+      "theoretical_loss": 4.943800578478745,
+      "tokens_seen": 77856768
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004931293881644935,
+      "loss": 3.4788,
+      "theoretical_loss": 4.9432034477334925,
+      "tokens_seen": 77922304
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004931193580742227,
+      "loss": 3.4963,
+      "theoretical_loss": 4.942606959474562,
+      "tokens_seen": 77987840
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004931093279839519,
+      "loss": 3.4853,
+      "theoretical_loss": 4.942011112471633,
+      "tokens_seen": 78053376
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004930992978936811,
+      "loss": 3.3058,
+      "theoretical_loss": 4.94141590549777,
+      "tokens_seen": 78118912
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004930892678034102,
+      "loss": 3.7623,
+      "theoretical_loss": 4.940821337329414,
+      "tokens_seen": 78184448
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004930792377131394,
+      "loss": 3.324,
+      "theoretical_loss": 4.940227406746368,
+      "tokens_seen": 78249984
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004930692076228686,
+      "loss": 3.595,
+      "theoretical_loss": 4.939634112531784,
+      "tokens_seen": 78315520
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004930591775325978,
+      "loss": 3.6571,
+      "theoretical_loss": 4.939041453472152,
+      "tokens_seen": 78381056
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.000493049147442327,
+      "loss": 3.546,
+      "theoretical_loss": 4.9384494283572895,
+      "tokens_seen": 78446592
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004930391173520562,
+      "loss": 3.7395,
+      "theoretical_loss": 4.937858035980327,
+      "tokens_seen": 78512128
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004930290872617853,
+      "loss": 3.5258,
+      "theoretical_loss": 4.9372672751377,
+      "tokens_seen": 78577664
+    },
+    {
+      "epoch": 0.02,
+      "objective/train/docs_used": 54900,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 2.9137701988220215,
+      "objective/train/theoretical_loss": 4.936677144629131,
+      "objective/train/tokens_used": 99103200,
+      "theoretical_loss": 4.936677144629131,
+      "tokens_seen": 78643200
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004930190571715146,
+      "loss": 3.2102,
+      "theoretical_loss": 4.936677144629131,
+      "tokens_seen": 78643200
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004930090270812437,
+      "loss": 3.7118,
+      "theoretical_loss": 4.936087643257625,
+      "tokens_seen": 78708736
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.000492998996990973,
+      "loss": 3.5876,
+      "theoretical_loss": 4.935498769829453,
+      "tokens_seen": 78774272
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004929889669007021,
+      "loss": 3.7919,
+      "theoretical_loss": 4.934910523154139,
+      "tokens_seen": 78839808
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004929789368104313,
+      "loss": 3.8801,
+      "theoretical_loss": 4.934322902044457,
+      "tokens_seen": 78905344
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004929689067201605,
+      "loss": 3.5096,
+      "theoretical_loss": 4.933735905316408,
+      "tokens_seen": 78970880
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004929588766298897,
+      "loss": 3.4217,
+      "theoretical_loss": 4.933149531789217,
+      "tokens_seen": 79036416
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004929488465396189,
+      "loss": 3.6953,
+      "theoretical_loss": 4.932563780285319,
+      "tokens_seen": 79101952
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004929388164493481,
+      "loss": 3.6487,
+      "theoretical_loss": 4.931978649630346,
+      "tokens_seen": 79167488
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004929287863590773,
+      "loss": 3.6641,
+      "theoretical_loss": 4.931394138653117,
+      "tokens_seen": 79233024
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004929187562688064,
+      "loss": 3.8094,
+      "theoretical_loss": 4.930810246185626,
+      "tokens_seen": 79298560
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004929087261785356,
+      "loss": 3.5257,
+      "theoretical_loss": 4.930226971063035,
+      "tokens_seen": 79364096
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004928986960882648,
+      "loss": 3.4985,
+      "theoretical_loss": 4.929644312123655,
+      "tokens_seen": 79429632
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.000492888665997994,
+      "loss": 3.3258,
+      "theoretical_loss": 4.929062268208941,
+      "tokens_seen": 79495168
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004928786359077232,
+      "loss": 3.2647,
+      "theoretical_loss": 4.92848083816348,
+      "tokens_seen": 79560704
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004928686058174523,
+      "loss": 3.8228,
+      "theoretical_loss": 4.927900020834979,
+      "tokens_seen": 79626240
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004928585757271815,
+      "loss": 3.3943,
+      "theoretical_loss": 4.927319815074252,
+      "tokens_seen": 79691776
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004928485456369107,
+      "loss": 3.4187,
+      "theoretical_loss": 4.926740219735214,
+      "tokens_seen": 79757312
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.00049283851554664,
+      "loss": 3.6324,
+      "theoretical_loss": 4.926161233674863,
+      "tokens_seen": 79822848
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004928284854563691,
+      "loss": 3.627,
+      "theoretical_loss": 4.925582855753279,
+      "tokens_seen": 79888384
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004928184553660984,
+      "loss": 3.4882,
+      "theoretical_loss": 4.925005084833603,
+      "tokens_seen": 79953920
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004928084252758274,
+      "loss": 3.8262,
+      "theoretical_loss": 4.924427919782033,
+      "tokens_seen": 80019456
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004927983951855567,
+      "loss": 3.5192,
+      "theoretical_loss": 4.923851359467811,
+      "tokens_seen": 80084992
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004927883650952859,
+      "loss": 3.714,
+      "theoretical_loss": 4.923275402763213,
+      "tokens_seen": 80150528
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004927783350050151,
+      "loss": 3.4187,
+      "theoretical_loss": 4.922700048543536,
+      "tokens_seen": 80216064
+    },
+    {
+      "epoch": 0.02,
+      "objective/train/docs_used": 56090,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 3.1410176753997803,
+      "objective/train/theoretical_loss": 4.922125295687091,
+      "objective/train/tokens_used": 100741600,
+      "theoretical_loss": 4.922125295687091,
+      "tokens_seen": 80281600
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004927683049147443,
+      "loss": 3.5733,
+      "theoretical_loss": 4.922125295687091,
+      "tokens_seen": 80281600
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004927582748244734,
+      "loss": 3.396,
+      "theoretical_loss": 4.92155114307519,
+      "tokens_seen": 80347136
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004927482447342026,
+      "loss": 3.6166,
+      "theoretical_loss": 4.9209775895921375,
+      "tokens_seen": 80412672
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004927382146439318,
+      "loss": 3.4859,
+      "theoretical_loss": 4.920404634125217,
+      "tokens_seen": 80478208
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.000492728184553661,
+      "loss": 3.2846,
+      "theoretical_loss": 4.919832275564684,
+      "tokens_seen": 80543744
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004927181544633902,
+      "loss": 3.653,
+      "theoretical_loss": 4.9192605128037545,
+      "tokens_seen": 80609280
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004927081243731193,
+      "loss": 3.5658,
+      "theoretical_loss": 4.918689344738594,
+      "tokens_seen": 80674816
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004926980942828485,
+      "loss": 3.5587,
+      "theoretical_loss": 4.918118770268305,
+      "tokens_seen": 80740352
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004926880641925777,
+      "loss": 3.3881,
+      "theoretical_loss": 4.917548788294924,
+      "tokens_seen": 80805888
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004926780341023069,
+      "loss": 3.4548,
+      "theoretical_loss": 4.916979397723403,
+      "tokens_seen": 80871424
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004926680040120361,
+      "loss": 3.6868,
+      "theoretical_loss": 4.916410597461606,
+      "tokens_seen": 80936960
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004926579739217654,
+      "loss": 3.6518,
+      "theoretical_loss": 4.9158423864202945,
+      "tokens_seen": 81002496
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004926479438314944,
+      "loss": 3.4168,
+      "theoretical_loss": 4.915274763513119,
+      "tokens_seen": 81068032
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004926379137412237,
+      "loss": 3.353,
+      "theoretical_loss": 4.914707727656609,
+      "tokens_seen": 81133568
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004926278836509528,
+      "loss": 3.3831,
+      "theoretical_loss": 4.914141277770164,
+      "tokens_seen": 81199104
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004926178535606821,
+      "loss": 3.1851,
+      "theoretical_loss": 4.913575412776042,
+      "tokens_seen": 81264640
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004926078234704113,
+      "loss": 3.7187,
+      "theoretical_loss": 4.91301013159935,
+      "tokens_seen": 81330176
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004925977933801404,
+      "loss": 3.5052,
+      "theoretical_loss": 4.912445433168036,
+      "tokens_seen": 81395712
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004925877632898696,
+      "loss": 3.5131,
+      "theoretical_loss": 4.911881316412876,
+      "tokens_seen": 81461248
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004925777331995988,
+      "loss": 3.7694,
+      "theoretical_loss": 4.911317780267467,
+      "tokens_seen": 81526784
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.000492567703109328,
+      "loss": 3.3832,
+      "theoretical_loss": 4.910754823668215,
+      "tokens_seen": 81592320
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004925576730190572,
+      "loss": 3.8546,
+      "theoretical_loss": 4.910192445554329,
+      "tokens_seen": 81657856
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004925476429287864,
+      "loss": 3.6127,
+      "theoretical_loss": 4.909630644867807,
+      "tokens_seen": 81723392
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004925376128385155,
+      "loss": 3.3545,
+      "theoretical_loss": 4.909069420553429,
+      "tokens_seen": 81788928
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004925275827482447,
+      "loss": 3.5589,
+      "theoretical_loss": 4.908508771558748,
+      "tokens_seen": 81854464
+    },
+    {
+      "epoch": 0.02,
+      "objective/train/docs_used": 57069,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 3.6827940940856934,
+      "objective/train/theoretical_loss": 4.907948696834077,
+      "objective/train/tokens_used": 102380000,
+      "theoretical_loss": 4.907948696834077,
+      "tokens_seen": 81920000
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004925175526579739,
+      "loss": 3.7195,
+      "theoretical_loss": 4.907948696834077,
+      "tokens_seen": 81920000
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004925075225677031,
+      "loss": 3.5548,
+      "theoretical_loss": 4.907389195332484,
+      "tokens_seen": 81985536
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004924974924774323,
+      "loss": 3.807,
+      "theoretical_loss": 4.906830266009781,
+      "tokens_seen": 82051072
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004924874623871615,
+      "loss": 3.4511,
+      "theoretical_loss": 4.90627190782451,
+      "tokens_seen": 82116608
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004924774322968907,
+      "loss": 3.7151,
+      "theoretical_loss": 4.905714119737945,
+      "tokens_seen": 82182144
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004924674022066199,
+      "loss": 3.5237,
+      "theoretical_loss": 4.9051569007140685,
+      "tokens_seen": 82247680
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004924573721163491,
+      "loss": 3.332,
+      "theoretical_loss": 4.904600249719573,
+      "tokens_seen": 82313216
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004924473420260783,
+      "loss": 3.504,
+      "theoretical_loss": 4.90404416572385,
+      "tokens_seen": 82378752
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004924373119358075,
+      "loss": 3.7408,
+      "theoretical_loss": 4.903488647698972,
+      "tokens_seen": 82444288
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004924272818455366,
+      "loss": 3.334,
+      "theoretical_loss": 4.902933694619698,
+      "tokens_seen": 82509824
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004924172517552658,
+      "loss": 3.2703,
+      "theoretical_loss": 4.902379305463451,
+      "tokens_seen": 82575360
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.000492407221664995,
+      "loss": 3.5645,
+      "theoretical_loss": 4.901825479210317,
+      "tokens_seen": 82640896
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004923971915747242,
+      "loss": 3.3278,
+      "theoretical_loss": 4.9012722148430345,
+      "tokens_seen": 82706432
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004923871614844534,
+      "loss": 3.6277,
+      "theoretical_loss": 4.900719511346983,
+      "tokens_seen": 82771968
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004923771313941825,
+      "loss": 3.6241,
+      "theoretical_loss": 4.900167367710176,
+      "tokens_seen": 82837504
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004923671013039117,
+      "loss": 3.5454,
+      "theoretical_loss": 4.899615782923251,
+      "tokens_seen": 82903040
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004923570712136409,
+      "loss": 3.6864,
+      "theoretical_loss": 4.899064755979464,
+      "tokens_seen": 82968576
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004923470411233702,
+      "loss": 3.3855,
+      "theoretical_loss": 4.898514285874674,
+      "tokens_seen": 83034112
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004923370110330993,
+      "loss": 3.4924,
+      "theoretical_loss": 4.897964371607341,
+      "tokens_seen": 83099648
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004923269809428286,
+      "loss": 3.4934,
+      "theoretical_loss": 4.897415012178515,
+      "tokens_seen": 83165184
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004923169508525576,
+      "loss": 3.3992,
+      "theoretical_loss": 4.896866206591823,
+      "tokens_seen": 83230720
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004923069207622869,
+      "loss": 3.5205,
+      "theoretical_loss": 4.896317953853467,
+      "tokens_seen": 83296256
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004922968906720161,
+      "loss": 3.624,
+      "theoretical_loss": 4.8957702529722145,
+      "tokens_seen": 83361792
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004922868605817453,
+      "loss": 3.5903,
+      "theoretical_loss": 4.895223102959381,
+      "tokens_seen": 83427328
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004922768304914745,
+      "loss": 3.7871,
+      "theoretical_loss": 4.894676502828836,
+      "tokens_seen": 83492864
+    },
+    {
+      "epoch": 0.03,
+      "objective/train/docs_used": 57750,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 3.804002046585083,
+      "objective/train/theoretical_loss": 4.894130451596979,
+      "objective/train/tokens_used": 104018400,
+      "theoretical_loss": 4.894130451596979,
+      "tokens_seen": 83558400
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004922668004012036,
+      "loss": 3.4882,
+      "theoretical_loss": 4.894130451596979,
+      "tokens_seen": 83558400
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004922567703109328,
+      "loss": 3.2586,
+      "theoretical_loss": 4.893584948282745,
+      "tokens_seen": 83623936
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.000492246740220662,
+      "loss": 3.5408,
+      "theoretical_loss": 4.893039991907587,
+      "tokens_seen": 83689472
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004922367101303912,
+      "loss": 3.6106,
+      "theoretical_loss": 4.892495581495468,
+      "tokens_seen": 83755008
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004922266800401204,
+      "loss": 3.2746,
+      "theoretical_loss": 4.8919517160728585,
+      "tokens_seen": 83820544
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004922166499498495,
+      "loss": 3.5637,
+      "theoretical_loss": 4.891408394668723,
+      "tokens_seen": 83886080
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004922066198595787,
+      "loss": 3.5945,
+      "theoretical_loss": 4.89086561631451,
+      "tokens_seen": 83951616
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004921965897693079,
+      "loss": 3.6297,
+      "theoretical_loss": 4.890323380044153,
+      "tokens_seen": 84017152
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004921865596790371,
+      "loss": 3.6036,
+      "theoretical_loss": 4.889781684894051,
+      "tokens_seen": 84082688
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004921765295887663,
+      "loss": 3.7009,
+      "theoretical_loss": 4.889240529903066,
+      "tokens_seen": 84148224
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004921664994984956,
+      "loss": 3.4759,
+      "theoretical_loss": 4.888699914112516,
+      "tokens_seen": 84213760
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004921564694082246,
+      "loss": 3.4833,
+      "theoretical_loss": 4.888159836566162,
+      "tokens_seen": 84279296
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004921464393179539,
+      "loss": 3.7335,
+      "theoretical_loss": 4.887620296310205,
+      "tokens_seen": 84344832
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.000492136409227683,
+      "loss": 3.8101,
+      "theoretical_loss": 4.887081292393276,
+      "tokens_seen": 84410368
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004921263791374123,
+      "loss": 3.6913,
+      "theoretical_loss": 4.886542823866424,
+      "tokens_seen": 84475904
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004921163490471415,
+      "loss": 3.5273,
+      "theoretical_loss": 4.886004889783116,
+      "tokens_seen": 84541440
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004921063189568706,
+      "loss": 3.5718,
+      "theoretical_loss": 4.885467489199222,
+      "tokens_seen": 84606976
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004920962888665998,
+      "loss": 3.6503,
+      "theoretical_loss": 4.884930621173009,
+      "tokens_seen": 84672512
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.000492086258776329,
+      "loss": 3.5557,
+      "theoretical_loss": 4.884394284765135,
+      "tokens_seen": 84738048
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004920762286860582,
+      "loss": 3.5114,
+      "theoretical_loss": 4.88385847903864,
+      "tokens_seen": 84803584
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004920661985957874,
+      "loss": 3.8069,
+      "theoretical_loss": 4.883323203058936,
+      "tokens_seen": 84869120
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004920561685055166,
+      "loss": 3.529,
+      "theoretical_loss": 4.882788455893802,
+      "tokens_seen": 84934656
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004920461384152457,
+      "loss": 3.5584,
+      "theoretical_loss": 4.882254236613376,
+      "tokens_seen": 85000192
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004920361083249749,
+      "loss": 3.7302,
+      "theoretical_loss": 4.881720544290144,
+      "tokens_seen": 85065728
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004920260782347041,
+      "loss": 3.4352,
+      "theoretical_loss": 4.881187377998938,
+      "tokens_seen": 85131264
+    },
+    {
+      "epoch": 0.03,
+      "objective/train/docs_used": 58904,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 3.6649792194366455,
+      "objective/train/theoretical_loss": 4.880654736816922,
+      "objective/train/tokens_used": 105656800,
+      "theoretical_loss": 4.880654736816922,
+      "tokens_seen": 85196800
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004920160481444333,
+      "loss": 3.7428,
+      "theoretical_loss": 4.880654736816922,
+      "tokens_seen": 85196800
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004920060180541625,
+      "loss": 3.3637,
+      "theoretical_loss": 4.880122619823589,
+      "tokens_seen": 85262336
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004919959879638916,
+      "loss": 3.4179,
+      "theoretical_loss": 4.87959102610075,
+      "tokens_seen": 85327872
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004919859578736209,
+      "loss": 3.3888,
+      "theoretical_loss": 4.879059954732529,
+      "tokens_seen": 85393408
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.00049197592778335,
+      "loss": 3.7427,
+      "theoretical_loss": 4.878529404805356,
+      "tokens_seen": 85458944
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004919658976930793,
+      "loss": 3.7709,
+      "theoretical_loss": 4.8779993754079545,
+      "tokens_seen": 85524480
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004919558676028084,
+      "loss": 3.5494,
+      "theoretical_loss": 4.877469865631338,
+      "tokens_seen": 85590016
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004919458375125377,
+      "loss": 3.5295,
+      "theoretical_loss": 4.876940874568804,
+      "tokens_seen": 85655552
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004919358074222668,
+      "loss": 3.5878,
+      "theoretical_loss": 4.876412401315923,
+      "tokens_seen": 85721088
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.000491925777331996,
+      "loss": 3.57,
+      "theoretical_loss": 4.8758844449705325,
+      "tokens_seen": 85786624
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004919157472417252,
+      "loss": 3.5246,
+      "theoretical_loss": 4.875357004632728,
+      "tokens_seen": 85852160
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004919057171514544,
+      "loss": 3.3963,
+      "theoretical_loss": 4.87483007940486,
+      "tokens_seen": 85917696
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004918956870611836,
+      "loss": 3.6586,
+      "theoretical_loss": 4.874303668391521,
+      "tokens_seen": 85983232
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004918856569709127,
+      "loss": 3.6073,
+      "theoretical_loss": 4.873777770699542,
+      "tokens_seen": 86048768
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004918756268806419,
+      "loss": 3.5692,
+      "theoretical_loss": 4.873252385437985,
+      "tokens_seen": 86114304
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004918655967903711,
+      "loss": 3.5077,
+      "theoretical_loss": 4.872727511718134,
+      "tokens_seen": 86179840
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004918555667001003,
+      "loss": 3.6584,
+      "theoretical_loss": 4.8722031486534885,
+      "tokens_seen": 86245376
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004918455366098295,
+      "loss": 3.4986,
+      "theoretical_loss": 4.8716792953597565,
+      "tokens_seen": 86310912
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004918355065195586,
+      "loss": 3.7466,
+      "theoretical_loss": 4.871155950954847,
+      "tokens_seen": 86376448
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004918254764292878,
+      "loss": 3.627,
+      "theoretical_loss": 4.870633114558865,
+      "tokens_seen": 86441984
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.000491815446339017,
+      "loss": 3.5282,
+      "theoretical_loss": 4.8701107852941,
+      "tokens_seen": 86507520
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004918054162487463,
+      "loss": 3.1841,
+      "theoretical_loss": 4.869588962285023,
+      "tokens_seen": 86573056
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004917953861584754,
+      "loss": 3.6871,
+      "theoretical_loss": 4.869067644658276,
+      "tokens_seen": 86638592
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004917853560682047,
+      "loss": 3.6656,
+      "theoretical_loss": 4.86854683154267,
+      "tokens_seen": 86704128
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004917753259779337,
+      "loss": 3.5156,
+      "theoretical_loss": 4.868026522069172,
+      "tokens_seen": 86769664
+    },
+    {
+      "epoch": 0.03,
+      "objective/train/docs_used": 59520,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 3.652179718017578,
+      "objective/train/theoretical_loss": 4.867506715370903,
+      "objective/train/tokens_used": 107295200,
+      "theoretical_loss": 4.867506715370903,
+      "tokens_seen": 86835200
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.000491765295887663,
+      "loss": 3.4928,
+      "theoretical_loss": 4.867506715370903,
+      "tokens_seen": 86835200
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004917552657973922,
+      "loss": 3.4802,
+      "theoretical_loss": 4.866987410583125,
+      "tokens_seen": 86900736
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004917452357071214,
+      "loss": 3.5752,
+      "theoretical_loss": 4.8664686068432434,
+      "tokens_seen": 86966272
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004917352056168506,
+      "loss": 3.5258,
+      "theoretical_loss": 4.8659503032907905,
+      "tokens_seen": 87031808
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004917251755265797,
+      "loss": 3.7263,
+      "theoretical_loss": 4.865432499067425,
+      "tokens_seen": 87097344
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004917151454363089,
+      "loss": 3.7067,
+      "theoretical_loss": 4.864915193316922,
+      "tokens_seen": 87162880
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004917051153460381,
+      "loss": 3.4247,
+      "theoretical_loss": 4.864398385185169,
+      "tokens_seen": 87228416
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004916950852557673,
+      "loss": 3.5719,
+      "theoretical_loss": 4.863882073820154,
+      "tokens_seen": 87293952
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004916850551654965,
+      "loss": 3.5349,
+      "theoretical_loss": 4.863366258371963,
+      "tokens_seen": 87359488
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004916750250752258,
+      "loss": 3.5815,
+      "theoretical_loss": 4.862850937992777,
+      "tokens_seen": 87425024
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004916649949849548,
+      "loss": 3.5233,
+      "theoretical_loss": 4.862336111836852,
+      "tokens_seen": 87490560
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004916549648946841,
+      "loss": 3.5181,
+      "theoretical_loss": 4.861821779060529,
+      "tokens_seen": 87556096
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004916449348044132,
+      "loss": 3.369,
+      "theoretical_loss": 4.861307938822215,
+      "tokens_seen": 87621632
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004916349047141425,
+      "loss": 3.3532,
+      "theoretical_loss": 4.86079459028238,
+      "tokens_seen": 87687168
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004916248746238717,
+      "loss": 3.4634,
+      "theoretical_loss": 4.860281732603555,
+      "tokens_seen": 87752704
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004916148445336008,
+      "loss": 3.64,
+      "theoretical_loss": 4.859769364950319,
+      "tokens_seen": 87818240
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.00049160481444333,
+      "loss": 3.5164,
+      "theoretical_loss": 4.859257486489293,
+      "tokens_seen": 87883776
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004915947843530592,
+      "loss": 3.5639,
+      "theoretical_loss": 4.858746096389139,
+      "tokens_seen": 87949312
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004915847542627884,
+      "loss": 3.4632,
+      "theoretical_loss": 4.8582351938205495,
+      "tokens_seen": 88014848
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004915747241725176,
+      "loss": 3.2569,
+      "theoretical_loss": 4.857724777956237,
+      "tokens_seen": 88080384
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004915646940822468,
+      "loss": 3.4019,
+      "theoretical_loss": 4.857214847970939,
+      "tokens_seen": 88145920
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004915546639919759,
+      "loss": 3.266,
+      "theoretical_loss": 4.856705403041397,
+      "tokens_seen": 88211456
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004915446339017051,
+      "loss": 3.7935,
+      "theoretical_loss": 4.8561964423463655,
+      "tokens_seen": 88276992
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004915346038114343,
+      "loss": 3.7385,
+      "theoretical_loss": 4.8556879650665925,
+      "tokens_seen": 88342528
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004915245737211635,
+      "loss": 3.3908,
+      "theoretical_loss": 4.855179970384819,
+      "tokens_seen": 88408064
+    },
+    {
+      "epoch": 0.03,
+      "objective/train/docs_used": 60543,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 3.110377788543701,
+      "objective/train/theoretical_loss": 4.8546724574857745,
+      "objective/train/tokens_used": 108933600,
+      "theoretical_loss": 4.8546724574857745,
+      "tokens_seen": 88473600
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004915145436308927,
+      "loss": 3.4761,
+      "theoretical_loss": 4.8546724574857745,
+      "tokens_seen": 88473600
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004915045135406218,
+      "loss": 3.442,
+      "theoretical_loss": 4.854165425556166,
+      "tokens_seen": 88539136
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004914944834503511,
+      "loss": 3.613,
+      "theoretical_loss": 4.853658873784675,
+      "tokens_seen": 88604672
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004914844533600802,
+      "loss": 3.7501,
+      "theoretical_loss": 4.853152801361952,
+      "tokens_seen": 88670208
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004914744232698095,
+      "loss": 3.4899,
+      "theoretical_loss": 4.852647207480604,
+      "tokens_seen": 88735744
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004914643931795386,
+      "loss": 3.6797,
+      "theoretical_loss": 4.8521420913352,
+      "tokens_seen": 88801280
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004914543630892679,
+      "loss": 3.2736,
+      "theoretical_loss": 4.851637452122251,
+      "tokens_seen": 88866816
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.000491444332998997,
+      "loss": 3.5697,
+      "theoretical_loss": 4.851133289040214,
+      "tokens_seen": 88932352
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004914343029087262,
+      "loss": 3.6628,
+      "theoretical_loss": 4.850629601289482,
+      "tokens_seen": 88997888
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004914242728184554,
+      "loss": 3.4603,
+      "theoretical_loss": 4.850126388072379,
+      "tokens_seen": 89063424
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004914142427281846,
+      "loss": 3.5286,
+      "theoretical_loss": 4.849623648593151,
+      "tokens_seen": 89128960
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004914042126379138,
+      "loss": 3.4864,
+      "theoretical_loss": 4.849121382057964,
+      "tokens_seen": 89194496
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004913941825476429,
+      "loss": 3.5269,
+      "theoretical_loss": 4.848619587674898,
+      "tokens_seen": 89260032
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004913841524573721,
+      "loss": 3.4857,
+      "theoretical_loss": 4.8481182646539365,
+      "tokens_seen": 89325568
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004913741223671013,
+      "loss": 3.6437,
+      "theoretical_loss": 4.847617412206963,
+      "tokens_seen": 89391104
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004913640922768305,
+      "loss": 3.4963,
+      "theoretical_loss": 4.847117029547758,
+      "tokens_seen": 89456640
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004913540621865597,
+      "loss": 3.5452,
+      "theoretical_loss": 4.846617115891988,
+      "tokens_seen": 89522176
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004913440320962888,
+      "loss": 3.2749,
+      "theoretical_loss": 4.8461176704572,
+      "tokens_seen": 89587712
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.000491334002006018,
+      "loss": 3.319,
+      "theoretical_loss": 4.845618692462826,
+      "tokens_seen": 89653248
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004913239719157472,
+      "loss": 3.4855,
+      "theoretical_loss": 4.845120181130158,
+      "tokens_seen": 89718784
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004913139418254765,
+      "loss": 3.7812,
+      "theoretical_loss": 4.844622135682359,
+      "tokens_seen": 89784320
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004913039117352056,
+      "loss": 3.1217,
+      "theoretical_loss": 4.84412455534445,
+      "tokens_seen": 89849856
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004912938816449349,
+      "loss": 3.3833,
+      "theoretical_loss": 4.843627439343306,
+      "tokens_seen": 89915392
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004912838515546639,
+      "loss": 3.5363,
+      "theoretical_loss": 4.843130786907647,
+      "tokens_seen": 89980928
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004912738214643932,
+      "loss": 3.4395,
+      "theoretical_loss": 4.842634597268036,
+      "tokens_seen": 90046464
+    },
+    {
+      "epoch": 0.03,
+      "objective/train/docs_used": 60961,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 3.3933932781219482,
+      "objective/train/theoretical_loss": 4.842138869656871,
+      "objective/train/tokens_used": 110572000,
+      "theoretical_loss": 4.842138869656871,
+      "tokens_seen": 90112000
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004912637913741224,
+      "loss": 3.5361,
+      "theoretical_loss": 4.842138869656871,
+      "tokens_seen": 90112000
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004912537612838516,
+      "loss": 3.1717,
+      "theoretical_loss": 4.841643603308384,
+      "tokens_seen": 90177536
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004912437311935808,
+      "loss": 3.6274,
+      "theoretical_loss": 4.841148797458627,
+      "tokens_seen": 90243072
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.00049123370110331,
+      "loss": 3.5958,
+      "theoretical_loss": 4.840654451345472,
+      "tokens_seen": 90308608
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004912236710130391,
+      "loss": 3.6702,
+      "theoretical_loss": 4.840160564208606,
+      "tokens_seen": 90374144
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004912136409227683,
+      "loss": 3.4133,
+      "theoretical_loss": 4.839667135289522,
+      "tokens_seen": 90439680
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004912036108324975,
+      "loss": 3.3868,
+      "theoretical_loss": 4.839174163831515,
+      "tokens_seen": 90505216
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004911935807422267,
+      "loss": 3.4971,
+      "theoretical_loss": 4.838681649079677,
+      "tokens_seen": 90570752
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004911835506519559,
+      "loss": 3.581,
+      "theoretical_loss": 4.838189590280891,
+      "tokens_seen": 90636288
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.000491173520561685,
+      "loss": 3.5547,
+      "theoretical_loss": 4.837697986683825,
+      "tokens_seen": 90701824
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004911634904714142,
+      "loss": 3.6512,
+      "theoretical_loss": 4.837206837538925,
+      "tokens_seen": 90767360
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004911534603811434,
+      "loss": 3.189,
+      "theoretical_loss": 4.836716142098417,
+      "tokens_seen": 90832896
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004911434302908726,
+      "loss": 3.4003,
+      "theoretical_loss": 4.836225899616288,
+      "tokens_seen": 90898432
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004911334002006019,
+      "loss": 3.5343,
+      "theoretical_loss": 4.835736109348293,
+      "tokens_seen": 90963968
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004911233701103309,
+      "loss": 3.491,
+      "theoretical_loss": 4.8352467705519455,
+      "tokens_seen": 91029504
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004911133400200602,
+      "loss": 3.543,
+      "theoretical_loss": 4.83475788248651,
+      "tokens_seen": 91095040
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004911033099297893,
+      "loss": 3.5829,
+      "theoretical_loss": 4.834269444412996,
+      "tokens_seen": 91160576
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004910932798395186,
+      "loss": 3.5038,
+      "theoretical_loss": 4.83378145559416,
+      "tokens_seen": 91226112
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004910832497492478,
+      "loss": 3.6003,
+      "theoretical_loss": 4.83329391529449,
+      "tokens_seen": 91291648
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.000491073219658977,
+      "loss": 3.5657,
+      "theoretical_loss": 4.832806822780208,
+      "tokens_seen": 91357184
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004910631895687061,
+      "loss": 3.3872,
+      "theoretical_loss": 4.832320177319261,
+      "tokens_seen": 91422720
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004910531594784353,
+      "loss": 3.4167,
+      "theoretical_loss": 4.831833978181314,
+      "tokens_seen": 91488256
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004910431293881645,
+      "loss": 3.2479,
+      "theoretical_loss": 4.831348224637752,
+      "tokens_seen": 91553792
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004910330992978937,
+      "loss": 3.5555,
+      "theoretical_loss": 4.830862915961665,
+      "tokens_seen": 91619328
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004910230692076229,
+      "loss": 3.4961,
+      "theoretical_loss": 4.830378051427852,
+      "tokens_seen": 91684864
+    },
+    {
+      "epoch": 0.03,
+      "objective/train/docs_used": 62148,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 3.0938072204589844,
+      "objective/train/theoretical_loss": 4.829893630312808,
+      "objective/train/tokens_used": 112210400,
+      "theoretical_loss": 4.829893630312808,
+      "tokens_seen": 91750400
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.000491013039117352,
+      "loss": 3.1482,
+      "theoretical_loss": 4.829893630312808,
+      "tokens_seen": 91750400
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004910030090270812,
+      "loss": 3.4885,
+      "theoretical_loss": 4.829409651894725,
+      "tokens_seen": 91815936
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004909929789368104,
+      "loss": 3.5385,
+      "theoretical_loss": 4.8289261154534815,
+      "tokens_seen": 91881472
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004909829488465397,
+      "loss": 3.2172,
+      "theoretical_loss": 4.828443020270642,
+      "tokens_seen": 91947008
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004909729187562688,
+      "loss": 3.351,
+      "theoretical_loss": 4.827960365629449,
+      "tokens_seen": 92012544
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004909628886659981,
+      "loss": 3.6262,
+      "theoretical_loss": 4.827478150814821,
+      "tokens_seen": 92078080
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004909528585757272,
+      "loss": 3.4536,
+      "theoretical_loss": 4.826996375113339,
+      "tokens_seen": 92143616
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004909428284854564,
+      "loss": 3.4569,
+      "theoretical_loss": 4.826515037813255,
+      "tokens_seen": 92209152
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004909327983951856,
+      "loss": 3.7922,
+      "theoretical_loss": 4.826034138204474,
+      "tokens_seen": 92274688
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004909227683049148,
+      "loss": 3.4368,
+      "theoretical_loss": 4.825553675578557,
+      "tokens_seen": 92340224
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.000490912738214644,
+      "loss": 3.3922,
+      "theoretical_loss": 4.825073649228713,
+      "tokens_seen": 92405760
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004909027081243731,
+      "loss": 3.5078,
+      "theoretical_loss": 4.824594058449794,
+      "tokens_seen": 92471296
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004908926780341023,
+      "loss": 3.3106,
+      "theoretical_loss": 4.824114902538292,
+      "tokens_seen": 92536832
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004908826479438315,
+      "loss": 3.136,
+      "theoretical_loss": 4.823636180792329,
+      "tokens_seen": 92602368
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004908726178535607,
+      "loss": 3.3286,
+      "theoretical_loss": 4.823157892511661,
+      "tokens_seen": 92667904
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004908625877632899,
+      "loss": 3.6266,
+      "theoretical_loss": 4.82268003699766,
+      "tokens_seen": 92733440
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.000490852557673019,
+      "loss": 3.5695,
+      "theoretical_loss": 4.8222026135533245,
+      "tokens_seen": 92798976
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004908425275827482,
+      "loss": 3.487,
+      "theoretical_loss": 4.821725621483262,
+      "tokens_seen": 92864512
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004908324974924774,
+      "loss": 3.503,
+      "theoretical_loss": 4.821249060093692,
+      "tokens_seen": 92930048
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004908224674022067,
+      "loss": 3.3758,
+      "theoretical_loss": 4.820772928692433,
+      "tokens_seen": 92995584
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004908124373119358,
+      "loss": 3.4497,
+      "theoretical_loss": 4.820297226588908,
+      "tokens_seen": 93061120
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004908024072216651,
+      "loss": 3.6942,
+      "theoretical_loss": 4.819821953094133,
+      "tokens_seen": 93126656
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004907923771313941,
+      "loss": 3.6533,
+      "theoretical_loss": 4.819347107520713,
+      "tokens_seen": 93192192
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004907823470411234,
+      "loss": 3.3973,
+      "theoretical_loss": 4.818872689182836,
+      "tokens_seen": 93257728
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004907723169508526,
+      "loss": 3.2205,
+      "theoretical_loss": 4.818398697396274,
+      "tokens_seen": 93323264
+    },
+    {
+      "epoch": 0.03,
+      "objective/train/docs_used": 62815,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 3.5069005489349365,
+      "objective/train/theoretical_loss": 4.817925131478374,
+      "objective/train/tokens_used": 113848800,
+      "theoretical_loss": 4.817925131478374,
+      "tokens_seen": 93388800
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004907622868605818,
+      "loss": 3.5578,
+      "theoretical_loss": 4.817925131478374,
+      "tokens_seen": 93388800
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.000490752256770311,
+      "loss": 3.5471,
+      "theoretical_loss": 4.817451990748047,
+      "tokens_seen": 93454336
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004907422266800401,
+      "loss": 3.6923,
+      "theoretical_loss": 4.81697927452578,
+      "tokens_seen": 93519872
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004907321965897693,
+      "loss": 3.5227,
+      "theoretical_loss": 4.816506982133613,
+      "tokens_seen": 93585408
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004907221664994985,
+      "loss": 3.2295,
+      "theoretical_loss": 4.816035112895147,
+      "tokens_seen": 93650944
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004907121364092277,
+      "loss": 3.1312,
+      "theoretical_loss": 4.815563666135532,
+      "tokens_seen": 93716480
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004907021063189569,
+      "loss": 3.6798,
+      "theoretical_loss": 4.815092641181467,
+      "tokens_seen": 93782016
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.000490692076228686,
+      "loss": 3.7669,
+      "theoretical_loss": 4.814622037361195,
+      "tokens_seen": 93847552
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004906820461384152,
+      "loss": 3.6417,
+      "theoretical_loss": 4.814151854004493,
+      "tokens_seen": 93913088
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004906720160481444,
+      "loss": 3.2937,
+      "theoretical_loss": 4.813682090442674,
+      "tokens_seen": 93978624
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004906619859578736,
+      "loss": 3.2833,
+      "theoretical_loss": 4.81321274600858,
+      "tokens_seen": 94044160
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004906519558676028,
+      "loss": 3.2788,
+      "theoretical_loss": 4.812743820036577,
+      "tokens_seen": 94109696
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004906419257773321,
+      "loss": 3.5618,
+      "theoretical_loss": 4.81227531186255,
+      "tokens_seen": 94175232
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004906318956870611,
+      "loss": 3.5329,
+      "theoretical_loss": 4.811807220823899,
+      "tokens_seen": 94240768
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004906218655967904,
+      "loss": 3.6156,
+      "theoretical_loss": 4.811339546259536,
+      "tokens_seen": 94306304
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004906118355065195,
+      "loss": 3.3777,
+      "theoretical_loss": 4.810872287509877,
+      "tokens_seen": 94371840
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004906018054162488,
+      "loss": 3.5883,
+      "theoretical_loss": 4.810405443916842,
+      "tokens_seen": 94437376
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.000490591775325978,
+      "loss": 3.6907,
+      "theoretical_loss": 4.809939014823849,
+      "tokens_seen": 94502912
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004905817452357072,
+      "loss": 3.6602,
+      "theoretical_loss": 4.809472999575805,
+      "tokens_seen": 94568448
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004905717151454363,
+      "loss": 3.6322,
+      "theoretical_loss": 4.809007397519108,
+      "tokens_seen": 94633984
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004905616850551655,
+      "loss": 3.4204,
+      "theoretical_loss": 4.80854220800164,
+      "tokens_seen": 94699520
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004905516549648947,
+      "loss": 3.5405,
+      "theoretical_loss": 4.808077430372762,
+      "tokens_seen": 94765056
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004905416248746239,
+      "loss": 3.4647,
+      "theoretical_loss": 4.807613063983311,
+      "tokens_seen": 94830592
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004905315947843531,
+      "loss": 3.7424,
+      "theoretical_loss": 4.807149108185593,
+      "tokens_seen": 94896128
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004905215646940822,
+      "loss": 3.3372,
+      "theoretical_loss": 4.806685562333383,
+      "tokens_seen": 94961664
+    },
+    {
+      "epoch": 0.03,
+      "objective/train/docs_used": 64162,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 3.0013229846954346,
+      "objective/train/theoretical_loss": 4.806222425781916,
+      "objective/train/tokens_used": 115487200,
+      "theoretical_loss": 4.806222425781916,
+      "tokens_seen": 95027200
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004905115346038114,
+      "loss": 3.4672,
+      "theoretical_loss": 4.806222425781916,
+      "tokens_seen": 95027200
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004905015045135406,
+      "loss": 3.4124,
+      "theoretical_loss": 4.805759697887886,
+      "tokens_seen": 95092736
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004904914744232698,
+      "loss": 3.5337,
+      "theoretical_loss": 4.80529737800944,
+      "tokens_seen": 95158272
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.000490481444332999,
+      "loss": 3.1987,
+      "theoretical_loss": 4.804835465506176,
+      "tokens_seen": 95223808
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004904714142427281,
+      "loss": 3.3264,
+      "theoretical_loss": 4.8043739597391335,
+      "tokens_seen": 95289344
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004904613841524574,
+      "loss": 3.421,
+      "theoretical_loss": 4.803912860070795,
+      "tokens_seen": 95354880
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004904513540621865,
+      "loss": 3.5238,
+      "theoretical_loss": 4.803452165865082,
+      "tokens_seen": 95420416
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004904413239719158,
+      "loss": 3.4263,
+      "theoretical_loss": 4.802991876487344,
+      "tokens_seen": 95485952
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004904312938816449,
+      "loss": 3.3975,
+      "theoretical_loss": 4.802531991304359,
+      "tokens_seen": 95551488
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004904212637913742,
+      "loss": 3.5391,
+      "theoretical_loss": 4.802072509684331,
+      "tokens_seen": 95617024
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004904112337011033,
+      "loss": 3.4524,
+      "theoretical_loss": 4.801613430996884,
+      "tokens_seen": 95682560
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004904012036108325,
+      "loss": 3.5944,
+      "theoretical_loss": 4.801154754613056,
+      "tokens_seen": 95748096
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004903911735205617,
+      "loss": 3.2372,
+      "theoretical_loss": 4.800696479905296,
+      "tokens_seen": 95813632
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004903811434302909,
+      "loss": 3.1815,
+      "theoretical_loss": 4.800238606247462,
+      "tokens_seen": 95879168
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004903711133400201,
+      "loss": 3.4315,
+      "theoretical_loss": 4.799781133014815,
+      "tokens_seen": 95944704
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004903610832497492,
+      "loss": 3.2851,
+      "theoretical_loss": 4.799324059584013,
+      "tokens_seen": 96010240
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004903510531594784,
+      "loss": 3.6757,
+      "theoretical_loss": 4.798867385333111,
+      "tokens_seen": 96075776
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004903410230692076,
+      "loss": 3.3841,
+      "theoretical_loss": 4.798411109641553,
+      "tokens_seen": 96141312
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004903309929789368,
+      "loss": 3.6343,
+      "theoretical_loss": 4.797955231890175,
+      "tokens_seen": 96206848
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.000490320962888666,
+      "loss": 3.3026,
+      "theoretical_loss": 4.797499751461188,
+      "tokens_seen": 96272384
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004903109327983952,
+      "loss": 3.3258,
+      "theoretical_loss": 4.797044667738188,
+      "tokens_seen": 96337920
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004903009027081243,
+      "loss": 3.4414,
+      "theoretical_loss": 4.796589980106143,
+      "tokens_seen": 96403456
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004902908726178535,
+      "loss": 3.2512,
+      "theoretical_loss": 4.796135687951393,
+      "tokens_seen": 96468992
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004902808425275828,
+      "loss": 3.4072,
+      "theoretical_loss": 4.795681790661644,
+      "tokens_seen": 96534528
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004902708124373119,
+      "loss": 3.4574,
+      "theoretical_loss": 4.795228287625964,
+      "tokens_seen": 96600064
+    },
+    {
+      "epoch": 0.03,
+      "objective/train/docs_used": 64775,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 3.715846538543701,
+      "objective/train/theoretical_loss": 4.794775178234782,
+      "objective/train/tokens_used": 117125600,
+      "theoretical_loss": 4.794775178234782,
+      "tokens_seen": 96665600
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004902607823470412,
+      "loss": 3.622,
+      "theoretical_loss": 4.794775178234782,
+      "tokens_seen": 96665600
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004902507522567703,
+      "loss": 3.3135,
+      "theoretical_loss": 4.794322461879881,
+      "tokens_seen": 96731136
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004902407221664995,
+      "loss": 3.3367,
+      "theoretical_loss": 4.793870137954395,
+      "tokens_seen": 96796672
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004902306920762287,
+      "loss": 3.3923,
+      "theoretical_loss": 4.793418205852804,
+      "tokens_seen": 96862208
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004902206619859579,
+      "loss": 3.2944,
+      "theoretical_loss": 4.7929666649709315,
+      "tokens_seen": 96927744
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004902106318956871,
+      "loss": 3.4677,
+      "theoretical_loss": 4.7925155147059435,
+      "tokens_seen": 96993280
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004902006018054163,
+      "loss": 3.5354,
+      "theoretical_loss": 4.792064754456337,
+      "tokens_seen": 97058816
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004901905717151454,
+      "loss": 3.4739,
+      "theoretical_loss": 4.791614383621943,
+      "tokens_seen": 97124352
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004901805416248746,
+      "loss": 3.4712,
+      "theoretical_loss": 4.79116440160392,
+      "tokens_seen": 97189888
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004901705115346038,
+      "loss": 3.6806,
+      "theoretical_loss": 4.790714807804751,
+      "tokens_seen": 97255424
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.000490160481444333,
+      "loss": 3.2497,
+      "theoretical_loss": 4.790265601628235,
+      "tokens_seen": 97320960
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004901504513540623,
+      "loss": 3.3692,
+      "theoretical_loss": 4.789816782479494,
+      "tokens_seen": 97386496
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004901404212637913,
+      "loss": 3.576,
+      "theoretical_loss": 4.789368349764956,
+      "tokens_seen": 97452032
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004901303911735206,
+      "loss": 3.4743,
+      "theoretical_loss": 4.788920302892365,
+      "tokens_seen": 97517568
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004901203610832497,
+      "loss": 3.1291,
+      "theoretical_loss": 4.7884726412707606,
+      "tokens_seen": 97583104
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.000490110330992979,
+      "loss": 3.5709,
+      "theoretical_loss": 4.788025364310492,
+      "tokens_seen": 97648640
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004901003009027082,
+      "loss": 3.3566,
+      "theoretical_loss": 4.787578471423203,
+      "tokens_seen": 97714176
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004900902708124374,
+      "loss": 3.6399,
+      "theoretical_loss": 4.787131962021827,
+      "tokens_seen": 97779712
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004900802407221665,
+      "loss": 3.577,
+      "theoretical_loss": 4.786685835520595,
+      "tokens_seen": 97845248
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004900702106318957,
+      "loss": 3.4237,
+      "theoretical_loss": 4.786240091335019,
+      "tokens_seen": 97910784
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004900601805416249,
+      "loss": 3.3724,
+      "theoretical_loss": 4.785794728881895,
+      "tokens_seen": 97976320
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004900501504513541,
+      "loss": 3.3342,
+      "theoretical_loss": 4.785349747579299,
+      "tokens_seen": 98041856
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004900401203610833,
+      "loss": 3.5862,
+      "theoretical_loss": 4.784905146846581,
+      "tokens_seen": 98107392
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004900300902708124,
+      "loss": 3.3465,
+      "theoretical_loss": 4.784460926104362,
+      "tokens_seen": 98172928
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004900200601805416,
+      "loss": 3.3709,
+      "theoretical_loss": 4.7840170847745345,
+      "tokens_seen": 98238464
+    },
+    {
+      "epoch": 0.03,
+      "objective/train/docs_used": 65599,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 3.469925880432129,
+      "objective/train/theoretical_loss": 4.783573622280253,
+      "objective/train/tokens_used": 118764000,
+      "theoretical_loss": 4.783573622280253,
+      "tokens_seen": 98304000
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004900100300902708,
+      "loss": 3.5123,
+      "theoretical_loss": 4.783573622280253,
+      "tokens_seen": 98304000
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.00049,
+      "loss": 3.1796,
+      "theoretical_loss": 4.7831305380459295,
+      "tokens_seen": 98369536
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004899899699097292,
+      "loss": 3.5071,
+      "theoretical_loss": 4.78268783149724,
+      "tokens_seen": 98435072
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004899799398194583,
+      "loss": 3.1618,
+      "theoretical_loss": 4.782245502061111,
+      "tokens_seen": 98500608
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004899699097291876,
+      "loss": 3.5861,
+      "theoretical_loss": 4.781803549165718,
+      "tokens_seen": 98566144
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004899598796389167,
+      "loss": 3.7046,
+      "theoretical_loss": 4.781361972240484,
+      "tokens_seen": 98631680
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.000489949849548646,
+      "loss": 3.4213,
+      "theoretical_loss": 4.780920770716076,
+      "tokens_seen": 98697216
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004899398194583751,
+      "loss": 3.5335,
+      "theoretical_loss": 4.780479944024398,
+      "tokens_seen": 98762752
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004899297893681044,
+      "loss": 3.3783,
+      "theoretical_loss": 4.7800394915985915,
+      "tokens_seen": 98828288
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004899197592778335,
+      "loss": 3.5404,
+      "theoretical_loss": 4.779599412873031,
+      "tokens_seen": 98893824
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004899097291875627,
+      "loss": 3.4117,
+      "theoretical_loss": 4.77915970728332,
+      "tokens_seen": 98959360
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004898996990972919,
+      "loss": 3.1311,
+      "theoretical_loss": 4.7787203742662845,
+      "tokens_seen": 99024896
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004898896690070211,
+      "loss": 3.3778,
+      "theoretical_loss": 4.778281413259977,
+      "tokens_seen": 99090432
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004898796389167503,
+      "loss": 3.3097,
+      "theoretical_loss": 4.777842823703665,
+      "tokens_seen": 99155968
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004898696088264794,
+      "loss": 3.4696,
+      "theoretical_loss": 4.777404605037833,
+      "tokens_seen": 99221504
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004898595787362086,
+      "loss": 3.5204,
+      "theoretical_loss": 4.7769667567041765,
+      "tokens_seen": 99287040
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004898495486459378,
+      "loss": 3.5602,
+      "theoretical_loss": 4.776529278145597,
+      "tokens_seen": 99352576
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.000489839518555667,
+      "loss": 3.4991,
+      "theoretical_loss": 4.776092168806207,
+      "tokens_seen": 99418112
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004898294884653962,
+      "loss": 3.6413,
+      "theoretical_loss": 4.775655428131313,
+      "tokens_seen": 99483648
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004898194583751254,
+      "loss": 3.6154,
+      "theoretical_loss": 4.775219055567424,
+      "tokens_seen": 99549184
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004898094282848545,
+      "loss": 3.4266,
+      "theoretical_loss": 4.7747830505622435,
+      "tokens_seen": 99614720
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004897993981945837,
+      "loss": 3.2347,
+      "theoretical_loss": 4.774347412564666,
+      "tokens_seen": 99680256
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.000489789368104313,
+      "loss": 3.5374,
+      "theoretical_loss": 4.773912141024771,
+      "tokens_seen": 99745792
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004897793380140421,
+      "loss": 3.3582,
+      "theoretical_loss": 4.773477235393826,
+      "tokens_seen": 99811328
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004897693079237714,
+      "loss": 3.4578,
+      "theoretical_loss": 4.773042695124279,
+      "tokens_seen": 99876864
+    },
+    {
+      "epoch": 0.03,
+      "objective/train/docs_used": 66410,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 4.062344551086426,
+      "objective/train/theoretical_loss": 4.772608519669756,
+      "objective/train/tokens_used": 120402400,
+      "theoretical_loss": 4.772608519669756,
+      "tokens_seen": 99942400
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004897592778335005,
+      "loss": 3.5402,
+      "theoretical_loss": 4.772608519669756,
+      "tokens_seen": 99942400
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004897492477432297,
+      "loss": 3.5992,
+      "theoretical_loss": 4.772174708485057,
+      "tokens_seen": 100007936
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004897392176529589,
+      "loss": 3.4125,
+      "theoretical_loss": 4.771741261026155,
+      "tokens_seen": 100073472
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004897291875626881,
+      "loss": 3.3972,
+      "theoretical_loss": 4.771308176750187,
+      "tokens_seen": 100139008
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004897191574724173,
+      "loss": 3.7219,
+      "theoretical_loss": 4.77087545511546,
+      "tokens_seen": 100204544
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004897091273821465,
+      "loss": 3.5976,
+      "theoretical_loss": 4.77044309558144,
+      "tokens_seen": 100270080
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004896990972918756,
+      "loss": 3.5781,
+      "theoretical_loss": 4.7700110976087515,
+      "tokens_seen": 100335616
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004896890672016048,
+      "loss": 3.4885,
+      "theoretical_loss": 4.769579460659173,
+      "tokens_seen": 100401152
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.000489679037111334,
+      "loss": 3.6787,
+      "theoretical_loss": 4.769148184195636,
+      "tokens_seen": 100466688
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004896690070210632,
+      "loss": 3.5478,
+      "theoretical_loss": 4.768717267682223,
+      "tokens_seen": 100532224
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004896589769307924,
+      "loss": 3.4238,
+      "theoretical_loss": 4.768286710584157,
+      "tokens_seen": 100597760
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004896489468405215,
+      "loss": 3.3217,
+      "theoretical_loss": 4.767856512367805,
+      "tokens_seen": 100663296
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004896389167502507,
+      "loss": 3.3783,
+      "theoretical_loss": 4.767426672500677,
+      "tokens_seen": 100728832
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004896288866599799,
+      "loss": 3.2931,
+      "theoretical_loss": 4.766997190451413,
+      "tokens_seen": 100794368
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004896188565697091,
+      "loss": 3.6434,
+      "theoretical_loss": 4.766568065689789,
+      "tokens_seen": 100859904
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004896088264794384,
+      "loss": 3.5874,
+      "theoretical_loss": 4.766139297686712,
+      "tokens_seen": 100925440
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004895987963891674,
+      "loss": 3.4533,
+      "theoretical_loss": 4.76571088591421,
+      "tokens_seen": 100990976
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004895887662988967,
+      "loss": 3.6816,
+      "theoretical_loss": 4.7652828298454395,
+      "tokens_seen": 101056512
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004895787362086259,
+      "loss": 3.3644,
+      "theoretical_loss": 4.764855128954675,
+      "tokens_seen": 101122048
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004895687061183551,
+      "loss": 3.2458,
+      "theoretical_loss": 4.764427782717309,
+      "tokens_seen": 101187584
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004895586760280843,
+      "loss": 3.3975,
+      "theoretical_loss": 4.764000790609848,
+      "tokens_seen": 101253120
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004895486459378135,
+      "loss": 3.3918,
+      "theoretical_loss": 4.763574152109907,
+      "tokens_seen": 101318656
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004895386158475426,
+      "loss": 3.4413,
+      "theoretical_loss": 4.763147866696213,
+      "tokens_seen": 101384192
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004895285857572718,
+      "loss": 3.154,
+      "theoretical_loss": 4.762721933848593,
+      "tokens_seen": 101449728
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.000489518555667001,
+      "loss": 3.367,
+      "theoretical_loss": 4.762296353047979,
+      "tokens_seen": 101515264
+    },
+    {
+      "epoch": 0.03,
+      "objective/train/docs_used": 66745,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 3.258596897125244,
+      "objective/train/theoretical_loss": 4.7618711237764035,
+      "objective/train/tokens_used": 122040800,
+      "theoretical_loss": 4.7618711237764035,
+      "tokens_seen": 101580800
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004895085255767302,
+      "loss": 3.3147,
+      "theoretical_loss": 4.7618711237764035,
+      "tokens_seen": 101580800
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004894984954864594,
+      "loss": 3.2477,
+      "theoretical_loss": 4.761446245516988,
+      "tokens_seen": 101646336
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004894884653961885,
+      "loss": 3.5315,
+      "theoretical_loss": 4.7610217177539536,
+      "tokens_seen": 101711872
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004894784353059178,
+      "loss": 3.3191,
+      "theoretical_loss": 4.760597539972606,
+      "tokens_seen": 101777408
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004894684052156469,
+      "loss": 3.0442,
+      "theoretical_loss": 4.760173711659341,
+      "tokens_seen": 101842944
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004894583751253762,
+      "loss": 3.4036,
+      "theoretical_loss": 4.759750232301635,
+      "tokens_seen": 101908480
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004894483450351053,
+      "loss": 3.3851,
+      "theoretical_loss": 4.759327101388048,
+      "tokens_seen": 101974016
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004894383149448346,
+      "loss": 3.5575,
+      "theoretical_loss": 4.7589043184082165,
+      "tokens_seen": 102039552
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004894282848545637,
+      "loss": 3.3678,
+      "theoretical_loss": 4.75848188285285,
+      "tokens_seen": 102105088
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004894182547642929,
+      "loss": 3.5807,
+      "theoretical_loss": 4.7580597942137315,
+      "tokens_seen": 102170624
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004894082246740221,
+      "loss": 3.4755,
+      "theoretical_loss": 4.757638051983713,
+      "tokens_seen": 102236160
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004893981945837513,
+      "loss": 3.308,
+      "theoretical_loss": 4.757216655656713,
+      "tokens_seen": 102301696
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004893881644934805,
+      "loss": 3.4312,
+      "theoretical_loss": 4.756795604727708,
+      "tokens_seen": 102367232
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004893781344032096,
+      "loss": 3.4421,
+      "theoretical_loss": 4.7563748986927425,
+      "tokens_seen": 102432768
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004893681043129388,
+      "loss": 3.3071,
+      "theoretical_loss": 4.755954537048913,
+      "tokens_seen": 102498304
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.000489358074222668,
+      "loss": 3.7184,
+      "theoretical_loss": 4.755534519294368,
+      "tokens_seen": 102563840
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004893480441323972,
+      "loss": 3.6644,
+      "theoretical_loss": 4.7551148449283165,
+      "tokens_seen": 102629376
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004893380140421264,
+      "loss": 3.3233,
+      "theoretical_loss": 4.754695513451004,
+      "tokens_seen": 102694912
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004893279839518556,
+      "loss": 3.6128,
+      "theoretical_loss": 4.754276524363733,
+      "tokens_seen": 102760448
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004893179538615847,
+      "loss": 3.4243,
+      "theoretical_loss": 4.75385787716884,
+      "tokens_seen": 102825984
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004893079237713139,
+      "loss": 3.6836,
+      "theoretical_loss": 4.7534395713697055,
+      "tokens_seen": 102891520
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004892978936810432,
+      "loss": 3.6634,
+      "theoretical_loss": 4.753021606470748,
+      "tokens_seen": 102957056
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004892878635907723,
+      "loss": 3.3886,
+      "theoretical_loss": 4.752603981977417,
+      "tokens_seen": 103022592
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004892778335005016,
+      "loss": 3.4605,
+      "theoretical_loss": 4.752186697396198,
+      "tokens_seen": 103088128
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004892678034102307,
+      "loss": 3.3429,
+      "theoretical_loss": 4.7517697522345985,
+      "tokens_seen": 103153664
+    },
+    {
+      "epoch": 0.03,
+      "objective/train/docs_used": 68050,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 3.391291379928589,
+      "objective/train/theoretical_loss": 4.7513531460011595,
+      "objective/train/tokens_used": 123679200,
+      "theoretical_loss": 4.7513531460011595,
+      "tokens_seen": 103219200
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004892577733199599,
+      "loss": 3.488,
+      "theoretical_loss": 4.7513531460011595,
+      "tokens_seen": 103219200
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004892477432296891,
+      "loss": 3.4412,
+      "theoretical_loss": 4.750936878205438,
+      "tokens_seen": 103284736
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004892377131394183,
+      "loss": 3.1551,
+      "theoretical_loss": 4.750520948358019,
+      "tokens_seen": 103350272
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004892276830491475,
+      "loss": 3.3288,
+      "theoretical_loss": 4.750105355970497,
+      "tokens_seen": 103415808
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004892176529588767,
+      "loss": 3.3249,
+      "theoretical_loss": 4.749690100555487,
+      "tokens_seen": 103481344
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004892076228686058,
+      "loss": 3.4856,
+      "theoretical_loss": 4.749275181626612,
+      "tokens_seen": 103546880
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.000489197592778335,
+      "loss": 3.447,
+      "theoretical_loss": 4.748860598698509,
+      "tokens_seen": 103612416
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004891875626880642,
+      "loss": 3.3712,
+      "theoretical_loss": 4.748446351286817,
+      "tokens_seen": 103677952
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004891775325977934,
+      "loss": 3.3355,
+      "theoretical_loss": 4.748032438908181,
+      "tokens_seen": 103743488
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004891675025075226,
+      "loss": 3.6823,
+      "theoretical_loss": 4.747618861080247,
+      "tokens_seen": 103809024
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004891574724172517,
+      "loss": 3.5451,
+      "theoretical_loss": 4.74720561732166,
+      "tokens_seen": 103874560
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004891474423269809,
+      "loss": 3.3377,
+      "theoretical_loss": 4.746792707152057,
+      "tokens_seen": 103940096
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004891374122367101,
+      "loss": 3.2533,
+      "theoretical_loss": 4.746380130092074,
+      "tokens_seen": 104005632
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004891273821464393,
+      "loss": 3.5114,
+      "theoretical_loss": 4.745967885663333,
+      "tokens_seen": 104071168
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004891173520561686,
+      "loss": 3.5891,
+      "theoretical_loss": 4.745555973388447,
+      "tokens_seen": 104136704
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004891073219658976,
+      "loss": 3.5146,
+      "theoretical_loss": 4.745144392791008,
+      "tokens_seen": 104202240
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004890972918756269,
+      "loss": 3.477,
+      "theoretical_loss": 4.744733143395598,
+      "tokens_seen": 104267776
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004890872617853561,
+      "loss": 3.5138,
+      "theoretical_loss": 4.744322224727773,
+      "tokens_seen": 104333312
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004890772316950853,
+      "loss": 3.4684,
+      "theoretical_loss": 4.743911636314069,
+      "tokens_seen": 104398848
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004890672016048145,
+      "loss": 3.3884,
+      "theoretical_loss": 4.743501377681995,
+      "tokens_seen": 104464384
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004890571715145437,
+      "loss": 3.2946,
+      "theoretical_loss": 4.743091448360034,
+      "tokens_seen": 104529920
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004890471414242728,
+      "loss": 3.362,
+      "theoretical_loss": 4.742681847877636,
+      "tokens_seen": 104595456
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.000489037111334002,
+      "loss": 3.1456,
+      "theoretical_loss": 4.742272575765218,
+      "tokens_seen": 104660992
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004890270812437312,
+      "loss": 3.6056,
+      "theoretical_loss": 4.741863631554163,
+      "tokens_seen": 104726528
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004890170511534604,
+      "loss": 3.5804,
+      "theoretical_loss": 4.741455014776813,
+      "tokens_seen": 104792064
+    },
+    {
+      "epoch": 0.03,
+      "objective/train/docs_used": 68417,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 3.796355724334717,
+      "objective/train/theoretical_loss": 4.741046724966468,
+      "objective/train/tokens_used": 125317600,
+      "theoretical_loss": 4.741046724966468,
+      "tokens_seen": 104857600
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004890070210631896,
+      "loss": 3.5708,
+      "theoretical_loss": 4.741046724966468,
+      "tokens_seen": 104857600
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004889969909729187,
+      "loss": 3.1131,
+      "theoretical_loss": 4.74063876165739,
+      "tokens_seen": 104923136
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004889869608826479,
+      "loss": 3.4213,
+      "theoretical_loss": 4.740231124384789,
+      "tokens_seen": 104988672
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004889769307923771,
+      "loss": 3.3977,
+      "theoretical_loss": 4.739823812684829,
+      "tokens_seen": 105054208
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004889669007021063,
+      "loss": 3.345,
+      "theoretical_loss": 4.739416826094621,
+      "tokens_seen": 105119744
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004889568706118355,
+      "loss": 3.4305,
+      "theoretical_loss": 4.739010164152224,
+      "tokens_seen": 105185280
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004889468405215647,
+      "loss": 3.393,
+      "theoretical_loss": 4.7386038263966395,
+      "tokens_seen": 105250816
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004889368104312939,
+      "loss": 3.4701,
+      "theoretical_loss": 4.73819781236781,
+      "tokens_seen": 105316352
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.000488926780341023,
+      "loss": 3.2626,
+      "theoretical_loss": 4.737792121606619,
+      "tokens_seen": 105381888
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004889167502507523,
+      "loss": 3.364,
+      "theoretical_loss": 4.737386753654883,
+      "tokens_seen": 105447424
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004889067201604815,
+      "loss": 3.4215,
+      "theoretical_loss": 4.736981708055353,
+      "tokens_seen": 105512960
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004888966900702107,
+      "loss": 3.2655,
+      "theoretical_loss": 4.7365769843517125,
+      "tokens_seen": 105578496
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004888866599799398,
+      "loss": 3.2771,
+      "theoretical_loss": 4.736172582088571,
+      "tokens_seen": 105644032
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.000488876629889669,
+      "loss": 3.3345,
+      "theoretical_loss": 4.735768500811469,
+      "tokens_seen": 105709568
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004888665997993982,
+      "loss": 3.4764,
+      "theoretical_loss": 4.735364740066865,
+      "tokens_seen": 105775104
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004888565697091274,
+      "loss": 3.7094,
+      "theoretical_loss": 4.734961299402143,
+      "tokens_seen": 105840640
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004888465396188566,
+      "loss": 3.4389,
+      "theoretical_loss": 4.734558178365605,
+      "tokens_seen": 105906176
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004888365095285858,
+      "loss": 3.5066,
+      "theoretical_loss": 4.734155376506466,
+      "tokens_seen": 105971712
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004888264794383149,
+      "loss": 3.2004,
+      "theoretical_loss": 4.733752893374863,
+      "tokens_seen": 106037248
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004888164493480441,
+      "loss": 3.9504,
+      "theoretical_loss": 4.733350728521835,
+      "tokens_seen": 106102784
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004888064192577733,
+      "loss": 3.3078,
+      "theoretical_loss": 4.732948881499336,
+      "tokens_seen": 106168320
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004887963891675025,
+      "loss": 3.258,
+      "theoretical_loss": 4.7325473518602275,
+      "tokens_seen": 106233856
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004887863590772317,
+      "loss": 3.4263,
+      "theoretical_loss": 4.732146139158271,
+      "tokens_seen": 106299392
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.000488776328986961,
+      "loss": 3.2168,
+      "theoretical_loss": 4.731745242948134,
+      "tokens_seen": 106364928
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.00048876629889669,
+      "loss": 3.7064,
+      "theoretical_loss": 4.731344662785381,
+      "tokens_seen": 106430464
+    },
+    {
+      "epoch": 0.03,
+      "objective/train/docs_used": 69666,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 3.1660373210906982,
+      "objective/train/theoretical_loss": 4.730944398226477,
+      "objective/train/tokens_used": 126956000,
+      "theoretical_loss": 4.730944398226477,
+      "tokens_seen": 106496000
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004887562688064193,
+      "loss": 3.4624,
+      "theoretical_loss": 4.730944398226477,
+      "tokens_seen": 106496000
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004887462387161484,
+      "loss": 3.2648,
+      "theoretical_loss": 4.730544448828777,
+      "tokens_seen": 106561536
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004887362086258777,
+      "loss": 3.3377,
+      "theoretical_loss": 4.730144814150533,
+      "tokens_seen": 106627072
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004887261785356069,
+      "loss": 3.182,
+      "theoretical_loss": 4.729745493750885,
+      "tokens_seen": 106692608
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.000488716148445336,
+      "loss": 3.5011,
+      "theoretical_loss": 4.729346487189863,
+      "tokens_seen": 106758144
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004887061183550652,
+      "loss": 3.4765,
+      "theoretical_loss": 4.728947794028379,
+      "tokens_seen": 106823680
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004886960882647944,
+      "loss": 3.5698,
+      "theoretical_loss": 4.728549413828231,
+      "tokens_seen": 106889216
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004886860581745236,
+      "loss": 3.5244,
+      "theoretical_loss": 4.728151346152098,
+      "tokens_seen": 106954752
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004886760280842528,
+      "loss": 3.2724,
+      "theoretical_loss": 4.727753590563535,
+      "tokens_seen": 107020288
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004886659979939819,
+      "loss": 3.2698,
+      "theoretical_loss": 4.727356146626976,
+      "tokens_seen": 107085824
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004886559679037111,
+      "loss": 3.3792,
+      "theoretical_loss": 4.726959013907727,
+      "tokens_seen": 107151360
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004886459378134403,
+      "loss": 3.0486,
+      "theoretical_loss": 4.7265621919719685,
+      "tokens_seen": 107216896
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004886359077231695,
+      "loss": 3.2127,
+      "theoretical_loss": 4.726165680386746,
+      "tokens_seen": 107282432
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004886258776328988,
+      "loss": 3.4488,
+      "theoretical_loss": 4.725769478719978,
+      "tokens_seen": 107347968
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004886158475426278,
+      "loss": 3.6296,
+      "theoretical_loss": 4.725373586540442,
+      "tokens_seen": 107413504
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004886058174523571,
+      "loss": 3.1497,
+      "theoretical_loss": 4.724978003417782,
+      "tokens_seen": 107479040
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004885957873620863,
+      "loss": 3.0499,
+      "theoretical_loss": 4.724582728922501,
+      "tokens_seen": 107544576
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004885857572718155,
+      "loss": 3.3971,
+      "theoretical_loss": 4.724187762625959,
+      "tokens_seen": 107610112
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004885757271815447,
+      "loss": 3.4614,
+      "theoretical_loss": 4.723793104100376,
+      "tokens_seen": 107675648
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004885656970912739,
+      "loss": 3.2867,
+      "theoretical_loss": 4.72339875291882,
+      "tokens_seen": 107741184
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.000488555667001003,
+      "loss": 3.5243,
+      "theoretical_loss": 4.723004708655214,
+      "tokens_seen": 107806720
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004885456369107322,
+      "loss": 3.471,
+      "theoretical_loss": 4.722610970884331,
+      "tokens_seen": 107872256
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004885356068204614,
+      "loss": 3.4896,
+      "theoretical_loss": 4.722217539181788,
+      "tokens_seen": 107937792
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004885255767301906,
+      "loss": 3.5163,
+      "theoretical_loss": 4.7218244131240485,
+      "tokens_seen": 108003328
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004885155466399198,
+      "loss": 3.1836,
+      "theoretical_loss": 4.721431592288419,
+      "tokens_seen": 108068864
+    },
+    {
+      "epoch": 0.03,
+      "objective/train/docs_used": 70233,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 3.18623423576355,
+      "objective/train/theoretical_loss": 4.721039076253046,
+      "objective/train/tokens_used": 128594400,
+      "theoretical_loss": 4.721039076253046,
+      "tokens_seen": 108134400
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.000488505516549649,
+      "loss": 3.2707,
+      "theoretical_loss": 4.721039076253046,
+      "tokens_seen": 108134400
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004884954864593781,
+      "loss": 3.3744,
+      "theoretical_loss": 4.720646864596914,
+      "tokens_seen": 108199936
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004884854563691073,
+      "loss": 3.2403,
+      "theoretical_loss": 4.720254956899844,
+      "tokens_seen": 108265472
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004884754262788365,
+      "loss": 3.4192,
+      "theoretical_loss": 4.719863352742491,
+      "tokens_seen": 108331008
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004884653961885657,
+      "loss": 3.2221,
+      "theoretical_loss": 4.719472051706341,
+      "tokens_seen": 108396544
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004884553660982949,
+      "loss": 3.4325,
+      "theoretical_loss": 4.71908105337371,
+      "tokens_seen": 108462080
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004884453360080241,
+      "loss": 3.3533,
+      "theoretical_loss": 4.718690357327743,
+      "tokens_seen": 108527616
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004884353059177532,
+      "loss": 3.5112,
+      "theoretical_loss": 4.718299963152407,
+      "tokens_seen": 108593152
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004884252758274825,
+      "loss": 3.2922,
+      "theoretical_loss": 4.7179098704324955,
+      "tokens_seen": 108658688
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004884152457372117,
+      "loss": 3.4903,
+      "theoretical_loss": 4.717520078753621,
+      "tokens_seen": 108724224
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004884052156469409,
+      "loss": 3.5257,
+      "theoretical_loss": 4.717130587702217,
+      "tokens_seen": 108789760
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.00048839518555667,
+      "loss": 3.371,
+      "theoretical_loss": 4.716741396865531,
+      "tokens_seen": 108855296
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004883851554663992,
+      "loss": 3.388,
+      "theoretical_loss": 4.716352505831628,
+      "tokens_seen": 108920832
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004883751253761284,
+      "loss": 3.5684,
+      "theoretical_loss": 4.715963914189384,
+      "tokens_seen": 108986368
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004883650952858576,
+      "loss": 3.3996,
+      "theoretical_loss": 4.715575621528487,
+      "tokens_seen": 109051904
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004883550651955868,
+      "loss": 3.1784,
+      "theoretical_loss": 4.71518762743943,
+      "tokens_seen": 109117440
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.000488345035105316,
+      "loss": 3.2919,
+      "theoretical_loss": 4.714799931513517,
+      "tokens_seen": 109182976
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004883350050150451,
+      "loss": 3.1558,
+      "theoretical_loss": 4.714412533342855,
+      "tokens_seen": 109248512
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004883249749247743,
+      "loss": 3.5333,
+      "theoretical_loss": 4.7140254325203506,
+      "tokens_seen": 109314048
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004883149448345035,
+      "loss": 3.6226,
+      "theoretical_loss": 4.713638628639713,
+      "tokens_seen": 109379584
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004883049147442327,
+      "loss": 3.2969,
+      "theoretical_loss": 4.713252121295449,
+      "tokens_seen": 109445120
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004882948846539619,
+      "loss": 3.6223,
+      "theoretical_loss": 4.712865910082863,
+      "tokens_seen": 109510656
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004882848545636911,
+      "loss": 3.4854,
+      "theoretical_loss": 4.712479994598052,
+      "tokens_seen": 109576192
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004882748244734203,
+      "loss": 3.4307,
+      "theoretical_loss": 4.7120943744379025,
+      "tokens_seen": 109641728
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.00048826479438314946,
+      "loss": 3.3526,
+      "theoretical_loss": 4.7117090492000955,
+      "tokens_seen": 109707264
+    },
+    {
+      "epoch": 0.03,
+      "objective/train/docs_used": 70991,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 3.7265214920043945,
+      "objective/train/theoretical_loss": 4.711324018483099,
+      "objective/train/tokens_used": 130232800,
+      "theoretical_loss": 4.711324018483099,
+      "tokens_seen": 109772800
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.00048825476429287864,
+      "loss": 3.4221,
+      "theoretical_loss": 4.711324018483099,
+      "tokens_seen": 109772800
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004882447342026078,
+      "loss": 3.156,
+      "theoretical_loss": 4.710939281886164,
+      "tokens_seen": 109838336
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.000488234704112337,
+      "loss": 3.5109,
+      "theoretical_loss": 4.71055483900933,
+      "tokens_seen": 109903872
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.00048822467402206624,
+      "loss": 3.6234,
+      "theoretical_loss": 4.710170689453415,
+      "tokens_seen": 109969408
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.00048821464393179536,
+      "loss": 3.3112,
+      "theoretical_loss": 4.709786832820017,
+      "tokens_seen": 110034944
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004882046138415246,
+      "loss": 3.3367,
+      "theoretical_loss": 4.7094032687115135,
+      "tokens_seen": 110100480
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004881945837512537,
+      "loss": 3.3711,
+      "theoretical_loss": 4.709019996731058,
+      "tokens_seen": 110166016
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.00048818455366098296,
+      "loss": 3.1866,
+      "theoretical_loss": 4.708637016482579,
+      "tokens_seen": 110231552
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.00048817452357071214,
+      "loss": 3.3009,
+      "theoretical_loss": 4.708254327570774,
+      "tokens_seen": 110297088
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004881644934804413,
+      "loss": 3.6798,
+      "theoretical_loss": 4.7078719296011124,
+      "tokens_seen": 110362624
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004881544633901705,
+      "loss": 3.5188,
+      "theoretical_loss": 4.7074898221798325,
+      "tokens_seen": 110428160
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.00048814443329989974,
+      "loss": 3.1,
+      "theoretical_loss": 4.707108004913936,
+      "tokens_seen": 110493696
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.00048813440320962887,
+      "loss": 3.5579,
+      "theoretical_loss": 4.706726477411193,
+      "tokens_seen": 110559232
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004881243731193581,
+      "loss": 3.3834,
+      "theoretical_loss": 4.706345239280133,
+      "tokens_seen": 110624768
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.00048811434302908723,
+      "loss": 3.3456,
+      "theoretical_loss": 4.705964290130046,
+      "tokens_seen": 110690304
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.00048810431293881646,
+      "loss": 3.3642,
+      "theoretical_loss": 4.705583629570983,
+      "tokens_seen": 110755840
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.00048809428284854564,
+      "loss": 3.2147,
+      "theoretical_loss": 4.705203257213746,
+      "tokens_seen": 110821376
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004880842527582748,
+      "loss": 3.4359,
+      "theoretical_loss": 4.7048231726699,
+      "tokens_seen": 110886912
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.000488074222668004,
+      "loss": 3.163,
+      "theoretical_loss": 4.7044433755517545,
+      "tokens_seen": 110952448
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004880641925777332,
+      "loss": 3.2166,
+      "theoretical_loss": 4.704063865472374,
+      "tokens_seen": 111017984
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.00048805416248746237,
+      "loss": 3.2815,
+      "theoretical_loss": 4.703684642045571,
+      "tokens_seen": 111083520
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004880441323971916,
+      "loss": 3.3855,
+      "theoretical_loss": 4.703305704885906,
+      "tokens_seen": 111149056
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.00048803410230692073,
+      "loss": 3.476,
+      "theoretical_loss": 4.7029270536086845,
+      "tokens_seen": 111214592
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.00048802407221664997,
+      "loss": 3.1811,
+      "theoretical_loss": 4.702548687829953,
+      "tokens_seen": 111280128
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.00048801404212637915,
+      "loss": 3.212,
+      "theoretical_loss": 4.702170607166503,
+      "tokens_seen": 111345664
+    },
+    {
+      "epoch": 0.03,
+      "objective/train/docs_used": 71566,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 3.1423094272613525,
+      "objective/train/theoretical_loss": 4.701792811235861,
+      "objective/train/tokens_used": 131871200,
+      "theoretical_loss": 4.701792811235861,
+      "tokens_seen": 111411200
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.00048800401203610833,
+      "loss": 3.3106,
+      "theoretical_loss": 4.701792811235861,
+      "tokens_seen": 111411200
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004879939819458375,
+      "loss": 3.3804,
+      "theoretical_loss": 4.701415299656295,
+      "tokens_seen": 111476736
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004879839518555667,
+      "loss": 3.3137,
+      "theoretical_loss": 4.701038072046808,
+      "tokens_seen": 111542272
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004879739217652959,
+      "loss": 3.4184,
+      "theoretical_loss": 4.7006611280271375,
+      "tokens_seen": 111607808
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004879638916750251,
+      "loss": 3.5134,
+      "theoretical_loss": 4.700284467217749,
+      "tokens_seen": 111673344
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004879538615847543,
+      "loss": 3.515,
+      "theoretical_loss": 4.699908089239845,
+      "tokens_seen": 111738880
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.00048794383149448347,
+      "loss": 3.4391,
+      "theoretical_loss": 4.6995319937153495,
+      "tokens_seen": 111804416
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.00048793380140421265,
+      "loss": 3.4928,
+      "theoretical_loss": 4.699156180266918,
+      "tokens_seen": 111869952
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.00048792377131394183,
+      "loss": 3.4204,
+      "theoretical_loss": 4.69878064851793,
+      "tokens_seen": 111935488
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.00048791374122367107,
+      "loss": 3.2864,
+      "theoretical_loss": 4.698405398092486,
+      "tokens_seen": 112001024
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004879037111334002,
+      "loss": 3.52,
+      "theoretical_loss": 4.698030428615411,
+      "tokens_seen": 112066560
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.00048789368104312943,
+      "loss": 3.2593,
+      "theoretical_loss": 4.6976557397122445,
+      "tokens_seen": 112132096
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.00048788365095285856,
+      "loss": 3.3313,
+      "theoretical_loss": 4.697281331009249,
+      "tokens_seen": 112197632
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004878736208625878,
+      "loss": 3.1676,
+      "theoretical_loss": 4.6969072021333975,
+      "tokens_seen": 112263168
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.00048786359077231697,
+      "loss": 3.4483,
+      "theoretical_loss": 4.696533352712381,
+      "tokens_seen": 112328704
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.00048785356068204615,
+      "loss": 3.3785,
+      "theoretical_loss": 4.696159782374602,
+      "tokens_seen": 112394240
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.00048784353059177533,
+      "loss": 3.0219,
+      "theoretical_loss": 4.695786490749171,
+      "tokens_seen": 112459776
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.00048783350050150457,
+      "loss": 3.3436,
+      "theoretical_loss": 4.6954134774659115,
+      "tokens_seen": 112525312
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004878234704112337,
+      "loss": 3.3403,
+      "theoretical_loss": 4.695040742155349,
+      "tokens_seen": 112590848
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.00048781344032096293,
+      "loss": 3.4199,
+      "theoretical_loss": 4.694668284448717,
+      "tokens_seen": 112656384
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.00048780341023069206,
+      "loss": 3.0947,
+      "theoretical_loss": 4.694296103977953,
+      "tokens_seen": 112721920
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004877933801404213,
+      "loss": 2.9975,
+      "theoretical_loss": 4.693924200375694,
+      "tokens_seen": 112787456
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004877833500501505,
+      "loss": 3.2263,
+      "theoretical_loss": 4.693552573275277,
+      "tokens_seen": 112852992
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.00048777331995987966,
+      "loss": 3.3911,
+      "theoretical_loss": 4.69318122231074,
+      "tokens_seen": 112918528
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.00048776328986960884,
+      "loss": 3.3579,
+      "theoretical_loss": 4.692810147116814,
+      "tokens_seen": 112984064
+    },
+    {
+      "epoch": 0.03,
+      "objective/train/docs_used": 72808,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 3.543510913848877,
+      "objective/train/theoretical_loss": 4.692439347328926,
+      "objective/train/tokens_used": 133509600,
+      "theoretical_loss": 4.692439347328926,
+      "tokens_seen": 113049600
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.000487753259779338,
+      "loss": 3.5704,
+      "theoretical_loss": 4.692439347328926,
+      "tokens_seen": 113049600
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004877432296890672,
+      "loss": 3.2623,
+      "theoretical_loss": 4.692068822583197,
+      "tokens_seen": 113115136
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.00048773319959879644,
+      "loss": 3.7086,
+      "theoretical_loss": 4.6916985725164375,
+      "tokens_seen": 113180672
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.00048772316950852556,
+      "loss": 3.2168,
+      "theoretical_loss": 4.691328596766148,
+      "tokens_seen": 113246208
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004877131394182548,
+      "loss": 3.5721,
+      "theoretical_loss": 4.690958894970518,
+      "tokens_seen": 113311744
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004877031093279839,
+      "loss": 3.5405,
+      "theoretical_loss": 4.690589466768422,
+      "tokens_seen": 113377280
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.00048769307923771316,
+      "loss": 3.4867,
+      "theoretical_loss": 4.69022031179942,
+      "tokens_seen": 113442816
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.00048768304914744234,
+      "loss": 3.3823,
+      "theoretical_loss": 4.689851429703753,
+      "tokens_seen": 113508352
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004876730190571715,
+      "loss": 3.242,
+      "theoretical_loss": 4.689482820122342,
+      "tokens_seen": 113573888
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004876629889669007,
+      "loss": 3.6015,
+      "theoretical_loss": 4.689114482696793,
+      "tokens_seen": 113639424
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.00048765295887662994,
+      "loss": 3.4168,
+      "theoretical_loss": 4.6887464170693836,
+      "tokens_seen": 113704960
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.00048764292878635907,
+      "loss": 3.4562,
+      "theoretical_loss": 4.6883786228830715,
+      "tokens_seen": 113770496
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004876328986960883,
+      "loss": 3.1948,
+      "theoretical_loss": 4.688011099781486,
+      "tokens_seen": 113836032
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.00048762286860581743,
+      "loss": 3.2999,
+      "theoretical_loss": 4.687643847408929,
+      "tokens_seen": 113901568
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.00048761283851554666,
+      "loss": 3.342,
+      "theoretical_loss": 4.687276865410377,
+      "tokens_seen": 113967104
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.00048760280842527584,
+      "loss": 3.4757,
+      "theoretical_loss": 4.686910153431472,
+      "tokens_seen": 114032640
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.000487592778335005,
+      "loss": 3.5307,
+      "theoretical_loss": 4.686543711118526,
+      "tokens_seen": 114098176
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004875827482447342,
+      "loss": 3.2597,
+      "theoretical_loss": 4.686177538118516,
+      "tokens_seen": 114163712
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004875727181544634,
+      "loss": 3.4569,
+      "theoretical_loss": 4.685811634079084,
+      "tokens_seen": 114229248
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.00048756268806419257,
+      "loss": 3.3531,
+      "theoretical_loss": 4.685445998648532,
+      "tokens_seen": 114294784
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004875526579739218,
+      "loss": 3.4034,
+      "theoretical_loss": 4.685080631475831,
+      "tokens_seen": 114360320
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.00048754262788365093,
+      "loss": 3.0558,
+      "theoretical_loss": 4.6847155322106016,
+      "tokens_seen": 114425856
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.00048753259779338017,
+      "loss": 3.4359,
+      "theoretical_loss": 4.6843507005031295,
+      "tokens_seen": 114491392
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.00048752256770310935,
+      "loss": 3.4293,
+      "theoretical_loss": 4.683986136004355,
+      "tokens_seen": 114556928
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.00048751253761283853,
+      "loss": 3.1354,
+      "theoretical_loss": 4.6836218383658705,
+      "tokens_seen": 114622464
+    },
+    {
+      "epoch": 0.03,
+      "objective/train/docs_used": 73349,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 2.8527109622955322,
+      "objective/train/theoretical_loss": 4.683257807239925,
+      "objective/train/tokens_used": 135148000,
+      "theoretical_loss": 4.683257807239925,
+      "tokens_seen": 114688000
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004875025075225677,
+      "loss": 3.1073,
+      "theoretical_loss": 4.683257807239925,
+      "tokens_seen": 114688000
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004874924774322969,
+      "loss": 3.096,
+      "theoretical_loss": 4.682894042279417,
+      "tokens_seen": 114753536
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.00048748244734202607,
+      "loss": 3.2853,
+      "theoretical_loss": 4.682530543137897,
+      "tokens_seen": 114819072
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004874724172517553,
+      "loss": 3.6298,
+      "theoretical_loss": 4.68216730946956,
+      "tokens_seen": 114884608
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.00048746238716148443,
+      "loss": 3.305,
+      "theoretical_loss": 4.6818043409292525,
+      "tokens_seen": 114950144
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.00048745235707121367,
+      "loss": 3.4946,
+      "theoretical_loss": 4.681441637172463,
+      "tokens_seen": 115015680
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004874423269809428,
+      "loss": 3.3972,
+      "theoretical_loss": 4.681079197855325,
+      "tokens_seen": 115081216
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.00048743229689067203,
+      "loss": 3.1421,
+      "theoretical_loss": 4.680717022634612,
+      "tokens_seen": 115146752
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004874222668004012,
+      "loss": 3.6661,
+      "theoretical_loss": 4.680355111167742,
+      "tokens_seen": 115212288
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004874122367101304,
+      "loss": 3.2117,
+      "theoretical_loss": 4.679993463112769,
+      "tokens_seen": 115277824
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004874022066198596,
+      "loss": 3.4083,
+      "theoretical_loss": 4.679632078128385,
+      "tokens_seen": 115343360
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.00048739217652958876,
+      "loss": 3.2653,
+      "theoretical_loss": 4.679270955873918,
+      "tokens_seen": 115408896
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.00048738214643931794,
+      "loss": 3.4833,
+      "theoretical_loss": 4.678910096009329,
+      "tokens_seen": 115474432
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048737211634904717,
+      "loss": 3.3117,
+      "theoretical_loss": 4.678549498195213,
+      "tokens_seen": 115539968
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004873620862587763,
+      "loss": 3.3255,
+      "theoretical_loss": 4.678189162092798,
+      "tokens_seen": 115605504
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048735205616850553,
+      "loss": 3.2552,
+      "theoretical_loss": 4.677829087363937,
+      "tokens_seen": 115671040
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004873420260782347,
+      "loss": 3.2912,
+      "theoretical_loss": 4.677469273671118,
+      "tokens_seen": 115736576
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004873319959879639,
+      "loss": 3.1695,
+      "theoretical_loss": 4.677109720677448,
+      "tokens_seen": 115802112
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004873219658976931,
+      "loss": 3.2226,
+      "theoretical_loss": 4.676750428046665,
+      "tokens_seen": 115867648
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048731193580742226,
+      "loss": 3.2803,
+      "theoretical_loss": 4.676391395443128,
+      "tokens_seen": 115933184
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048730190571715144,
+      "loss": 3.5568,
+      "theoretical_loss": 4.6760326225318165,
+      "tokens_seen": 115998720
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004872918756268807,
+      "loss": 3.339,
+      "theoretical_loss": 4.675674108978335,
+      "tokens_seen": 116064256
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004872818455366098,
+      "loss": 3.1442,
+      "theoretical_loss": 4.675315854448903,
+      "tokens_seen": 116129792
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048727181544633904,
+      "loss": 3.1936,
+      "theoretical_loss": 4.674957858610361,
+      "tokens_seen": 116195328
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048726178535606816,
+      "loss": 3.2333,
+      "theoretical_loss": 4.674600121130163,
+      "tokens_seen": 116260864
+    },
+    {
+      "epoch": 0.04,
+      "objective/train/docs_used": 74658,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 2.9736719131469727,
+      "objective/train/theoretical_loss": 4.674242641676377,
+      "objective/train/tokens_used": 136786400,
+      "theoretical_loss": 4.674242641676377,
+      "tokens_seen": 116326400
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004872517552657974,
+      "loss": 3.3693,
+      "theoretical_loss": 4.674242641676377,
+      "tokens_seen": 116326400
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004872417251755266,
+      "loss": 3.3527,
+      "theoretical_loss": 4.673885419917689,
+      "tokens_seen": 116391936
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048723169508525576,
+      "loss": 3.5978,
+      "theoretical_loss": 4.673528455523392,
+      "tokens_seen": 116457472
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.000487221664994985,
+      "loss": 3.416,
+      "theoretical_loss": 4.673171748163389,
+      "tokens_seen": 116523008
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004872116349047141,
+      "loss": 3.3723,
+      "theoretical_loss": 4.672815297508194,
+      "tokens_seen": 116588544
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048720160481444336,
+      "loss": 3.1412,
+      "theoretical_loss": 4.672459103228929,
+      "tokens_seen": 116654080
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048719157472417254,
+      "loss": 3.6399,
+      "theoretical_loss": 4.672103164997319,
+      "tokens_seen": 116719616
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004871815446339017,
+      "loss": 3.3651,
+      "theoretical_loss": 4.671747482485696,
+      "tokens_seen": 116785152
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004871715145436309,
+      "loss": 3.3461,
+      "theoretical_loss": 4.671392055366992,
+      "tokens_seen": 116850688
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048716148445336014,
+      "loss": 3.2057,
+      "theoretical_loss": 4.671036883314743,
+      "tokens_seen": 116916224
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048715145436308927,
+      "loss": 3.1564,
+      "theoretical_loss": 4.670681966003084,
+      "tokens_seen": 116981760
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004871414242728185,
+      "loss": 3.3224,
+      "theoretical_loss": 4.670327303106748,
+      "tokens_seen": 117047296
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048713139418254763,
+      "loss": 3.5285,
+      "theoretical_loss": 4.6699728943010665,
+      "tokens_seen": 117112832
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048712136409227686,
+      "loss": 3.2677,
+      "theoretical_loss": 4.669618739261967,
+      "tokens_seen": 117178368
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048711133400200604,
+      "loss": 3.1055,
+      "theoretical_loss": 4.669264837665969,
+      "tokens_seen": 117243904
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004871013039117352,
+      "loss": 3.3293,
+      "theoretical_loss": 4.668911189190185,
+      "tokens_seen": 117309440
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004870912738214644,
+      "loss": 3.3489,
+      "theoretical_loss": 4.668557793512322,
+      "tokens_seen": 117374976
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004870812437311936,
+      "loss": 3.1125,
+      "theoretical_loss": 4.668204650310676,
+      "tokens_seen": 117440512
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048707121364092277,
+      "loss": 3.2515,
+      "theoretical_loss": 4.667851759264128,
+      "tokens_seen": 117506048
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.000487061183550652,
+      "loss": 3.3586,
+      "theoretical_loss": 4.667499120052151,
+      "tokens_seen": 117571584
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048705115346038113,
+      "loss": 3.2803,
+      "theoretical_loss": 4.6671467323548,
+      "tokens_seen": 117637120
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048704112337011037,
+      "loss": 3.3058,
+      "theoretical_loss": 4.666794595852718,
+      "tokens_seen": 117702656
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048703109327983955,
+      "loss": 3.2752,
+      "theoretical_loss": 4.666442710227129,
+      "tokens_seen": 117768192
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048702106318956873,
+      "loss": 3.3295,
+      "theoretical_loss": 4.666091075159839,
+      "tokens_seen": 117833728
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004870110330992979,
+      "loss": 3.5682,
+      "theoretical_loss": 4.665739690333233,
+      "tokens_seen": 117899264
+    },
+    {
+      "epoch": 0.04,
+      "objective/train/docs_used": 75891,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 2.9445178508758545,
+      "objective/train/theoretical_loss": 4.665388555430277,
+      "objective/train/tokens_used": 138424800,
+      "theoretical_loss": 4.665388555430277,
+      "tokens_seen": 117964800
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004870010030090271,
+      "loss": 3.4408,
+      "theoretical_loss": 4.665388555430277,
+      "tokens_seen": 117964800
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048699097291875627,
+      "loss": 3.42,
+      "theoretical_loss": 4.665037670134513,
+      "tokens_seen": 118030336
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004869809428284855,
+      "loss": 3.4118,
+      "theoretical_loss": 4.66468703413006,
+      "tokens_seen": 118095872
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048697091273821463,
+      "loss": 2.8531,
+      "theoretical_loss": 4.664336647101612,
+      "tokens_seen": 118161408
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048696088264794387,
+      "loss": 3.5486,
+      "theoretical_loss": 4.663986508734435,
+      "tokens_seen": 118226944
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.000486950852557673,
+      "loss": 3.3137,
+      "theoretical_loss": 4.663636618714368,
+      "tokens_seen": 118292480
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048694082246740223,
+      "loss": 3.2493,
+      "theoretical_loss": 4.663286976727821,
+      "tokens_seen": 118358016
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004869307923771314,
+      "loss": 3.4941,
+      "theoretical_loss": 4.6629375824617725,
+      "tokens_seen": 118423552
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004869207622868606,
+      "loss": 3.1022,
+      "theoretical_loss": 4.6625884356037695,
+      "tokens_seen": 118489088
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004869107321965898,
+      "loss": 3.2497,
+      "theoretical_loss": 4.662239535841925,
+      "tokens_seen": 118554624
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048690070210631896,
+      "loss": 3.465,
+      "theoretical_loss": 4.661890882864919,
+      "tokens_seen": 118620160
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048689067201604814,
+      "loss": 3.3506,
+      "theoretical_loss": 4.661542476361992,
+      "tokens_seen": 118685696
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048688064192577737,
+      "loss": 3.2192,
+      "theoretical_loss": 4.661194316022952,
+      "tokens_seen": 118751232
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004868706118355065,
+      "loss": 3.274,
+      "theoretical_loss": 4.660846401538165,
+      "tokens_seen": 118816768
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048686058174523573,
+      "loss": 3.1325,
+      "theoretical_loss": 4.6604987325985565,
+      "tokens_seen": 118882304
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004868505516549649,
+      "loss": 3.241,
+      "theoretical_loss": 4.660151308895614,
+      "tokens_seen": 118947840
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004868405215646941,
+      "loss": 3.457,
+      "theoretical_loss": 4.659804130121378,
+      "tokens_seen": 119013376
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004868304914744233,
+      "loss": 3.2732,
+      "theoretical_loss": 4.659457195968448,
+      "tokens_seen": 119078912
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048682046138415246,
+      "loss": 3.1738,
+      "theoretical_loss": 4.659110506129977,
+      "tokens_seen": 119144448
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048681043129388164,
+      "loss": 3.2882,
+      "theoretical_loss": 4.6587640602996725,
+      "tokens_seen": 119209984
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004868004012036109,
+      "loss": 3.4992,
+      "theoretical_loss": 4.658417858171794,
+      "tokens_seen": 119275520
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048679037111334,
+      "loss": 3.2217,
+      "theoretical_loss": 4.6580718994411505,
+      "tokens_seen": 119341056
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048678034102306924,
+      "loss": 3.3294,
+      "theoretical_loss": 4.657726183803101,
+      "tokens_seen": 119406592
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048677031093279836,
+      "loss": 3.0235,
+      "theoretical_loss": 4.657380710953554,
+      "tokens_seen": 119472128
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004867602808425276,
+      "loss": 3.2772,
+      "theoretical_loss": 4.657035480588962,
+      "tokens_seen": 119537664
+    },
+    {
+      "epoch": 0.04,
+      "objective/train/docs_used": 76546,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 3.836437225341797,
+      "objective/train/theoretical_loss": 4.656690492406327,
+      "objective/train/tokens_used": 140063200,
+      "theoretical_loss": 4.656690492406327,
+      "tokens_seen": 119603200
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004867502507522568,
+      "loss": 3.395,
+      "theoretical_loss": 4.656690492406327,
+      "tokens_seen": 119603200
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048674022066198596,
+      "loss": 3.2332,
+      "theoretical_loss": 4.6563457461031925,
+      "tokens_seen": 119668736
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048673019057171514,
+      "loss": 3.3067,
+      "theoretical_loss": 4.656001241377647,
+      "tokens_seen": 119734272
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004867201604814443,
+      "loss": 3.4375,
+      "theoretical_loss": 4.655656977928318,
+      "tokens_seen": 119799808
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004867101303911735,
+      "loss": 3.3728,
+      "theoretical_loss": 4.655312955454378,
+      "tokens_seen": 119865344
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048670010030090274,
+      "loss": 3.4504,
+      "theoretical_loss": 4.654969173655536,
+      "tokens_seen": 119930880
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048669007021063187,
+      "loss": 3.168,
+      "theoretical_loss": 4.654625632232037,
+      "tokens_seen": 119996416
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004866800401203611,
+      "loss": 3.1419,
+      "theoretical_loss": 4.654282330884667,
+      "tokens_seen": 120061952
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004866700100300903,
+      "loss": 3.483,
+      "theoretical_loss": 4.653939269314744,
+      "tokens_seen": 120127488
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048665997993981947,
+      "loss": 3.1995,
+      "theoretical_loss": 4.653596447224125,
+      "tokens_seen": 120193024
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048664994984954865,
+      "loss": 3.0174,
+      "theoretical_loss": 4.653253864315193,
+      "tokens_seen": 120258560
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048663991975927783,
+      "loss": 3.2561,
+      "theoretical_loss": 4.6529115202908695,
+      "tokens_seen": 120324096
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.000486629889669007,
+      "loss": 3.3328,
+      "theoretical_loss": 4.652569414854604,
+      "tokens_seen": 120389632
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048661985957873624,
+      "loss": 3.2924,
+      "theoretical_loss": 4.652227547710373,
+      "tokens_seen": 120455168
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048660982948846537,
+      "loss": 3.2271,
+      "theoretical_loss": 4.651885918562684,
+      "tokens_seen": 120520704
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004865997993981946,
+      "loss": 3.2564,
+      "theoretical_loss": 4.65154452711657,
+      "tokens_seen": 120586240
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048658976930792373,
+      "loss": 3.2625,
+      "theoretical_loss": 4.651203373077593,
+      "tokens_seen": 120651776
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048657973921765297,
+      "loss": 3.1227,
+      "theoretical_loss": 4.650862456151833,
+      "tokens_seen": 120717312
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048656970912738215,
+      "loss": 3.2546,
+      "theoretical_loss": 4.650521776045898,
+      "tokens_seen": 120782848
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048655967903711133,
+      "loss": 3.2324,
+      "theoretical_loss": 4.650181332466918,
+      "tokens_seen": 120848384
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004865496489468405,
+      "loss": 3.3792,
+      "theoretical_loss": 4.64984112512254,
+      "tokens_seen": 120913920
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048653961885656975,
+      "loss": 3.4118,
+      "theoretical_loss": 4.6495011537209345,
+      "tokens_seen": 120979456
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004865295887662989,
+      "loss": 3.3581,
+      "theoretical_loss": 4.649161417970789,
+      "tokens_seen": 121044992
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004865195586760281,
+      "loss": 3.4152,
+      "theoretical_loss": 4.648821917581308,
+      "tokens_seen": 121110528
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048650952858575724,
+      "loss": 3.4244,
+      "theoretical_loss": 4.648482652262211,
+      "tokens_seen": 121176064
+    },
+    {
+      "epoch": 0.04,
+      "objective/train/docs_used": 77942,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 2.9632604122161865,
+      "objective/train/theoretical_loss": 4.648143621723734,
+      "objective/train/tokens_used": 141701600,
+      "theoretical_loss": 4.648143621723734,
+      "tokens_seen": 121241600
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048649949849548647,
+      "loss": 3.2822,
+      "theoretical_loss": 4.648143621723734,
+      "tokens_seen": 121241600
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048648946840521565,
+      "loss": 3.0832,
+      "theoretical_loss": 4.6478048256766265,
+      "tokens_seen": 121307136
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048647943831494483,
+      "loss": 3.3079,
+      "theoretical_loss": 4.647466263832148,
+      "tokens_seen": 121372672
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048646940822467407,
+      "loss": 3.2723,
+      "theoretical_loss": 4.647127935902072,
+      "tokens_seen": 121438208
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004864593781344032,
+      "loss": 3.5748,
+      "theoretical_loss": 4.646789841598681,
+      "tokens_seen": 121503744
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048644934804413243,
+      "loss": 3.1642,
+      "theoretical_loss": 4.646451980634765,
+      "tokens_seen": 121569280
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004864393179538616,
+      "loss": 3.1145,
+      "theoretical_loss": 4.646114352723625,
+      "tokens_seen": 121634816
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004864292878635908,
+      "loss": 3.5071,
+      "theoretical_loss": 4.645776957579065,
+      "tokens_seen": 121700352
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048641925777332,
+      "loss": 3.4015,
+      "theoretical_loss": 4.645439794915395,
+      "tokens_seen": 121765888
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048640922768304916,
+      "loss": 3.2914,
+      "theoretical_loss": 4.6451028644474315,
+      "tokens_seen": 121831424
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048639919759277834,
+      "loss": 3.2374,
+      "theoretical_loss": 4.644766165890491,
+      "tokens_seen": 121896960
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048638916750250757,
+      "loss": 3.2349,
+      "theoretical_loss": 4.644429698960394,
+      "tokens_seen": 121962496
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004863791374122367,
+      "loss": 3.4859,
+      "theoretical_loss": 4.644093463373462,
+      "tokens_seen": 122028032
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048636910732196593,
+      "loss": 3.2108,
+      "theoretical_loss": 4.643757458846514,
+      "tokens_seen": 122093568
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004863590772316951,
+      "loss": 3.2694,
+      "theoretical_loss": 4.643421685096866,
+      "tokens_seen": 122159104
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004863490471414243,
+      "loss": 3.525,
+      "theoretical_loss": 4.643086141842337,
+      "tokens_seen": 122224640
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004863390170511535,
+      "loss": 3.4927,
+      "theoretical_loss": 4.642750828801238,
+      "tokens_seen": 122290176
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048632898696088266,
+      "loss": 3.4461,
+      "theoretical_loss": 4.642415745692375,
+      "tokens_seen": 122355712
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048631895687061184,
+      "loss": 3.3788,
+      "theoretical_loss": 4.6420808922350485,
+      "tokens_seen": 122421248
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004863089267803411,
+      "loss": 3.3053,
+      "theoretical_loss": 4.641746268149052,
+      "tokens_seen": 122486784
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004862988966900702,
+      "loss": 3.2416,
+      "theoretical_loss": 4.64141187315467,
+      "tokens_seen": 122552320
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048628886659979944,
+      "loss": 3.0169,
+      "theoretical_loss": 4.641077706972679,
+      "tokens_seen": 122617856
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048627883650952857,
+      "loss": 3.3138,
+      "theoretical_loss": 4.6407437693243425,
+      "tokens_seen": 122683392
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004862688064192578,
+      "loss": 3.1577,
+      "theoretical_loss": 4.640410059931414,
+      "tokens_seen": 122748928
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.000486258776328987,
+      "loss": 3.3863,
+      "theoretical_loss": 4.640076578516132,
+      "tokens_seen": 122814464
+    },
+    {
+      "epoch": 0.04,
+      "objective/train/docs_used": 78541,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 2.7341437339782715,
+      "objective/train/theoretical_loss": 4.639743324801225,
+      "objective/train/tokens_used": 143340000,
+      "theoretical_loss": 4.639743324801225,
+      "tokens_seen": 122880000
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048624874623871616,
+      "loss": 3.1381,
+      "theoretical_loss": 4.639743324801225,
+      "tokens_seen": 122880000
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048623871614844534,
+      "loss": 3.3316,
+      "theoretical_loss": 4.639410298509901,
+      "tokens_seen": 122945536
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004862286860581745,
+      "loss": 3.2356,
+      "theoretical_loss": 4.639077499365856,
+      "tokens_seen": 123011072
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004862186559679037,
+      "loss": 3.355,
+      "theoretical_loss": 4.638744927093268,
+      "tokens_seen": 123076608
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048620862587763294,
+      "loss": 3.3971,
+      "theoretical_loss": 4.6384125814167945,
+      "tokens_seen": 123142144
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048619859578736207,
+      "loss": 3.0734,
+      "theoretical_loss": 4.638080462061575,
+      "tokens_seen": 123207680
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004861885656970913,
+      "loss": 3.1079,
+      "theoretical_loss": 4.63774856875323,
+      "tokens_seen": 123273216
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004861785356068205,
+      "loss": 3.3234,
+      "theoretical_loss": 4.637416901217852,
+      "tokens_seen": 123338752
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048616850551654967,
+      "loss": 3.3152,
+      "theoretical_loss": 4.63708545918202,
+      "tokens_seen": 123404288
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048615847542627885,
+      "loss": 3.345,
+      "theoretical_loss": 4.636754242372782,
+      "tokens_seen": 123469824
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048614844533600803,
+      "loss": 3.3265,
+      "theoretical_loss": 4.636423250517664,
+      "tokens_seen": 123535360
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004861384152457372,
+      "loss": 3.3053,
+      "theoretical_loss": 4.636092483344665,
+      "tokens_seen": 123600896
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048612838515546644,
+      "loss": 3.3878,
+      "theoretical_loss": 4.635761940582256,
+      "tokens_seen": 123666432
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048611835506519557,
+      "loss": 3.3957,
+      "theoretical_loss": 4.635431621959384,
+      "tokens_seen": 123731968
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004861083249749248,
+      "loss": 3.4768,
+      "theoretical_loss": 4.63510152720546,
+      "tokens_seen": 123797504
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048609829488465393,
+      "loss": 3.2736,
+      "theoretical_loss": 4.634771656050372,
+      "tokens_seen": 123863040
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048608826479438317,
+      "loss": 3.1239,
+      "theoretical_loss": 4.63444200822447,
+      "tokens_seen": 123928576
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048607823470411235,
+      "loss": 3.2963,
+      "theoretical_loss": 4.6341125834585775,
+      "tokens_seen": 123994112
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048606820461384153,
+      "loss": 3.4747,
+      "theoretical_loss": 4.633783381483978,
+      "tokens_seen": 124059648
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004860581745235707,
+      "loss": 3.374,
+      "theoretical_loss": 4.633454402032427,
+      "tokens_seen": 124125184
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048604814443329995,
+      "loss": 3.4227,
+      "theoretical_loss": 4.63312564483614,
+      "tokens_seen": 124190720
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004860381143430291,
+      "loss": 3.6141,
+      "theoretical_loss": 4.632797109627798,
+      "tokens_seen": 124256256
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004860280842527583,
+      "loss": 3.2804,
+      "theoretical_loss": 4.632468796140543,
+      "tokens_seen": 124321792
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048601805416248744,
+      "loss": 3.4935,
+      "theoretical_loss": 4.632140704107979,
+      "tokens_seen": 124387328
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048600802407221667,
+      "loss": 3.3112,
+      "theoretical_loss": 4.631812833264169,
+      "tokens_seen": 124452864
+    },
+    {
+      "epoch": 0.04,
+      "objective/train/docs_used": 79709,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 3.385240316390991,
+      "objective/train/theoretical_loss": 4.631485183343639,
+      "objective/train/tokens_used": 144978400,
+      "theoretical_loss": 4.631485183343639,
+      "tokens_seen": 124518400
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048599799398194585,
+      "loss": 3.2349,
+      "theoretical_loss": 4.631485183343639,
+      "tokens_seen": 124518400
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048598796389167503,
+      "loss": 3.3546,
+      "theoretical_loss": 4.631157754081366,
+      "tokens_seen": 124583936
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004859779338014042,
+      "loss": 3.088,
+      "theoretical_loss": 4.630830545212792,
+      "tokens_seen": 124649472
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004859679037111334,
+      "loss": 3.4382,
+      "theoretical_loss": 4.63050355647381,
+      "tokens_seen": 124715008
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004859578736208626,
+      "loss": 3.3753,
+      "theoretical_loss": 4.630176787600771,
+      "tokens_seen": 124780544
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004859478435305918,
+      "loss": 3.1749,
+      "theoretical_loss": 4.6298502383304765,
+      "tokens_seen": 124846080
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048593781344032094,
+      "loss": 3.1883,
+      "theoretical_loss": 4.629523908400184,
+      "tokens_seen": 124911616
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004859277833500502,
+      "loss": 3.4012,
+      "theoretical_loss": 4.6291977975476035,
+      "tokens_seen": 124977152
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004859177532597793,
+      "loss": 3.1556,
+      "theoretical_loss": 4.628871905510893,
+      "tokens_seen": 125042688
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048590772316950854,
+      "loss": 3.5815,
+      "theoretical_loss": 4.628546232028661,
+      "tokens_seen": 125108224
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004858976930792377,
+      "loss": 3.2682,
+      "theoretical_loss": 4.62822077683997,
+      "tokens_seen": 125173760
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004858876629889669,
+      "loss": 3.685,
+      "theoretical_loss": 4.627895539684323,
+      "tokens_seen": 125239296
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004858776328986961,
+      "loss": 3.2897,
+      "theoretical_loss": 4.627570520301673,
+      "tokens_seen": 125304832
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004858676028084253,
+      "loss": 3.1734,
+      "theoretical_loss": 4.627245718432423,
+      "tokens_seen": 125370368
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048585757271815444,
+      "loss": 3.0723,
+      "theoretical_loss": 4.626921133817413,
+      "tokens_seen": 125435904
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004858475426278837,
+      "loss": 3.1019,
+      "theoretical_loss": 4.626596766197936,
+      "tokens_seen": 125501440
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004858375125376128,
+      "loss": 3.21,
+      "theoretical_loss": 4.62627261531572,
+      "tokens_seen": 125566976
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048582748244734204,
+      "loss": 3.1586,
+      "theoretical_loss": 4.625948680912939,
+      "tokens_seen": 125632512
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004858174523570712,
+      "loss": 3.1464,
+      "theoretical_loss": 4.625624962732207,
+      "tokens_seen": 125698048
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004858074222668004,
+      "loss": 3.5956,
+      "theoretical_loss": 4.62530146051658,
+      "tokens_seen": 125763584
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004857973921765296,
+      "loss": 3.2982,
+      "theoretical_loss": 4.624978174009549,
+      "tokens_seen": 125829120
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048578736208625877,
+      "loss": 2.8431,
+      "theoretical_loss": 4.624655102955048,
+      "tokens_seen": 125894656
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048577733199598795,
+      "loss": 3.2039,
+      "theoretical_loss": 4.624332247097442,
+      "tokens_seen": 125960192
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004857673019057172,
+      "loss": 3.255,
+      "theoretical_loss": 4.624009606181539,
+      "tokens_seen": 126025728
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004857572718154463,
+      "loss": 3.3476,
+      "theoretical_loss": 4.623687179952577,
+      "tokens_seen": 126091264
+    },
+    {
+      "epoch": 0.04,
+      "objective/train/docs_used": 80984,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 3.0424399375915527,
+      "objective/train/theoretical_loss": 4.623364968156229,
+      "objective/train/tokens_used": 146616800,
+      "theoretical_loss": 4.623364968156229,
+      "tokens_seen": 126156800
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048574724172517554,
+      "loss": 3.1198,
+      "theoretical_loss": 4.623364968156229,
+      "tokens_seen": 126156800
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048573721163490467,
+      "loss": 3.2627,
+      "theoretical_loss": 4.623042970538604,
+      "tokens_seen": 126222336
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004857271815446339,
+      "loss": 3.2619,
+      "theoretical_loss": 4.622721186846241,
+      "tokens_seen": 126287872
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048571715145436314,
+      "loss": 3.077,
+      "theoretical_loss": 4.62239961682611,
+      "tokens_seen": 126353408
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048570712136409227,
+      "loss": 3.4283,
+      "theoretical_loss": 4.622078260225611,
+      "tokens_seen": 126418944
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004856970912738215,
+      "loss": 3.1504,
+      "theoretical_loss": 4.621757116792576,
+      "tokens_seen": 126484480
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004856870611835507,
+      "loss": 3.2759,
+      "theoretical_loss": 4.621436186275262,
+      "tokens_seen": 126550016
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048567703109327987,
+      "loss": 3.2942,
+      "theoretical_loss": 4.6211154684223565,
+      "tokens_seen": 126615552
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048566700100300905,
+      "loss": 3.462,
+      "theoretical_loss": 4.6207949629829725,
+      "tokens_seen": 126681088
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048565697091273823,
+      "loss": 2.9686,
+      "theoretical_loss": 4.620474669706647,
+      "tokens_seen": 126746624
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004856469408224674,
+      "loss": 3.3596,
+      "theoretical_loss": 4.620154588343343,
+      "tokens_seen": 126812160
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048563691073219664,
+      "loss": 3.3793,
+      "theoretical_loss": 4.619834718643448,
+      "tokens_seen": 126877696
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048562688064192577,
+      "loss": 3.327,
+      "theoretical_loss": 4.619515060357771,
+      "tokens_seen": 126943232
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.000485616850551655,
+      "loss": 3.5938,
+      "theoretical_loss": 4.619195613237542,
+      "tokens_seen": 127008768
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048560682046138413,
+      "loss": 3.1359,
+      "theoretical_loss": 4.6188763770344154,
+      "tokens_seen": 127074304
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048559679037111337,
+      "loss": 3.6939,
+      "theoretical_loss": 4.618557351500461,
+      "tokens_seen": 127139840
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048558676028084255,
+      "loss": 3.3604,
+      "theoretical_loss": 4.618238536388171,
+      "tokens_seen": 127205376
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048557673019057173,
+      "loss": 3.4903,
+      "theoretical_loss": 4.617919931450454,
+      "tokens_seen": 127270912
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004855667001003009,
+      "loss": 3.3289,
+      "theoretical_loss": 4.617601536440637,
+      "tokens_seen": 127336448
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048555667001003015,
+      "loss": 3.1247,
+      "theoretical_loss": 4.617283351112462,
+      "tokens_seen": 127401984
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004855466399197593,
+      "loss": 3.5764,
+      "theoretical_loss": 4.616965375220088,
+      "tokens_seen": 127467520
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004855366098294885,
+      "loss": 3.419,
+      "theoretical_loss": 4.616647608518086,
+      "tokens_seen": 127533056
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048552657973921764,
+      "loss": 3.2447,
+      "theoretical_loss": 4.616330050761444,
+      "tokens_seen": 127598592
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048551654964894687,
+      "loss": 3.0136,
+      "theoretical_loss": 4.616012701705559,
+      "tokens_seen": 127664128
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048550651955867605,
+      "loss": 3.1339,
+      "theoretical_loss": 4.615695561106241,
+      "tokens_seen": 127729664
+    },
+    {
+      "epoch": 0.04,
+      "objective/train/docs_used": 81703,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 3.6864616870880127,
+      "objective/train/theoretical_loss": 4.6153786287197125,
+      "objective/train/tokens_used": 148255200,
+      "theoretical_loss": 4.6153786287197125,
+      "tokens_seen": 127795200
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048549648946840523,
+      "loss": 3.2948,
+      "theoretical_loss": 4.6153786287197125,
+      "tokens_seen": 127795200
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004854864593781344,
+      "loss": 3.0792,
+      "theoretical_loss": 4.615061904302603,
+      "tokens_seen": 127860736
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004854764292878636,
+      "loss": 3.1573,
+      "theoretical_loss": 4.614745387611955,
+      "tokens_seen": 127926272
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004854663991975928,
+      "loss": 3.4941,
+      "theoretical_loss": 4.614429078405214,
+      "tokens_seen": 127991808
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.000485456369107322,
+      "loss": 3.2639,
+      "theoretical_loss": 4.6141129764402375,
+      "tokens_seen": 128057344
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048544633901705114,
+      "loss": 3.2058,
+      "theoretical_loss": 4.613797081475287,
+      "tokens_seen": 128122880
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004854363089267804,
+      "loss": 3.4029,
+      "theoretical_loss": 4.613481393269028,
+      "tokens_seen": 128188416
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004854262788365095,
+      "loss": 3.5993,
+      "theoretical_loss": 4.613165911580533,
+      "tokens_seen": 128253952
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048541624874623874,
+      "loss": 3.1374,
+      "theoretical_loss": 4.612850636169279,
+      "tokens_seen": 128319488
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004854062186559679,
+      "loss": 3.3721,
+      "theoretical_loss": 4.612535566795142,
+      "tokens_seen": 128385024
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004853961885656971,
+      "loss": 3.1656,
+      "theoretical_loss": 4.612220703218403,
+      "tokens_seen": 128450560
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004853861584754263,
+      "loss": 3.4822,
+      "theoretical_loss": 4.611906045199744,
+      "tokens_seen": 128516096
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004853761283851555,
+      "loss": 3.1366,
+      "theoretical_loss": 4.611591592500245,
+      "tokens_seen": 128581632
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048536609829488464,
+      "loss": 3.3858,
+      "theoretical_loss": 4.611277344881389,
+      "tokens_seen": 128647168
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004853560682046139,
+      "loss": 3.1729,
+      "theoretical_loss": 4.610963302105051,
+      "tokens_seen": 128712704
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.000485346038114343,
+      "loss": 3.2684,
+      "theoretical_loss": 4.610649463933511,
+      "tokens_seen": 128778240
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048533600802407224,
+      "loss": 3.1187,
+      "theoretical_loss": 4.61033583012944,
+      "tokens_seen": 128843776
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004853259779338014,
+      "loss": 3.2702,
+      "theoretical_loss": 4.610022400455909,
+      "tokens_seen": 128909312
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004853159478435306,
+      "loss": 3.1401,
+      "theoretical_loss": 4.6097091746763805,
+      "tokens_seen": 128974848
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004853059177532598,
+      "loss": 3.0625,
+      "theoretical_loss": 4.609396152554714,
+      "tokens_seen": 129040384
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048529588766298897,
+      "loss": 3.4469,
+      "theoretical_loss": 4.60908333385516,
+      "tokens_seen": 129105920
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048528585757271815,
+      "loss": 3.2416,
+      "theoretical_loss": 4.608770718342361,
+      "tokens_seen": 129171456
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004852758274824474,
+      "loss": 3.2241,
+      "theoretical_loss": 4.6084583057813555,
+      "tokens_seen": 129236992
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004852657973921765,
+      "loss": 3.2384,
+      "theoretical_loss": 4.608146095937567,
+      "tokens_seen": 129302528
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048525576730190574,
+      "loss": 3.4009,
+      "theoretical_loss": 4.607834088576811,
+      "tokens_seen": 129368064
+    },
+    {
+      "epoch": 0.04,
+      "objective/train/docs_used": 82831,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 3.1748886108398438,
+      "objective/train/theoretical_loss": 4.607522283465294,
+      "objective/train/tokens_used": 149893600,
+      "theoretical_loss": 4.607522283465294,
+      "tokens_seen": 129433600
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048524573721163487,
+      "loss": 3.2513,
+      "theoretical_loss": 4.607522283465294,
+      "tokens_seen": 129433600
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004852357071213641,
+      "loss": 3.3024,
+      "theoretical_loss": 4.607210680369608,
+      "tokens_seen": 129499136
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004852256770310933,
+      "loss": 3.5175,
+      "theoretical_loss": 4.606899279056734,
+      "tokens_seen": 129564672
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048521564694082247,
+      "loss": 3.1854,
+      "theoretical_loss": 4.606588079294038,
+      "tokens_seen": 129630208
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048520561685055165,
+      "loss": 3.1448,
+      "theoretical_loss": 4.606277080849273,
+      "tokens_seen": 129695744
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004851955867602809,
+      "loss": 3.2784,
+      "theoretical_loss": 4.605966283490575,
+      "tokens_seen": 129761280
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048518555667001,
+      "loss": 3.055,
+      "theoretical_loss": 4.6056556869864655,
+      "tokens_seen": 129826816
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048517552657973925,
+      "loss": 3.4625,
+      "theoretical_loss": 4.60534529110585,
+      "tokens_seen": 129892352
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004851654964894684,
+      "loss": 3.2055,
+      "theoretical_loss": 4.605035095618014,
+      "tokens_seen": 129957888
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004851554663991976,
+      "loss": 3.3952,
+      "theoretical_loss": 4.604725100292626,
+      "tokens_seen": 130023424
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004851454363089268,
+      "loss": 3.193,
+      "theoretical_loss": 4.604415304899735,
+      "tokens_seen": 130088960
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048513540621865597,
+      "loss": 3.3349,
+      "theoretical_loss": 4.604105709209769,
+      "tokens_seen": 130154496
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048512537612838515,
+      "loss": 3.4078,
+      "theoretical_loss": 4.603796312993536,
+      "tokens_seen": 130220032
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048511534603811433,
+      "loss": 3.3374,
+      "theoretical_loss": 4.603487116022222,
+      "tokens_seen": 130285568
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004851053159478435,
+      "loss": 3.2205,
+      "theoretical_loss": 4.60317811806739,
+      "tokens_seen": 130351104
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048509528585757275,
+      "loss": 3.3053,
+      "theoretical_loss": 4.60286931890098,
+      "tokens_seen": 130416640
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004850852557673019,
+      "loss": 3.3346,
+      "theoretical_loss": 4.602560718295308,
+      "tokens_seen": 130482176
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004850752256770311,
+      "loss": 3.3864,
+      "theoretical_loss": 4.602252316023064,
+      "tokens_seen": 130547712
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048506519558676024,
+      "loss": 3.4327,
+      "theoretical_loss": 4.601944111857312,
+      "tokens_seen": 130613248
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004850551654964895,
+      "loss": 3.2583,
+      "theoretical_loss": 4.601636105571492,
+      "tokens_seen": 130678784
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048504513540621866,
+      "loss": 3.4605,
+      "theoretical_loss": 4.601328296939414,
+      "tokens_seen": 130744320
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048503510531594784,
+      "loss": 3.2544,
+      "theoretical_loss": 4.601020685735259,
+      "tokens_seen": 130809856
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.000485025075225677,
+      "loss": 3.2836,
+      "theoretical_loss": 4.600713271733582,
+      "tokens_seen": 130875392
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048501504513540625,
+      "loss": 3.4586,
+      "theoretical_loss": 4.600406054709307,
+      "tokens_seen": 130940928
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004850050150451354,
+      "loss": 3.3115,
+      "theoretical_loss": 4.600099034437726,
+      "tokens_seen": 131006464
+    },
+    {
+      "epoch": 0.04,
+      "objective/train/docs_used": 83204,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 3.2750132083892822,
+      "objective/train/theoretical_loss": 4.5997922106945,
+      "objective/train/tokens_used": 151532000,
+      "theoretical_loss": 4.5997922106945,
+      "tokens_seen": 131072000
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004849949849548646,
+      "loss": 3.3318,
+      "theoretical_loss": 4.5997922106945,
+      "tokens_seen": 131072000
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004849849548645938,
+      "loss": 3.5144,
+      "theoretical_loss": 4.5994855832556585,
+      "tokens_seen": 131137536
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.000484974924774323,
+      "loss": 3.4244,
+      "theoretical_loss": 4.5991791518976,
+      "tokens_seen": 131203072
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004849648946840522,
+      "loss": 3.2389,
+      "theoretical_loss": 4.5988729163970845,
+      "tokens_seen": 131268608
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048495486459378134,
+      "loss": 3.3639,
+      "theoretical_loss": 4.59856687653124,
+      "tokens_seen": 131334144
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004849448345035106,
+      "loss": 3.2156,
+      "theoretical_loss": 4.598261032077561,
+      "tokens_seen": 131399680
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004849348044132397,
+      "loss": 3.5008,
+      "theoretical_loss": 4.597955382813903,
+      "tokens_seen": 131465216
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048492477432296894,
+      "loss": 3.4594,
+      "theoretical_loss": 4.597649928518486,
+      "tokens_seen": 131530752
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004849147442326981,
+      "loss": 3.3812,
+      "theoretical_loss": 4.597344668969892,
+      "tokens_seen": 131596288
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004849047141424273,
+      "loss": 3.3369,
+      "theoretical_loss": 4.597039603947064,
+      "tokens_seen": 131661824
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004848946840521565,
+      "loss": 3.2768,
+      "theoretical_loss": 4.596734733229306,
+      "tokens_seen": 131727360
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004848846539618857,
+      "loss": 2.9459,
+      "theoretical_loss": 4.596430056596285,
+      "tokens_seen": 131792896
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048487462387161484,
+      "loss": 3.0683,
+      "theoretical_loss": 4.596125573828021,
+      "tokens_seen": 131858432
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004848645937813441,
+      "loss": 3.3636,
+      "theoretical_loss": 4.595821284704899,
+      "tokens_seen": 131923968
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004848545636910732,
+      "loss": 3.2567,
+      "theoretical_loss": 4.5955171890076585,
+      "tokens_seen": 131989504
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048484453360080244,
+      "loss": 3.5107,
+      "theoretical_loss": 4.595213286517394,
+      "tokens_seen": 132055040
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004848345035105316,
+      "loss": 3.3222,
+      "theoretical_loss": 4.594909577015562,
+      "tokens_seen": 132120576
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004848244734202608,
+      "loss": 3.4088,
+      "theoretical_loss": 4.594606060283969,
+      "tokens_seen": 132186112
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048481444332999,
+      "loss": 3.3058,
+      "theoretical_loss": 4.594302736104778,
+      "tokens_seen": 132251648
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048480441323971917,
+      "loss": 3.3761,
+      "theoretical_loss": 4.593999604260507,
+      "tokens_seen": 132317184
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048479438314944835,
+      "loss": 3.115,
+      "theoretical_loss": 4.593696664534026,
+      "tokens_seen": 132382720
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004847843530591776,
+      "loss": 3.0734,
+      "theoretical_loss": 4.593393916708557,
+      "tokens_seen": 132448256
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004847743229689067,
+      "loss": 3.4007,
+      "theoretical_loss": 4.593091360567676,
+      "tokens_seen": 132513792
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048476429287863594,
+      "loss": 3.4221,
+      "theoretical_loss": 4.59278899589531,
+      "tokens_seen": 132579328
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048475426278836507,
+      "loss": 3.3316,
+      "theoretical_loss": 4.592486822475731,
+      "tokens_seen": 132644864
+    },
+    {
+      "epoch": 0.04,
+      "objective/train/docs_used": 84183,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 3.4309723377227783,
+      "objective/train/theoretical_loss": 4.592184840093566,
+      "objective/train/tokens_used": 153170400,
+      "theoretical_loss": 4.592184840093566,
+      "tokens_seen": 132710400
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004847442326980943,
+      "loss": 3.1609,
+      "theoretical_loss": 4.592184840093566,
+      "tokens_seen": 132710400
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004847342026078235,
+      "loss": 3.3487,
+      "theoretical_loss": 4.59188304853379,
+      "tokens_seen": 132775936
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048472417251755267,
+      "loss": 3.2624,
+      "theoretical_loss": 4.591581447581723,
+      "tokens_seen": 132841472
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048471414242728185,
+      "loss": 3.0568,
+      "theoretical_loss": 4.591280037023036,
+      "tokens_seen": 132907008
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004847041123370111,
+      "loss": 3.3944,
+      "theoretical_loss": 4.590978816643744,
+      "tokens_seen": 132972544
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004846940822467402,
+      "loss": 3.375,
+      "theoretical_loss": 4.590677786230208,
+      "tokens_seen": 133038080
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048468405215646945,
+      "loss": 3.1915,
+      "theoretical_loss": 4.590376945569136,
+      "tokens_seen": 133103616
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004846740220661986,
+      "loss": 3.2734,
+      "theoretical_loss": 4.590076294447578,
+      "tokens_seen": 133169152
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004846639919759278,
+      "loss": 3.1604,
+      "theoretical_loss": 4.589775832652929,
+      "tokens_seen": 133234688
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.000484653961885657,
+      "loss": 3.1837,
+      "theoretical_loss": 4.589475559972927,
+      "tokens_seen": 133300224
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048464393179538617,
+      "loss": 3.3031,
+      "theoretical_loss": 4.589175476195651,
+      "tokens_seen": 133365760
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048463390170511535,
+      "loss": 3.1795,
+      "theoretical_loss": 4.588875581109523,
+      "tokens_seen": 133431296
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048462387161484453,
+      "loss": 3.2039,
+      "theoretical_loss": 4.588575874503304,
+      "tokens_seen": 133496832
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004846138415245737,
+      "loss": 3.4852,
+      "theoretical_loss": 4.588276356166096,
+      "tokens_seen": 133562368
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048460381143430295,
+      "loss": 3.3087,
+      "theoretical_loss": 4.587977025887342,
+      "tokens_seen": 133627904
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004845937813440321,
+      "loss": 3.1137,
+      "theoretical_loss": 4.58767788345682,
+      "tokens_seen": 133693440
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004845837512537613,
+      "loss": 3.2587,
+      "theoretical_loss": 4.5873789286646485,
+      "tokens_seen": 133758976
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048457372116349044,
+      "loss": 3.3583,
+      "theoretical_loss": 4.587080161301283,
+      "tokens_seen": 133824512
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004845636910732197,
+      "loss": 3.2645,
+      "theoretical_loss": 4.586781581157513,
+      "tokens_seen": 133890048
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048455366098294886,
+      "loss": 3.289,
+      "theoretical_loss": 4.58648318802447,
+      "tokens_seen": 133955584
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048454363089267804,
+      "loss": 2.9687,
+      "theoretical_loss": 4.586184981693611,
+      "tokens_seen": 134021120
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004845336008024072,
+      "loss": 3.1314,
+      "theoretical_loss": 4.585886961956737,
+      "tokens_seen": 134086656
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048452357071213645,
+      "loss": 3.6532,
+      "theoretical_loss": 4.585589128605976,
+      "tokens_seen": 134152192
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004845135406218656,
+      "loss": 3.1767,
+      "theoretical_loss": 4.585291481433792,
+      "tokens_seen": 134217728
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004845035105315948,
+      "loss": 3.1699,
+      "theoretical_loss": 4.584994020232981,
+      "tokens_seen": 134283264
+    },
+    {
+      "epoch": 0.04,
+      "objective/train/docs_used": 84861,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 3.49385142326355,
+      "objective/train/theoretical_loss": 4.58469674479667,
+      "objective/train/tokens_used": 154808800,
+      "theoretical_loss": 4.58469674479667,
+      "tokens_seen": 134348800
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048449348044132394,
+      "loss": 3.3437,
+      "theoretical_loss": 4.58469674479667,
+      "tokens_seen": 134348800
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004844834503510532,
+      "loss": 3.3897,
+      "theoretical_loss": 4.584399654918316,
+      "tokens_seen": 134414336
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048447342026078236,
+      "loss": 3.2379,
+      "theoretical_loss": 4.584102750391709,
+      "tokens_seen": 134479872
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048446339017051154,
+      "loss": 3.4963,
+      "theoretical_loss": 4.583806031010965,
+      "tokens_seen": 134545408
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004844533600802407,
+      "loss": 3.4515,
+      "theoretical_loss": 4.5835094965705325,
+      "tokens_seen": 134610944
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004844433299899699,
+      "loss": 3.323,
+      "theoretical_loss": 4.5832131468651855,
+      "tokens_seen": 134676480
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004844332998996991,
+      "loss": 3.0751,
+      "theoretical_loss": 4.582916981690026,
+      "tokens_seen": 134742016
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004844232698094283,
+      "loss": 3.2973,
+      "theoretical_loss": 4.582621000840484,
+      "tokens_seen": 134807552
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048441323971915745,
+      "loss": 3.3443,
+      "theoretical_loss": 4.582325204112314,
+      "tokens_seen": 134873088
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004844032096288867,
+      "loss": 3.2763,
+      "theoretical_loss": 4.582029591301595,
+      "tokens_seen": 134938624
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048439317953861586,
+      "loss": 3.3043,
+      "theoretical_loss": 4.581734162204734,
+      "tokens_seen": 135004160
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048438314944834504,
+      "loss": 3.2092,
+      "theoretical_loss": 4.5814389166184615,
+      "tokens_seen": 135069696
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004843731193580742,
+      "loss": 3.2379,
+      "theoretical_loss": 4.581143854339828,
+      "tokens_seen": 135135232
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004843630892678034,
+      "loss": 3.2659,
+      "theoretical_loss": 4.580848975166209,
+      "tokens_seen": 135200768
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004843530591775326,
+      "loss": 3.0079,
+      "theoretical_loss": 4.580554278895304,
+      "tokens_seen": 135266304
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004843430290872618,
+      "loss": 3.2396,
+      "theoretical_loss": 4.58025976532513,
+      "tokens_seen": 135331840
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048433299899699095,
+      "loss": 3.2323,
+      "theoretical_loss": 4.579965434254028,
+      "tokens_seen": 135397376
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004843229689067202,
+      "loss": 3.0602,
+      "theoretical_loss": 4.579671285480657,
+      "tokens_seen": 135462912
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004843129388164493,
+      "loss": 3.2404,
+      "theoretical_loss": 4.579377318803996,
+      "tokens_seen": 135528448
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048430290872617855,
+      "loss": 3.0519,
+      "theoretical_loss": 4.579083534023344,
+      "tokens_seen": 135593984
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048429287863590773,
+      "loss": 3.3365,
+      "theoretical_loss": 4.578789930938315,
+      "tokens_seen": 135659520
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004842828485456369,
+      "loss": 3.2243,
+      "theoretical_loss": 4.5784965093488434,
+      "tokens_seen": 135725056
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004842728184553661,
+      "loss": 3.1942,
+      "theoretical_loss": 4.578203269055179,
+      "tokens_seen": 135790592
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048426278836509527,
+      "loss": 3.4092,
+      "theoretical_loss": 4.577910209857889,
+      "tokens_seen": 135856128
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048425275827482445,
+      "loss": 3.1487,
+      "theoretical_loss": 4.577617331557854,
+      "tokens_seen": 135921664
+    },
+    {
+      "epoch": 0.04,
+      "objective/train/docs_used": 86086,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 3.4546592235565186,
+      "objective/train/theoretical_loss": 4.577324633956271,
+      "objective/train/tokens_used": 156447200,
+      "theoretical_loss": 4.577324633956271,
+      "tokens_seen": 135987200
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004842427281845537,
+      "loss": 3.3413,
+      "theoretical_loss": 4.577324633956271,
+      "tokens_seen": 135987200
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048423269809428287,
+      "loss": 3.4106,
+      "theoretical_loss": 4.5770321168546495,
+      "tokens_seen": 136052736
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048422266800401205,
+      "loss": 3.4241,
+      "theoretical_loss": 4.576739780054815,
+      "tokens_seen": 136118272
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004842126379137413,
+      "loss": 3.254,
+      "theoretical_loss": 4.576447623358905,
+      "tokens_seen": 136183808
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004842026078234704,
+      "loss": 3.0941,
+      "theoretical_loss": 4.576155646569367,
+      "tokens_seen": 136249344
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048419257773319965,
+      "loss": 3.3943,
+      "theoretical_loss": 4.575863849488963,
+      "tokens_seen": 136314880
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004841825476429288,
+      "loss": 3.2206,
+      "theoretical_loss": 4.575572231920763,
+      "tokens_seen": 136380416
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.000484172517552658,
+      "loss": 3.1229,
+      "theoretical_loss": 4.575280793668151,
+      "tokens_seen": 136445952
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004841624874623872,
+      "loss": 2.9849,
+      "theoretical_loss": 4.574989534534817,
+      "tokens_seen": 136511488
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048415245737211637,
+      "loss": 3.2635,
+      "theoretical_loss": 4.574698454324764,
+      "tokens_seen": 136577024
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048414242728184555,
+      "loss": 3.1675,
+      "theoretical_loss": 4.574407552842299,
+      "tokens_seen": 136642560
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048413239719157473,
+      "loss": 3.5906,
+      "theoretical_loss": 4.57411682989204,
+      "tokens_seen": 136708096
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004841223671013039,
+      "loss": 3.2567,
+      "theoretical_loss": 4.57382628527891,
+      "tokens_seen": 136773632
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048411233701103315,
+      "loss": 3.5297,
+      "theoretical_loss": 4.573535918808141,
+      "tokens_seen": 136839168
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004841023069207623,
+      "loss": 3.3214,
+      "theoretical_loss": 4.573245730285268,
+      "tokens_seen": 136904704
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004840922768304915,
+      "loss": 3.2527,
+      "theoretical_loss": 4.572955719516132,
+      "tokens_seen": 136970240
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048408224674022064,
+      "loss": 3.0917,
+      "theoretical_loss": 4.572665886306881,
+      "tokens_seen": 137035776
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004840722166499499,
+      "loss": 3.0382,
+      "theoretical_loss": 4.572376230463965,
+      "tokens_seen": 137101312
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048406218655967906,
+      "loss": 3.3253,
+      "theoretical_loss": 4.572086751794135,
+      "tokens_seen": 137166848
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048405215646940824,
+      "loss": 3.4228,
+      "theoretical_loss": 4.571797450104451,
+      "tokens_seen": 137232384
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004840421263791374,
+      "loss": 3.4935,
+      "theoretical_loss": 4.571508325202268,
+      "tokens_seen": 137297920
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048403209628886665,
+      "loss": 3.2935,
+      "theoretical_loss": 4.5712193768952485,
+      "tokens_seen": 137363456
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004840220661985958,
+      "loss": 3.0949,
+      "theoretical_loss": 4.570930604991352,
+      "tokens_seen": 137428992
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.000484012036108325,
+      "loss": 3.0731,
+      "theoretical_loss": 4.57064200929884,
+      "tokens_seen": 137494528
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048400200601805414,
+      "loss": 3.2526,
+      "theoretical_loss": 4.570353589626273,
+      "tokens_seen": 137560064
+    },
+    {
+      "epoch": 0.04,
+      "objective/train/docs_used": 86654,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 3.495647668838501,
+      "objective/train/theoretical_loss": 4.570065345782512,
+      "objective/train/tokens_used": 158085600,
+      "theoretical_loss": 4.570065345782512,
+      "tokens_seen": 137625600
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004839919759277834,
+      "loss": 3.3585,
+      "theoretical_loss": 4.570065345782512,
+      "tokens_seen": 137625600
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048398194583751256,
+      "loss": 3.331,
+      "theoretical_loss": 4.569777277576713,
+      "tokens_seen": 137691136
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048397191574724174,
+      "loss": 3.2636,
+      "theoretical_loss": 4.569489384818334,
+      "tokens_seen": 137756672
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004839618856569709,
+      "loss": 3.3772,
+      "theoretical_loss": 4.569201667317128,
+      "tokens_seen": 137822208
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004839518555667001,
+      "loss": 3.1082,
+      "theoretical_loss": 4.568914124883145,
+      "tokens_seen": 137887744
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004839418254764293,
+      "loss": 3.567,
+      "theoretical_loss": 4.568626757326731,
+      "tokens_seen": 137953280
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004839317953861585,
+      "loss": 3.3668,
+      "theoretical_loss": 4.568339564458528,
+      "tokens_seen": 138018816
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048392176529588765,
+      "loss": 3.2898,
+      "theoretical_loss": 4.568052546089472,
+      "tokens_seen": 138084352
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004839117352056169,
+      "loss": 3.2154,
+      "theoretical_loss": 4.567765702030792,
+      "tokens_seen": 138149888
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048390170511534606,
+      "loss": 3.3787,
+      "theoretical_loss": 4.567479032094015,
+      "tokens_seen": 138215424
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048389167502507524,
+      "loss": 3.2051,
+      "theoretical_loss": 4.567192536090956,
+      "tokens_seen": 138280960
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004838816449348044,
+      "loss": 3.2675,
+      "theoretical_loss": 4.566906213833725,
+      "tokens_seen": 138346496
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004838716148445336,
+      "loss": 3.4467,
+      "theoretical_loss": 4.566620065134725,
+      "tokens_seen": 138412032
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004838615847542628,
+      "loss": 3.3219,
+      "theoretical_loss": 4.566334089806647,
+      "tokens_seen": 138477568
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.000483851554663992,
+      "loss": 3.3479,
+      "theoretical_loss": 4.566048287662475,
+      "tokens_seen": 138543104
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048384152457372115,
+      "loss": 3.3473,
+      "theoretical_loss": 4.565762658515482,
+      "tokens_seen": 138608640
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004838314944834504,
+      "loss": 3.3177,
+      "theoretical_loss": 4.565477202179232,
+      "tokens_seen": 138674176
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004838214643931795,
+      "loss": 3.2072,
+      "theoretical_loss": 4.565191918467576,
+      "tokens_seen": 138739712
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048381143430290875,
+      "loss": 3.4187,
+      "theoretical_loss": 4.564906807194655,
+      "tokens_seen": 138805248
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048380140421263793,
+      "loss": 2.8605,
+      "theoretical_loss": 4.564621868174897,
+      "tokens_seen": 138870784
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004837913741223671,
+      "loss": 3.0615,
+      "theoretical_loss": 4.564337101223017,
+      "tokens_seen": 138936320
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004837813440320963,
+      "loss": 3.3648,
+      "theoretical_loss": 4.564052506154015,
+      "tokens_seen": 139001856
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048377131394182547,
+      "loss": 3.051,
+      "theoretical_loss": 4.563768082783182,
+      "tokens_seen": 139067392
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048376128385155465,
+      "loss": 3.1675,
+      "theoretical_loss": 4.563483830926089,
+      "tokens_seen": 139132928
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004837512537612839,
+      "loss": 3.1113,
+      "theoretical_loss": 4.563199750398592,
+      "tokens_seen": 139198464
+    },
+    {
+      "epoch": 0.04,
+      "objective/train/docs_used": 87912,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 3.470696210861206,
+      "objective/train/theoretical_loss": 4.562915841016838,
+      "objective/train/tokens_used": 159724000,
+      "theoretical_loss": 4.562915841016838,
+      "tokens_seen": 139264000
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.000483741223671013,
+      "loss": 3.5703,
+      "theoretical_loss": 4.562915841016838,
+      "tokens_seen": 139264000
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048373119358074225,
+      "loss": 3.2538,
+      "theoretical_loss": 4.5626321025972505,
+      "tokens_seen": 139329536
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048372116349047143,
+      "loss": 3.2385,
+      "theoretical_loss": 4.562348534956538,
+      "tokens_seen": 139395072
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004837111334002006,
+      "loss": 3.4436,
+      "theoretical_loss": 4.562065137911691,
+      "tokens_seen": 139460608
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004837011033099298,
+      "loss": 3.1403,
+      "theoretical_loss": 4.561781911279987,
+      "tokens_seen": 139526144
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.000483691073219659,
+      "loss": 3.1669,
+      "theoretical_loss": 4.5614988548789785,
+      "tokens_seen": 139591680
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048368104312938816,
+      "loss": 3.1656,
+      "theoretical_loss": 4.561215968526502,
+      "tokens_seen": 139657216
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004836710130391174,
+      "loss": 3.2068,
+      "theoretical_loss": 4.560933252040675,
+      "tokens_seen": 139722752
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004836609829488465,
+      "loss": 3.2876,
+      "theoretical_loss": 4.56065070523989,
+      "tokens_seen": 139788288
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048365095285857575,
+      "loss": 3.7268,
+      "theoretical_loss": 4.560368327942827,
+      "tokens_seen": 139853824
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004836409227683049,
+      "loss": 3.0936,
+      "theoretical_loss": 4.560086119968435,
+      "tokens_seen": 139919360
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004836308926780341,
+      "loss": 3.362,
+      "theoretical_loss": 4.559804081135949,
+      "tokens_seen": 139984896
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004836208625877633,
+      "loss": 3.4023,
+      "theoretical_loss": 4.559522211264877,
+      "tokens_seen": 140050432
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004836108324974925,
+      "loss": 3.299,
+      "theoretical_loss": 4.559240510175006,
+      "tokens_seen": 140115968
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048360080240722166,
+      "loss": 3.5194,
+      "theoretical_loss": 4.558958977686398,
+      "tokens_seen": 140181504
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048359077231695084,
+      "loss": 3.1333,
+      "theoretical_loss": 4.558677613619393,
+      "tokens_seen": 140247040
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048358074222668,
+      "loss": 3.1792,
+      "theoretical_loss": 4.558396417794603,
+      "tokens_seen": 140312576
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048357071213640926,
+      "loss": 3.2967,
+      "theoretical_loss": 4.558115390032916,
+      "tokens_seen": 140378112
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004835606820461384,
+      "loss": 3.2453,
+      "theoretical_loss": 4.557834530155497,
+      "tokens_seen": 140443648
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004835506519558676,
+      "loss": 3.2309,
+      "theoretical_loss": 4.5575538379837806,
+      "tokens_seen": 140509184
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004835406218655968,
+      "loss": 3.1676,
+      "theoretical_loss": 4.557273313339477,
+      "tokens_seen": 140574720
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.000483530591775326,
+      "loss": 3.3045,
+      "theoretical_loss": 4.556992956044569,
+      "tokens_seen": 140640256
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048352056168505516,
+      "loss": 3.4098,
+      "theoretical_loss": 4.55671276592131,
+      "tokens_seen": 140705792
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048351053159478434,
+      "loss": 3.2187,
+      "theoretical_loss": 4.556432742792225,
+      "tokens_seen": 140771328
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004835005015045135,
+      "loss": 3.2732,
+      "theoretical_loss": 4.55615288648011,
+      "tokens_seen": 140836864
+    },
+    {
+      "epoch": 0.04,
+      "objective/train/docs_used": 88907,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 3.168332576751709,
+      "objective/train/theoretical_loss": 4.555873196808033,
+      "objective/train/tokens_used": 161362400,
+      "theoretical_loss": 4.555873196808033,
+      "tokens_seen": 140902400
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048349047141424276,
+      "loss": 3.2827,
+      "theoretical_loss": 4.555873196808033,
+      "tokens_seen": 140902400
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048348044132397194,
+      "loss": 3.0851,
+      "theoretical_loss": 4.555593673599329,
+      "tokens_seen": 140967936
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004834704112337011,
+      "loss": 3.2459,
+      "theoretical_loss": 4.555314316677606,
+      "tokens_seen": 141033472
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004834603811434303,
+      "loss": 3.1295,
+      "theoretical_loss": 4.555035125866738,
+      "tokens_seen": 141099008
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004834503510531595,
+      "loss": 3.2718,
+      "theoretical_loss": 4.5547561009908675,
+      "tokens_seen": 141164544
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004834403209628887,
+      "loss": 3.4012,
+      "theoretical_loss": 4.554477241874405,
+      "tokens_seen": 141230080
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048343029087261785,
+      "loss": 3.2815,
+      "theoretical_loss": 4.554198548342029,
+      "tokens_seen": 141295616
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004834202607823471,
+      "loss": 3.3752,
+      "theoretical_loss": 4.553920020218683,
+      "tokens_seen": 141361152
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048341023069207626,
+      "loss": 3.4304,
+      "theoretical_loss": 4.55364165732958,
+      "tokens_seen": 141426688
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048340020060180544,
+      "loss": 3.06,
+      "theoretical_loss": 4.553363459500193,
+      "tokens_seen": 141492224
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004833901705115346,
+      "loss": 3.4717,
+      "theoretical_loss": 4.553085426556265,
+      "tokens_seen": 141557760
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004833801404212638,
+      "loss": 3.2863,
+      "theoretical_loss": 4.552807558323801,
+      "tokens_seen": 141623296
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.000483370110330993,
+      "loss": 3.5702,
+      "theoretical_loss": 4.552529854629072,
+      "tokens_seen": 141688832
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004833600802407222,
+      "loss": 3.3102,
+      "theoretical_loss": 4.552252315298611,
+      "tokens_seen": 141754368
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048335005015045135,
+      "loss": 3.2317,
+      "theoretical_loss": 4.551974940159212,
+      "tokens_seen": 141819904
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004833400200601806,
+      "loss": 3.3219,
+      "theoretical_loss": 4.551697729037938,
+      "tokens_seen": 141885440
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004833299899699097,
+      "loss": 3.4492,
+      "theoretical_loss": 4.551420681762107,
+      "tokens_seen": 141950976
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048331995987963895,
+      "loss": 3.2021,
+      "theoretical_loss": 4.551143798159299,
+      "tokens_seen": 142016512
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048330992978936813,
+      "loss": 3.2159,
+      "theoretical_loss": 4.550867078057361,
+      "tokens_seen": 142082048
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004832998996990973,
+      "loss": 3.2044,
+      "theoretical_loss": 4.550590521284393,
+      "tokens_seen": 142147584
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004832898696088265,
+      "loss": 3.1647,
+      "theoretical_loss": 4.550314127668761,
+      "tokens_seen": 142213120
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048327983951855567,
+      "loss": 3.4173,
+      "theoretical_loss": 4.550037897039086,
+      "tokens_seen": 142278656
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048326980942828485,
+      "loss": 3.0888,
+      "theoretical_loss": 4.54976182922425,
+      "tokens_seen": 142344192
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004832597793380141,
+      "loss": 3.0527,
+      "theoretical_loss": 4.549485924053392,
+      "tokens_seen": 142409728
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004832497492477432,
+      "loss": 3.3659,
+      "theoretical_loss": 4.54921018135591,
+      "tokens_seen": 142475264
+    },
+    {
+      "epoch": 0.04,
+      "objective/train/docs_used": 89401,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 3.601848602294922,
+      "objective/train/theoretical_loss": 4.548934600961459,
+      "objective/train/tokens_used": 163000800,
+      "theoretical_loss": 4.548934600961459,
+      "tokens_seen": 142540800
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048323971915747245,
+      "loss": 3.4687,
+      "theoretical_loss": 4.548934600961459,
+      "tokens_seen": 142540800
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048322968906720163,
+      "loss": 3.2753,
+      "theoretical_loss": 4.54865918269995,
+      "tokens_seen": 142606336
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004832196589769308,
+      "loss": 3.1228,
+      "theoretical_loss": 4.548383926401552,
+      "tokens_seen": 142671872
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048320962888666,
+      "loss": 3.0876,
+      "theoretical_loss": 4.548108831896688,
+      "tokens_seen": 142737408
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004831995987963892,
+      "loss": 3.44,
+      "theoretical_loss": 4.547833899016037,
+      "tokens_seen": 142802944
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048318956870611836,
+      "loss": 3.0777,
+      "theoretical_loss": 4.547559127590535,
+      "tokens_seen": 142868480
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004831795386158476,
+      "loss": 3.5091,
+      "theoretical_loss": 4.547284517451367,
+      "tokens_seen": 142934016
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004831695085255767,
+      "loss": 3.3175,
+      "theoretical_loss": 4.547010068429978,
+      "tokens_seen": 142999552
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048315947843530595,
+      "loss": 3.1291,
+      "theoretical_loss": 4.546735780358061,
+      "tokens_seen": 143065088
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004831494483450351,
+      "loss": 3.1034,
+      "theoretical_loss": 4.546461653067565,
+      "tokens_seen": 143130624
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004831394182547643,
+      "loss": 3.0849,
+      "theoretical_loss": 4.54618768639069,
+      "tokens_seen": 143196160
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004831293881644935,
+      "loss": 3.0665,
+      "theoretical_loss": 4.545913880159889,
+      "tokens_seen": 143261696
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004831193580742227,
+      "loss": 3.2806,
+      "theoretical_loss": 4.545640234207866,
+      "tokens_seen": 143327232
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048310932798395186,
+      "loss": 3.3249,
+      "theoretical_loss": 4.545366748367574,
+      "tokens_seen": 143392768
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048309929789368104,
+      "loss": 3.4583,
+      "theoretical_loss": 4.545093422472218,
+      "tokens_seen": 143458304
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004830892678034102,
+      "loss": 3.5278,
+      "theoretical_loss": 4.544820256355253,
+      "tokens_seen": 143523840
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048307923771313946,
+      "loss": 3.0959,
+      "theoretical_loss": 4.544547249850384,
+      "tokens_seen": 143589376
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004830692076228686,
+      "loss": 3.4636,
+      "theoretical_loss": 4.544274402791562,
+      "tokens_seen": 143654912
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004830591775325978,
+      "loss": 3.1348,
+      "theoretical_loss": 4.544001715012989,
+      "tokens_seen": 143720448
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.000483049147442327,
+      "loss": 3.3713,
+      "theoretical_loss": 4.543729186349115,
+      "tokens_seen": 143785984
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004830391173520562,
+      "loss": 3.334,
+      "theoretical_loss": 4.5434568166346345,
+      "tokens_seen": 143851520
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048302908726178536,
+      "loss": 3.2556,
+      "theoretical_loss": 4.543184605704493,
+      "tokens_seen": 143917056
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048301905717151454,
+      "loss": 3.24,
+      "theoretical_loss": 4.542912553393879,
+      "tokens_seen": 143982592
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004830090270812437,
+      "loss": 3.0543,
+      "theoretical_loss": 4.542640659538229,
+      "tokens_seen": 144048128
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048299899699097296,
+      "loss": 3.3306,
+      "theoretical_loss": 4.542368923973226,
+      "tokens_seen": 144113664
+    },
+    {
+      "epoch": 0.04,
+      "objective/train/docs_used": 90537,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 3.076249361038208,
+      "objective/train/theoretical_loss": 4.542097346534794,
+      "objective/train/tokens_used": 164639200,
+      "theoretical_loss": 4.542097346534794,
+      "tokens_seen": 144179200
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004829889669007021,
+      "loss": 3.2985,
+      "theoretical_loss": 4.542097346534794,
+      "tokens_seen": 144179200
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004829789368104313,
+      "loss": 3.2073,
+      "theoretical_loss": 4.541825927059106,
+      "tokens_seen": 144244736
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048296890672016045,
+      "loss": 3.3592,
+      "theoretical_loss": 4.541554665382575,
+      "tokens_seen": 144310272
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004829588766298897,
+      "loss": 3.5993,
+      "theoretical_loss": 4.541283561341862,
+      "tokens_seen": 144375808
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048294884653961887,
+      "loss": 3.0916,
+      "theoretical_loss": 4.541012614773868,
+      "tokens_seen": 144441344
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048293881644934805,
+      "loss": 3.2114,
+      "theoretical_loss": 4.540741825515736,
+      "tokens_seen": 144506880
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048292878635907723,
+      "loss": 3.4082,
+      "theoretical_loss": 4.540471193404855,
+      "tokens_seen": 144572416
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048291875626880646,
+      "loss": 3.1695,
+      "theoretical_loss": 4.54020071827885,
+      "tokens_seen": 144637952
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004829087261785356,
+      "loss": 3.4008,
+      "theoretical_loss": 4.539930399975596,
+      "tokens_seen": 144703488
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004828986960882648,
+      "loss": 3.1431,
+      "theoretical_loss": 4.539660238333198,
+      "tokens_seen": 144769024
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048288866599799395,
+      "loss": 3.3694,
+      "theoretical_loss": 4.53939023319001,
+      "tokens_seen": 144834560
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004828786359077232,
+      "loss": 3.3767,
+      "theoretical_loss": 4.539120384384621,
+      "tokens_seen": 144900096
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048286860581745237,
+      "loss": 3.5769,
+      "theoretical_loss": 4.538850691755861,
+      "tokens_seen": 144965632
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048285857572718155,
+      "loss": 3.3304,
+      "theoretical_loss": 4.538581155142799,
+      "tokens_seen": 145031168
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048284854563691073,
+      "loss": 3.1185,
+      "theoretical_loss": 4.538311774384743,
+      "tokens_seen": 145096704
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004828385155466399,
+      "loss": 3.4686,
+      "theoretical_loss": 4.538042549321238,
+      "tokens_seen": 145162240
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004828284854563691,
+      "loss": 3.1808,
+      "theoretical_loss": 4.537773479792067,
+      "tokens_seen": 145227776
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048281845536609833,
+      "loss": 3.2259,
+      "theoretical_loss": 4.537504565637248,
+      "tokens_seen": 145293312
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048280842527582746,
+      "loss": 3.3492,
+      "theoretical_loss": 4.537235806697041,
+      "tokens_seen": 145358848
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004827983951855567,
+      "loss": 3.4474,
+      "theoretical_loss": 4.536967202811935,
+      "tokens_seen": 145424384
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004827883650952858,
+      "loss": 3.2921,
+      "theoretical_loss": 4.536698753822661,
+      "tokens_seen": 145489920
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048277833500501505,
+      "loss": 3.4976,
+      "theoretical_loss": 4.53643045957018,
+      "tokens_seen": 145555456
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048276830491474423,
+      "loss": 3.4497,
+      "theoretical_loss": 4.536162319895695,
+      "tokens_seen": 145620992
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004827582748244734,
+      "loss": 3.4119,
+      "theoretical_loss": 4.535894334640634,
+      "tokens_seen": 145686528
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004827482447342026,
+      "loss": 3.0995,
+      "theoretical_loss": 4.535626503646666,
+      "tokens_seen": 145752064
+    },
+    {
+      "epoch": 0.04,
+      "objective/train/docs_used": 91176,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 2.9223389625549316,
+      "objective/train/theoretical_loss": 4.535358826755691,
+      "objective/train/tokens_used": 166277600,
+      "theoretical_loss": 4.535358826755691,
+      "tokens_seen": 145817600
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048273821464393183,
+      "loss": 3.2178,
+      "theoretical_loss": 4.535358826755691,
+      "tokens_seen": 145817600
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.000482728184553661,
+      "loss": 3.4704,
+      "theoretical_loss": 4.535091303809842,
+      "tokens_seen": 145883136
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004827181544633902,
+      "loss": 3.2936,
+      "theoretical_loss": 4.534823934651484,
+      "tokens_seen": 145948672
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004827081243731194,
+      "loss": 3.1424,
+      "theoretical_loss": 4.5345567191232155,
+      "tokens_seen": 146014208
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048269809428284856,
+      "loss": 3.3168,
+      "theoretical_loss": 4.534289657067865,
+      "tokens_seen": 146079744
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004826880641925778,
+      "loss": 2.9011,
+      "theoretical_loss": 4.5340227483284945,
+      "tokens_seen": 146145280
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004826780341023069,
+      "loss": 3.2725,
+      "theoretical_loss": 4.533755992748395,
+      "tokens_seen": 146210816
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048266800401203615,
+      "loss": 3.2624,
+      "theoretical_loss": 4.533489390171088,
+      "tokens_seen": 146276352
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004826579739217653,
+      "loss": 3.4202,
+      "theoretical_loss": 4.533222940440325,
+      "tokens_seen": 146341888
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004826479438314945,
+      "loss": 3.1386,
+      "theoretical_loss": 4.532956643400087,
+      "tokens_seen": 146407424
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004826379137412237,
+      "loss": 3.3923,
+      "theoretical_loss": 4.532690498894585,
+      "tokens_seen": 146472960
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004826278836509529,
+      "loss": 3.4023,
+      "theoretical_loss": 4.5324245067682565,
+      "tokens_seen": 146538496
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048261785356068206,
+      "loss": 3.1156,
+      "theoretical_loss": 4.532158666865769,
+      "tokens_seen": 146604032
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048260782347041124,
+      "loss": 3.1851,
+      "theoretical_loss": 4.5318929790320155,
+      "tokens_seen": 146669568
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004825977933801404,
+      "loss": 3.0943,
+      "theoretical_loss": 4.53162744311212,
+      "tokens_seen": 146735104
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048258776328986966,
+      "loss": 3.392,
+      "theoretical_loss": 4.53136205895143,
+      "tokens_seen": 146800640
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004825777331995988,
+      "loss": 3.4047,
+      "theoretical_loss": 4.531096826395519,
+      "tokens_seen": 146866176
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.000482567703109328,
+      "loss": 3.2556,
+      "theoretical_loss": 4.5308317452901905,
+      "tokens_seen": 146931712
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004825576730190572,
+      "loss": 3.1769,
+      "theoretical_loss": 4.530566815481469,
+      "tokens_seen": 146997248
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004825476429287864,
+      "loss": 3.2533,
+      "theoretical_loss": 4.530302036815607,
+      "tokens_seen": 147062784
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048253761283851556,
+      "loss": 3.3405,
+      "theoretical_loss": 4.53003740913908,
+      "tokens_seen": 147128320
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048252758274824474,
+      "loss": 3.3032,
+      "theoretical_loss": 4.529772932298588,
+      "tokens_seen": 147193856
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004825175526579739,
+      "loss": 3.4738,
+      "theoretical_loss": 4.529508606141057,
+      "tokens_seen": 147259392
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048250752256770316,
+      "loss": 3.2304,
+      "theoretical_loss": 4.529244430513632,
+      "tokens_seen": 147324928
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004824974924774323,
+      "loss": 3.2967,
+      "theoretical_loss": 4.528980405263686,
+      "tokens_seen": 147390464
+    },
+    {
+      "epoch": 0.04,
+      "objective/train/docs_used": 92532,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 3.4305601119995117,
+      "objective/train/theoretical_loss": 4.528716530238812,
+      "objective/train/tokens_used": 167916000,
+      "theoretical_loss": 4.528716530238812,
+      "tokens_seen": 147456000
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004824874623871615,
+      "loss": 3.4659,
+      "theoretical_loss": 4.528716530238812,
+      "tokens_seen": 147456000
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048247743229689065,
+      "loss": 3.2645,
+      "theoretical_loss": 4.528452805286824,
+      "tokens_seen": 147521536
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004824674022066199,
+      "loss": 3.2269,
+      "theoretical_loss": 4.528189230255759,
+      "tokens_seen": 147587072
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048245737211634907,
+      "loss": 3.2869,
+      "theoretical_loss": 4.527925804993875,
+      "tokens_seen": 147652608
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048244734202607825,
+      "loss": 3.3874,
+      "theoretical_loss": 4.5276625293496515,
+      "tokens_seen": 147718144
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048243731193580743,
+      "loss": 3.1961,
+      "theoretical_loss": 4.527399403171788,
+      "tokens_seen": 147783680
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048242728184553666,
+      "loss": 3.3541,
+      "theoretical_loss": 4.5271364263092035,
+      "tokens_seen": 147849216
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004824172517552658,
+      "loss": 3.2284,
+      "theoretical_loss": 4.526873598611038,
+      "tokens_seen": 147914752
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.000482407221664995,
+      "loss": 3.1859,
+      "theoretical_loss": 4.52661091992665,
+      "tokens_seen": 147980288
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048239719157472415,
+      "loss": 3.3306,
+      "theoretical_loss": 4.526348390105614,
+      "tokens_seen": 148045824
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004823871614844534,
+      "loss": 3.0245,
+      "theoretical_loss": 4.526086008997728,
+      "tokens_seen": 148111360
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048237713139418257,
+      "loss": 3.1419,
+      "theoretical_loss": 4.525823776453004,
+      "tokens_seen": 148176896
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048236710130391175,
+      "loss": 3.0672,
+      "theoretical_loss": 4.525561692321673,
+      "tokens_seen": 148242432
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048235707121364093,
+      "loss": 3.259,
+      "theoretical_loss": 4.525299756454182,
+      "tokens_seen": 148307968
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004823470411233701,
+      "loss": 3.3995,
+      "theoretical_loss": 4.525037968701198,
+      "tokens_seen": 148373504
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004823370110330993,
+      "loss": 3.1846,
+      "theoretical_loss": 4.5247763289136,
+      "tokens_seen": 148439040
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00048232698094282853,
+      "loss": 3.3233,
+      "theoretical_loss": 4.524514836942485,
+      "tokens_seen": 148504576
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00048231695085255766,
+      "loss": 3.1244,
+      "theoretical_loss": 4.524253492639165,
+      "tokens_seen": 148570112
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004823069207622869,
+      "loss": 3.1294,
+      "theoretical_loss": 4.523992295855169,
+      "tokens_seen": 148635648
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.000482296890672016,
+      "loss": 3.4438,
+      "theoretical_loss": 4.523731246442237,
+      "tokens_seen": 148701184
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00048228686058174525,
+      "loss": 3.2735,
+      "theoretical_loss": 4.523470344252326,
+      "tokens_seen": 148766720
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00048227683049147443,
+      "loss": 3.2357,
+      "theoretical_loss": 4.523209589137606,
+      "tokens_seen": 148832256
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004822668004012036,
+      "loss": 3.3941,
+      "theoretical_loss": 4.522948980950461,
+      "tokens_seen": 148897792
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004822567703109328,
+      "loss": 3.0532,
+      "theoretical_loss": 4.522688519543487,
+      "tokens_seen": 148963328
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00048224674022066203,
+      "loss": 3.1432,
+      "theoretical_loss": 4.5224282047694935,
+      "tokens_seen": 149028864
+    },
+    {
+      "epoch": 0.05,
+      "objective/train/docs_used": 93022,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 3.5269510746002197,
+      "objective/train/theoretical_loss": 4.522168036481502,
+      "objective/train/tokens_used": 169554400,
+      "theoretical_loss": 4.522168036481502,
+      "tokens_seen": 149094400
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00048223671013039116,
+      "loss": 3.5101,
+      "theoretical_loss": 4.522168036481502,
+      "tokens_seen": 149094400
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004822266800401204,
+      "loss": 3.2561,
+      "theoretical_loss": 4.521908014532746,
+      "tokens_seen": 149159936
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004822166499498495,
+      "loss": 3.3344,
+      "theoretical_loss": 4.52164813877667,
+      "tokens_seen": 149225472
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00048220661985957876,
+      "loss": 3.2209,
+      "theoretical_loss": 4.52138840906693,
+      "tokens_seen": 149291008
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00048219658976930794,
+      "loss": 3.1369,
+      "theoretical_loss": 4.521128825257392,
+      "tokens_seen": 149356544
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004821865596790371,
+      "loss": 3.1751,
+      "theoretical_loss": 4.520869387202134,
+      "tokens_seen": 149422080
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004821765295887663,
+      "loss": 2.9797,
+      "theoretical_loss": 4.52061009475544,
+      "tokens_seen": 149487616
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004821664994984955,
+      "loss": 3.196,
+      "theoretical_loss": 4.52035094777181,
+      "tokens_seen": 149553152
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00048215646940822466,
+      "loss": 3.3068,
+      "theoretical_loss": 4.520091946105946,
+      "tokens_seen": 149618688
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004821464393179539,
+      "loss": 3.4288,
+      "theoretical_loss": 4.519833089612764,
+      "tokens_seen": 149684224
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.000482136409227683,
+      "loss": 3.2192,
+      "theoretical_loss": 4.519574378147384,
+      "tokens_seen": 149749760
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00048212637913741226,
+      "loss": 3.291,
+      "theoretical_loss": 4.519315811565137,
+      "tokens_seen": 149815296
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004821163490471414,
+      "loss": 3.3752,
+      "theoretical_loss": 4.51905738972156,
+      "tokens_seen": 149880832
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004821063189568706,
+      "loss": 3.6035,
+      "theoretical_loss": 4.518799112472401,
+      "tokens_seen": 149946368
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004820962888665998,
+      "loss": 3.5239,
+      "theoretical_loss": 4.518540979673606,
+      "tokens_seen": 150011904
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.000482086258776329,
+      "loss": 3.1651,
+      "theoretical_loss": 4.518282991181337,
+      "tokens_seen": 150077440
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00048207622868605816,
+      "loss": 3.1406,
+      "theoretical_loss": 4.518025146851956,
+      "tokens_seen": 150142976
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004820661985957874,
+      "loss": 3.2538,
+      "theoretical_loss": 4.517767446542033,
+      "tokens_seen": 150208512
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00048205616850551653,
+      "loss": 3.1976,
+      "theoretical_loss": 4.517509890108343,
+      "tokens_seen": 150274048
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00048204613841524576,
+      "loss": 3.1097,
+      "theoretical_loss": 4.5172524774078635,
+      "tokens_seen": 150339584
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004820361083249749,
+      "loss": 3.3081,
+      "theoretical_loss": 4.5169952082977805,
+      "tokens_seen": 150405120
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004820260782347041,
+      "loss": 3.2403,
+      "theoretical_loss": 4.51673808263548,
+      "tokens_seen": 150470656
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004820160481444333,
+      "loss": 3.0648,
+      "theoretical_loss": 4.516481100278555,
+      "tokens_seen": 150536192
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004820060180541625,
+      "loss": 3.1256,
+      "theoretical_loss": 4.516224261084799,
+      "tokens_seen": 150601728
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00048199598796389167,
+      "loss": 3.0304,
+      "theoretical_loss": 4.515967564912211,
+      "tokens_seen": 150667264
+    },
+    {
+      "epoch": 0.05,
+      "objective/train/docs_used": 94173,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 3.2401325702667236,
+      "objective/train/theoretical_loss": 4.515711011618992,
+      "objective/train/tokens_used": 171192800,
+      "theoretical_loss": 4.515711011618992,
+      "tokens_seen": 150732800
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00048198595787362085,
+      "loss": 3.2269,
+      "theoretical_loss": 4.515711011618992,
+      "tokens_seen": 150732800
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004819759277833501,
+      "loss": 3.1091,
+      "theoretical_loss": 4.515454601063542,
+      "tokens_seen": 150798336
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00048196589769307927,
+      "loss": 3.2182,
+      "theoretical_loss": 4.5151983331044665,
+      "tokens_seen": 150863872
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00048195586760280845,
+      "loss": 3.2107,
+      "theoretical_loss": 4.514942207600572,
+      "tokens_seen": 150929408
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00048194583751253763,
+      "loss": 3.1827,
+      "theoretical_loss": 4.514686224410863,
+      "tokens_seen": 150994944
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00048193580742226686,
+      "loss": 3.3203,
+      "theoretical_loss": 4.514430383394549,
+      "tokens_seen": 151060480
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.000481925777331996,
+      "loss": 3.0874,
+      "theoretical_loss": 4.514174684411037,
+      "tokens_seen": 151126016
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004819157472417252,
+      "loss": 3.2352,
+      "theoretical_loss": 4.513919127319935,
+      "tokens_seen": 151191552
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00048190571715145435,
+      "loss": 3.3179,
+      "theoretical_loss": 4.513663711981049,
+      "tokens_seen": 151257088
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004818956870611836,
+      "loss": 3.266,
+      "theoretical_loss": 4.513408438254386,
+      "tokens_seen": 151322624
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00048188565697091277,
+      "loss": 3.1682,
+      "theoretical_loss": 4.5131533060001505,
+      "tokens_seen": 151388160
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00048187562688064195,
+      "loss": 3.2016,
+      "theoretical_loss": 4.512898315078747,
+      "tokens_seen": 151453696
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00048186559679037113,
+      "loss": 3.2221,
+      "theoretical_loss": 4.512643465350775,
+      "tokens_seen": 151519232
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004818555667001003,
+      "loss": 3.394,
+      "theoretical_loss": 4.512388756677037,
+      "tokens_seen": 151584768
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004818455366098295,
+      "loss": 3.3002,
+      "theoretical_loss": 4.5121341889185285,
+      "tokens_seen": 151650304
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00048183550651955873,
+      "loss": 3.131,
+      "theoretical_loss": 4.511879761936442,
+      "tokens_seen": 151715840
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00048182547642928786,
+      "loss": 3.3703,
+      "theoretical_loss": 4.511625475592169,
+      "tokens_seen": 151781376
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004818154463390171,
+      "loss": 3.2669,
+      "theoretical_loss": 4.511371329747295,
+      "tokens_seen": 151846912
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004818054162487462,
+      "loss": 3.0424,
+      "theoretical_loss": 4.511117324263603,
+      "tokens_seen": 151912448
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00048179538615847545,
+      "loss": 3.6565,
+      "theoretical_loss": 4.5108634590030725,
+      "tokens_seen": 151977984
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00048178535606820463,
+      "loss": 3.411,
+      "theoretical_loss": 4.510609733827873,
+      "tokens_seen": 152043520
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004817753259779338,
+      "loss": 3.1894,
+      "theoretical_loss": 4.5103561486003745,
+      "tokens_seen": 152109056
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.000481765295887663,
+      "loss": 3.053,
+      "theoretical_loss": 4.51010270318314,
+      "tokens_seen": 152174592
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00048175526579739223,
+      "loss": 3.1504,
+      "theoretical_loss": 4.509849397438925,
+      "tokens_seen": 152240128
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00048174523570712136,
+      "loss": 3.2262,
+      "theoretical_loss": 4.509596231230679,
+      "tokens_seen": 152305664
+    },
+    {
+      "epoch": 0.05,
+      "objective/train/docs_used": 94867,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 3.429643392562866,
+      "objective/train/theoretical_loss": 4.509343204421548,
+      "objective/train/tokens_used": 172831200,
+      "theoretical_loss": 4.509343204421548,
+      "tokens_seen": 152371200
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004817352056168506,
+      "loss": 3.4209,
+      "theoretical_loss": 4.509343204421548,
+      "tokens_seen": 152371200
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004817251755265797,
+      "loss": 3.2597,
+      "theoretical_loss": 4.509090316874865,
+      "tokens_seen": 152436736
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00048171514543630896,
+      "loss": 3.2857,
+      "theoretical_loss": 4.508837568454162,
+      "tokens_seen": 152502272
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00048170511534603814,
+      "loss": 3.5512,
+      "theoretical_loss": 4.508584959023159,
+      "tokens_seen": 152567808
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004816950852557673,
+      "loss": 3.3878,
+      "theoretical_loss": 4.50833248844577,
+      "tokens_seen": 152633344
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004816850551654965,
+      "loss": 3.3808,
+      "theoretical_loss": 4.5080801565861,
+      "tokens_seen": 152698880
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004816750250752257,
+      "loss": 3.1163,
+      "theoretical_loss": 4.507827963308447,
+      "tokens_seen": 152764416
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00048166499498495486,
+      "loss": 3.2056,
+      "theoretical_loss": 4.507575908477296,
+      "tokens_seen": 152829952
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004816549648946841,
+      "loss": 3.1235,
+      "theoretical_loss": 4.507323991957325,
+      "tokens_seen": 152895488
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004816449348044132,
+      "loss": 3.439,
+      "theoretical_loss": 4.507072213613402,
+      "tokens_seen": 152961024
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00048163490471414246,
+      "loss": 3.2998,
+      "theoretical_loss": 4.506820573310586,
+      "tokens_seen": 153026560
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004816248746238716,
+      "loss": 3.5354,
+      "theoretical_loss": 4.506569070914123,
+      "tokens_seen": 153092096
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004816148445336008,
+      "loss": 3.1693,
+      "theoretical_loss": 4.506317706289451,
+      "tokens_seen": 153157632
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00048160481444333,
+      "loss": 3.3531,
+      "theoretical_loss": 4.506066479302193,
+      "tokens_seen": 153223168
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004815947843530592,
+      "loss": 3.3859,
+      "theoretical_loss": 4.505815389818165,
+      "tokens_seen": 153288704
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00048158475426278837,
+      "loss": 3.2548,
+      "theoretical_loss": 4.505564437703368,
+      "tokens_seen": 153354240
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004815747241725176,
+      "loss": 3.1131,
+      "theoretical_loss": 4.5053136228239925,
+      "tokens_seen": 153419776
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00048156469408224673,
+      "loss": 3.3206,
+      "theoretical_loss": 4.505062945046413,
+      "tokens_seen": 153485312
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00048155466399197596,
+      "loss": 3.3988,
+      "theoretical_loss": 4.504812404237197,
+      "tokens_seen": 153550848
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004815446339017051,
+      "loss": 3.1752,
+      "theoretical_loss": 4.504562000263093,
+      "tokens_seen": 153616384
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004815346038114343,
+      "loss": 3.1504,
+      "theoretical_loss": 4.504311732991039,
+      "tokens_seen": 153681920
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004815245737211635,
+      "loss": 3.0018,
+      "theoretical_loss": 4.504061602288159,
+      "tokens_seen": 153747456
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004815145436308927,
+      "loss": 2.9879,
+      "theoretical_loss": 4.503811608021761,
+      "tokens_seen": 153812992
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00048150451354062187,
+      "loss": 3.1938,
+      "theoretical_loss": 4.50356175005934,
+      "tokens_seen": 153878528
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00048149448345035105,
+      "loss": 3.5199,
+      "theoretical_loss": 4.503312028268576,
+      "tokens_seen": 153944064
+    },
+    {
+      "epoch": 0.05,
+      "objective/train/docs_used": 95853,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 3.367169141769409,
+      "objective/train/theoretical_loss": 4.503062442517334,
+      "objective/train/tokens_used": 174469600,
+      "theoretical_loss": 4.503062442517334,
+      "tokens_seen": 154009600
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00048148445336008023,
+      "loss": 3.257,
+      "theoretical_loss": 4.503062442517334,
+      "tokens_seen": 154009600
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00048147442326980947,
+      "loss": 3.4149,
+      "theoretical_loss": 4.502812992673661,
+      "tokens_seen": 154075136
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004814643931795386,
+      "loss": 3.3831,
+      "theoretical_loss": 4.50256367860579,
+      "tokens_seen": 154140672
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00048145436308926783,
+      "loss": 3.2127,
+      "theoretical_loss": 4.502314500182138,
+      "tokens_seen": 154206208
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00048144433299899696,
+      "loss": 3.0253,
+      "theoretical_loss": 4.502065457271304,
+      "tokens_seen": 154271744
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004814343029087262,
+      "loss": 3.522,
+      "theoretical_loss": 4.501816549742072,
+      "tokens_seen": 154337280
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00048142427281845537,
+      "loss": 3.031,
+      "theoretical_loss": 4.501567777463406,
+      "tokens_seen": 154402816
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00048141424272818455,
+      "loss": 3.33,
+      "theoretical_loss": 4.501319140304456,
+      "tokens_seen": 154468352
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00048140421263791373,
+      "loss": 3.0943,
+      "theoretical_loss": 4.50107063813455,
+      "tokens_seen": 154533888
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00048139418254764297,
+      "loss": 2.9423,
+      "theoretical_loss": 4.5008222708232,
+      "tokens_seen": 154599424
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004813841524573721,
+      "loss": 3.2762,
+      "theoretical_loss": 4.500574038240099,
+      "tokens_seen": 154664960
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00048137412236710133,
+      "loss": 3.1341,
+      "theoretical_loss": 4.500325940255123,
+      "tokens_seen": 154730496
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00048136409227683046,
+      "loss": 3.323,
+      "theoretical_loss": 4.500077976738324,
+      "tokens_seen": 154796032
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004813540621865597,
+      "loss": 3.2968,
+      "theoretical_loss": 4.499830147559939,
+      "tokens_seen": 154861568
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004813440320962889,
+      "loss": 3.3287,
+      "theoretical_loss": 4.499582452590383,
+      "tokens_seen": 154927104
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00048133400200601806,
+      "loss": 3.4478,
+      "theoretical_loss": 4.499334891700251,
+      "tokens_seen": 154992640
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00048132397191574724,
+      "loss": 3.1801,
+      "theoretical_loss": 4.499087464760317,
+      "tokens_seen": 155058176
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004813139418254764,
+      "loss": 3.0457,
+      "theoretical_loss": 4.498840171641534,
+      "tokens_seen": 155123712
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004813039117352056,
+      "loss": 2.9473,
+      "theoretical_loss": 4.498593012215036,
+      "tokens_seen": 155189248
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00048129388164493483,
+      "loss": 3.1367,
+      "theoretical_loss": 4.498345986352133,
+      "tokens_seen": 155254784
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00048128385155466396,
+      "loss": 3.2789,
+      "theoretical_loss": 4.4980990939243135,
+      "tokens_seen": 155320320
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004812738214643932,
+      "loss": 3.265,
+      "theoretical_loss": 4.497852334803245,
+      "tokens_seen": 155385856
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004812637913741223,
+      "loss": 3.2818,
+      "theoretical_loss": 4.497605708860771,
+      "tokens_seen": 155451392
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00048125376128385156,
+      "loss": 3.148,
+      "theoretical_loss": 4.497359215968914,
+      "tokens_seen": 155516928
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00048124373119358074,
+      "loss": 3.1139,
+      "theoretical_loss": 4.497112855999871,
+      "tokens_seen": 155582464
+    },
+    {
+      "epoch": 0.05,
+      "objective/train/docs_used": 96994,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 3.577995777130127,
+      "objective/train/theoretical_loss": 4.496866628826018,
+      "objective/train/tokens_used": 176108000,
+      "theoretical_loss": 4.496866628826018,
+      "tokens_seen": 155648000
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004812337011033099,
+      "loss": 3.366,
+      "theoretical_loss": 4.496866628826018,
+      "tokens_seen": 155648000
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00048122367101303916,
+      "loss": 3.2348,
+      "theoretical_loss": 4.496620534319904,
+      "tokens_seen": 155713536
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00048121364092276834,
+      "loss": 3.1337,
+      "theoretical_loss": 4.496374572354256,
+      "tokens_seen": 155779072
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004812036108324975,
+      "loss": 3.4091,
+      "theoretical_loss": 4.496128742801979,
+      "tokens_seen": 155844608
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004811935807422267,
+      "loss": 3.3832,
+      "theoretical_loss": 4.495883045536148,
+      "tokens_seen": 155910144
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004811835506519559,
+      "loss": 3.1527,
+      "theoretical_loss": 4.495637480430018,
+      "tokens_seen": 155975680
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00048117352056168506,
+      "loss": 3.3667,
+      "theoretical_loss": 4.495392047357015,
+      "tokens_seen": 156041216
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004811634904714143,
+      "loss": 3.1819,
+      "theoretical_loss": 4.4951467461907395,
+      "tokens_seen": 156106752
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004811534603811434,
+      "loss": 3.2472,
+      "theoretical_loss": 4.494901576804969,
+      "tokens_seen": 156172288
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00048114343029087266,
+      "loss": 3.3371,
+      "theoretical_loss": 4.4946565390736515,
+      "tokens_seen": 156237824
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004811334002006018,
+      "loss": 3.3338,
+      "theoretical_loss": 4.494411632870911,
+      "tokens_seen": 156303360
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.000481123370110331,
+      "loss": 3.3003,
+      "theoretical_loss": 4.494166858071041,
+      "tokens_seen": 156368896
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004811133400200602,
+      "loss": 3.0684,
+      "theoretical_loss": 4.493922214548512,
+      "tokens_seen": 156434432
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004811033099297894,
+      "loss": 3.1874,
+      "theoretical_loss": 4.493677702177964,
+      "tokens_seen": 156499968
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00048109327983951857,
+      "loss": 3.2209,
+      "theoretical_loss": 4.493433320834209,
+      "tokens_seen": 156565504
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004810832497492478,
+      "loss": 3.3108,
+      "theoretical_loss": 4.493189070392235,
+      "tokens_seen": 156631040
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00048107321965897693,
+      "loss": 3.0176,
+      "theoretical_loss": 4.492944950727196,
+      "tokens_seen": 156696576
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00048106318956870616,
+      "loss": 3.0247,
+      "theoretical_loss": 4.492700961714419,
+      "tokens_seen": 156762112
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004810531594784353,
+      "loss": 3.1494,
+      "theoretical_loss": 4.492457103229405,
+      "tokens_seen": 156827648
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004810431293881645,
+      "loss": 3.4656,
+      "theoretical_loss": 4.492213375147822,
+      "tokens_seen": 156893184
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004810330992978937,
+      "loss": 3.1795,
+      "theoretical_loss": 4.491969777345509,
+      "tokens_seen": 156958720
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004810230692076229,
+      "loss": 3.2572,
+      "theoretical_loss": 4.491726309698475,
+      "tokens_seen": 157024256
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00048101303911735207,
+      "loss": 3.074,
+      "theoretical_loss": 4.491482972082901,
+      "tokens_seen": 157089792
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00048100300902708125,
+      "loss": 3.3692,
+      "theoretical_loss": 4.491239764375134,
+      "tokens_seen": 157155328
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00048099297893681043,
+      "loss": 3.166,
+      "theoretical_loss": 4.490996686451692,
+      "tokens_seen": 157220864
+    },
+    {
+      "epoch": 0.05,
+      "objective/train/docs_used": 97532,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 3.0817697048187256,
+      "objective/train/theoretical_loss": 4.4907537381892615,
+      "objective/train/tokens_used": 177746400,
+      "theoretical_loss": 4.4907537381892615,
+      "tokens_seen": 157286400
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00048098294884653967,
+      "loss": 3.0902,
+      "theoretical_loss": 4.4907537381892615,
+      "tokens_seen": 157286400
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004809729187562688,
+      "loss": 3.3092,
+      "theoretical_loss": 4.490510919464697,
+      "tokens_seen": 157351936
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00048096288866599803,
+      "loss": 3.1099,
+      "theoretical_loss": 4.4902682301550225,
+      "tokens_seen": 157417472
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00048095285857572716,
+      "loss": 2.9646,
+      "theoretical_loss": 4.4900256701374275,
+      "tokens_seen": 157483008
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004809428284854564,
+      "loss": 3.1576,
+      "theoretical_loss": 4.489783239289271,
+      "tokens_seen": 157548544
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00048093279839518557,
+      "loss": 3.1089,
+      "theoretical_loss": 4.48954093748808,
+      "tokens_seen": 157614080
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00048092276830491475,
+      "loss": 3.4034,
+      "theoretical_loss": 4.489298764611546,
+      "tokens_seen": 157679616
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00048091273821464393,
+      "loss": 2.9096,
+      "theoretical_loss": 4.48905672053753,
+      "tokens_seen": 157745152
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00048090270812437317,
+      "loss": 3.1601,
+      "theoretical_loss": 4.488814805144055,
+      "tokens_seen": 157810688
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004808926780341023,
+      "loss": 3.1648,
+      "theoretical_loss": 4.4885730183093155,
+      "tokens_seen": 157876224
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00048088264794383153,
+      "loss": 3.3537,
+      "theoretical_loss": 4.488331359911669,
+      "tokens_seen": 157941760
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00048087261785356066,
+      "loss": 3.1861,
+      "theoretical_loss": 4.488089829829638,
+      "tokens_seen": 158007296
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004808625877632899,
+      "loss": 3.1146,
+      "theoretical_loss": 4.487848427941911,
+      "tokens_seen": 158072832
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004808525576730191,
+      "loss": 3.4209,
+      "theoretical_loss": 4.487607154127343,
+      "tokens_seen": 158138368
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00048084252758274826,
+      "loss": 3.4348,
+      "theoretical_loss": 4.487366008264949,
+      "tokens_seen": 158203904
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00048083249749247744,
+      "loss": 3.3548,
+      "theoretical_loss": 4.487124990233915,
+      "tokens_seen": 158269440
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004808224674022066,
+      "loss": 3.2445,
+      "theoretical_loss": 4.486884099913585,
+      "tokens_seen": 158334976
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004808124373119358,
+      "loss": 3.5389,
+      "theoretical_loss": 4.486643337183471,
+      "tokens_seen": 158400512
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00048080240722166503,
+      "loss": 2.9913,
+      "theoretical_loss": 4.486402701923245,
+      "tokens_seen": 158466048
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00048079237713139416,
+      "loss": 3.1903,
+      "theoretical_loss": 4.486162194012745,
+      "tokens_seen": 158531584
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004807823470411234,
+      "loss": 3.3898,
+      "theoretical_loss": 4.48592181333197,
+      "tokens_seen": 158597120
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004807723169508525,
+      "loss": 3.134,
+      "theoretical_loss": 4.485681559761082,
+      "tokens_seen": 158662656
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00048076228686058176,
+      "loss": 3.1577,
+      "theoretical_loss": 4.485441433180408,
+      "tokens_seen": 158728192
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00048075225677031094,
+      "loss": 3.0772,
+      "theoretical_loss": 4.485201433470432,
+      "tokens_seen": 158793728
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004807422266800401,
+      "loss": 3.3051,
+      "theoretical_loss": 4.484961560511804,
+      "tokens_seen": 158859264
+    },
+    {
+      "epoch": 0.05,
+      "objective/train/docs_used": 98333,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 3.562363386154175,
+      "objective/train/theoretical_loss": 4.484721814185333,
+      "objective/train/tokens_used": 179384800,
+      "theoretical_loss": 4.484721814185333,
+      "tokens_seen": 158924800
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004807321965897693,
+      "loss": 3.3407,
+      "theoretical_loss": 4.484721814185333,
+      "tokens_seen": 158924800
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00048072216649949854,
+      "loss": 3.439,
+      "theoretical_loss": 4.48448219437199,
+      "tokens_seen": 158990336
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00048071213640922766,
+      "loss": 3.5068,
+      "theoretical_loss": 4.484242700952908,
+      "tokens_seen": 159055872
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004807021063189569,
+      "loss": 3.4014,
+      "theoretical_loss": 4.48400333380938,
+      "tokens_seen": 159121408
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.000480692076228686,
+      "loss": 3.3618,
+      "theoretical_loss": 4.483764092822857,
+      "tokens_seen": 159186944
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00048068204613841526,
+      "loss": 3.2863,
+      "theoretical_loss": 4.483524977874953,
+      "tokens_seen": 159252480
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00048067201604814444,
+      "loss": 3.2132,
+      "theoretical_loss": 4.48328598884744,
+      "tokens_seen": 159318016
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004806619859578736,
+      "loss": 3.2951,
+      "theoretical_loss": 4.483047125622251,
+      "tokens_seen": 159383552
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004806519558676028,
+      "loss": 3.3459,
+      "theoretical_loss": 4.482808388081477,
+      "tokens_seen": 159449088
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.000480641925777332,
+      "loss": 3.1496,
+      "theoretical_loss": 4.482569776107368,
+      "tokens_seen": 159514624
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00048063189568706117,
+      "loss": 3.2453,
+      "theoretical_loss": 4.482331289582334,
+      "tokens_seen": 159580160
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004806218655967904,
+      "loss": 3.1819,
+      "theoretical_loss": 4.482092928388941,
+      "tokens_seen": 159645696
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00048061183550651953,
+      "loss": 3.2609,
+      "theoretical_loss": 4.4818546924099145,
+      "tokens_seen": 159711232
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00048060180541624877,
+      "loss": 3.2747,
+      "theoretical_loss": 4.481616581528138,
+      "tokens_seen": 159776768
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00048059177532597795,
+      "loss": 3.1228,
+      "theoretical_loss": 4.481378595626652,
+      "tokens_seen": 159842304
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00048058174523570713,
+      "loss": 3.0437,
+      "theoretical_loss": 4.481140734588655,
+      "tokens_seen": 159907840
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004805717151454363,
+      "loss": 3.3693,
+      "theoretical_loss": 4.4809029982975,
+      "tokens_seen": 159973376
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004805616850551655,
+      "loss": 3.4456,
+      "theoretical_loss": 4.480665386636701,
+      "tokens_seen": 160038912
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00048055165496489467,
+      "loss": 3.1342,
+      "theoretical_loss": 4.480427899489925,
+      "tokens_seen": 160104448
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004805416248746239,
+      "loss": 2.9679,
+      "theoretical_loss": 4.480190536740995,
+      "tokens_seen": 160169984
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00048053159478435303,
+      "loss": 3.4332,
+      "theoretical_loss": 4.479953298273893,
+      "tokens_seen": 160235520
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00048052156469408227,
+      "loss": 3.2521,
+      "theoretical_loss": 4.479716183972753,
+      "tokens_seen": 160301056
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004805115346038114,
+      "loss": 3.5047,
+      "theoretical_loss": 4.4794791937218665,
+      "tokens_seen": 160366592
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00048050150451354063,
+      "loss": 3.3783,
+      "theoretical_loss": 4.47924232740568,
+      "tokens_seen": 160432128
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004804914744232698,
+      "loss": 3.3839,
+      "theoretical_loss": 4.479005584908793,
+      "tokens_seen": 160497664
+    },
+    {
+      "epoch": 0.05,
+      "objective/train/docs_used": 99019,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 3.4908287525177,
+      "objective/train/theoretical_loss": 4.478768966115963,
+      "objective/train/tokens_used": 181023200,
+      "theoretical_loss": 4.478768966115963,
+      "tokens_seen": 160563200
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.000480481444332999,
+      "loss": 3.1208,
+      "theoretical_loss": 4.478768966115963,
+      "tokens_seen": 160563200
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00048047141424272823,
+      "loss": 3.2379,
+      "theoretical_loss": 4.478532470912096,
+      "tokens_seen": 160628736
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00048046138415245736,
+      "loss": 3.0995,
+      "theoretical_loss": 4.478296099182261,
+      "tokens_seen": 160694272
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004804513540621866,
+      "loss": 3.2414,
+      "theoretical_loss": 4.47805985081167,
+      "tokens_seen": 160759808
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00048044132397191577,
+      "loss": 3.1472,
+      "theoretical_loss": 4.477823725685695,
+      "tokens_seen": 160825344
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00048043129388164495,
+      "loss": 3.2584,
+      "theoretical_loss": 4.477587723689862,
+      "tokens_seen": 160890880
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00048042126379137413,
+      "loss": 3.2151,
+      "theoretical_loss": 4.477351844709846,
+      "tokens_seen": 160956416
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00048041123370110337,
+      "loss": 3.464,
+      "theoretical_loss": 4.477116088631476,
+      "tokens_seen": 161021952
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004804012036108325,
+      "loss": 3.2482,
+      "theoretical_loss": 4.476880455340735,
+      "tokens_seen": 161087488
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00048039117352056173,
+      "loss": 3.347,
+      "theoretical_loss": 4.4766449447237555,
+      "tokens_seen": 161153024
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00048038114343029086,
+      "loss": 3.2773,
+      "theoretical_loss": 4.476409556666825,
+      "tokens_seen": 161218560
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004803711133400201,
+      "loss": 3.0332,
+      "theoretical_loss": 4.476174291056379,
+      "tokens_seen": 161284096
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004803610832497493,
+      "loss": 3.2546,
+      "theoretical_loss": 4.475939147779008,
+      "tokens_seen": 161349632
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00048035105315947846,
+      "loss": 3.0797,
+      "theoretical_loss": 4.47570412672145,
+      "tokens_seen": 161415168
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00048034102306920764,
+      "loss": 3.1193,
+      "theoretical_loss": 4.475469227770596,
+      "tokens_seen": 161480704
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004803309929789368,
+      "loss": 3.222,
+      "theoretical_loss": 4.475234450813487,
+      "tokens_seen": 161546240
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.000480320962888666,
+      "loss": 3.3821,
+      "theoretical_loss": 4.474999795737315,
+      "tokens_seen": 161611776
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00048031093279839523,
+      "loss": 3.17,
+      "theoretical_loss": 4.47476526242942,
+      "tokens_seen": 161677312
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00048030090270812436,
+      "loss": 3.0239,
+      "theoretical_loss": 4.474530850777294,
+      "tokens_seen": 161742848
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004802908726178536,
+      "loss": 3.2871,
+      "theoretical_loss": 4.474296560668577,
+      "tokens_seen": 161808384
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004802808425275827,
+      "loss": 3.0498,
+      "theoretical_loss": 4.474062391991058,
+      "tokens_seen": 161873920
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00048027081243731196,
+      "loss": 3.0457,
+      "theoretical_loss": 4.473828344632677,
+      "tokens_seen": 161939456
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00048026078234704114,
+      "loss": 3.114,
+      "theoretical_loss": 4.473594418481522,
+      "tokens_seen": 162004992
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004802507522567703,
+      "loss": 3.3295,
+      "theoretical_loss": 4.473360613425827,
+      "tokens_seen": 162070528
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004802407221664995,
+      "loss": 3.046,
+      "theoretical_loss": 4.473126929353977,
+      "tokens_seen": 162136064
+    },
+    {
+      "epoch": 0.05,
+      "objective/train/docs_used": 100000,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 2.690746545791626,
+      "objective/train/theoretical_loss": 4.472893366154503,
+      "objective/train/tokens_used": 182661600,
+      "theoretical_loss": 4.472893366154503,
+      "tokens_seen": 162201600
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00048023069207622874,
+      "loss": 2.9009,
+      "theoretical_loss": 4.472893366154503,
+      "tokens_seen": 162201600
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00048022066198595786,
+      "loss": 3.0755,
+      "theoretical_loss": 4.472659923716087,
+      "tokens_seen": 162267136
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004802106318956871,
+      "loss": 3.0381,
+      "theoretical_loss": 4.472426601927554,
+      "tokens_seen": 162332672
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00048020060180541623,
+      "loss": 3.0383,
+      "theoretical_loss": 4.472193400677878,
+      "tokens_seen": 162398208
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00048019057171514546,
+      "loss": 3.207,
+      "theoretical_loss": 4.471960319856182,
+      "tokens_seen": 162463744
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00048018054162487464,
+      "loss": 3.0581,
+      "theoretical_loss": 4.471727359351734,
+      "tokens_seen": 162529280
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004801705115346038,
+      "loss": 3.323,
+      "theoretical_loss": 4.4714945190539455,
+      "tokens_seen": 162594816
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.000480160481444333,
+      "loss": 3.3514,
+      "theoretical_loss": 4.47126179885238,
+      "tokens_seen": 162660352
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004801504513540622,
+      "loss": 3.0515,
+      "theoretical_loss": 4.471029198636741,
+      "tokens_seen": 162725888
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00048014042126379137,
+      "loss": 2.9545,
+      "theoretical_loss": 4.470796718296884,
+      "tokens_seen": 162791424
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004801303911735206,
+      "loss": 3.0233,
+      "theoretical_loss": 4.470564357722804,
+      "tokens_seen": 162856960
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00048012036108324973,
+      "loss": 3.258,
+      "theoretical_loss": 4.470332116804644,
+      "tokens_seen": 162922496
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00048011033099297897,
+      "loss": 3.2388,
+      "theoretical_loss": 4.470099995432692,
+      "tokens_seen": 162988032
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00048010030090270815,
+      "loss": 3.0379,
+      "theoretical_loss": 4.469867993497379,
+      "tokens_seen": 163053568
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00048009027081243733,
+      "loss": 3.0891,
+      "theoretical_loss": 4.469636110889283,
+      "tokens_seen": 163119104
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004800802407221665,
+      "loss": 3.2988,
+      "theoretical_loss": 4.4694043474991245,
+      "tokens_seen": 163184640
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004800702106318957,
+      "loss": 3.0297,
+      "theoretical_loss": 4.469172703217767,
+      "tokens_seen": 163250176
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00048006018054162487,
+      "loss": 3.0857,
+      "theoretical_loss": 4.46894117793622,
+      "tokens_seen": 163315712
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004800501504513541,
+      "loss": 3.1744,
+      "theoretical_loss": 4.468709771545634,
+      "tokens_seen": 163381248
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00048004012036108323,
+      "loss": 3.0103,
+      "theoretical_loss": 4.4684784839373055,
+      "tokens_seen": 163446784
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00048003009027081247,
+      "loss": 3.2331,
+      "theoretical_loss": 4.468247315002671,
+      "tokens_seen": 163512320
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004800200601805416,
+      "loss": 3.1392,
+      "theoretical_loss": 4.468016264633311,
+      "tokens_seen": 163577856
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00048001003009027083,
+      "loss": 3.1491,
+      "theoretical_loss": 4.4677853327209505,
+      "tokens_seen": 163643392
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00048,
+      "loss": 3.3932,
+      "theoretical_loss": 4.467554519157452,
+      "tokens_seen": 163708928
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004799899699097292,
+      "loss": 3.0553,
+      "theoretical_loss": 4.467323823834824,
+      "tokens_seen": 163774464
+    },
+    {
+      "debugging/Self-BLEU-5": 0.4946583130096161,
+      "debugging/distinct-1-grams": 0.815790683224271,
+      "debugging/distinct-2-grams": 0.9751385384195338,
+      "debugging/entropy-1-grams": 5.884314010514224,
+      "debugging/entropy-2-grams": 6.564196933680893,
+      "debugging/length": 580.8,
+      "debugging/num_segments": 10,
+      "epoch": 0.05,
+      "objective/train/docs_used": 101353,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 3.959970474243164,
+      "objective/train/theoretical_loss": 4.467093246645215,
+      "objective/train/tokens_used": 184300000,
+      "theoretical_loss": 4.467093246645215,
+      "tokens_seen": 163840000
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004799799398194584,
+      "loss": 3.3758,
+      "theoretical_loss": 4.467093246645215,
+      "tokens_seen": 163840000
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00047996990972918756,
+      "loss": 3.0913,
+      "theoretical_loss": 4.466862787480915,
+      "tokens_seen": 163905536
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00047995987963891674,
+      "loss": 3.201,
+      "theoretical_loss": 4.466632446234355,
+      "tokens_seen": 163971072
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00047994984954864597,
+      "loss": 3.1657,
+      "theoretical_loss": 4.466402222798108,
+      "tokens_seen": 164036608
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004799398194583751,
+      "loss": 3.0971,
+      "theoretical_loss": 4.466172117064889,
+      "tokens_seen": 164102144
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00047992978936810433,
+      "loss": 3.0109,
+      "theoretical_loss": 4.465942128927549,
+      "tokens_seen": 164167680
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004799197592778335,
+      "loss": 3.2315,
+      "theoretical_loss": 4.4657122582790825,
+      "tokens_seen": 164233216
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004799097291875627,
+      "loss": 3.2653,
+      "theoretical_loss": 4.465482505012624,
+      "tokens_seen": 164298752
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004798996990972919,
+      "loss": 3.359,
+      "theoretical_loss": 4.465252869021448,
+      "tokens_seen": 164364288
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00047988966900702106,
+      "loss": 3.0349,
+      "theoretical_loss": 4.465023350198967,
+      "tokens_seen": 164429824
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00047987963891675024,
+      "loss": 3.4322,
+      "theoretical_loss": 4.464793948438733,
+      "tokens_seen": 164495360
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004798696088264795,
+      "loss": 3.1553,
+      "theoretical_loss": 4.464564663634439,
+      "tokens_seen": 164560896
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004798595787362086,
+      "loss": 3.3483,
+      "theoretical_loss": 4.464335495679914,
+      "tokens_seen": 164626432
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00047984954864593784,
+      "loss": 3.2043,
+      "theoretical_loss": 4.464106444469129,
+      "tokens_seen": 164691968
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00047983951855566696,
+      "loss": 3.1635,
+      "theoretical_loss": 4.463877509896189,
+      "tokens_seen": 164757504
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004798294884653962,
+      "loss": 3.298,
+      "theoretical_loss": 4.463648691855342,
+      "tokens_seen": 164823040
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004798194583751254,
+      "loss": 3.3382,
+      "theoretical_loss": 4.46341999024097,
+      "tokens_seen": 164888576
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00047980942828485456,
+      "loss": 3.3223,
+      "theoretical_loss": 4.463191404947595,
+      "tokens_seen": 164954112
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00047979939819458374,
+      "loss": 3.2243,
+      "theoretical_loss": 4.462962935869875,
+      "tokens_seen": 165019648
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004797893681043129,
+      "loss": 3.1862,
+      "theoretical_loss": 4.4627345829026055,
+      "tokens_seen": 165085184
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004797793380140421,
+      "loss": 3.2557,
+      "theoretical_loss": 4.462506345940719,
+      "tokens_seen": 165150720
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00047976930792377134,
+      "loss": 2.9256,
+      "theoretical_loss": 4.462278224879286,
+      "tokens_seen": 165216256
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00047975927783350047,
+      "loss": 2.9158,
+      "theoretical_loss": 4.462050219613511,
+      "tokens_seen": 165281792
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004797492477432297,
+      "loss": 3.2504,
+      "theoretical_loss": 4.461822330038737,
+      "tokens_seen": 165347328
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004797392176529589,
+      "loss": 3.042,
+      "theoretical_loss": 4.461594556050444,
+      "tokens_seen": 165412864
+    },
+    {
+      "epoch": 0.05,
+      "objective/train/docs_used": 101896,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 3.066077470779419,
+      "objective/train/theoretical_loss": 4.4613668975442415,
+      "objective/train/tokens_used": 185938400,
+      "theoretical_loss": 4.4613668975442415,
+      "tokens_seen": 165478400
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00047972918756268807,
+      "loss": 3.1283,
+      "theoretical_loss": 4.4613668975442415,
+      "tokens_seen": 165478400
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004797191574724173,
+      "loss": 3.3684,
+      "theoretical_loss": 4.4611393544158835,
+      "tokens_seen": 165543936
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00047970912738214643,
+      "loss": 3.3009,
+      "theoretical_loss": 4.460911926561252,
+      "tokens_seen": 165609472
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00047969909729187566,
+      "loss": 3.4005,
+      "theoretical_loss": 4.460684613876367,
+      "tokens_seen": 165675008
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00047968906720160484,
+      "loss": 3.1465,
+      "theoretical_loss": 4.460457416257386,
+      "tokens_seen": 165740544
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.000479679037111334,
+      "loss": 2.865,
+      "theoretical_loss": 4.460230333600595,
+      "tokens_seen": 165806080
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004796690070210632,
+      "loss": 3.2027,
+      "theoretical_loss": 4.46000336580242,
+      "tokens_seen": 165871616
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004796589769307924,
+      "loss": 3.3585,
+      "theoretical_loss": 4.459776512759419,
+      "tokens_seen": 165937152
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00047964894684052157,
+      "loss": 3.2687,
+      "theoretical_loss": 4.459549774368284,
+      "tokens_seen": 166002688
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004796389167502508,
+      "loss": 3.4614,
+      "theoretical_loss": 4.45932315052584,
+      "tokens_seen": 166068224
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00047962888665997993,
+      "loss": 2.9803,
+      "theoretical_loss": 4.459096641129047,
+      "tokens_seen": 166133760
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00047961885656970917,
+      "loss": 3.0739,
+      "theoretical_loss": 4.458870246074998,
+      "tokens_seen": 166199296
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00047960882647943835,
+      "loss": 3.285,
+      "theoretical_loss": 4.458643965260917,
+      "tokens_seen": 166264832
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00047959879638916753,
+      "loss": 3.1638,
+      "theoretical_loss": 4.458417798584164,
+      "tokens_seen": 166330368
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004795887662988967,
+      "loss": 2.905,
+      "theoretical_loss": 4.45819174594223,
+      "tokens_seen": 166395904
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004795787362086259,
+      "loss": 3.2569,
+      "theoretical_loss": 4.457965807232739,
+      "tokens_seen": 166461440
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00047956870611835507,
+      "loss": 3.299,
+      "theoretical_loss": 4.457739982353445,
+      "tokens_seen": 166526976
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004795586760280843,
+      "loss": 2.9863,
+      "theoretical_loss": 4.457514271202237,
+      "tokens_seen": 166592512
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00047954864593781343,
+      "loss": 3.2391,
+      "theoretical_loss": 4.457288673677134,
+      "tokens_seen": 166658048
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00047953861584754267,
+      "loss": 3.3358,
+      "theoretical_loss": 4.457063189676287,
+      "tokens_seen": 166723584
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004795285857572718,
+      "loss": 3.2532,
+      "theoretical_loss": 4.456837819097978,
+      "tokens_seen": 166789120
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00047951855566700103,
+      "loss": 3.1618,
+      "theoretical_loss": 4.45661256184062,
+      "tokens_seen": 166854656
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004795085255767302,
+      "loss": 3.0743,
+      "theoretical_loss": 4.456387417802757,
+      "tokens_seen": 166920192
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004794984954864594,
+      "loss": 3.3956,
+      "theoretical_loss": 4.456162386883065,
+      "tokens_seen": 166985728
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004794884653961886,
+      "loss": 3.1311,
+      "theoretical_loss": 4.455937468980348,
+      "tokens_seen": 167051264
+    },
+    {
+      "epoch": 0.05,
+      "objective/train/docs_used": 103097,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 3.102741003036499,
+      "objective/train/theoretical_loss": 4.455712663993541,
+      "objective/train/tokens_used": 187576800,
+      "theoretical_loss": 4.455712663993541,
+      "tokens_seen": 167116800
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00047947843530591776,
+      "loss": 3.2024,
+      "theoretical_loss": 4.455712663993541,
+      "tokens_seen": 167116800
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00047946840521564694,
+      "loss": 3.4233,
+      "theoretical_loss": 4.455487971821709,
+      "tokens_seen": 167182336
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00047945837512537617,
+      "loss": 3.1938,
+      "theoretical_loss": 4.4552633923640474,
+      "tokens_seen": 167247872
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004794483450351053,
+      "loss": 3.1579,
+      "theoretical_loss": 4.455038925519882,
+      "tokens_seen": 167313408
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00047943831494483453,
+      "loss": 3.2525,
+      "theoretical_loss": 4.454814571188665,
+      "tokens_seen": 167378944
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004794282848545637,
+      "loss": 3.0859,
+      "theoretical_loss": 4.454590329269979,
+      "tokens_seen": 167444480
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004794182547642929,
+      "loss": 3.4173,
+      "theoretical_loss": 4.454366199663537,
+      "tokens_seen": 167510016
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004794082246740221,
+      "loss": 3.1989,
+      "theoretical_loss": 4.4541421822691785,
+      "tokens_seen": 167575552
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00047939819458375126,
+      "loss": 3.2261,
+      "theoretical_loss": 4.453918276986872,
+      "tokens_seen": 167641088
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00047938816449348044,
+      "loss": 3.3857,
+      "theoretical_loss": 4.453694483716717,
+      "tokens_seen": 167706624
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004793781344032097,
+      "loss": 3.2033,
+      "theoretical_loss": 4.453470802358938,
+      "tokens_seen": 167772160
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004793681043129388,
+      "loss": 3.3288,
+      "theoretical_loss": 4.453247232813885,
+      "tokens_seen": 167837696
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00047935807422266804,
+      "loss": 3.3873,
+      "theoretical_loss": 4.453023774982041,
+      "tokens_seen": 167903232
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00047934804413239716,
+      "loss": 3.027,
+      "theoretical_loss": 4.4528004287640135,
+      "tokens_seen": 167968768
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004793380140421264,
+      "loss": 2.93,
+      "theoretical_loss": 4.452577194060538,
+      "tokens_seen": 168034304
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004793279839518556,
+      "loss": 3.338,
+      "theoretical_loss": 4.452354070772476,
+      "tokens_seen": 168099840
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00047931795386158476,
+      "loss": 3.0985,
+      "theoretical_loss": 4.452131058800816,
+      "tokens_seen": 168165376
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00047930792377131394,
+      "loss": 3.0258,
+      "theoretical_loss": 4.451908158046674,
+      "tokens_seen": 168230912
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004792978936810431,
+      "loss": 3.2144,
+      "theoretical_loss": 4.451685368411292,
+      "tokens_seen": 168296448
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004792878635907723,
+      "loss": 3.1754,
+      "theoretical_loss": 4.4514626897960365,
+      "tokens_seen": 168361984
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00047927783350050154,
+      "loss": 2.8836,
+      "theoretical_loss": 4.451240122102401,
+      "tokens_seen": 168427520
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00047926780341023067,
+      "loss": 3.2178,
+      "theoretical_loss": 4.451017665232007,
+      "tokens_seen": 168493056
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004792577733199599,
+      "loss": 2.9942,
+      "theoretical_loss": 4.4507953190865965,
+      "tokens_seen": 168558592
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004792477432296891,
+      "loss": 3.2664,
+      "theoretical_loss": 4.450573083568042,
+      "tokens_seen": 168624128
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00047923771313941827,
+      "loss": 2.9989,
+      "theoretical_loss": 4.450350958578337,
+      "tokens_seen": 168689664
+    },
+    {
+      "epoch": 0.05,
+      "objective/train/docs_used": 104347,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 2.790850877761841,
+      "objective/train/theoretical_loss": 4.4501289440196015,
+      "objective/train/tokens_used": 189215200,
+      "theoretical_loss": 4.4501289440196015,
+      "tokens_seen": 168755200
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00047922768304914745,
+      "loss": 3.2164,
+      "theoretical_loss": 4.4501289440196015,
+      "tokens_seen": 168755200
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00047921765295887663,
+      "loss": 3.3929,
+      "theoretical_loss": 4.449907039794081,
+      "tokens_seen": 168820736
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004792076228686058,
+      "loss": 3.2061,
+      "theoretical_loss": 4.449685245804145,
+      "tokens_seen": 168886272
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00047919759277833504,
+      "loss": 2.9032,
+      "theoretical_loss": 4.449463561952285,
+      "tokens_seen": 168951808
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00047918756268806417,
+      "loss": 3.1277,
+      "theoretical_loss": 4.44924198814112,
+      "tokens_seen": 169017344
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004791775325977934,
+      "loss": 3.2685,
+      "theoretical_loss": 4.44902052427339,
+      "tokens_seen": 169082880
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00047916750250752253,
+      "loss": 3.0655,
+      "theoretical_loss": 4.448799170251961,
+      "tokens_seen": 169148416
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00047915747241725177,
+      "loss": 3.3278,
+      "theoretical_loss": 4.448577925979821,
+      "tokens_seen": 169213952
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00047914744232698095,
+      "loss": 3.2436,
+      "theoretical_loss": 4.44835679136008,
+      "tokens_seen": 169279488
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00047913741223671013,
+      "loss": 3.2687,
+      "theoretical_loss": 4.448135766295974,
+      "tokens_seen": 169345024
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004791273821464393,
+      "loss": 3.1304,
+      "theoretical_loss": 4.447914850690859,
+      "tokens_seen": 169410560
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00047911735205616855,
+      "loss": 3.199,
+      "theoretical_loss": 4.447694044448216,
+      "tokens_seen": 169476096
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004791073219658977,
+      "loss": 3.2522,
+      "theoretical_loss": 4.447473347471645,
+      "tokens_seen": 169541632
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004790972918756269,
+      "loss": 3.1969,
+      "theoretical_loss": 4.447252759664874,
+      "tokens_seen": 169607168
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00047908726178535604,
+      "loss": 3.0039,
+      "theoretical_loss": 4.4470322809317455,
+      "tokens_seen": 169672704
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00047907723169508527,
+      "loss": 3.5408,
+      "theoretical_loss": 4.44681191117623,
+      "tokens_seen": 169738240
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00047906720160481445,
+      "loss": 3.2186,
+      "theoretical_loss": 4.446591650302416,
+      "tokens_seen": 169803776
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00047905717151454363,
+      "loss": 2.9781,
+      "theoretical_loss": 4.446371498214516,
+      "tokens_seen": 169869312
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004790471414242728,
+      "loss": 3.4193,
+      "theoretical_loss": 4.446151454816862,
+      "tokens_seen": 169934848
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.000479037111334002,
+      "loss": 3.0771,
+      "theoretical_loss": 4.445931520013906,
+      "tokens_seen": 170000384
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004790270812437312,
+      "loss": 3.2464,
+      "theoretical_loss": 4.445711693710223,
+      "tokens_seen": 170065920
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004790170511534604,
+      "loss": 3.2014,
+      "theoretical_loss": 4.4454919758105085,
+      "tokens_seen": 170131456
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00047900702106318954,
+      "loss": 3.3265,
+      "theoretical_loss": 4.445272366219575,
+      "tokens_seen": 170196992
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004789969909729188,
+      "loss": 3.073,
+      "theoretical_loss": 4.44505286484236,
+      "tokens_seen": 170262528
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004789869608826479,
+      "loss": 2.936,
+      "theoretical_loss": 4.4448334715839195,
+      "tokens_seen": 170328064
+    },
+    {
+      "epoch": 0.05,
+      "objective/train/docs_used": 105037,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 2.8210434913635254,
+      "objective/train/theoretical_loss": 4.444614186349425,
+      "objective/train/tokens_used": 190853600,
+      "theoretical_loss": 4.444614186349425,
+      "tokens_seen": 170393600
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00047897693079237714,
+      "loss": 2.9571,
+      "theoretical_loss": 4.444614186349425,
+      "tokens_seen": 170393600
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00047896690070210637,
+      "loss": 2.9778,
+      "theoretical_loss": 4.444395009044175,
+      "tokens_seen": 170459136
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004789568706118355,
+      "loss": 3.2967,
+      "theoretical_loss": 4.444175939573581,
+      "tokens_seen": 170524672
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00047894684052156473,
+      "loss": 2.8109,
+      "theoretical_loss": 4.443956977843177,
+      "tokens_seen": 170590208
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004789368104312939,
+      "loss": 3.4383,
+      "theoretical_loss": 4.4437381237586155,
+      "tokens_seen": 170655744
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004789267803410231,
+      "loss": 3.1635,
+      "theoretical_loss": 4.443519377225666,
+      "tokens_seen": 170721280
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004789167502507523,
+      "loss": 3.1526,
+      "theoretical_loss": 4.443300738150221,
+      "tokens_seen": 170786816
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00047890672016048146,
+      "loss": 3.3384,
+      "theoretical_loss": 4.4430822064382856,
+      "tokens_seen": 170852352
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00047889669007021064,
+      "loss": 3.4135,
+      "theoretical_loss": 4.442863781995989,
+      "tokens_seen": 170917888
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004788866599799399,
+      "loss": 3.0138,
+      "theoretical_loss": 4.442645464729573,
+      "tokens_seen": 170983424
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.000478876629889669,
+      "loss": 3.2596,
+      "theoretical_loss": 4.4424272545454,
+      "tokens_seen": 171048960
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00047886659979939824,
+      "loss": 3.2555,
+      "theoretical_loss": 4.442209151349951,
+      "tokens_seen": 171114496
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00047885656970912736,
+      "loss": 3.3042,
+      "theoretical_loss": 4.4419911550498234,
+      "tokens_seen": 171180032
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004788465396188566,
+      "loss": 3.2061,
+      "theoretical_loss": 4.441773265551731,
+      "tokens_seen": 171245568
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004788365095285858,
+      "loss": 3.0708,
+      "theoretical_loss": 4.441555482762507,
+      "tokens_seen": 171311104
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00047882647943831496,
+      "loss": 3.2816,
+      "theoretical_loss": 4.441337806589097,
+      "tokens_seen": 171376640
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00047881644934804414,
+      "loss": 3.1222,
+      "theoretical_loss": 4.441120236938568,
+      "tokens_seen": 171442176
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004788064192577733,
+      "loss": 3.263,
+      "theoretical_loss": 4.440902773718101,
+      "tokens_seen": 171507712
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004787963891675025,
+      "loss": 2.9574,
+      "theoretical_loss": 4.440685416834994,
+      "tokens_seen": 171573248
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00047878635907723174,
+      "loss": 3.4099,
+      "theoretical_loss": 4.440468166196662,
+      "tokens_seen": 171638784
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00047877632898696087,
+      "loss": 3.2833,
+      "theoretical_loss": 4.440251021710634,
+      "tokens_seen": 171704320
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004787662988966901,
+      "loss": 3.2847,
+      "theoretical_loss": 4.440033983284556,
+      "tokens_seen": 171769856
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004787562688064193,
+      "loss": 3.3134,
+      "theoretical_loss": 4.43981705082619,
+      "tokens_seen": 171835392
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00047874623871614847,
+      "loss": 3.1022,
+      "theoretical_loss": 4.439600224243412,
+      "tokens_seen": 171900928
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00047873620862587765,
+      "loss": 3.1256,
+      "theoretical_loss": 4.439383503444214,
+      "tokens_seen": 171966464
+    },
+    {
+      "epoch": 0.05,
+      "objective/train/docs_used": 106163,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 3.448519468307495,
+      "objective/train/theoretical_loss": 4.439166888336702,
+      "objective/train/tokens_used": 192492000,
+      "theoretical_loss": 4.439166888336702,
+      "tokens_seen": 172032000
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00047872617853560683,
+      "loss": 3.3453,
+      "theoretical_loss": 4.439166888336702,
+      "tokens_seen": 172032000
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.000478716148445336,
+      "loss": 3.0654,
+      "theoretical_loss": 4.438950378829098,
+      "tokens_seen": 172097536
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00047870611835506524,
+      "loss": 3.3586,
+      "theoretical_loss": 4.43873397482974,
+      "tokens_seen": 172163072
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00047869608826479437,
+      "loss": 3.1219,
+      "theoretical_loss": 4.438517676247075,
+      "tokens_seen": 172228608
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004786860581745236,
+      "loss": 3.0914,
+      "theoretical_loss": 4.4383014829896705,
+      "tokens_seen": 172294144
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00047867602808425273,
+      "loss": 3.1461,
+      "theoretical_loss": 4.4380853949662065,
+      "tokens_seen": 172359680
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00047866599799398197,
+      "loss": 3.0581,
+      "theoretical_loss": 4.4378694120854725,
+      "tokens_seen": 172425216
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00047865596790371115,
+      "loss": 3.0474,
+      "theoretical_loss": 4.437653534256377,
+      "tokens_seen": 172490752
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00047864593781344033,
+      "loss": 3.207,
+      "theoretical_loss": 4.437437761387939,
+      "tokens_seen": 172556288
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004786359077231695,
+      "loss": 3.1347,
+      "theoretical_loss": 4.437222093389293,
+      "tokens_seen": 172621824
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00047862587763289875,
+      "loss": 3.0996,
+      "theoretical_loss": 4.437006530169684,
+      "tokens_seen": 172687360
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004786158475426279,
+      "loss": 3.255,
+      "theoretical_loss": 4.436791071638472,
+      "tokens_seen": 172752896
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004786058174523571,
+      "loss": 3.3615,
+      "theoretical_loss": 4.43657571770513,
+      "tokens_seen": 172818432
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00047859578736208624,
+      "loss": 3.3575,
+      "theoretical_loss": 4.436360468279242,
+      "tokens_seen": 172883968
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00047858575727181547,
+      "loss": 3.1795,
+      "theoretical_loss": 4.4361453232705035,
+      "tokens_seen": 172949504
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00047857572718154465,
+      "loss": 3.2246,
+      "theoretical_loss": 4.435930282588725,
+      "tokens_seen": 173015040
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00047856569709127383,
+      "loss": 3.1589,
+      "theoretical_loss": 4.435715346143828,
+      "tokens_seen": 173080576
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.000478555667001003,
+      "loss": 3.1079,
+      "theoretical_loss": 4.435500513845845,
+      "tokens_seen": 173146112
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004785456369107322,
+      "loss": 3.0886,
+      "theoretical_loss": 4.435285785604922,
+      "tokens_seen": 173211648
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004785356068204614,
+      "loss": 3.3142,
+      "theoretical_loss": 4.435071161331313,
+      "tokens_seen": 173277184
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004785255767301906,
+      "loss": 3.3968,
+      "theoretical_loss": 4.434856640935388,
+      "tokens_seen": 173342720
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00047851554663991974,
+      "loss": 3.1791,
+      "theoretical_loss": 4.434642224327625,
+      "tokens_seen": 173408256
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.000478505516549649,
+      "loss": 3.1941,
+      "theoretical_loss": 4.434427911418613,
+      "tokens_seen": 173473792
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004784954864593781,
+      "loss": 3.1568,
+      "theoretical_loss": 4.434213702119052,
+      "tokens_seen": 173539328
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00047848545636910734,
+      "loss": 3.3522,
+      "theoretical_loss": 4.433999596339755,
+      "tokens_seen": 173604864
+    },
+    {
+      "epoch": 0.05,
+      "objective/train/docs_used": 106755,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 2.8909780979156494,
+      "objective/train/theoretical_loss": 4.433785593991642,
+      "objective/train/tokens_used": 194130400,
+      "theoretical_loss": 4.433785593991642,
+      "tokens_seen": 173670400
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004784754262788365,
+      "loss": 3.2649,
+      "theoretical_loss": 4.433785593991642,
+      "tokens_seen": 173670400
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004784653961885657,
+      "loss": 3.2448,
+      "theoretical_loss": 4.433571694985744,
+      "tokens_seen": 173735936
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004784553660982949,
+      "loss": 3.2039,
+      "theoretical_loss": 4.433357899233204,
+      "tokens_seen": 173801472
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004784453360080241,
+      "loss": 3.0269,
+      "theoretical_loss": 4.433144206645274,
+      "tokens_seen": 173867008
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00047843530591775324,
+      "loss": 3.0917,
+      "theoretical_loss": 4.432930617133313,
+      "tokens_seen": 173932544
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004784252758274825,
+      "loss": 3.1783,
+      "theoretical_loss": 4.432717130608793,
+      "tokens_seen": 173998080
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004784152457372116,
+      "loss": 3.1232,
+      "theoretical_loss": 4.432503746983295,
+      "tokens_seen": 174063616
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00047840521564694084,
+      "loss": 3.2631,
+      "theoretical_loss": 4.432290466168507,
+      "tokens_seen": 174129152
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00047839518555667,
+      "loss": 3.2136,
+      "theoretical_loss": 4.432077288076228,
+      "tokens_seen": 174194688
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004783851554663992,
+      "loss": 3.2796,
+      "theoretical_loss": 4.431864212618365,
+      "tokens_seen": 174260224
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004783751253761284,
+      "loss": 3.4694,
+      "theoretical_loss": 4.431651239706934,
+      "tokens_seen": 174325760
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00047836509528585756,
+      "loss": 3.0788,
+      "theoretical_loss": 4.431438369254058,
+      "tokens_seen": 174391296
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00047835506519558675,
+      "loss": 3.0872,
+      "theoretical_loss": 4.43122560117197,
+      "tokens_seen": 174456832
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.000478345035105316,
+      "loss": 2.9119,
+      "theoretical_loss": 4.431012935373012,
+      "tokens_seen": 174522368
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004783350050150451,
+      "loss": 3.2154,
+      "theoretical_loss": 4.430800371769631,
+      "tokens_seen": 174587904
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00047832497492477434,
+      "loss": 3.2817,
+      "theoretical_loss": 4.430587910274383,
+      "tokens_seen": 174653440
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00047831494483450347,
+      "loss": 3.1007,
+      "theoretical_loss": 4.430375550799933,
+      "tokens_seen": 174718976
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004783049147442327,
+      "loss": 3.3202,
+      "theoretical_loss": 4.430163293259053,
+      "tokens_seen": 174784512
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004782948846539619,
+      "loss": 3.2775,
+      "theoretical_loss": 4.42995113756462,
+      "tokens_seen": 174850048
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00047828485456369107,
+      "loss": 3.4485,
+      "theoretical_loss": 4.42973908362962,
+      "tokens_seen": 174915584
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00047827482447342025,
+      "loss": 3.1023,
+      "theoretical_loss": 4.429527131367147,
+      "tokens_seen": 174981120
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004782647943831495,
+      "loss": 3.0822,
+      "theoretical_loss": 4.429315280690398,
+      "tokens_seen": 175046656
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004782547642928786,
+      "loss": 3.2901,
+      "theoretical_loss": 4.4291035315126805,
+      "tokens_seen": 175112192
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00047824473420260785,
+      "loss": 3.2442,
+      "theoretical_loss": 4.428891883747406,
+      "tokens_seen": 175177728
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.000478234704112337,
+      "loss": 3.166,
+      "theoretical_loss": 4.428680337308094,
+      "tokens_seen": 175243264
+    },
+    {
+      "epoch": 0.05,
+      "objective/train/docs_used": 107724,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 3.6482720375061035,
+      "objective/train/theoretical_loss": 4.428468892108368,
+      "objective/train/tokens_used": 195768800,
+      "theoretical_loss": 4.428468892108368,
+      "tokens_seen": 175308800
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004782246740220662,
+      "loss": 3.3804,
+      "theoretical_loss": 4.428468892108368,
+      "tokens_seen": 175308800
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00047821464393179544,
+      "loss": 3.4307,
+      "theoretical_loss": 4.428257548061959,
+      "tokens_seen": 175374336
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00047820461384152457,
+      "loss": 3.2556,
+      "theoretical_loss": 4.428046305082704,
+      "tokens_seen": 175439872
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004781945837512538,
+      "loss": 3.3203,
+      "theoretical_loss": 4.427835163084543,
+      "tokens_seen": 175505408
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00047818455366098293,
+      "loss": 3.1723,
+      "theoretical_loss": 4.427624121981523,
+      "tokens_seen": 175570944
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00047817452357071217,
+      "loss": 3.1751,
+      "theoretical_loss": 4.427413181687799,
+      "tokens_seen": 175636480
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00047816449348044135,
+      "loss": 2.9857,
+      "theoretical_loss": 4.427202342117624,
+      "tokens_seen": 175702016
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00047815446339017053,
+      "loss": 3.1727,
+      "theoretical_loss": 4.426991603185365,
+      "tokens_seen": 175767552
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004781444332998997,
+      "loss": 3.121,
+      "theoretical_loss": 4.426780964805486,
+      "tokens_seen": 175833088
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00047813440320962895,
+      "loss": 3.3494,
+      "theoretical_loss": 4.4265704268925585,
+      "tokens_seen": 175898624
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004781243731193581,
+      "loss": 3.313,
+      "theoretical_loss": 4.426359989361259,
+      "tokens_seen": 175964160
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004781143430290873,
+      "loss": 3.3073,
+      "theoretical_loss": 4.426149652126369,
+      "tokens_seen": 176029696
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00047810431293881644,
+      "loss": 3.0604,
+      "theoretical_loss": 4.42593941510277,
+      "tokens_seen": 176095232
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00047809428284854567,
+      "loss": 2.7989,
+      "theoretical_loss": 4.42572927820545,
+      "tokens_seen": 176160768
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00047808425275827485,
+      "loss": 3.2073,
+      "theoretical_loss": 4.4255192413495035,
+      "tokens_seen": 176226304
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00047807422266800403,
+      "loss": 3.0824,
+      "theoretical_loss": 4.425309304450122,
+      "tokens_seen": 176291840
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004780641925777332,
+      "loss": 3.1597,
+      "theoretical_loss": 4.4250994674226085,
+      "tokens_seen": 176357376
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004780541624874624,
+      "loss": 3.1731,
+      "theoretical_loss": 4.424889730182361,
+      "tokens_seen": 176422912
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004780441323971916,
+      "loss": 3.3307,
+      "theoretical_loss": 4.424680092644886,
+      "tokens_seen": 176488448
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004780341023069208,
+      "loss": 3.0847,
+      "theoretical_loss": 4.42447055472579,
+      "tokens_seen": 176553984
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00047802407221664994,
+      "loss": 3.3959,
+      "theoretical_loss": 4.4242611163407854,
+      "tokens_seen": 176619520
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004780140421263792,
+      "loss": 3.2021,
+      "theoretical_loss": 4.424051777405684,
+      "tokens_seen": 176685056
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004780040120361083,
+      "loss": 3.0791,
+      "theoretical_loss": 4.423842537836402,
+      "tokens_seen": 176750592
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00047799398194583754,
+      "loss": 3.112,
+      "theoretical_loss": 4.423633397548956,
+      "tokens_seen": 176816128
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004779839518555667,
+      "loss": 3.0609,
+      "theoretical_loss": 4.423424356459467,
+      "tokens_seen": 176881664
+    },
+    {
+      "epoch": 0.05,
+      "objective/train/docs_used": 108296,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 3.4872632026672363,
+      "objective/train/theoretical_loss": 4.423215414484155,
+      "objective/train/tokens_used": 197407200,
+      "theoretical_loss": 4.423215414484155,
+      "tokens_seen": 176947200
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004779739217652959,
+      "loss": 3.3284,
+      "theoretical_loss": 4.423215414484155,
+      "tokens_seen": 176947200
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004779638916750251,
+      "loss": 2.839,
+      "theoretical_loss": 4.423006571539346,
+      "tokens_seen": 177012736
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004779538615847543,
+      "loss": 3.1684,
+      "theoretical_loss": 4.422797827541462,
+      "tokens_seen": 177078272
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00047794383149448344,
+      "loss": 3.1424,
+      "theoretical_loss": 4.422589182407032,
+      "tokens_seen": 177143808
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004779338014042127,
+      "loss": 3.2199,
+      "theoretical_loss": 4.422380636052681,
+      "tokens_seen": 177209344
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004779237713139418,
+      "loss": 2.9687,
+      "theoretical_loss": 4.422172188395141,
+      "tokens_seen": 177274880
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00047791374122367104,
+      "loss": 3.0297,
+      "theoretical_loss": 4.421963839351239,
+      "tokens_seen": 177340416
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004779037111334002,
+      "loss": 3.1726,
+      "theoretical_loss": 4.421755588837907,
+      "tokens_seen": 177405952
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004778936810431294,
+      "loss": 3.3507,
+      "theoretical_loss": 4.421547436772176,
+      "tokens_seen": 177471488
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004778836509528586,
+      "loss": 3.2595,
+      "theoretical_loss": 4.421339383071178,
+      "tokens_seen": 177537024
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00047787362086258776,
+      "loss": 3.3193,
+      "theoretical_loss": 4.421131427652145,
+      "tokens_seen": 177602560
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00047786359077231695,
+      "loss": 3.2873,
+      "theoretical_loss": 4.420923570432407,
+      "tokens_seen": 177668096
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004778535606820462,
+      "loss": 3.1947,
+      "theoretical_loss": 4.420715811329398,
+      "tokens_seen": 177733632
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004778435305917753,
+      "loss": 3.2383,
+      "theoretical_loss": 4.420508150260651,
+      "tokens_seen": 177799168
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00047783350050150454,
+      "loss": 3.1387,
+      "theoretical_loss": 4.420300587143795,
+      "tokens_seen": 177864704
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00047782347041123367,
+      "loss": 2.8977,
+      "theoretical_loss": 4.420093121896563,
+      "tokens_seen": 177930240
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004778134403209629,
+      "loss": 3.1988,
+      "theoretical_loss": 4.419885754436784,
+      "tokens_seen": 177995776
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004778034102306921,
+      "loss": 3.0306,
+      "theoretical_loss": 4.4196784846823896,
+      "tokens_seen": 178061312
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00047779338014042127,
+      "loss": 3.1273,
+      "theoretical_loss": 4.419471312551408,
+      "tokens_seen": 178126848
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00047778335005015045,
+      "loss": 2.9744,
+      "theoretical_loss": 4.419264237961966,
+      "tokens_seen": 178192384
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004777733199598797,
+      "loss": 3.0912,
+      "theoretical_loss": 4.41905726083229,
+      "tokens_seen": 178257920
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004777632898696088,
+      "loss": 3.2138,
+      "theoretical_loss": 4.418850381080706,
+      "tokens_seen": 178323456
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00047775325977933805,
+      "loss": 3.2271,
+      "theoretical_loss": 4.4186435986256365,
+      "tokens_seen": 178388992
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004777432296890672,
+      "loss": 3.0616,
+      "theoretical_loss": 4.418436913385604,
+      "tokens_seen": 178454528
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004777331995987964,
+      "loss": 3.2644,
+      "theoretical_loss": 4.418230325279228,
+      "tokens_seen": 178520064
+    },
+    {
+      "epoch": 0.05,
+      "objective/train/docs_used": 109542,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 3.0579349994659424,
+      "objective/train/theoretical_loss": 4.418023834225226,
+      "objective/train/tokens_used": 199045600,
+      "theoretical_loss": 4.418023834225226,
+      "tokens_seen": 178585600
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004777231695085256,
+      "loss": 3.1012,
+      "theoretical_loss": 4.418023834225226,
+      "tokens_seen": 178585600
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00047771313941825477,
+      "loss": 3.1625,
+      "theoretical_loss": 4.417817440142414,
+      "tokens_seen": 178651136
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00047770310932798395,
+      "loss": 3.0349,
+      "theoretical_loss": 4.417611142949705,
+      "tokens_seen": 178716672
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00047769307923771313,
+      "loss": 3.2227,
+      "theoretical_loss": 4.417404942566111,
+      "tokens_seen": 178782208
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004776830491474423,
+      "loss": 3.0844,
+      "theoretical_loss": 4.417198838910737,
+      "tokens_seen": 178847744
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00047767301905717155,
+      "loss": 3.3573,
+      "theoretical_loss": 4.4169928319027925,
+      "tokens_seen": 178913280
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004776629889669007,
+      "loss": 3.297,
+      "theoretical_loss": 4.416786921461577,
+      "tokens_seen": 178978816
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004776529588766299,
+      "loss": 3.2404,
+      "theoretical_loss": 4.41658110750649,
+      "tokens_seen": 179044352
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00047764292878635904,
+      "loss": 3.2353,
+      "theoretical_loss": 4.416375389957029,
+      "tokens_seen": 179109888
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004776328986960883,
+      "loss": 3.2372,
+      "theoretical_loss": 4.416169768732786,
+      "tokens_seen": 179175424
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00047762286860581746,
+      "loss": 3.1656,
+      "theoretical_loss": 4.415964243753448,
+      "tokens_seen": 179240960
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00047761283851554664,
+      "loss": 2.8394,
+      "theoretical_loss": 4.415758814938803,
+      "tokens_seen": 179306496
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004776028084252758,
+      "loss": 3.3786,
+      "theoretical_loss": 4.415553482208733,
+      "tokens_seen": 179372032
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00047759277833500505,
+      "loss": 3.2626,
+      "theoretical_loss": 4.415348245483212,
+      "tokens_seen": 179437568
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004775827482447342,
+      "loss": 3.2584,
+      "theoretical_loss": 4.415143104682317,
+      "tokens_seen": 179503104
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004775727181544634,
+      "loss": 3.2606,
+      "theoretical_loss": 4.4149380597262144,
+      "tokens_seen": 179568640
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00047756268806419254,
+      "loss": 3.3959,
+      "theoretical_loss": 4.41473311053517,
+      "tokens_seen": 179634176
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004775526579739218,
+      "loss": 3.1822,
+      "theoretical_loss": 4.414528257029543,
+      "tokens_seen": 179699712
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00047754262788365096,
+      "loss": 3.3075,
+      "theoretical_loss": 4.414323499129791,
+      "tokens_seen": 179765248
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00047753259779338014,
+      "loss": 3.1764,
+      "theoretical_loss": 4.414118836756462,
+      "tokens_seen": 179830784
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004775225677031093,
+      "loss": 3.2029,
+      "theoretical_loss": 4.4139142698302,
+      "tokens_seen": 179896320
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004775125376128385,
+      "loss": 3.0117,
+      "theoretical_loss": 4.413709798271748,
+      "tokens_seen": 179961856
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004775025075225677,
+      "loss": 3.0247,
+      "theoretical_loss": 4.41350542200194,
+      "tokens_seen": 180027392
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004774924774322969,
+      "loss": 3.2042,
+      "theoretical_loss": 4.413301140941703,
+      "tokens_seen": 180092928
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00047748244734202605,
+      "loss": 3.1478,
+      "theoretical_loss": 4.413096955012063,
+      "tokens_seen": 180158464
+    },
+    {
+      "epoch": 0.05,
+      "objective/train/docs_used": 110066,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 2.8472869396209717,
+      "objective/train/theoretical_loss": 4.412892864134137,
+      "objective/train/tokens_used": 200684000,
+      "theoretical_loss": 4.412892864134137,
+      "tokens_seen": 180224000
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004774724172517553,
+      "loss": 3.0943,
+      "theoretical_loss": 4.412892864134137,
+      "tokens_seen": 180224000
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004774623871614845,
+      "loss": 2.8881,
+      "theoretical_loss": 4.412688868229136,
+      "tokens_seen": 180289536
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00047745235707121364,
+      "loss": 3.3796,
+      "theoretical_loss": 4.412484967218368,
+      "tokens_seen": 180355072
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004774423269809429,
+      "loss": 3.234,
+      "theoretical_loss": 4.4122811610232295,
+      "tokens_seen": 180420608
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.000477432296890672,
+      "loss": 3.1805,
+      "theoretical_loss": 4.412077449565216,
+      "tokens_seen": 180486144
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00047742226680040124,
+      "loss": 3.4462,
+      "theoretical_loss": 4.411873832765913,
+      "tokens_seen": 180551680
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004774122367101304,
+      "loss": 3.1975,
+      "theoretical_loss": 4.4116703105470005,
+      "tokens_seen": 180617216
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004774022066198596,
+      "loss": 3.1883,
+      "theoretical_loss": 4.411466882830252,
+      "tokens_seen": 180682752
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004773921765295888,
+      "loss": 3.1599,
+      "theoretical_loss": 4.411263549537534,
+      "tokens_seen": 180748288
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00047738214643931797,
+      "loss": 3.1214,
+      "theoretical_loss": 4.411060310590805,
+      "tokens_seen": 180813824
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00047737211634904715,
+      "loss": 3.0375,
+      "theoretical_loss": 4.410857165912116,
+      "tokens_seen": 180879360
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004773620862587764,
+      "loss": 3.0394,
+      "theoretical_loss": 4.410654115423613,
+      "tokens_seen": 180944896
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004773520561685055,
+      "loss": 3.0477,
+      "theoretical_loss": 4.410451159047533,
+      "tokens_seen": 181010432
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00047734202607823474,
+      "loss": 3.2956,
+      "theoretical_loss": 4.410248296706204,
+      "tokens_seen": 181075968
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00047733199598796387,
+      "loss": 3.1011,
+      "theoretical_loss": 4.410045528322048,
+      "tokens_seen": 181141504
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004773219658976931,
+      "loss": 3.1455,
+      "theoretical_loss": 4.409842853817577,
+      "tokens_seen": 181207040
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004773119358074223,
+      "loss": 3.1878,
+      "theoretical_loss": 4.409640273115399,
+      "tokens_seen": 181272576
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00047730190571715147,
+      "loss": 2.8744,
+      "theoretical_loss": 4.409437786138211,
+      "tokens_seen": 181338112
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00047729187562688065,
+      "loss": 3.1718,
+      "theoretical_loss": 4.4092353928087995,
+      "tokens_seen": 181403648
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004772818455366099,
+      "loss": 3.1126,
+      "theoretical_loss": 4.409033093050046,
+      "tokens_seen": 181469184
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.000477271815446339,
+      "loss": 3.4133,
+      "theoretical_loss": 4.408830886784922,
+      "tokens_seen": 181534720
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.00047726178535606825,
+      "loss": 3.2217,
+      "theoretical_loss": 4.40862877393649,
+      "tokens_seen": 181600256
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004772517552657974,
+      "loss": 3.1425,
+      "theoretical_loss": 4.408426754427904,
+      "tokens_seen": 181665792
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004772417251755266,
+      "loss": 3.009,
+      "theoretical_loss": 4.408224828182409,
+      "tokens_seen": 181731328
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004772316950852558,
+      "loss": 3.0163,
+      "theoretical_loss": 4.40802299512334,
+      "tokens_seen": 181796864
+    },
+    {
+      "epoch": 0.06,
+      "objective/train/docs_used": 111369,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 3.439749002456665,
+      "objective/train/theoretical_loss": 4.407821255174123,
+      "objective/train/tokens_used": 202322400,
+      "theoretical_loss": 4.407821255174123,
+      "tokens_seen": 181862400
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.00047722166499498497,
+      "loss": 3.1176,
+      "theoretical_loss": 4.407821255174123,
+      "tokens_seen": 181862400
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.00047721163490471415,
+      "loss": 3.3191,
+      "theoretical_loss": 4.407619608258274,
+      "tokens_seen": 181927936
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.00047720160481444333,
+      "loss": 3.1351,
+      "theoretical_loss": 4.407418054299402,
+      "tokens_seen": 181993472
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004771915747241725,
+      "loss": 3.2347,
+      "theoretical_loss": 4.407216593221202,
+      "tokens_seen": 182059008
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.00047718154463390175,
+      "loss": 3.182,
+      "theoretical_loss": 4.407015224947463,
+      "tokens_seen": 182124544
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004771715145436309,
+      "loss": 3.3172,
+      "theoretical_loss": 4.40681394940206,
+      "tokens_seen": 182190080
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004771614844533601,
+      "loss": 3.3428,
+      "theoretical_loss": 4.406612766508961,
+      "tokens_seen": 182255616
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.00047715145436308924,
+      "loss": 3.1138,
+      "theoretical_loss": 4.4064116761922225,
+      "tokens_seen": 182321152
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004771414242728185,
+      "loss": 2.9224,
+      "theoretical_loss": 4.406210678375992,
+      "tokens_seen": 182386688
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.00047713139418254766,
+      "loss": 3.0207,
+      "theoretical_loss": 4.406009772984502,
+      "tokens_seen": 182452224
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.00047712136409227684,
+      "loss": 2.9416,
+      "theoretical_loss": 4.40580895994208,
+      "tokens_seen": 182517760
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.000477111334002006,
+      "loss": 3.1591,
+      "theoretical_loss": 4.405608239173138,
+      "tokens_seen": 182583296
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.00047710130391173525,
+      "loss": 3.142,
+      "theoretical_loss": 4.405407610602181,
+      "tokens_seen": 182648832
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004770912738214644,
+      "loss": 3.2257,
+      "theoretical_loss": 4.405207074153797,
+      "tokens_seen": 182714368
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004770812437311936,
+      "loss": 3.0585,
+      "theoretical_loss": 4.40500662975267,
+      "tokens_seen": 182779904
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.00047707121364092274,
+      "loss": 3.0379,
+      "theoretical_loss": 4.4048062773235666,
+      "tokens_seen": 182845440
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.000477061183550652,
+      "loss": 3.018,
+      "theoretical_loss": 4.404606016791345,
+      "tokens_seen": 182910976
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.00047705115346038116,
+      "loss": 3.1405,
+      "theoretical_loss": 4.404405848080951,
+      "tokens_seen": 182976512
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.00047704112337011034,
+      "loss": 2.9844,
+      "theoretical_loss": 4.404205771117417,
+      "tokens_seen": 183042048
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004770310932798395,
+      "loss": 3.0418,
+      "theoretical_loss": 4.404005785825866,
+      "tokens_seen": 183107584
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004770210631895687,
+      "loss": 3.2314,
+      "theoretical_loss": 4.403805892131508,
+      "tokens_seen": 183173120
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004770110330992979,
+      "loss": 3.2265,
+      "theoretical_loss": 4.40360608995964,
+      "tokens_seen": 183238656
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004770010030090271,
+      "loss": 3.0284,
+      "theoretical_loss": 4.403406379235646,
+      "tokens_seen": 183304192
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.00047699097291875625,
+      "loss": 3.0799,
+      "theoretical_loss": 4.403206759884999,
+      "tokens_seen": 183369728
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004769809428284855,
+      "loss": 3.0723,
+      "theoretical_loss": 4.403007231833261,
+      "tokens_seen": 183435264
+    },
+    {
+      "epoch": 0.06,
+      "objective/train/docs_used": 111925,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 3.625623941421509,
+      "objective/train/theoretical_loss": 4.402807795006074,
+      "objective/train/tokens_used": 203960800,
+      "theoretical_loss": 4.402807795006074,
+      "tokens_seen": 183500800
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.00047697091273821466,
+      "loss": 3.2865,
+      "theoretical_loss": 4.402807795006074,
+      "tokens_seen": 183500800
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.00047696088264794384,
+      "loss": 2.8014,
+      "theoretical_loss": 4.402608449329178,
+      "tokens_seen": 183566336
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.000476950852557673,
+      "loss": 3.007,
+      "theoretical_loss": 4.40240919472839,
+      "tokens_seen": 183631872
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004769408224674022,
+      "loss": 3.0607,
+      "theoretical_loss": 4.40221003112962,
+      "tokens_seen": 183697408
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004769307923771314,
+      "loss": 3.393,
+      "theoretical_loss": 4.4020109584588605,
+      "tokens_seen": 183762944
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004769207622868606,
+      "loss": 3.3108,
+      "theoretical_loss": 4.401811976642195,
+      "tokens_seen": 183828480
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.00047691073219658975,
+      "loss": 2.9578,
+      "theoretical_loss": 4.4016130856057885,
+      "tokens_seen": 183894016
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.000476900702106319,
+      "loss": 3.2152,
+      "theoretical_loss": 4.401414285275898,
+      "tokens_seen": 183959552
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004768906720160481,
+      "loss": 3.2252,
+      "theoretical_loss": 4.401215575578861,
+      "tokens_seen": 184025088
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.00047688064192577735,
+      "loss": 3.0185,
+      "theoretical_loss": 4.4010169564411035,
+      "tokens_seen": 184090624
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.00047687061183550653,
+      "loss": 3.0891,
+      "theoretical_loss": 4.4008184277891385,
+      "tokens_seen": 184156160
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004768605817452357,
+      "loss": 3.1172,
+      "theoretical_loss": 4.400619989549563,
+      "tokens_seen": 184221696
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004768505516549649,
+      "loss": 3.0661,
+      "theoretical_loss": 4.400421641649061,
+      "tokens_seen": 184287232
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.00047684052156469407,
+      "loss": 2.9647,
+      "theoretical_loss": 4.400223384014399,
+      "tokens_seen": 184352768
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.00047683049147442325,
+      "loss": 3.0501,
+      "theoretical_loss": 4.400025216572435,
+      "tokens_seen": 184418304
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004768204613841525,
+      "loss": 2.9332,
+      "theoretical_loss": 4.399827139250106,
+      "tokens_seen": 184483840
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004768104312938816,
+      "loss": 3.2431,
+      "theoretical_loss": 4.399629151974436,
+      "tokens_seen": 184549376
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.00047680040120361085,
+      "loss": 2.9531,
+      "theoretical_loss": 4.399431254672535,
+      "tokens_seen": 184614912
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.00047679037111334003,
+      "loss": 3.1576,
+      "theoretical_loss": 4.399233447271598,
+      "tokens_seen": 184680448
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004767803410230692,
+      "loss": 3.1937,
+      "theoretical_loss": 4.399035729698903,
+      "tokens_seen": 184745984
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004767703109327984,
+      "loss": 3.0762,
+      "theoretical_loss": 4.398838101881815,
+      "tokens_seen": 184811520
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004767602808425276,
+      "loss": 3.1412,
+      "theoretical_loss": 4.39864056374778,
+      "tokens_seen": 184877056
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.00047675025075225676,
+      "loss": 3.2723,
+      "theoretical_loss": 4.398443115224333,
+      "tokens_seen": 184942592
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.000476740220661986,
+      "loss": 3.3477,
+      "theoretical_loss": 4.398245756239088,
+      "tokens_seen": 185008128
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004767301905717151,
+      "loss": 2.7878,
+      "theoretical_loss": 4.398048486719748,
+      "tokens_seen": 185073664
+    },
+    {
+      "epoch": 0.06,
+      "objective/train/docs_used": 113096,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 2.7102208137512207,
+      "objective/train/theoretical_loss": 4.397851306594097,
+      "objective/train/tokens_used": 205599200,
+      "theoretical_loss": 4.397851306594097,
+      "tokens_seen": 185139200
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.00047672016048144435,
+      "loss": 2.8609,
+      "theoretical_loss": 4.397851306594097,
+      "tokens_seen": 185139200
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.00047671013039117353,
+      "loss": 3.0009,
+      "theoretical_loss": 4.397654215790003,
+      "tokens_seen": 185204736
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004767001003009027,
+      "loss": 3.029,
+      "theoretical_loss": 4.397457214235419,
+      "tokens_seen": 185270272
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.00047669007021063195,
+      "loss": 3.292,
+      "theoretical_loss": 4.397260301858379,
+      "tokens_seen": 185335808
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004766800401203611,
+      "loss": 3.1928,
+      "theoretical_loss": 4.397063478587005,
+      "tokens_seen": 185401344
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004766700100300903,
+      "loss": 3.3045,
+      "theoretical_loss": 4.396866744349498,
+      "tokens_seen": 185466880
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.00047665997993981944,
+      "loss": 3.1277,
+      "theoretical_loss": 4.396670099074143,
+      "tokens_seen": 185532416
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004766499498495487,
+      "loss": 3.0829,
+      "theoretical_loss": 4.39647354268931,
+      "tokens_seen": 185597952
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.00047663991975927786,
+      "loss": 3.2087,
+      "theoretical_loss": 4.39627707512345,
+      "tokens_seen": 185663488
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.00047662988966900704,
+      "loss": 2.8462,
+      "theoretical_loss": 4.3960806963050985,
+      "tokens_seen": 185729024
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004766198595787362,
+      "loss": 3.0272,
+      "theoretical_loss": 4.395884406162873,
+      "tokens_seen": 185794560
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.00047660982948846545,
+      "loss": 3.2038,
+      "theoretical_loss": 4.395688204625471,
+      "tokens_seen": 185860096
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004765997993981946,
+      "loss": 2.8081,
+      "theoretical_loss": 4.395492091621677,
+      "tokens_seen": 185925632
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004765897693079238,
+      "loss": 3.3035,
+      "theoretical_loss": 4.395296067080356,
+      "tokens_seen": 185991168
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.00047657973921765294,
+      "loss": 3.1633,
+      "theoretical_loss": 4.3951001309304525,
+      "tokens_seen": 186056704
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004765697091273822,
+      "loss": 3.145,
+      "theoretical_loss": 4.394904283100997,
+      "tokens_seen": 186122240
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.00047655967903711136,
+      "loss": 3.2733,
+      "theoretical_loss": 4.394708523521101,
+      "tokens_seen": 186187776
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.00047654964894684054,
+      "loss": 3.1495,
+      "theoretical_loss": 4.394512852119956,
+      "tokens_seen": 186253312
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004765396188565697,
+      "loss": 3.1897,
+      "theoretical_loss": 4.3943172688268355,
+      "tokens_seen": 186318848
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004765295887662989,
+      "loss": 3.0105,
+      "theoretical_loss": 4.394121773571097,
+      "tokens_seen": 186384384
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004765195586760281,
+      "loss": 3.1493,
+      "theoretical_loss": 4.3939263662821775,
+      "tokens_seen": 186449920
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004765095285857573,
+      "loss": 2.9624,
+      "theoretical_loss": 4.393731046889595,
+      "tokens_seen": 186515456
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.00047649949849548645,
+      "loss": 3.3138,
+      "theoretical_loss": 4.3935358153229505,
+      "tokens_seen": 186580992
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004764894684052157,
+      "loss": 3.1593,
+      "theoretical_loss": 4.3933406715119245,
+      "tokens_seen": 186646528
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.00047647943831494486,
+      "loss": 3.0466,
+      "theoretical_loss": 4.393145615386279,
+      "tokens_seen": 186712064
+    },
+    {
+      "epoch": 0.06,
+      "objective/train/docs_used": 113525,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 2.944962978363037,
+      "objective/train/theoretical_loss": 4.392950646875857,
+      "objective/train/tokens_used": 207237600,
+      "theoretical_loss": 4.392950646875857,
+      "tokens_seen": 186777600
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.00047646940822467404,
+      "loss": 3.0963,
+      "theoretical_loss": 4.392950646875857,
+      "tokens_seen": 186777600
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004764593781344032,
+      "loss": 3.0612,
+      "theoretical_loss": 4.392755765910581,
+      "tokens_seen": 186843136
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004764493480441324,
+      "loss": 3.1482,
+      "theoretical_loss": 4.3925609724204575,
+      "tokens_seen": 186908672
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004764393179538616,
+      "loss": 3.2602,
+      "theoretical_loss": 4.392366266335569,
+      "tokens_seen": 186974208
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004764292878635908,
+      "loss": 3.1856,
+      "theoretical_loss": 4.392171647586081,
+      "tokens_seen": 187039744
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.00047641925777331995,
+      "loss": 3.0132,
+      "theoretical_loss": 4.39197711610224,
+      "tokens_seen": 187105280
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004764092276830492,
+      "loss": 3.152,
+      "theoretical_loss": 4.391782671814369,
+      "tokens_seen": 187170816
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004763991975927783,
+      "loss": 3.2149,
+      "theoretical_loss": 4.391588314652875,
+      "tokens_seen": 187236352
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.00047638916750250755,
+      "loss": 3.2574,
+      "theoretical_loss": 4.3913940445482424,
+      "tokens_seen": 187301888
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.00047637913741223673,
+      "loss": 3.1899,
+      "theoretical_loss": 4.391199861431037,
+      "tokens_seen": 187367424
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004763691073219659,
+      "loss": 3.0975,
+      "theoretical_loss": 4.3910057652319034,
+      "tokens_seen": 187432960
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004763590772316951,
+      "loss": 3.2897,
+      "theoretical_loss": 4.390811755881566,
+      "tokens_seen": 187498496
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.00047634904714142427,
+      "loss": 3.3427,
+      "theoretical_loss": 4.390617833310827,
+      "tokens_seen": 187564032
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.00047633901705115345,
+      "loss": 3.181,
+      "theoretical_loss": 4.390423997450571,
+      "tokens_seen": 187629568
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004763289869608827,
+      "loss": 3.1201,
+      "theoretical_loss": 4.3902302482317594,
+      "tokens_seen": 187695104
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004763189568706118,
+      "loss": 3.12,
+      "theoretical_loss": 4.390036585585432,
+      "tokens_seen": 187760640
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.00047630892678034105,
+      "loss": 2.9343,
+      "theoretical_loss": 4.3898430094427106,
+      "tokens_seen": 187826176
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.00047629889669007023,
+      "loss": 3.0305,
+      "theoretical_loss": 4.389649519734793,
+      "tokens_seen": 187891712
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004762888665997994,
+      "loss": 3.0496,
+      "theoretical_loss": 4.3894561163929575,
+      "tokens_seen": 187957248
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004762788365095286,
+      "loss": 3.2563,
+      "theoretical_loss": 4.389262799348559,
+      "tokens_seen": 188022784
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004762688064192578,
+      "loss": 3.2352,
+      "theoretical_loss": 4.389069568533033,
+      "tokens_seen": 188088320
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.00047625877632898696,
+      "loss": 3.1003,
+      "theoretical_loss": 4.388876423877893,
+      "tokens_seen": 188153856
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004762487462387162,
+      "loss": 3.1905,
+      "theoretical_loss": 4.388683365314728,
+      "tokens_seen": 188219392
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004762387161484453,
+      "loss": 3.3903,
+      "theoretical_loss": 4.388490392775209,
+      "tokens_seen": 188284928
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.00047622868605817455,
+      "loss": 3.2064,
+      "theoretical_loss": 4.3882975061910825,
+      "tokens_seen": 188350464
+    },
+    {
+      "epoch": 0.06,
+      "objective/train/docs_used": 114652,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 2.638829469680786,
+      "objective/train/theoretical_loss": 4.3881047054941735,
+      "objective/train/tokens_used": 208876000,
+      "theoretical_loss": 4.3881047054941735,
+      "tokens_seen": 188416000
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004762186559679037,
+      "loss": 3.0819,
+      "theoretical_loss": 4.3881047054941735,
+      "tokens_seen": 188416000
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004762086258776329,
+      "loss": 2.914,
+      "theoretical_loss": 4.387911990616386,
+      "tokens_seen": 188481536
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004761985957873621,
+      "loss": 3.038,
+      "theoretical_loss": 4.387719361489699,
+      "tokens_seen": 188547072
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004761885656970913,
+      "loss": 2.9676,
+      "theoretical_loss": 4.3875268180461715,
+      "tokens_seen": 188612608
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.00047617853560682046,
+      "loss": 3.2529,
+      "theoretical_loss": 4.387334360217939,
+      "tokens_seen": 188678144
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.00047616850551654964,
+      "loss": 3.1968,
+      "theoretical_loss": 4.3871419879372135,
+      "tokens_seen": 188743680
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004761584754262788,
+      "loss": 3.3051,
+      "theoretical_loss": 4.3869497011362855,
+      "tokens_seen": 188809216
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.00047614844533600806,
+      "loss": 3.1021,
+      "theoretical_loss": 4.386757499747521,
+      "tokens_seen": 188874752
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004761384152457372,
+      "loss": 2.8122,
+      "theoretical_loss": 4.386565383703366,
+      "tokens_seen": 188940288
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004761283851554664,
+      "loss": 3.308,
+      "theoretical_loss": 4.38637335293634,
+      "tokens_seen": 189005824
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004761183550651956,
+      "loss": 3.2071,
+      "theoretical_loss": 4.3861814073790395,
+      "tokens_seen": 189071360
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004761083249749248,
+      "loss": 3.0571,
+      "theoretical_loss": 4.38598954696414,
+      "tokens_seen": 189136896
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.00047609829488465396,
+      "loss": 3.2339,
+      "theoretical_loss": 4.385797771624393,
+      "tokens_seen": 189202432
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.00047608826479438314,
+      "loss": 2.9145,
+      "theoretical_loss": 4.385606081292623,
+      "tokens_seen": 189267968
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004760782347041123,
+      "loss": 2.9952,
+      "theoretical_loss": 4.385414475901737,
+      "tokens_seen": 189333504
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.00047606820461384156,
+      "loss": 3.4306,
+      "theoretical_loss": 4.385222955384711,
+      "tokens_seen": 189399040
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004760581745235707,
+      "loss": 3.2663,
+      "theoretical_loss": 4.385031519674601,
+      "tokens_seen": 189464576
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004760481444332999,
+      "loss": 3.267,
+      "theoretical_loss": 4.38484016870454,
+      "tokens_seen": 189530112
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.00047603811434302905,
+      "loss": 3.1402,
+      "theoretical_loss": 4.384648902407734,
+      "tokens_seen": 189595648
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004760280842527583,
+      "loss": 3.3339,
+      "theoretical_loss": 4.384457720717466,
+      "tokens_seen": 189661184
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.00047601805416248746,
+      "loss": 3.3079,
+      "theoretical_loss": 4.384266623567096,
+      "tokens_seen": 189726720
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.00047600802407221665,
+      "loss": 3.0988,
+      "theoretical_loss": 4.384075610890057,
+      "tokens_seen": 189792256
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004759979939819458,
+      "loss": 3.1992,
+      "theoretical_loss": 4.3838846826198585,
+      "tokens_seen": 189857792
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.00047598796389167506,
+      "loss": 3.1131,
+      "theoretical_loss": 4.383693838690085,
+      "tokens_seen": 189923328
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004759779338014042,
+      "loss": 3.2468,
+      "theoretical_loss": 4.383503079034397,
+      "tokens_seen": 189988864
+    },
+    {
+      "epoch": 0.06,
+      "objective/train/docs_used": 115651,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 2.9877991676330566,
+      "objective/train/theoretical_loss": 4.383312403586528,
+      "objective/train/tokens_used": 210514400,
+      "theoretical_loss": 4.383312403586528,
+      "tokens_seen": 190054400
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004759679037111334,
+      "loss": 3.0685,
+      "theoretical_loss": 4.383312403586528,
+      "tokens_seen": 190054400
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004759578736208626,
+      "loss": 2.9333,
+      "theoretical_loss": 4.38312181228029,
+      "tokens_seen": 190119936
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004759478435305918,
+      "loss": 3.366,
+      "theoretical_loss": 4.382931305049566,
+      "tokens_seen": 190185472
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.000475937813440321,
+      "loss": 3.087,
+      "theoretical_loss": 4.382740881828316,
+      "tokens_seen": 190251008
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.00047592778335005015,
+      "loss": 3.0998,
+      "theoretical_loss": 4.3825505425505735,
+      "tokens_seen": 190316544
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004759177532597794,
+      "loss": 3.172,
+      "theoretical_loss": 4.382360287150449,
+      "tokens_seen": 190382080
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004759077231695085,
+      "loss": 3.2168,
+      "theoretical_loss": 4.382170115562122,
+      "tokens_seen": 190447616
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.00047589769307923775,
+      "loss": 3.2619,
+      "theoretical_loss": 4.381980027719852,
+      "tokens_seen": 190513152
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.00047588766298896693,
+      "loss": 3.1796,
+      "theoretical_loss": 4.3817900235579685,
+      "tokens_seen": 190578688
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004758776328986961,
+      "loss": 3.0825,
+      "theoretical_loss": 4.381600103010879,
+      "tokens_seen": 190644224
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004758676028084253,
+      "loss": 2.9786,
+      "theoretical_loss": 4.381410266013061,
+      "tokens_seen": 190709760
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.00047585757271815447,
+      "loss": 3.2466,
+      "theoretical_loss": 4.381220512499068,
+      "tokens_seen": 190775296
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.00047584754262788365,
+      "loss": 3.0826,
+      "theoretical_loss": 4.381030842403527,
+      "tokens_seen": 190840832
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004758375125376129,
+      "loss": 3.2852,
+      "theoretical_loss": 4.38084125566114,
+      "tokens_seen": 190906368
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.000475827482447342,
+      "loss": 3.3628,
+      "theoretical_loss": 4.380651752206679,
+      "tokens_seen": 190971904
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.00047581745235707125,
+      "loss": 2.9071,
+      "theoretical_loss": 4.380462331974991,
+      "tokens_seen": 191037440
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.00047580742226680043,
+      "loss": 2.9733,
+      "theoretical_loss": 4.380272994900998,
+      "tokens_seen": 191102976
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004757973921765296,
+      "loss": 3.2946,
+      "theoretical_loss": 4.380083740919693,
+      "tokens_seen": 191168512
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004757873620862588,
+      "loss": 3.0393,
+      "theoretical_loss": 4.379894569966144,
+      "tokens_seen": 191234048
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.000475777331995988,
+      "loss": 3.2391,
+      "theoretical_loss": 4.37970548197549,
+      "tokens_seen": 191299584
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.00047576730190571716,
+      "loss": 3.0312,
+      "theoretical_loss": 4.3795164768829435,
+      "tokens_seen": 191365120
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004757572718154464,
+      "loss": 3.0777,
+      "theoretical_loss": 4.37932755462379,
+      "tokens_seen": 191430656
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004757472417251755,
+      "loss": 3.3053,
+      "theoretical_loss": 4.379138715133389,
+      "tokens_seen": 191496192
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.00047573721163490475,
+      "loss": 3.0129,
+      "theoretical_loss": 4.378949958347171,
+      "tokens_seen": 191561728
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004757271815446339,
+      "loss": 3.1993,
+      "theoretical_loss": 4.378761284200638,
+      "tokens_seen": 191627264
+    },
+    {
+      "epoch": 0.06,
+      "objective/train/docs_used": 116251,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 2.686540126800537,
+      "objective/train/theoretical_loss": 4.3785726926293655,
+      "objective/train/tokens_used": 212152800,
+      "theoretical_loss": 4.3785726926293655,
+      "tokens_seen": 191692800
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004757171514543631,
+      "loss": 2.8721,
+      "theoretical_loss": 4.3785726926293655,
+      "tokens_seen": 191692800
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004757071213640923,
+      "loss": 3.144,
+      "theoretical_loss": 4.378384183569002,
+      "tokens_seen": 191758336
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004756970912738215,
+      "loss": 3.2052,
+      "theoretical_loss": 4.378195756955267,
+      "tokens_seen": 191823872
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.00047568706118355066,
+      "loss": 3.192,
+      "theoretical_loss": 4.378007412723953,
+      "tokens_seen": 191889408
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.00047567703109327984,
+      "loss": 2.9825,
+      "theoretical_loss": 4.377819150810923,
+      "tokens_seen": 191954944
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.000475667001003009,
+      "loss": 2.9849,
+      "theoretical_loss": 4.377630971152113,
+      "tokens_seen": 192020480
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.00047565697091273826,
+      "loss": 3.2592,
+      "theoretical_loss": 4.377442873683529,
+      "tokens_seen": 192086016
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004756469408224674,
+      "loss": 3.0667,
+      "theoretical_loss": 4.377254858341252,
+      "tokens_seen": 192151552
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004756369107321966,
+      "loss": 3.194,
+      "theoretical_loss": 4.377066925061431,
+      "tokens_seen": 192217088
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004756268806419258,
+      "loss": 3.2189,
+      "theoretical_loss": 4.376879073780288,
+      "tokens_seen": 192282624
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.000475616850551655,
+      "loss": 3.0755,
+      "theoretical_loss": 4.376691304434117,
+      "tokens_seen": 192348160
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.00047560682046138416,
+      "loss": 3.1954,
+      "theoretical_loss": 4.376503616959281,
+      "tokens_seen": 192413696
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.00047559679037111334,
+      "loss": 3.233,
+      "theoretical_loss": 4.376316011292214,
+      "tokens_seen": 192479232
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004755867602808425,
+      "loss": 3.0876,
+      "theoretical_loss": 4.376128487369426,
+      "tokens_seen": 192544768
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.00047557673019057176,
+      "loss": 2.9403,
+      "theoretical_loss": 4.375941045127492,
+      "tokens_seen": 192610304
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004755667001003009,
+      "loss": 3.3601,
+      "theoretical_loss": 4.375753684503059,
+      "tokens_seen": 192675840
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004755566700100301,
+      "loss": 2.9828,
+      "theoretical_loss": 4.3755664054328465,
+      "tokens_seen": 192741376
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.00047554663991975925,
+      "loss": 2.8375,
+      "theoretical_loss": 4.375379207853644,
+      "tokens_seen": 192806912
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004755366098294885,
+      "loss": 3.323,
+      "theoretical_loss": 4.375192091702311,
+      "tokens_seen": 192872448
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.00047552657973921766,
+      "loss": 3.0783,
+      "theoretical_loss": 4.375005056915777,
+      "tokens_seen": 192937984
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.00047551654964894685,
+      "loss": 2.974,
+      "theoretical_loss": 4.374818103431042,
+      "tokens_seen": 193003520
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.00047550651955867603,
+      "loss": 3.0758,
+      "theoretical_loss": 4.374631231185177,
+      "tokens_seen": 193069056
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.00047549648946840526,
+      "loss": 3.0464,
+      "theoretical_loss": 4.374444440115322,
+      "tokens_seen": 193134592
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004754864593781344,
+      "loss": 3.1286,
+      "theoretical_loss": 4.374257730158687,
+      "tokens_seen": 193200128
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004754764292878636,
+      "loss": 3.1677,
+      "theoretical_loss": 4.374071101252554,
+      "tokens_seen": 193265664
+    },
+    {
+      "epoch": 0.06,
+      "objective/train/docs_used": 117431,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 2.7149760723114014,
+      "objective/train/theoretical_loss": 4.373884553334271,
+      "objective/train/tokens_used": 213791200,
+      "theoretical_loss": 4.373884553334271,
+      "tokens_seen": 193331200
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.00047546639919759275,
+      "loss": 3.1442,
+      "theoretical_loss": 4.373884553334271,
+      "tokens_seen": 193331200
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.000475456369107322,
+      "loss": 3.1432,
+      "theoretical_loss": 4.373698086341258,
+      "tokens_seen": 193396736
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.00047544633901705117,
+      "loss": 3.2882,
+      "theoretical_loss": 4.373511700211004,
+      "tokens_seen": 193462272
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.00047543630892678035,
+      "loss": 3.1277,
+      "theoretical_loss": 4.373325394881069,
+      "tokens_seen": 193527808
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.00047542627883650953,
+      "loss": 2.8819,
+      "theoretical_loss": 4.373139170289079,
+      "tokens_seen": 193593344
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004754162487462387,
+      "loss": 3.1914,
+      "theoretical_loss": 4.3729530263727305,
+      "tokens_seen": 193658880
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004754062186559679,
+      "loss": 3.0863,
+      "theoretical_loss": 4.372766963069793,
+      "tokens_seen": 193724416
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.00047539618856569713,
+      "loss": 3.1611,
+      "theoretical_loss": 4.3725809803181,
+      "tokens_seen": 193789952
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.00047538615847542625,
+      "loss": 3.1407,
+      "theoretical_loss": 4.372395078055554,
+      "tokens_seen": 193855488
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004753761283851555,
+      "loss": 3.2841,
+      "theoretical_loss": 4.372209256220131,
+      "tokens_seen": 193921024
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004753660982948846,
+      "loss": 3.2507,
+      "theoretical_loss": 4.3720235147498725,
+      "tokens_seen": 193986560
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.00047535606820461385,
+      "loss": 3.2756,
+      "theoretical_loss": 4.371837853582887,
+      "tokens_seen": 194052096
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.00047534603811434303,
+      "loss": 2.9721,
+      "theoretical_loss": 4.3716522726573555,
+      "tokens_seen": 194117632
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004753360080240722,
+      "loss": 3.1121,
+      "theoretical_loss": 4.371466771911524,
+      "tokens_seen": 194183168
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004753259779338014,
+      "loss": 3.07,
+      "theoretical_loss": 4.371281351283709,
+      "tokens_seen": 194248704
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.00047531594784353063,
+      "loss": 3.0433,
+      "theoretical_loss": 4.371096010712295,
+      "tokens_seen": 194314240
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.00047530591775325976,
+      "loss": 3.1014,
+      "theoretical_loss": 4.370910750135733,
+      "tokens_seen": 194379776
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.000475295887662989,
+      "loss": 3.3776,
+      "theoretical_loss": 4.370725569492545,
+      "tokens_seen": 194445312
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004752858575727181,
+      "loss": 3.2206,
+      "theoretical_loss": 4.370540468721318,
+      "tokens_seen": 194510848
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.00047527582748244736,
+      "loss": 3.1334,
+      "theoretical_loss": 4.370355447760708,
+      "tokens_seen": 194576384
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.00047526579739217654,
+      "loss": 3.1063,
+      "theoretical_loss": 4.370170506549439,
+      "tokens_seen": 194641920
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004752557673019057,
+      "loss": 3.133,
+      "theoretical_loss": 4.369985645026302,
+      "tokens_seen": 194707456
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004752457372116349,
+      "loss": 3.1316,
+      "theoretical_loss": 4.369800863130157,
+      "tokens_seen": 194772992
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004752357071213641,
+      "loss": 3.0326,
+      "theoretical_loss": 4.36961616079993,
+      "tokens_seen": 194838528
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.00047522567703109326,
+      "loss": 3.2395,
+      "theoretical_loss": 4.3694315379746165,
+      "tokens_seen": 194904064
+    },
+    {
+      "epoch": 0.06,
+      "objective/train/docs_used": 118646,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 2.6308515071868896,
+      "objective/train/theoretical_loss": 4.369246994593274,
+      "objective/train/tokens_used": 215429600,
+      "theoretical_loss": 4.369246994593274,
+      "tokens_seen": 194969600
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004752156469408225,
+      "loss": 3.2026,
+      "theoretical_loss": 4.369246994593274,
+      "tokens_seen": 194969600
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004752056168505517,
+      "loss": 3.152,
+      "theoretical_loss": 4.369062530595034,
+      "tokens_seen": 195035136
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.00047519558676028086,
+      "loss": 2.9739,
+      "theoretical_loss": 4.368878145919091,
+      "tokens_seen": 195100672
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.00047518555667001004,
+      "loss": 3.0244,
+      "theoretical_loss": 4.368693840504706,
+      "tokens_seen": 195166208
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004751755265797392,
+      "loss": 2.816,
+      "theoretical_loss": 4.368509614291209,
+      "tokens_seen": 195231744
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.00047516549648946846,
+      "loss": 3.0789,
+      "theoretical_loss": 4.368325467217998,
+      "tokens_seen": 195297280
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004751554663991976,
+      "loss": 3.1642,
+      "theoretical_loss": 4.368141399224534,
+      "tokens_seen": 195362816
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004751454363089268,
+      "loss": 2.9522,
+      "theoretical_loss": 4.367957410250345,
+      "tokens_seen": 195428352
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.000475135406218656,
+      "loss": 3.0048,
+      "theoretical_loss": 4.367773500235029,
+      "tokens_seen": 195493888
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004751253761283852,
+      "loss": 3.172,
+      "theoretical_loss": 4.367589669118248,
+      "tokens_seen": 195559424
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.00047511534603811436,
+      "loss": 3.02,
+      "theoretical_loss": 4.367405916839729,
+      "tokens_seen": 195624960
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.00047510531594784354,
+      "loss": 3.0327,
+      "theoretical_loss": 4.367222243339268,
+      "tokens_seen": 195690496
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004750952858575727,
+      "loss": 3.3324,
+      "theoretical_loss": 4.367038648556725,
+      "tokens_seen": 195756032
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.00047508525576730196,
+      "loss": 3.2043,
+      "theoretical_loss": 4.366855132432028,
+      "tokens_seen": 195821568
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004750752256770311,
+      "loss": 3.2153,
+      "theoretical_loss": 4.366671694905169,
+      "tokens_seen": 195887104
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004750651955867603,
+      "loss": 3.2576,
+      "theoretical_loss": 4.366488335916207,
+      "tokens_seen": 195952640
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.00047505516549648945,
+      "loss": 2.9306,
+      "theoretical_loss": 4.366305055405267,
+      "tokens_seen": 196018176
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004750451354062187,
+      "loss": 3.163,
+      "theoretical_loss": 4.366121853312537,
+      "tokens_seen": 196083712
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.00047503510531594787,
+      "loss": 3.2586,
+      "theoretical_loss": 4.365938729578274,
+      "tokens_seen": 196149248
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.00047502507522567705,
+      "loss": 3.313,
+      "theoretical_loss": 4.365755684142799,
+      "tokens_seen": 196214784
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.00047501504513540623,
+      "loss": 3.2721,
+      "theoretical_loss": 4.365572716946499,
+      "tokens_seen": 196280320
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.00047500501504513546,
+      "loss": 3.3105,
+      "theoretical_loss": 4.365389827929825,
+      "tokens_seen": 196345856
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004749949849548646,
+      "loss": 3.2141,
+      "theoretical_loss": 4.365207017033293,
+      "tokens_seen": 196411392
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004749849548645938,
+      "loss": 3.2766,
+      "theoretical_loss": 4.365024284197486,
+      "tokens_seen": 196476928
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.00047497492477432295,
+      "loss": 2.9995,
+      "theoretical_loss": 4.364841629363051,
+      "tokens_seen": 196542464
+    },
+    {
+      "epoch": 0.06,
+      "objective/train/docs_used": 119101,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 3.2888681888580322,
+      "objective/train/theoretical_loss": 4.364659052470699,
+      "objective/train/tokens_used": 217068000,
+      "theoretical_loss": 4.364659052470699,
+      "tokens_seen": 196608000
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004749648946840522,
+      "loss": 3.1609,
+      "theoretical_loss": 4.364659052470699,
+      "tokens_seen": 196608000
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.00047495486459378137,
+      "loss": 3.1296,
+      "theoretical_loss": 4.364476553461208,
+      "tokens_seen": 196673536
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.00047494483450351055,
+      "loss": 3.062,
+      "theoretical_loss": 4.364294132275419,
+      "tokens_seen": 196739072
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.00047493480441323973,
+      "loss": 3.1813,
+      "theoretical_loss": 4.364111788854237,
+      "tokens_seen": 196804608
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004749247743229689,
+      "loss": 3.0742,
+      "theoretical_loss": 4.3639295231386335,
+      "tokens_seen": 196870144
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004749147442326981,
+      "loss": 3.1893,
+      "theoretical_loss": 4.363747335069642,
+      "tokens_seen": 196935680
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.00047490471414242733,
+      "loss": 3.2753,
+      "theoretical_loss": 4.363565224588364,
+      "tokens_seen": 197001216
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.00047489468405215646,
+      "loss": 3.1104,
+      "theoretical_loss": 4.363383191635961,
+      "tokens_seen": 197066752
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004748846539618857,
+      "loss": 3.0305,
+      "theoretical_loss": 4.363201236153663,
+      "tokens_seen": 197132288
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004748746238716148,
+      "loss": 3.0246,
+      "theoretical_loss": 4.363019358082759,
+      "tokens_seen": 197197824
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.00047486459378134405,
+      "loss": 3.1562,
+      "theoretical_loss": 4.362837557364608,
+      "tokens_seen": 197263360
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.00047485456369107323,
+      "loss": 3.1837,
+      "theoretical_loss": 4.362655833940627,
+      "tokens_seen": 197328896
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004748445336008024,
+      "loss": 3.2525,
+      "theoretical_loss": 4.3624741877523,
+      "tokens_seen": 197394432
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004748345035105316,
+      "loss": 3.1681,
+      "theoretical_loss": 4.362292618741174,
+      "tokens_seen": 197459968
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.00047482447342026083,
+      "loss": 3.4425,
+      "theoretical_loss": 4.362111126848862,
+      "tokens_seen": 197525504
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.00047481444332998996,
+      "loss": 3.3758,
+      "theoretical_loss": 4.3619297120170355,
+      "tokens_seen": 197591040
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004748044132397192,
+      "loss": 3.1236,
+      "theoretical_loss": 4.361748374187433,
+      "tokens_seen": 197656576
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004747943831494483,
+      "loss": 2.9103,
+      "theoretical_loss": 4.361567113301858,
+      "tokens_seen": 197722112
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.00047478435305917756,
+      "loss": 3.3328,
+      "theoretical_loss": 4.3613859293021715,
+      "tokens_seen": 197787648
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.00047477432296890674,
+      "loss": 3.2619,
+      "theoretical_loss": 4.361204822130304,
+      "tokens_seen": 197853184
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004747642928786359,
+      "loss": 2.9688,
+      "theoretical_loss": 4.361023791728243,
+      "tokens_seen": 197918720
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004747542627883651,
+      "loss": 2.8798,
+      "theoretical_loss": 4.360842838038045,
+      "tokens_seen": 197984256
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004747442326980943,
+      "loss": 3.164,
+      "theoretical_loss": 4.360661961001824,
+      "tokens_seen": 198049792
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.00047473420260782346,
+      "loss": 3.2726,
+      "theoretical_loss": 4.360481160561761,
+      "tokens_seen": 198115328
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004747241725175527,
+      "loss": 3.2962,
+      "theoretical_loss": 4.360300436660099,
+      "tokens_seen": 198180864
+    },
+    {
+      "epoch": 0.06,
+      "objective/train/docs_used": 119264,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 3.4219841957092285,
+      "objective/train/theoretical_loss": 4.36011978923914,
+      "objective/train/tokens_used": 218706400,
+      "theoretical_loss": 4.36011978923914,
+      "tokens_seen": 198246400
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004747141424272818,
+      "loss": 3.2649,
+      "theoretical_loss": 4.36011978923914,
+      "tokens_seen": 198246400
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.00047470411233701106,
+      "loss": 3.1434,
+      "theoretical_loss": 4.359939218241253,
+      "tokens_seen": 198311936
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004746940822467402,
+      "loss": 3.3527,
+      "theoretical_loss": 4.359758723608866,
+      "tokens_seen": 198377472
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004746840521564694,
+      "loss": 3.2825,
+      "theoretical_loss": 4.359578305284474,
+      "tokens_seen": 198443008
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004746740220661986,
+      "loss": 3.3552,
+      "theoretical_loss": 4.359397963210629,
+      "tokens_seen": 198508544
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004746639919759278,
+      "loss": 3.2728,
+      "theoretical_loss": 4.3592176973299495,
+      "tokens_seen": 198574080
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.00047465396188565696,
+      "loss": 2.9474,
+      "theoretical_loss": 4.359037507585111,
+      "tokens_seen": 198639616
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004746439317953862,
+      "loss": 2.9234,
+      "theoretical_loss": 4.358857393918857,
+      "tokens_seen": 198705152
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004746339017051153,
+      "loss": 2.9143,
+      "theoretical_loss": 4.358677356273988,
+      "tokens_seen": 198770688
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.00047462387161484456,
+      "loss": 3.1565,
+      "theoretical_loss": 4.358497394593371,
+      "tokens_seen": 198836224
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004746138415245737,
+      "loss": 3.118,
+      "theoretical_loss": 4.35831750881993,
+      "tokens_seen": 198901760
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004746038114343029,
+      "loss": 3.2261,
+      "theoretical_loss": 4.358137698896653,
+      "tokens_seen": 198967296
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004745937813440321,
+      "loss": 3.1764,
+      "theoretical_loss": 4.35795796476659,
+      "tokens_seen": 199032832
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004745837512537613,
+      "loss": 3.3871,
+      "theoretical_loss": 4.357778306372852,
+      "tokens_seen": 199098368
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.00047457372116349047,
+      "loss": 3.1078,
+      "theoretical_loss": 4.357598723658612,
+      "tokens_seen": 199163904
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.00047456369107321965,
+      "loss": 3.0791,
+      "theoretical_loss": 4.357419216567102,
+      "tokens_seen": 199229440
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.00047455366098294883,
+      "loss": 3.2517,
+      "theoretical_loss": 4.357239785041618,
+      "tokens_seen": 199294976
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.00047454363089267807,
+      "loss": 3.176,
+      "theoretical_loss": 4.357060429025516,
+      "tokens_seen": 199360512
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004745336008024072,
+      "loss": 3.1573,
+      "theoretical_loss": 4.356881148462213,
+      "tokens_seen": 199426048
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.00047452357071213643,
+      "loss": 2.9391,
+      "theoretical_loss": 4.356701943295186,
+      "tokens_seen": 199491584
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.00047451354062186555,
+      "loss": 3.0178,
+      "theoretical_loss": 4.356522813467976,
+      "tokens_seen": 199557120
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004745035105315948,
+      "loss": 3.008,
+      "theoretical_loss": 4.356343758924181,
+      "tokens_seen": 199622656
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.00047449348044132397,
+      "loss": 3.088,
+      "theoretical_loss": 4.356164779607463,
+      "tokens_seen": 199688192
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.00047448345035105315,
+      "loss": 3.0246,
+      "theoretical_loss": 4.355985875461542,
+      "tokens_seen": 199753728
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004744734202607824,
+      "loss": 3.2933,
+      "theoretical_loss": 4.3558070464302,
+      "tokens_seen": 199819264
+    },
+    {
+      "epoch": 0.06,
+      "objective/train/docs_used": 120457,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 2.7158637046813965,
+      "objective/train/theoretical_loss": 4.355628292457278,
+      "objective/train/tokens_used": 220344800,
+      "theoretical_loss": 4.355628292457278,
+      "tokens_seen": 199884800
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.00047446339017051157,
+      "loss": 3.2748,
+      "theoretical_loss": 4.355628292457278,
+      "tokens_seen": 199884800
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.00047445336008024075,
+      "loss": 3.3399,
+      "theoretical_loss": 4.355449613486681,
+      "tokens_seen": 199950336
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.00047444332998996993,
+      "loss": 2.9307,
+      "theoretical_loss": 4.355271009462371,
+      "tokens_seen": 200015872
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004744332998996991,
+      "loss": 2.8099,
+      "theoretical_loss": 4.355092480328369,
+      "tokens_seen": 200081408
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004744232698094283,
+      "loss": 3.2346,
+      "theoretical_loss": 4.35491402602876,
+      "tokens_seen": 200146944
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.00047441323971915753,
+      "loss": 3.1045,
+      "theoretical_loss": 4.354735646507687,
+      "tokens_seen": 200212480
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.00047440320962888666,
+      "loss": 3.1639,
+      "theoretical_loss": 4.354557341709354,
+      "tokens_seen": 200278016
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004743931795386159,
+      "loss": 3.1702,
+      "theoretical_loss": 4.354379111578022,
+      "tokens_seen": 200343552
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.000474383149448345,
+      "loss": 3.1362,
+      "theoretical_loss": 4.3542009560580155,
+      "tokens_seen": 200409088
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.00047437311935807425,
+      "loss": 3.1808,
+      "theoretical_loss": 4.354022875093716,
+      "tokens_seen": 200474624
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.00047436308926780343,
+      "loss": 3.2076,
+      "theoretical_loss": 4.353844868629566,
+      "tokens_seen": 200540160
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004743530591775326,
+      "loss": 3.1964,
+      "theoretical_loss": 4.353666936610068,
+      "tokens_seen": 200605696
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004743430290872618,
+      "loss": 3.0344,
+      "theoretical_loss": 4.3534890789797815,
+      "tokens_seen": 200671232
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.00047433299899699103,
+      "loss": 3.0296,
+      "theoretical_loss": 4.353311295683328,
+      "tokens_seen": 200736768
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.00047432296890672016,
+      "loss": 3.0211,
+      "theoretical_loss": 4.353133586665388,
+      "tokens_seen": 200802304
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004743129388164494,
+      "loss": 3.2548,
+      "theoretical_loss": 4.352955951870699,
+      "tokens_seen": 200867840
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004743029087261785,
+      "loss": 3.09,
+      "theoretical_loss": 4.352778391244061,
+      "tokens_seen": 200933376
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.00047429287863590776,
+      "loss": 2.9689,
+      "theoretical_loss": 4.352600904730329,
+      "tokens_seen": 200998912
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.00047428284854563694,
+      "loss": 3.4101,
+      "theoretical_loss": 4.35242349227442,
+      "tokens_seen": 201064448
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004742728184553661,
+      "loss": 3.288,
+      "theoretical_loss": 4.35224615382131,
+      "tokens_seen": 201129984
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004742627883650953,
+      "loss": 3.1146,
+      "theoretical_loss": 4.352068889316032,
+      "tokens_seen": 201195520
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004742527582748245,
+      "loss": 3.0871,
+      "theoretical_loss": 4.351891698703679,
+      "tokens_seen": 201261056
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.00047424272818455366,
+      "loss": 3.2115,
+      "theoretical_loss": 4.351714581929402,
+      "tokens_seen": 201326592
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004742326980942829,
+      "loss": 2.8483,
+      "theoretical_loss": 4.351537538938409,
+      "tokens_seen": 201392128
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.000474222668004012,
+      "loss": 3.0209,
+      "theoretical_loss": 4.351360569675971,
+      "tokens_seen": 201457664
+    },
+    {
+      "epoch": 0.06,
+      "objective/train/docs_used": 121711,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 3.200042724609375,
+      "objective/train/theoretical_loss": 4.351183674087412,
+      "objective/train/tokens_used": 221983200,
+      "theoretical_loss": 4.351183674087412,
+      "tokens_seen": 201523200
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.00047421263791374126,
+      "loss": 3.0898,
+      "theoretical_loss": 4.351183674087412,
+      "tokens_seen": 201523200
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004742026078234704,
+      "loss": 3.0786,
+      "theoretical_loss": 4.35100685211812,
+      "tokens_seen": 201588736
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004741925777331996,
+      "loss": 3.155,
+      "theoretical_loss": 4.350830103713535,
+      "tokens_seen": 201654272
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004741825476429288,
+      "loss": 3.1987,
+      "theoretical_loss": 4.350653428819159,
+      "tokens_seen": 201719808
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.000474172517552658,
+      "loss": 3.0476,
+      "theoretical_loss": 4.350476827380553,
+      "tokens_seen": 201785344
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.00047416248746238716,
+      "loss": 3.157,
+      "theoretical_loss": 4.350300299343332,
+      "tokens_seen": 201850880
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004741524573721164,
+      "loss": 3.0758,
+      "theoretical_loss": 4.350123844653173,
+      "tokens_seen": 201916416
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004741424272818455,
+      "loss": 3.2307,
+      "theoretical_loss": 4.349947463255808,
+      "tokens_seen": 201981952
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.00047413239719157476,
+      "loss": 3.1247,
+      "theoretical_loss": 4.349771155097029,
+      "tokens_seen": 202047488
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004741223671013039,
+      "loss": 3.1268,
+      "theoretical_loss": 4.349594920122682,
+      "tokens_seen": 202113024
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004741123370110331,
+      "loss": 3.1097,
+      "theoretical_loss": 4.349418758278675,
+      "tokens_seen": 202178560
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004741023069207623,
+      "loss": 3.139,
+      "theoretical_loss": 4.349242669510971,
+      "tokens_seen": 202244096
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004740922768304915,
+      "loss": 3.1241,
+      "theoretical_loss": 4.3490666537655915,
+      "tokens_seen": 202309632
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.00047408224674022067,
+      "loss": 3.1542,
+      "theoretical_loss": 4.348890710988615,
+      "tokens_seen": 202375168
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.00047407221664994985,
+      "loss": 3.2895,
+      "theoretical_loss": 4.348714841126174,
+      "tokens_seen": 202440704
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.00047406218655967903,
+      "loss": 3.1676,
+      "theoretical_loss": 4.348539044124465,
+      "tokens_seen": 202506240
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.00047405215646940827,
+      "loss": 2.9643,
+      "theoretical_loss": 4.348363319929735,
+      "tokens_seen": 202571776
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004740421263791374,
+      "loss": 3.0266,
+      "theoretical_loss": 4.348187668488293,
+      "tokens_seen": 202637312
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.00047403209628886663,
+      "loss": 3.0482,
+      "theoretical_loss": 4.3480120897465016,
+      "tokens_seen": 202702848
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.00047402206619859575,
+      "loss": 3.044,
+      "theoretical_loss": 4.347836583650782,
+      "tokens_seen": 202768384
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.000474012036108325,
+      "loss": 3.1396,
+      "theoretical_loss": 4.347661150147611,
+      "tokens_seen": 202833920
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.00047400200601805417,
+      "loss": 3.2979,
+      "theoretical_loss": 4.347485789183523,
+      "tokens_seen": 202899456
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.00047399197592778335,
+      "loss": 3.1927,
+      "theoretical_loss": 4.34731050070511,
+      "tokens_seen": 202964992
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.00047398194583751253,
+      "loss": 3.0788,
+      "theoretical_loss": 4.347135284659017,
+      "tokens_seen": 203030528
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.00047397191574724177,
+      "loss": 3.0052,
+      "theoretical_loss": 4.346960140991949,
+      "tokens_seen": 203096064
+    },
+    {
+      "epoch": 0.06,
+      "objective/train/docs_used": 122225,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 3.3157236576080322,
+      "objective/train/theoretical_loss": 4.346785069650667,
+      "objective/train/tokens_used": 223621600,
+      "theoretical_loss": 4.346785069650667,
+      "tokens_seen": 203161600
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004739618856569709,
+      "loss": 2.9543,
+      "theoretical_loss": 4.346785069650667,
+      "tokens_seen": 203161600
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.00047395185556670013,
+      "loss": 3.0566,
+      "theoretical_loss": 4.346610070581987,
+      "tokens_seen": 203227136
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.00047394182547642926,
+      "loss": 2.9878,
+      "theoretical_loss": 4.34643514373278,
+      "tokens_seen": 203292672
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004739317953861585,
+      "loss": 3.0709,
+      "theoretical_loss": 4.346260289049978,
+      "tokens_seen": 203358208
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004739217652958877,
+      "loss": 3.1681,
+      "theoretical_loss": 4.346085506480564,
+      "tokens_seen": 203423744
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.00047391173520561686,
+      "loss": 3.2437,
+      "theoretical_loss": 4.34591079597158,
+      "tokens_seen": 203489280
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.00047390170511534604,
+      "loss": 3.3256,
+      "theoretical_loss": 4.345736157470122,
+      "tokens_seen": 203554816
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004738916750250752,
+      "loss": 3.1524,
+      "theoretical_loss": 4.345561590923343,
+      "tokens_seen": 203620352
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004738816449348044,
+      "loss": 3.3006,
+      "theoretical_loss": 4.345387096278453,
+      "tokens_seen": 203685888
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.00047387161484453363,
+      "loss": 3.088,
+      "theoretical_loss": 4.345212673482715,
+      "tokens_seen": 203751424
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.00047386158475426276,
+      "loss": 3.139,
+      "theoretical_loss": 4.345038322483449,
+      "tokens_seen": 203816960
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.000473851554663992,
+      "loss": 3.1496,
+      "theoretical_loss": 4.34486404322803,
+      "tokens_seen": 203882496
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004738415245737211,
+      "loss": 3.1446,
+      "theoretical_loss": 4.34468983566389,
+      "tokens_seen": 203948032
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.00047383149448345036,
+      "loss": 3.4082,
+      "theoretical_loss": 4.3445156997385155,
+      "tokens_seen": 204013568
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.00047382146439317954,
+      "loss": 3.1106,
+      "theoretical_loss": 4.344341635399447,
+      "tokens_seen": 204079104
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004738114343029087,
+      "loss": 2.9751,
+      "theoretical_loss": 4.344167642594282,
+      "tokens_seen": 204144640
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004738014042126379,
+      "loss": 3.0756,
+      "theoretical_loss": 4.343993721270673,
+      "tokens_seen": 204210176
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.00047379137412236714,
+      "loss": 3.1045,
+      "theoretical_loss": 4.343819871376326,
+      "tokens_seen": 204275712
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.00047378134403209626,
+      "loss": 3.1058,
+      "theoretical_loss": 4.343646092859004,
+      "tokens_seen": 204341248
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004737713139418255,
+      "loss": 2.8941,
+      "theoretical_loss": 4.343472385666525,
+      "tokens_seen": 204406784
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004737612838515546,
+      "loss": 3.2849,
+      "theoretical_loss": 4.343298749746759,
+      "tokens_seen": 204472320
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.00047375125376128386,
+      "loss": 3.0801,
+      "theoretical_loss": 4.343125185047634,
+      "tokens_seen": 204537856
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.00047374122367101304,
+      "loss": 3.5135,
+      "theoretical_loss": 4.342951691517131,
+      "tokens_seen": 204603392
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004737311935807422,
+      "loss": 3.2053,
+      "theoretical_loss": 4.342778269103286,
+      "tokens_seen": 204668928
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.00047372116349047146,
+      "loss": 3.0911,
+      "theoretical_loss": 4.342604917754191,
+      "tokens_seen": 204734464
+    },
+    {
+      "epoch": 0.06,
+      "objective/train/docs_used": 123173,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 3.030834436416626,
+      "objective/train/theoretical_loss": 4.342431637417991,
+      "objective/train/tokens_used": 225260000,
+      "theoretical_loss": 4.342431637417991,
+      "tokens_seen": 204800000
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004737111334002006,
+      "loss": 3.1185,
+      "theoretical_loss": 4.342431637417991,
+      "tokens_seen": 204800000
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004737011033099298,
+      "loss": 3.0803,
+      "theoretical_loss": 4.342258428042883,
+      "tokens_seen": 204865536
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.000473691073219659,
+      "loss": 3.2108,
+      "theoretical_loss": 4.342085289577124,
+      "tokens_seen": 204931072
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004736810431293882,
+      "loss": 3.2812,
+      "theoretical_loss": 4.341912221969021,
+      "tokens_seen": 204996608
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.00047367101303911736,
+      "loss": 3.1871,
+      "theoretical_loss": 4.341739225166936,
+      "tokens_seen": 205062144
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004736609829488466,
+      "loss": 3.1366,
+      "theoretical_loss": 4.341566299119286,
+      "tokens_seen": 205127680
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004736509528585757,
+      "loss": 3.1494,
+      "theoretical_loss": 4.341393443774542,
+      "tokens_seen": 205193216
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.00047364092276830496,
+      "loss": 3.0442,
+      "theoretical_loss": 4.341220659081228,
+      "tokens_seen": 205258752
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004736308926780341,
+      "loss": 3.1534,
+      "theoretical_loss": 4.341047944987921,
+      "tokens_seen": 205324288
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004736208625877633,
+      "loss": 3.1698,
+      "theoretical_loss": 4.340875301443257,
+      "tokens_seen": 205389824
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004736108324974925,
+      "loss": 3.1333,
+      "theoretical_loss": 4.340702728395918,
+      "tokens_seen": 205455360
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004736008024072217,
+      "loss": 3.1996,
+      "theoretical_loss": 4.340530225794646,
+      "tokens_seen": 205520896
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.00047359077231695087,
+      "loss": 3.3751,
+      "theoretical_loss": 4.340357793588233,
+      "tokens_seen": 205586432
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.00047358074222668005,
+      "loss": 3.3156,
+      "theoretical_loss": 4.340185431725527,
+      "tokens_seen": 205651968
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.00047357071213640923,
+      "loss": 3.3006,
+      "theoretical_loss": 4.3400131401554285,
+      "tokens_seen": 205717504
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.00047356068204613847,
+      "loss": 3.1734,
+      "theoretical_loss": 4.33984091882689,
+      "tokens_seen": 205783040
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004735506519558676,
+      "loss": 2.9499,
+      "theoretical_loss": 4.339668767688918,
+      "tokens_seen": 205848576
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.00047354062186559683,
+      "loss": 2.998,
+      "theoretical_loss": 4.339496686690575,
+      "tokens_seen": 205914112
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.00047353059177532595,
+      "loss": 3.2018,
+      "theoretical_loss": 4.339324675780972,
+      "tokens_seen": 205979648
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004735205616850552,
+      "loss": 3.0006,
+      "theoretical_loss": 4.339152734909277,
+      "tokens_seen": 206045184
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.00047351053159478437,
+      "loss": 3.1032,
+      "theoretical_loss": 4.338980864024709,
+      "tokens_seen": 206110720
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.00047350050150451355,
+      "loss": 3.1907,
+      "theoretical_loss": 4.33880906307654,
+      "tokens_seen": 206176256
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.00047349047141424273,
+      "loss": 3.1745,
+      "theoretical_loss": 4.338637332014096,
+      "tokens_seen": 206241792
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.00047348044132397197,
+      "loss": 3.1799,
+      "theoretical_loss": 4.338465670786754,
+      "tokens_seen": 206307328
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004734704112337011,
+      "loss": 3.1274,
+      "theoretical_loss": 4.338294079343946,
+      "tokens_seen": 206372864
+    },
+    {
+      "epoch": 0.06,
+      "objective/train/docs_used": 123915,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 2.9498279094696045,
+      "objective/train/theoretical_loss": 4.338122557635156,
+      "objective/train/tokens_used": 226898400,
+      "theoretical_loss": 4.338122557635156,
+      "tokens_seen": 206438400
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.00047346038114343033,
+      "loss": 3.1472,
+      "theoretical_loss": 4.338122557635156,
+      "tokens_seen": 206438400
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.00047345035105315946,
+      "loss": 3.5027,
+      "theoretical_loss": 4.3379511056099185,
+      "tokens_seen": 206503936
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004734403209628887,
+      "loss": 3.3358,
+      "theoretical_loss": 4.337779723217823,
+      "tokens_seen": 206569472
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004734302908726179,
+      "loss": 3.3308,
+      "theoretical_loss": 4.33760841040851,
+      "tokens_seen": 206635008
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.00047342026078234706,
+      "loss": 3.2363,
+      "theoretical_loss": 4.337437167131674,
+      "tokens_seen": 206700544
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.00047341023069207624,
+      "loss": 3.339,
+      "theoretical_loss": 4.33726599333706,
+      "tokens_seen": 206766080
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004734002006018054,
+      "loss": 3.0416,
+      "theoretical_loss": 4.337094888974467,
+      "tokens_seen": 206831616
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004733901705115346,
+      "loss": 3.0678,
+      "theoretical_loss": 4.336923853993743,
+      "tokens_seen": 206897152
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.00047338014042126383,
+      "loss": 3.1236,
+      "theoretical_loss": 4.336752888344792,
+      "tokens_seen": 206962688
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.00047337011033099296,
+      "loss": 3.1134,
+      "theoretical_loss": 4.336581991977568,
+      "tokens_seen": 207028224
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004733600802407222,
+      "loss": 2.9821,
+      "theoretical_loss": 4.336411164842078,
+      "tokens_seen": 207093760
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004733500501504513,
+      "loss": 3.0005,
+      "theoretical_loss": 4.336240406888377,
+      "tokens_seen": 207159296
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.00047334002006018056,
+      "loss": 3.0117,
+      "theoretical_loss": 4.336069718066579,
+      "tokens_seen": 207224832
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.00047332998996990974,
+      "loss": 2.9441,
+      "theoretical_loss": 4.335899098326843,
+      "tokens_seen": 207290368
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004733199598796389,
+      "loss": 3.0821,
+      "theoretical_loss": 4.335728547619382,
+      "tokens_seen": 207355904
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004733099297893681,
+      "loss": 3.1602,
+      "theoretical_loss": 4.335558065894464,
+      "tokens_seen": 207421440
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.00047329989969909734,
+      "loss": 3.2261,
+      "theoretical_loss": 4.335387653102402,
+      "tokens_seen": 207486976
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.00047328986960882646,
+      "loss": 3.2569,
+      "theoretical_loss": 4.335217309193567,
+      "tokens_seen": 207552512
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004732798395185557,
+      "loss": 3.0878,
+      "theoretical_loss": 4.335047034118375,
+      "tokens_seen": 207618048
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004732698094282848,
+      "loss": 3.0245,
+      "theoretical_loss": 4.334876827827299,
+      "tokens_seen": 207683584
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.00047325977933801406,
+      "loss": 3.0552,
+      "theoretical_loss": 4.33470669027086,
+      "tokens_seen": 207749120
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.00047324974924774324,
+      "loss": 3.0553,
+      "theoretical_loss": 4.334536621399632,
+      "tokens_seen": 207814656
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004732397191574724,
+      "loss": 3.1419,
+      "theoretical_loss": 4.3343666211642375,
+      "tokens_seen": 207880192
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004732296890672016,
+      "loss": 2.9697,
+      "theoretical_loss": 4.334196689515354,
+      "tokens_seen": 207945728
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004732196589769308,
+      "loss": 3.4403,
+      "theoretical_loss": 4.334026826403705,
+      "tokens_seen": 208011264
+    },
+    {
+      "epoch": 0.06,
+      "objective/train/docs_used": 125279,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 3.2541067600250244,
+      "objective/train/theoretical_loss": 4.333857031780069,
+      "objective/train/tokens_used": 228536800,
+      "theoretical_loss": 4.333857031780069,
+      "tokens_seen": 208076800
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.00047320962888665997,
+      "loss": 3.1755,
+      "theoretical_loss": 4.333857031780069,
+      "tokens_seen": 208076800
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004731995987963892,
+      "loss": 3.3347,
+      "theoretical_loss": 4.333687305595276,
+      "tokens_seen": 208142336
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.00047318956870611833,
+      "loss": 3.1738,
+      "theoretical_loss": 4.3335176478002015,
+      "tokens_seen": 208207872
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.00047317953861584757,
+      "loss": 3.0407,
+      "theoretical_loss": 4.333348058345776,
+      "tokens_seen": 208273408
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.00047316950852557675,
+      "loss": 3.0055,
+      "theoretical_loss": 4.33317853718298,
+      "tokens_seen": 208338944
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.00047315947843530593,
+      "loss": 3.1805,
+      "theoretical_loss": 4.333009084262844,
+      "tokens_seen": 208404480
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004731494483450351,
+      "loss": 2.9499,
+      "theoretical_loss": 4.33283969953645,
+      "tokens_seen": 208470016
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004731394182547643,
+      "loss": 3.0383,
+      "theoretical_loss": 4.332670382954928,
+      "tokens_seen": 208535552
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.00047312938816449347,
+      "loss": 3.1877,
+      "theoretical_loss": 4.33250113446946,
+      "tokens_seen": 208601088
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004731193580742227,
+      "loss": 3.1835,
+      "theoretical_loss": 4.332331954031279,
+      "tokens_seen": 208666624
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.00047310932798395183,
+      "loss": 3.2001,
+      "theoretical_loss": 4.332162841591668,
+      "tokens_seen": 208732160
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.00047309929789368107,
+      "loss": 3.2484,
+      "theoretical_loss": 4.331993797101958,
+      "tokens_seen": 208797696
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004730892678034102,
+      "loss": 3.37,
+      "theoretical_loss": 4.331824820513534,
+      "tokens_seen": 208863232
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.00047307923771313943,
+      "loss": 3.1849,
+      "theoretical_loss": 4.331655911777826,
+      "tokens_seen": 208928768
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004730692076228686,
+      "loss": 3.219,
+      "theoretical_loss": 4.331487070846319,
+      "tokens_seen": 208994304
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004730591775325978,
+      "loss": 3.3387,
+      "theoretical_loss": 4.331318297670545,
+      "tokens_seen": 209059840
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.000473049147442327,
+      "loss": 3.1956,
+      "theoretical_loss": 4.3311495922020855,
+      "tokens_seen": 209125376
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.00047303911735205615,
+      "loss": 3.1449,
+      "theoretical_loss": 4.330980954392573,
+      "tokens_seen": 209190912
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.00047302908726178534,
+      "loss": 2.8264,
+      "theoretical_loss": 4.33081238419369,
+      "tokens_seen": 209256448
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.00047301905717151457,
+      "loss": 3.0265,
+      "theoretical_loss": 4.330643881557168,
+      "tokens_seen": 209321984
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004730090270812437,
+      "loss": 3.2387,
+      "theoretical_loss": 4.330475446434788,
+      "tokens_seen": 209387520
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.00047299899699097293,
+      "loss": 3.2265,
+      "theoretical_loss": 4.330307078778379,
+      "tokens_seen": 209453056
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004729889669007021,
+      "loss": 3.2092,
+      "theoretical_loss": 4.330138778539823,
+      "tokens_seen": 209518592
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004729789368104313,
+      "loss": 3.2283,
+      "theoretical_loss": 4.329970545671048,
+      "tokens_seen": 209584128
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.00047296890672016053,
+      "loss": 3.2865,
+      "theoretical_loss": 4.3298023801240335,
+      "tokens_seen": 209649664
+    },
+    {
+      "epoch": 0.06,
+      "objective/train/docs_used": 125881,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 3.1459879875183105,
+      "objective/train/theoretical_loss": 4.329634281850807,
+      "objective/train/tokens_used": 230175200,
+      "theoretical_loss": 4.329634281850807,
+      "tokens_seen": 209715200
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.00047295887662988966,
+      "loss": 3.168,
+      "theoretical_loss": 4.329634281850807,
+      "tokens_seen": 209715200
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004729488465396189,
+      "loss": 3.1116,
+      "theoretical_loss": 4.329466250803446,
+      "tokens_seen": 209780736
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004729388164493481,
+      "loss": 3.1933,
+      "theoretical_loss": 4.329298286934075,
+      "tokens_seen": 209846272
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.00047292878635907726,
+      "loss": 2.9711,
+      "theoretical_loss": 4.329130390194871,
+      "tokens_seen": 209911808
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.00047291875626880644,
+      "loss": 3.0108,
+      "theoretical_loss": 4.3289625605380575,
+      "tokens_seen": 209977344
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004729087261785356,
+      "loss": 3.1506,
+      "theoretical_loss": 4.328794797915908,
+      "tokens_seen": 210042880
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004728986960882648,
+      "loss": 3.237,
+      "theoretical_loss": 4.328627102280742,
+      "tokens_seen": 210108416
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.00047288866599799403,
+      "loss": 3.1071,
+      "theoretical_loss": 4.328459473584933,
+      "tokens_seen": 210173952
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.00047287863590772316,
+      "loss": 3.2433,
+      "theoretical_loss": 4.328291911780899,
+      "tokens_seen": 210239488
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004728686058174524,
+      "loss": 3.207,
+      "theoretical_loss": 4.328124416821107,
+      "tokens_seen": 210305024
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004728585757271815,
+      "loss": 3.0205,
+      "theoretical_loss": 4.327956988658076,
+      "tokens_seen": 210370560
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.00047284854563691076,
+      "loss": 3.2139,
+      "theoretical_loss": 4.3277896272443686,
+      "tokens_seen": 210436096
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.00047283851554663994,
+      "loss": 3.0787,
+      "theoretical_loss": 4.3276223325325995,
+      "tokens_seen": 210501632
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004728284854563691,
+      "loss": 3.1398,
+      "theoretical_loss": 4.327455104475431,
+      "tokens_seen": 210567168
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004728184553660983,
+      "loss": 3.3631,
+      "theoretical_loss": 4.327287943025572,
+      "tokens_seen": 210632704
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.00047280842527582754,
+      "loss": 2.9762,
+      "theoretical_loss": 4.327120848135781,
+      "tokens_seen": 210698240
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.00047279839518555666,
+      "loss": 3.067,
+      "theoretical_loss": 4.326953819758867,
+      "tokens_seen": 210763776
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004727883650952859,
+      "loss": 3.2263,
+      "theoretical_loss": 4.326786857847681,
+      "tokens_seen": 210829312
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.000472778335005015,
+      "loss": 3.3092,
+      "theoretical_loss": 4.326619962355128,
+      "tokens_seen": 210894848
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.00047276830491474426,
+      "loss": 3.4328,
+      "theoretical_loss": 4.326453133234159,
+      "tokens_seen": 210960384
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.00047275827482447344,
+      "loss": 3.1644,
+      "theoretical_loss": 4.326286370437772,
+      "tokens_seen": 211025920
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004727482447342026,
+      "loss": 3.1984,
+      "theoretical_loss": 4.326119673919013,
+      "tokens_seen": 211091456
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004727382146439318,
+      "loss": 3.0863,
+      "theoretical_loss": 4.325953043630978,
+      "tokens_seen": 211156992
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.000472728184553661,
+      "loss": 3.0964,
+      "theoretical_loss": 4.3257864795268075,
+      "tokens_seen": 211222528
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.00047271815446339017,
+      "loss": 3.0099,
+      "theoretical_loss": 4.325619981559692,
+      "tokens_seen": 211288064
+    },
+    {
+      "epoch": 0.06,
+      "objective/train/docs_used": 127261,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 3.0648367404937744,
+      "objective/train/theoretical_loss": 4.325453549682869,
+      "objective/train/tokens_used": 231813600,
+      "theoretical_loss": 4.325453549682869,
+      "tokens_seen": 211353600
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004727081243731194,
+      "loss": 3.116,
+      "theoretical_loss": 4.325453549682869,
+      "tokens_seen": 211353600
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.00047269809428284853,
+      "loss": 3.0499,
+      "theoretical_loss": 4.325287183849625,
+      "tokens_seen": 211419136
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.00047268806419257777,
+      "loss": 3.0799,
+      "theoretical_loss": 4.325120884013289,
+      "tokens_seen": 211484672
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.00047267803410230695,
+      "loss": 3.1633,
+      "theoretical_loss": 4.324954650127244,
+      "tokens_seen": 211550208
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.00047266800401203613,
+      "loss": 2.8267,
+      "theoretical_loss": 4.324788482144916,
+      "tokens_seen": 211615744
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004726579739217653,
+      "loss": 2.9392,
+      "theoretical_loss": 4.324622380019778,
+      "tokens_seen": 211681280
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004726479438314945,
+      "loss": 3.1858,
+      "theoretical_loss": 4.3244563437053545,
+      "tokens_seen": 211746816
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.00047263791374122367,
+      "loss": 3.1446,
+      "theoretical_loss": 4.324290373155213,
+      "tokens_seen": 211812352
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004726278836509529,
+      "loss": 3.259,
+      "theoretical_loss": 4.324124468322971,
+      "tokens_seen": 211877888
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.00047261785356068203,
+      "loss": 3.195,
+      "theoretical_loss": 4.32395862916229,
+      "tokens_seen": 211943424
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.00047260782347041127,
+      "loss": 3.0971,
+      "theoretical_loss": 4.32379285562688,
+      "tokens_seen": 212008960
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004725977933801404,
+      "loss": 3.1254,
+      "theoretical_loss": 4.323627147670498,
+      "tokens_seen": 212074496
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.00047258776328986963,
+      "loss": 3.0839,
+      "theoretical_loss": 4.323461505246949,
+      "tokens_seen": 212140032
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004725777331995988,
+      "loss": 2.8079,
+      "theoretical_loss": 4.323295928310084,
+      "tokens_seen": 212205568
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.000472567703109328,
+      "loss": 3.1229,
+      "theoretical_loss": 4.3231304168137985,
+      "tokens_seen": 212271104
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004725576730190572,
+      "loss": 3.1019,
+      "theoretical_loss": 4.322964970712039,
+      "tokens_seen": 212336640
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.00047254764292878636,
+      "loss": 3.272,
+      "theoretical_loss": 4.322799589958794,
+      "tokens_seen": 212402176
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.00047253761283851554,
+      "loss": 2.7217,
+      "theoretical_loss": 4.3226342745081014,
+      "tokens_seen": 212467712
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.00047252758274824477,
+      "loss": 3.0247,
+      "theoretical_loss": 4.322469024314046,
+      "tokens_seen": 212533248
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004725175526579739,
+      "loss": 3.2237,
+      "theoretical_loss": 4.322303839330757,
+      "tokens_seen": 212598784
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.00047250752256770313,
+      "loss": 3.3425,
+      "theoretical_loss": 4.322138719512411,
+      "tokens_seen": 212664320
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004724974924774323,
+      "loss": 3.2206,
+      "theoretical_loss": 4.321973664813231,
+      "tokens_seen": 212729856
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004724874623871615,
+      "loss": 3.2825,
+      "theoretical_loss": 4.321808675187486,
+      "tokens_seen": 212795392
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004724774322968907,
+      "loss": 2.8949,
+      "theoretical_loss": 4.321643750589493,
+      "tokens_seen": 212860928
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.00047246740220661986,
+      "loss": 3.0597,
+      "theoretical_loss": 4.32147889097361,
+      "tokens_seen": 212926464
+    },
+    {
+      "epoch": 0.06,
+      "objective/train/docs_used": 127839,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 2.5590968132019043,
+      "objective/train/theoretical_loss": 4.321314096294248,
+      "objective/train/tokens_used": 233452000,
+      "theoretical_loss": 4.321314096294248,
+      "tokens_seen": 212992000
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.00047245737211634904,
+      "loss": 2.9514,
+      "theoretical_loss": 4.321314096294248,
+      "tokens_seen": 212992000
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004724473420260783,
+      "loss": 3.0759,
+      "theoretical_loss": 4.321149366505858,
+      "tokens_seen": 213057536
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004724373119358074,
+      "loss": 2.9617,
+      "theoretical_loss": 4.32098470156294,
+      "tokens_seen": 213123072
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.00047242728184553664,
+      "loss": 2.9706,
+      "theoretical_loss": 4.320820101420041,
+      "tokens_seen": 213188608
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.00047241725175526576,
+      "loss": 3.3507,
+      "theoretical_loss": 4.320655566031751,
+      "tokens_seen": 213254144
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.000472407221664995,
+      "loss": 3.0787,
+      "theoretical_loss": 4.320491095352706,
+      "tokens_seen": 213319680
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004723971915747242,
+      "loss": 3.2185,
+      "theoretical_loss": 4.320326689337589,
+      "tokens_seen": 213385216
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.00047238716148445336,
+      "loss": 3.2852,
+      "theoretical_loss": 4.320162347941129,
+      "tokens_seen": 213450752
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.00047237713139418254,
+      "loss": 3.0982,
+      "theoretical_loss": 4.3199980711181,
+      "tokens_seen": 213516288
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004723671013039117,
+      "loss": 3.1012,
+      "theoretical_loss": 4.319833858823321,
+      "tokens_seen": 213581824
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004723570712136409,
+      "loss": 2.945,
+      "theoretical_loss": 4.319669711011656,
+      "tokens_seen": 213647360
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.00047234704112337014,
+      "loss": 3.1732,
+      "theoretical_loss": 4.319505627638016,
+      "tokens_seen": 213712896
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.00047233701103309927,
+      "loss": 3.1215,
+      "theoretical_loss": 4.319341608657357,
+      "tokens_seen": 213778432
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004723269809428285,
+      "loss": 3.3584,
+      "theoretical_loss": 4.319177654024679,
+      "tokens_seen": 213843968
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004723169508525577,
+      "loss": 3.0733,
+      "theoretical_loss": 4.319013763695029,
+      "tokens_seen": 213909504
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.00047230692076228686,
+      "loss": 3.2797,
+      "theoretical_loss": 4.318849937623497,
+      "tokens_seen": 213975040
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.00047229689067201605,
+      "loss": 2.9947,
+      "theoretical_loss": 4.318686175765221,
+      "tokens_seen": 214040576
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004722868605817452,
+      "loss": 3.0984,
+      "theoretical_loss": 4.31852247807538,
+      "tokens_seen": 214106112
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004722768304914744,
+      "loss": 2.8851,
+      "theoretical_loss": 4.318358844509202,
+      "tokens_seen": 214171648
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.00047226680040120364,
+      "loss": 3.0447,
+      "theoretical_loss": 4.318195275021959,
+      "tokens_seen": 214237184
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.00047225677031093277,
+      "loss": 3.2885,
+      "theoretical_loss": 4.318031769568966,
+      "tokens_seen": 214302720
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.000472246740220662,
+      "loss": 3.0849,
+      "theoretical_loss": 4.317868328105585,
+      "tokens_seen": 214368256
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.00047223671013039113,
+      "loss": 3.052,
+      "theoretical_loss": 4.3177049505872205,
+      "tokens_seen": 214433792
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.00047222668004012037,
+      "loss": 3.0657,
+      "theoretical_loss": 4.317541636969324,
+      "tokens_seen": 214499328
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004722166499498496,
+      "loss": 2.9079,
+      "theoretical_loss": 4.31737838720739,
+      "tokens_seen": 214564864
+    },
+    {
+      "epoch": 0.07,
+      "objective/train/docs_used": 128852,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 3.337009906768799,
+      "objective/train/theoretical_loss": 4.317215201256958,
+      "objective/train/tokens_used": 235090400,
+      "theoretical_loss": 4.317215201256958,
+      "tokens_seen": 214630400
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00047220661985957873,
+      "loss": 3.1137,
+      "theoretical_loss": 4.317215201256958,
+      "tokens_seen": 214630400
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00047219658976930797,
+      "loss": 3.0925,
+      "theoretical_loss": 4.317052079073613,
+      "tokens_seen": 214695936
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00047218655967903715,
+      "loss": 3.1037,
+      "theoretical_loss": 4.316889020612983,
+      "tokens_seen": 214761472
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00047217652958876633,
+      "loss": 2.9058,
+      "theoretical_loss": 4.316726025830741,
+      "tokens_seen": 214827008
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004721664994984955,
+      "loss": 3.1171,
+      "theoretical_loss": 4.316563094682604,
+      "tokens_seen": 214892544
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004721564694082247,
+      "loss": 3.2248,
+      "theoretical_loss": 4.316400227124335,
+      "tokens_seen": 214958080
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00047214643931795387,
+      "loss": 3.1588,
+      "theoretical_loss": 4.3162374231117395,
+      "tokens_seen": 215023616
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004721364092276831,
+      "loss": 3.2238,
+      "theoretical_loss": 4.316074682600666,
+      "tokens_seen": 215089152
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00047212637913741223,
+      "loss": 3.3115,
+      "theoretical_loss": 4.315912005547011,
+      "tokens_seen": 215154688
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00047211634904714147,
+      "loss": 2.9668,
+      "theoretical_loss": 4.315749391906712,
+      "tokens_seen": 215220224
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004721063189568706,
+      "loss": 3.0336,
+      "theoretical_loss": 4.31558684163575,
+      "tokens_seen": 215285760
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00047209628886659983,
+      "loss": 3.1727,
+      "theoretical_loss": 4.315424354690153,
+      "tokens_seen": 215351296
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.000472086258776329,
+      "loss": 3.0615,
+      "theoretical_loss": 4.31526193102599,
+      "tokens_seen": 215416832
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004720762286860582,
+      "loss": 2.9988,
+      "theoretical_loss": 4.315099570599374,
+      "tokens_seen": 215482368
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004720661985957874,
+      "loss": 3.0711,
+      "theoretical_loss": 4.314937273366466,
+      "tokens_seen": 215547904
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00047205616850551656,
+      "loss": 3.2474,
+      "theoretical_loss": 4.314775039283463,
+      "tokens_seen": 215613440
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00047204613841524574,
+      "loss": 3.2283,
+      "theoretical_loss": 4.314612868306614,
+      "tokens_seen": 215678976
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00047203610832497497,
+      "loss": 3.1768,
+      "theoretical_loss": 4.314450760392204,
+      "tokens_seen": 215744512
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004720260782347041,
+      "loss": 3.23,
+      "theoretical_loss": 4.314288715496568,
+      "tokens_seen": 215810048
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00047201604814443333,
+      "loss": 3.443,
+      "theoretical_loss": 4.314126733576082,
+      "tokens_seen": 215875584
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004720060180541625,
+      "loss": 2.9834,
+      "theoretical_loss": 4.3139648145871625,
+      "tokens_seen": 215941120
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004719959879638917,
+      "loss": 3.3007,
+      "theoretical_loss": 4.313802958486274,
+      "tokens_seen": 216006656
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004719859578736209,
+      "loss": 3.2257,
+      "theoretical_loss": 4.313641165229922,
+      "tokens_seen": 216072192
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00047197592778335006,
+      "loss": 2.9371,
+      "theoretical_loss": 4.313479434774655,
+      "tokens_seen": 216137728
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00047196589769307924,
+      "loss": 3.1597,
+      "theoretical_loss": 4.313317767077065,
+      "tokens_seen": 216203264
+    },
+    {
+      "epoch": 0.07,
+      "objective/train/docs_used": 129467,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 3.151235342025757,
+      "objective/train/theoretical_loss": 4.3131561620937875,
+      "objective/train/tokens_used": 236728800,
+      "theoretical_loss": 4.3131561620937875,
+      "tokens_seen": 216268800
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004719558676028085,
+      "loss": 3.1703,
+      "theoretical_loss": 4.3131561620937875,
+      "tokens_seen": 216268800
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004719458375125376,
+      "loss": 3.1118,
+      "theoretical_loss": 4.312994619781502,
+      "tokens_seen": 216334336
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00047193580742226684,
+      "loss": 3.2158,
+      "theoretical_loss": 4.312833140096929,
+      "tokens_seen": 216399872
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00047192577733199596,
+      "loss": 3.1632,
+      "theoretical_loss": 4.312671722996835,
+      "tokens_seen": 216465408
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004719157472417252,
+      "loss": 3.0144,
+      "theoretical_loss": 4.312510368438025,
+      "tokens_seen": 216530944
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004719057171514544,
+      "loss": 2.9652,
+      "theoretical_loss": 4.31234907637735,
+      "tokens_seen": 216596480
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00047189568706118356,
+      "loss": 3.0804,
+      "theoretical_loss": 4.312187846771704,
+      "tokens_seen": 216662016
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00047188565697091274,
+      "loss": 3.221,
+      "theoretical_loss": 4.312026679578022,
+      "tokens_seen": 216727552
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004718756268806419,
+      "loss": 3.2019,
+      "theoretical_loss": 4.311865574753283,
+      "tokens_seen": 216793088
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004718655967903711,
+      "loss": 2.9076,
+      "theoretical_loss": 4.311704532254509,
+      "tokens_seen": 216858624
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00047185556670010034,
+      "loss": 3.1173,
+      "theoretical_loss": 4.311543552038762,
+      "tokens_seen": 216924160
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00047184553660982947,
+      "loss": 2.9789,
+      "theoretical_loss": 4.31138263406315,
+      "tokens_seen": 216989696
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004718355065195587,
+      "loss": 3.0793,
+      "theoretical_loss": 4.311221778284821,
+      "tokens_seen": 217055232
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004718254764292879,
+      "loss": 3.1944,
+      "theoretical_loss": 4.311060984660967,
+      "tokens_seen": 217120768
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00047181544633901706,
+      "loss": 2.9635,
+      "theoretical_loss": 4.310900253148821,
+      "tokens_seen": 217186304
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00047180541624874625,
+      "loss": 3.0845,
+      "theoretical_loss": 4.31073958370566,
+      "tokens_seen": 217251840
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004717953861584754,
+      "loss": 3.3273,
+      "theoretical_loss": 4.3105789762888005,
+      "tokens_seen": 217317376
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004717853560682046,
+      "loss": 3.1426,
+      "theoretical_loss": 4.310418430855604,
+      "tokens_seen": 217382912
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00047177532597793384,
+      "loss": 3.1465,
+      "theoretical_loss": 4.310257947363474,
+      "tokens_seen": 217448448
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00047176529588766297,
+      "loss": 3.1651,
+      "theoretical_loss": 4.310097525769854,
+      "tokens_seen": 217513984
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004717552657973922,
+      "loss": 3.406,
+      "theoretical_loss": 4.309937166032231,
+      "tokens_seen": 217579520
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00047174523570712133,
+      "loss": 3.3688,
+      "theoretical_loss": 4.309776868108134,
+      "tokens_seen": 217645056
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00047173520561685057,
+      "loss": 3.2583,
+      "theoretical_loss": 4.309616631955134,
+      "tokens_seen": 217710592
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00047172517552657975,
+      "loss": 3.2181,
+      "theoretical_loss": 4.309456457530843,
+      "tokens_seen": 217776128
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00047171514543630893,
+      "loss": 3.0626,
+      "theoretical_loss": 4.309296344792916,
+      "tokens_seen": 217841664
+    },
+    {
+      "epoch": 0.07,
+      "objective/train/docs_used": 130589,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 3.174252510070801,
+      "objective/train/theoretical_loss": 4.309136293699048,
+      "objective/train/tokens_used": 238367200,
+      "theoretical_loss": 4.309136293699048,
+      "tokens_seen": 217907200
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004717051153460381,
+      "loss": 3.0759,
+      "theoretical_loss": 4.309136293699048,
+      "tokens_seen": 217907200
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00047169508525576735,
+      "loss": 3.0866,
+      "theoretical_loss": 4.308976304206978,
+      "tokens_seen": 217972736
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004716850551654965,
+      "loss": 3.0519,
+      "theoretical_loss": 4.3088163762744855,
+      "tokens_seen": 218038272
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004716750250752257,
+      "loss": 2.938,
+      "theoretical_loss": 4.308656509859392,
+      "tokens_seen": 218103808
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00047166499498495484,
+      "loss": 2.9371,
+      "theoretical_loss": 4.308496704919559,
+      "tokens_seen": 218169344
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00047165496489468407,
+      "loss": 2.8471,
+      "theoretical_loss": 4.308336961412892,
+      "tokens_seen": 218234880
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00047164493480441325,
+      "loss": 3.1918,
+      "theoretical_loss": 4.308177279297336,
+      "tokens_seen": 218300416
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00047163490471414243,
+      "loss": 3.0599,
+      "theoretical_loss": 4.308017658530879,
+      "tokens_seen": 218365952
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004716248746238716,
+      "loss": 3.1856,
+      "theoretical_loss": 4.307858099071549,
+      "tokens_seen": 218431488
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004716148445336008,
+      "loss": 2.7223,
+      "theoretical_loss": 4.307698600877416,
+      "tokens_seen": 218497024
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00047160481444333,
+      "loss": 3.1367,
+      "theoretical_loss": 4.30753916390659,
+      "tokens_seen": 218562560
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004715947843530592,
+      "loss": 3.048,
+      "theoretical_loss": 4.307379788117224,
+      "tokens_seen": 218628096
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00047158475426278834,
+      "loss": 3.1323,
+      "theoretical_loss": 4.3072204734675115,
+      "tokens_seen": 218693632
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004715747241725176,
+      "loss": 2.804,
+      "theoretical_loss": 4.307061219915687,
+      "tokens_seen": 218759168
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004715646940822467,
+      "loss": 3.1227,
+      "theoretical_loss": 4.306902027420025,
+      "tokens_seen": 218824704
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00047155466399197594,
+      "loss": 3.3008,
+      "theoretical_loss": 4.306742895938844,
+      "tokens_seen": 218890240
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004715446339017051,
+      "loss": 3.1554,
+      "theoretical_loss": 4.306583825430499,
+      "tokens_seen": 218955776
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004715346038114343,
+      "loss": 2.8274,
+      "theoretical_loss": 4.30642481585339,
+      "tokens_seen": 219021312
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004715245737211635,
+      "loss": 3.1366,
+      "theoretical_loss": 4.306265867165955,
+      "tokens_seen": 219086848
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004715145436308927,
+      "loss": 2.9215,
+      "theoretical_loss": 4.306106979326674,
+      "tokens_seen": 219152384
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00047150451354062184,
+      "loss": 3.2813,
+      "theoretical_loss": 4.305948152294068,
+      "tokens_seen": 219217920
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004714944834503511,
+      "loss": 2.9276,
+      "theoretical_loss": 4.305789386026698,
+      "tokens_seen": 219283456
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004714844533600802,
+      "loss": 3.0313,
+      "theoretical_loss": 4.305630680483166,
+      "tokens_seen": 219348992
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00047147442326980944,
+      "loss": 3.2763,
+      "theoretical_loss": 4.305472035622114,
+      "tokens_seen": 219414528
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004714643931795387,
+      "loss": 3.1934,
+      "theoretical_loss": 4.305313451402226,
+      "tokens_seen": 219480064
+    },
+    {
+      "epoch": 0.07,
+      "objective/train/docs_used": 131942,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 3.265974283218384,
+      "objective/train/theoretical_loss": 4.305154927782223,
+      "objective/train/tokens_used": 240005600,
+      "theoretical_loss": 4.305154927782223,
+      "tokens_seen": 219545600
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004714543630892678,
+      "loss": 3.0968,
+      "theoretical_loss": 4.305154927782223,
+      "tokens_seen": 219545600
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00047144433299899704,
+      "loss": 3.047,
+      "theoretical_loss": 4.304996464720872,
+      "tokens_seen": 219611136
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00047143430290872616,
+      "loss": 3.1081,
+      "theoretical_loss": 4.304838062176974,
+      "tokens_seen": 219676672
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004714242728184554,
+      "loss": 3.1186,
+      "theoretical_loss": 4.304679720109376,
+      "tokens_seen": 219742208
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004714142427281846,
+      "loss": 3.161,
+      "theoretical_loss": 4.304521438476961,
+      "tokens_seen": 219807744
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00047140421263791376,
+      "loss": 3.122,
+      "theoretical_loss": 4.304363217238654,
+      "tokens_seen": 219873280
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00047139418254764294,
+      "loss": 3.1317,
+      "theoretical_loss": 4.304205056353421,
+      "tokens_seen": 219938816
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004713841524573721,
+      "loss": 3.0803,
+      "theoretical_loss": 4.304046955780267,
+      "tokens_seen": 220004352
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004713741223671013,
+      "loss": 3.1409,
+      "theoretical_loss": 4.303888915478236,
+      "tokens_seen": 220069888
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00047136409227683054,
+      "loss": 3.1707,
+      "theoretical_loss": 4.303730935406414,
+      "tokens_seen": 220135424
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00047135406218655967,
+      "loss": 2.9751,
+      "theoretical_loss": 4.303573015523927,
+      "tokens_seen": 220200960
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004713440320962889,
+      "loss": 3.177,
+      "theoretical_loss": 4.3034151557899385,
+      "tokens_seen": 220266496
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004713340020060181,
+      "loss": 3.054,
+      "theoretical_loss": 4.303257356163654,
+      "tokens_seen": 220332032
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00047132397191574726,
+      "loss": 3.4153,
+      "theoretical_loss": 4.303099616604318,
+      "tokens_seen": 220397568
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00047131394182547645,
+      "loss": 3.0444,
+      "theoretical_loss": 4.302941937071216,
+      "tokens_seen": 220463104
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00047130391173520563,
+      "loss": 2.9421,
+      "theoretical_loss": 4.3027843175236695,
+      "tokens_seen": 220528640
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004712938816449348,
+      "loss": 3.1708,
+      "theoretical_loss": 4.302626757921045,
+      "tokens_seen": 220594176
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00047128385155466404,
+      "loss": 3.3514,
+      "theoretical_loss": 4.302469258222744,
+      "tokens_seen": 220659712
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00047127382146439317,
+      "loss": 3.2391,
+      "theoretical_loss": 4.30231181838821,
+      "tokens_seen": 220725248
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004712637913741224,
+      "loss": 3.122,
+      "theoretical_loss": 4.302154438376926,
+      "tokens_seen": 220790784
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00047125376128385153,
+      "loss": 3.0711,
+      "theoretical_loss": 4.301997118148412,
+      "tokens_seen": 220856320
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00047124373119358077,
+      "loss": 3.2042,
+      "theoretical_loss": 4.301839857662231,
+      "tokens_seen": 220921856
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00047123370110330995,
+      "loss": 2.8998,
+      "theoretical_loss": 4.301682656877983,
+      "tokens_seen": 220987392
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00047122367101303913,
+      "loss": 3.3909,
+      "theoretical_loss": 4.301525515755307,
+      "tokens_seen": 221052928
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004712136409227683,
+      "loss": 3.0117,
+      "theoretical_loss": 4.301368434253883,
+      "tokens_seen": 221118464
+    },
+    {
+      "epoch": 0.07,
+      "objective/train/docs_used": 132442,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 3.3066797256469727,
+      "objective/train/theoretical_loss": 4.301211412333429,
+      "objective/train/tokens_used": 241644000,
+      "theoretical_loss": 4.301211412333429,
+      "tokens_seen": 221184000
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00047120361083249755,
+      "loss": 3.1236,
+      "theoretical_loss": 4.301211412333429,
+      "tokens_seen": 221184000
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004711935807422267,
+      "loss": 3.0539,
+      "theoretical_loss": 4.301054449953702,
+      "tokens_seen": 221249536
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004711835506519559,
+      "loss": 3.2493,
+      "theoretical_loss": 4.300897547074499,
+      "tokens_seen": 221315072
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00047117352056168504,
+      "loss": 3.494,
+      "theoretical_loss": 4.300740703655655,
+      "tokens_seen": 221380608
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00047116349047141427,
+      "loss": 2.8181,
+      "theoretical_loss": 4.300583919657045,
+      "tokens_seen": 221446144
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00047115346038114345,
+      "loss": 3.1062,
+      "theoretical_loss": 4.300427195038582,
+      "tokens_seen": 221511680
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00047114343029087263,
+      "loss": 3.2018,
+      "theoretical_loss": 4.300270529760219,
+      "tokens_seen": 221577216
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004711334002006018,
+      "loss": 2.9325,
+      "theoretical_loss": 4.300113923781946,
+      "tokens_seen": 221642752
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.000471123370110331,
+      "loss": 3.0023,
+      "theoretical_loss": 4.2999573770637936,
+      "tokens_seen": 221708288
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004711133400200602,
+      "loss": 3.1196,
+      "theoretical_loss": 4.299800889565831,
+      "tokens_seen": 221773824
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004711033099297894,
+      "loss": 3.0607,
+      "theoretical_loss": 4.299644461248165,
+      "tokens_seen": 221839360
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00047109327983951854,
+      "loss": 3.2191,
+      "theoretical_loss": 4.299488092070942,
+      "tokens_seen": 221904896
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004710832497492478,
+      "loss": 3.0538,
+      "theoretical_loss": 4.299331781994347,
+      "tokens_seen": 221970432
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004710732196589769,
+      "loss": 3.073,
+      "theoretical_loss": 4.2991755309786015,
+      "tokens_seen": 222035968
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00047106318956870614,
+      "loss": 2.9409,
+      "theoretical_loss": 4.299019338983969,
+      "tokens_seen": 222101504
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004710531594784353,
+      "loss": 3.0287,
+      "theoretical_loss": 4.298863205970751,
+      "tokens_seen": 222167040
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004710431293881645,
+      "loss": 3.1599,
+      "theoretical_loss": 4.298707131899281,
+      "tokens_seen": 222232576
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004710330992978937,
+      "loss": 3.146,
+      "theoretical_loss": 4.298551116729941,
+      "tokens_seen": 222298112
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004710230692076229,
+      "loss": 3.085,
+      "theoretical_loss": 4.298395160423144,
+      "tokens_seen": 222363648
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00047101303911735204,
+      "loss": 3.09,
+      "theoretical_loss": 4.298239262939344,
+      "tokens_seen": 222429184
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004710030090270813,
+      "loss": 3.0419,
+      "theoretical_loss": 4.2980834242390324,
+      "tokens_seen": 222494720
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004709929789368104,
+      "loss": 3.2707,
+      "theoretical_loss": 4.29792764428274,
+      "tokens_seen": 222560256
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00047098294884653964,
+      "loss": 3.2489,
+      "theoretical_loss": 4.297771923031033,
+      "tokens_seen": 222625792
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004709729187562688,
+      "loss": 2.9268,
+      "theoretical_loss": 4.297616260444519,
+      "tokens_seen": 222691328
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.000470962888665998,
+      "loss": 3.1398,
+      "theoretical_loss": 4.297460656483842,
+      "tokens_seen": 222756864
+    },
+    {
+      "epoch": 0.07,
+      "objective/train/docs_used": 133596,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 2.741494655609131,
+      "objective/train/theoretical_loss": 4.297305111109683,
+      "objective/train/tokens_used": 243282400,
+      "theoretical_loss": 4.297305111109683,
+      "tokens_seen": 222822400
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004709528585757272,
+      "loss": 2.7095,
+      "theoretical_loss": 4.297305111109683,
+      "tokens_seen": 222822400
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00047094282848545636,
+      "loss": 2.9503,
+      "theoretical_loss": 4.297149624282762,
+      "tokens_seen": 222887936
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00047093279839518555,
+      "loss": 3.1134,
+      "theoretical_loss": 4.296994195963838,
+      "tokens_seen": 222953472
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004709227683049148,
+      "loss": 2.8174,
+      "theoretical_loss": 4.296838826113706,
+      "tokens_seen": 223019008
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004709127382146439,
+      "loss": 2.7694,
+      "theoretical_loss": 4.296683514693199,
+      "tokens_seen": 223084544
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00047090270812437314,
+      "loss": 3.1462,
+      "theoretical_loss": 4.296528261663189,
+      "tokens_seen": 223150080
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00047089267803410227,
+      "loss": 3.1611,
+      "theoretical_loss": 4.296373066984585,
+      "tokens_seen": 223215616
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004708826479438315,
+      "loss": 3.0784,
+      "theoretical_loss": 4.296217930618332,
+      "tokens_seen": 223281152
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004708726178535607,
+      "loss": 3.2461,
+      "theoretical_loss": 4.2960628525254165,
+      "tokens_seen": 223346688
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00047086258776328987,
+      "loss": 3.3592,
+      "theoretical_loss": 4.295907832666858,
+      "tokens_seen": 223412224
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00047085255767301905,
+      "loss": 3.134,
+      "theoretical_loss": 4.2957528710037165,
+      "tokens_seen": 223477760
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004708425275827483,
+      "loss": 2.793,
+      "theoretical_loss": 4.2955979674970886,
+      "tokens_seen": 223543296
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004708324974924774,
+      "loss": 3.1087,
+      "theoretical_loss": 4.295443122108108,
+      "tokens_seen": 223608832
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00047082246740220665,
+      "loss": 3.2745,
+      "theoretical_loss": 4.295288334797946,
+      "tokens_seen": 223674368
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004708124373119358,
+      "loss": 3.0884,
+      "theoretical_loss": 4.295133605527812,
+      "tokens_seen": 223739904
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.000470802407221665,
+      "loss": 3.0021,
+      "theoretical_loss": 4.294978934258951,
+      "tokens_seen": 223805440
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004707923771313942,
+      "loss": 2.9157,
+      "theoretical_loss": 4.294824320952646,
+      "tokens_seen": 223870976
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00047078234704112337,
+      "loss": 3.3173,
+      "theoretical_loss": 4.294669765570219,
+      "tokens_seen": 223936512
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00047077231695085255,
+      "loss": 3.1127,
+      "theoretical_loss": 4.294515268073026,
+      "tokens_seen": 224002048
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00047076228686058173,
+      "loss": 2.8344,
+      "theoretical_loss": 4.294360828422461,
+      "tokens_seen": 224067584
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004707522567703109,
+      "loss": 3.0905,
+      "theoretical_loss": 4.294206446579956,
+      "tokens_seen": 224133120
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00047074222668004015,
+      "loss": 3.1892,
+      "theoretical_loss": 4.29405212250698,
+      "tokens_seen": 224198656
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004707321965897693,
+      "loss": 2.8706,
+      "theoretical_loss": 4.293897856165039,
+      "tokens_seen": 224264192
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004707221664994985,
+      "loss": 3.2766,
+      "theoretical_loss": 4.293743647515674,
+      "tokens_seen": 224329728
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00047071213640922775,
+      "loss": 3.1984,
+      "theoretical_loss": 4.293589496520464,
+      "tokens_seen": 224395264
+    },
+    {
+      "epoch": 0.07,
+      "objective/train/docs_used": 134273,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 3.291004180908203,
+      "objective/train/theoretical_loss": 4.293435403141026,
+      "objective/train/tokens_used": 244920800,
+      "theoretical_loss": 4.293435403141026,
+      "tokens_seen": 224460800
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004707021063189569,
+      "loss": 3.2134,
+      "theoretical_loss": 4.293435403141026,
+      "tokens_seen": 224460800
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004706920762286861,
+      "loss": 3.107,
+      "theoretical_loss": 4.293281367339013,
+      "tokens_seen": 224526336
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00047068204613841524,
+      "loss": 3.2989,
+      "theoretical_loss": 4.293127389076113,
+      "tokens_seen": 224591872
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00047067201604814447,
+      "loss": 3.4678,
+      "theoretical_loss": 4.292973468314053,
+      "tokens_seen": 224657408
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00047066198595787365,
+      "loss": 3.0958,
+      "theoretical_loss": 4.292819605014596,
+      "tokens_seen": 224722944
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00047065195586760283,
+      "loss": 2.9754,
+      "theoretical_loss": 4.292665799139541,
+      "tokens_seen": 224788480
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.000470641925777332,
+      "loss": 3.2233,
+      "theoretical_loss": 4.292512050650723,
+      "tokens_seen": 224854016
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004706318956870612,
+      "loss": 2.9816,
+      "theoretical_loss": 4.292358359510015,
+      "tokens_seen": 224919552
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004706218655967904,
+      "loss": 3.2302,
+      "theoretical_loss": 4.292204725679326,
+      "tokens_seen": 224985088
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004706118355065196,
+      "loss": 3.1335,
+      "theoretical_loss": 4.2920511491206,
+      "tokens_seen": 225050624
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00047060180541624874,
+      "loss": 3.1664,
+      "theoretical_loss": 4.2918976297958205,
+      "tokens_seen": 225116160
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.000470591775325978,
+      "loss": 3.0454,
+      "theoretical_loss": 4.2917441676670025,
+      "tokens_seen": 225181696
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004705817452357071,
+      "loss": 3.0739,
+      "theoretical_loss": 4.291590762696202,
+      "tokens_seen": 225247232
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00047057171514543634,
+      "loss": 3.1721,
+      "theoretical_loss": 4.291437414845509,
+      "tokens_seen": 225312768
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004705616850551655,
+      "loss": 3.1347,
+      "theoretical_loss": 4.291284124077049,
+      "tokens_seen": 225378304
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004705516549648947,
+      "loss": 2.9492,
+      "theoretical_loss": 4.291130890352985,
+      "tokens_seen": 225443840
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004705416248746239,
+      "loss": 3.1897,
+      "theoretical_loss": 4.290977713635515,
+      "tokens_seen": 225509376
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004705315947843531,
+      "loss": 3.2739,
+      "theoretical_loss": 4.2908245938868745,
+      "tokens_seen": 225574912
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00047052156469408224,
+      "loss": 3.3183,
+      "theoretical_loss": 4.290671531069334,
+      "tokens_seen": 225640448
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004705115346038115,
+      "loss": 3.1922,
+      "theoretical_loss": 4.290518525145199,
+      "tokens_seen": 225705984
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004705015045135406,
+      "loss": 3.282,
+      "theoretical_loss": 4.290365576076812,
+      "tokens_seen": 225771520
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00047049147442326984,
+      "loss": 3.2065,
+      "theoretical_loss": 4.290212683826551,
+      "tokens_seen": 225837056
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.000470481444332999,
+      "loss": 3.251,
+      "theoretical_loss": 4.290059848356833,
+      "tokens_seen": 225902592
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004704714142427282,
+      "loss": 2.9908,
+      "theoretical_loss": 4.289907069630104,
+      "tokens_seen": 225968128
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004704613841524574,
+      "loss": 3.2375,
+      "theoretical_loss": 4.28975434760885,
+      "tokens_seen": 226033664
+    },
+    {
+      "epoch": 0.07,
+      "objective/train/docs_used": 135613,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 2.803276538848877,
+      "objective/train/theoretical_loss": 4.2896016822555945,
+      "objective/train/tokens_used": 246559200,
+      "theoretical_loss": 4.2896016822555945,
+      "tokens_seen": 226099200
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00047045135406218656,
+      "loss": 3.0062,
+      "theoretical_loss": 4.2896016822555945,
+      "tokens_seen": 226099200
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00047044132397191575,
+      "loss": 3.1042,
+      "theoretical_loss": 4.289449073532892,
+      "tokens_seen": 226164736
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.000470431293881645,
+      "loss": 3.3344,
+      "theoretical_loss": 4.289296521403337,
+      "tokens_seen": 226230272
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004704212637913741,
+      "loss": 3.1889,
+      "theoretical_loss": 4.289144025829555,
+      "tokens_seen": 226295808
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00047041123370110334,
+      "loss": 2.9189,
+      "theoretical_loss": 4.288991586774211,
+      "tokens_seen": 226361344
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00047040120361083247,
+      "loss": 3.2327,
+      "theoretical_loss": 4.288839204200003,
+      "tokens_seen": 226426880
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004703911735205617,
+      "loss": 2.9544,
+      "theoretical_loss": 4.288686878069666,
+      "tokens_seen": 226492416
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004703811434302909,
+      "loss": 3.1703,
+      "theoretical_loss": 4.288534608345969,
+      "tokens_seen": 226557952
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00047037111334002007,
+      "loss": 3.1722,
+      "theoretical_loss": 4.288382394991717,
+      "tokens_seen": 226623488
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00047036108324974925,
+      "loss": 3.0562,
+      "theoretical_loss": 4.288230237969751,
+      "tokens_seen": 226689024
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004703510531594785,
+      "loss": 3.074,
+      "theoretical_loss": 4.288078137242946,
+      "tokens_seen": 226754560
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004703410230692076,
+      "loss": 2.9752,
+      "theoretical_loss": 4.287926092774212,
+      "tokens_seen": 226820096
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00047033099297893685,
+      "loss": 2.8926,
+      "theoretical_loss": 4.287774104526496,
+      "tokens_seen": 226885632
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.000470320962888666,
+      "loss": 3.2959,
+      "theoretical_loss": 4.287622172462777,
+      "tokens_seen": 226951168
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004703109327983952,
+      "loss": 3.1867,
+      "theoretical_loss": 4.287470296546073,
+      "tokens_seen": 227016704
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004703009027081244,
+      "loss": 3.0586,
+      "theoretical_loss": 4.287318476739433,
+      "tokens_seen": 227082240
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00047029087261785357,
+      "loss": 3.1358,
+      "theoretical_loss": 4.287166713005945,
+      "tokens_seen": 227147776
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00047028084252758275,
+      "loss": 3.1048,
+      "theoretical_loss": 4.287015005308728,
+      "tokens_seen": 227213312
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00047027081243731193,
+      "loss": 3.1529,
+      "theoretical_loss": 4.286863353610939,
+      "tokens_seen": 227278848
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004702607823470411,
+      "loss": 3.0266,
+      "theoretical_loss": 4.286711757875768,
+      "tokens_seen": 227344384
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00047025075225677035,
+      "loss": 3.1715,
+      "theoretical_loss": 4.28656021806644,
+      "tokens_seen": 227409920
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004702407221664995,
+      "loss": 3.0616,
+      "theoretical_loss": 4.286408734146216,
+      "tokens_seen": 227475456
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004702306920762287,
+      "loss": 3.1768,
+      "theoretical_loss": 4.28625730607839,
+      "tokens_seen": 227540992
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00047022066198595784,
+      "loss": 3.1572,
+      "theoretical_loss": 4.286105933826292,
+      "tokens_seen": 227606528
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004702106318956871,
+      "loss": 3.07,
+      "theoretical_loss": 4.285954617353285,
+      "tokens_seen": 227672064
+    },
+    {
+      "epoch": 0.07,
+      "objective/train/docs_used": 136262,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 2.876788377761841,
+      "objective/train/theoretical_loss": 4.28580335662277,
+      "objective/train/tokens_used": 248197600,
+      "theoretical_loss": 4.28580335662277,
+      "tokens_seen": 227737600
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00047020060180541626,
+      "loss": 3.0746,
+      "theoretical_loss": 4.28580335662277,
+      "tokens_seen": 227737600
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00047019057171514544,
+      "loss": 3.2485,
+      "theoretical_loss": 4.285652151598178,
+      "tokens_seen": 227803136
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004701805416248746,
+      "loss": 3.2203,
+      "theoretical_loss": 4.285501002242977,
+      "tokens_seen": 227868672
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00047017051153460385,
+      "loss": 3.0961,
+      "theoretical_loss": 4.285349908520671,
+      "tokens_seen": 227934208
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.000470160481444333,
+      "loss": 3.1869,
+      "theoretical_loss": 4.285198870394796,
+      "tokens_seen": 227999744
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004701504513540622,
+      "loss": 3.154,
+      "theoretical_loss": 4.28504788782892,
+      "tokens_seen": 228065280
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00047014042126379134,
+      "loss": 3.1927,
+      "theoretical_loss": 4.284896960786653,
+      "tokens_seen": 228130816
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004701303911735206,
+      "loss": 3.2737,
+      "theoretical_loss": 4.284746089231631,
+      "tokens_seen": 228196352
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00047012036108324976,
+      "loss": 3.299,
+      "theoretical_loss": 4.28459527312753,
+      "tokens_seen": 228261888
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00047011033099297894,
+      "loss": 2.9585,
+      "theoretical_loss": 4.284444512438057,
+      "tokens_seen": 228327424
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004701003009027081,
+      "loss": 3.1119,
+      "theoretical_loss": 4.2842938071269545,
+      "tokens_seen": 228392960
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004700902708124373,
+      "loss": 3.165,
+      "theoretical_loss": 4.284143157157999,
+      "tokens_seen": 228458496
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004700802407221665,
+      "loss": 3.1586,
+      "theoretical_loss": 4.2839925624949995,
+      "tokens_seen": 228524032
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004700702106318957,
+      "loss": 3.1651,
+      "theoretical_loss": 4.283842023101803,
+      "tokens_seen": 228589568
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00047006018054162484,
+      "loss": 3.2229,
+      "theoretical_loss": 4.283691538942286,
+      "tokens_seen": 228655104
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004700501504513541,
+      "loss": 3.192,
+      "theoretical_loss": 4.28354110998036,
+      "tokens_seen": 228720640
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00047004012036108326,
+      "loss": 3.03,
+      "theoretical_loss": 4.283390736179975,
+      "tokens_seen": 228786176
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00047003009027081244,
+      "loss": 3.3334,
+      "theoretical_loss": 4.283240417505107,
+      "tokens_seen": 228851712
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004700200601805416,
+      "loss": 2.9563,
+      "theoretical_loss": 4.2830901539197725,
+      "tokens_seen": 228917248
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004700100300902708,
+      "loss": 3.0645,
+      "theoretical_loss": 4.282939945388019,
+      "tokens_seen": 228982784
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00047,
+      "loss": 3.2184,
+      "theoretical_loss": 4.2827897918739275,
+      "tokens_seen": 229048320
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004699899699097292,
+      "loss": 3.3057,
+      "theoretical_loss": 4.282639693341613,
+      "tokens_seen": 229113856
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00046997993981945835,
+      "loss": 3.179,
+      "theoretical_loss": 4.282489649755225,
+      "tokens_seen": 229179392
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004699699097291876,
+      "loss": 3.0861,
+      "theoretical_loss": 4.282339661078946,
+      "tokens_seen": 229244928
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00046995987963891676,
+      "loss": 3.2011,
+      "theoretical_loss": 4.282189727276991,
+      "tokens_seen": 229310464
+    },
+    {
+      "epoch": 0.07,
+      "objective/train/docs_used": 137363,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 3.0329909324645996,
+      "objective/train/theoretical_loss": 4.282039848313611,
+      "objective/train/tokens_used": 249836000,
+      "theoretical_loss": 4.282039848313611,
+      "tokens_seen": 229376000
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00046994984954864595,
+      "loss": 3.003,
+      "theoretical_loss": 4.282039848313611,
+      "tokens_seen": 229376000
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004699398194583752,
+      "loss": 3.2598,
+      "theoretical_loss": 4.281890024153089,
+      "tokens_seen": 229441536
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004699297893681043,
+      "loss": 3.1051,
+      "theoretical_loss": 4.2817402547597405,
+      "tokens_seen": 229507072
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00046991975927783354,
+      "loss": 3.2666,
+      "theoretical_loss": 4.281590540097916,
+      "tokens_seen": 229572608
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00046990972918756267,
+      "loss": 3.1706,
+      "theoretical_loss": 4.281440880131999,
+      "tokens_seen": 229638144
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004698996990972919,
+      "loss": 2.882,
+      "theoretical_loss": 4.2812912748264065,
+      "tokens_seen": 229703680
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004698896690070211,
+      "loss": 3.2516,
+      "theoretical_loss": 4.281141724145588,
+      "tokens_seen": 229769216
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00046987963891675027,
+      "loss": 3.1911,
+      "theoretical_loss": 4.280992228054027,
+      "tokens_seen": 229834752
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00046986960882647945,
+      "loss": 3.337,
+      "theoretical_loss": 4.28084278651624,
+      "tokens_seen": 229900288
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004698595787362087,
+      "loss": 3.1908,
+      "theoretical_loss": 4.280693399496776,
+      "tokens_seen": 229965824
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004698495486459378,
+      "loss": 3.0165,
+      "theoretical_loss": 4.280544066960219,
+      "tokens_seen": 230031360
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00046983951855566705,
+      "loss": 2.9808,
+      "theoretical_loss": 4.280394788871183,
+      "tokens_seen": 230096896
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004698294884653962,
+      "loss": 3.0817,
+      "theoretical_loss": 4.280245565194319,
+      "tokens_seen": 230162432
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004698194583751254,
+      "loss": 3.208,
+      "theoretical_loss": 4.280096395894308,
+      "tokens_seen": 230227968
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004698094282848546,
+      "loss": 3.1627,
+      "theoretical_loss": 4.279947280935864,
+      "tokens_seen": 230293504
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00046979939819458377,
+      "loss": 3.0551,
+      "theoretical_loss": 4.279798220283736,
+      "tokens_seen": 230359040
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00046978936810431295,
+      "loss": 3.0795,
+      "theoretical_loss": 4.279649213902704,
+      "tokens_seen": 230424576
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00046977933801404213,
+      "loss": 3.3708,
+      "theoretical_loss": 4.2795002617575815,
+      "tokens_seen": 230490112
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004697693079237713,
+      "loss": 3.1081,
+      "theoretical_loss": 4.279351363813215,
+      "tokens_seen": 230555648
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00046975927783350055,
+      "loss": 2.8502,
+      "theoretical_loss": 4.2792025200344845,
+      "tokens_seen": 230621184
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004697492477432297,
+      "loss": 3.1672,
+      "theoretical_loss": 4.279053730386301,
+      "tokens_seen": 230686720
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004697392176529589,
+      "loss": 3.1429,
+      "theoretical_loss": 4.278904994833609,
+      "tokens_seen": 230752256
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00046972918756268804,
+      "loss": 3.0444,
+      "theoretical_loss": 4.278756313341386,
+      "tokens_seen": 230817792
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004697191574724173,
+      "loss": 3.0223,
+      "theoretical_loss": 4.278607685874642,
+      "tokens_seen": 230883328
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00046970912738214646,
+      "loss": 2.8679,
+      "theoretical_loss": 4.278459112398418,
+      "tokens_seen": 230948864
+    },
+    {
+      "epoch": 0.07,
+      "objective/train/docs_used": 138640,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 3.336690664291382,
+      "objective/train/theoretical_loss": 4.278310592877791,
+      "objective/train/tokens_used": 251474400,
+      "theoretical_loss": 4.278310592877791,
+      "tokens_seen": 231014400
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00046969909729187564,
+      "loss": 2.9576,
+      "theoretical_loss": 4.278310592877791,
+      "tokens_seen": 231014400
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004696890672016048,
+      "loss": 2.9152,
+      "theoretical_loss": 4.2781621272778665,
+      "tokens_seen": 231079936
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00046967903711133405,
+      "loss": 3.0227,
+      "theoretical_loss": 4.278013715563786,
+      "tokens_seen": 231145472
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004696690070210632,
+      "loss": 3.1015,
+      "theoretical_loss": 4.27786535770072,
+      "tokens_seen": 231211008
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004696589769307924,
+      "loss": 3.1436,
+      "theoretical_loss": 4.277717053653875,
+      "tokens_seen": 231276544
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00046964894684052154,
+      "loss": 3.424,
+      "theoretical_loss": 4.277568803388486,
+      "tokens_seen": 231342080
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004696389167502508,
+      "loss": 2.9966,
+      "theoretical_loss": 4.277420606869824,
+      "tokens_seen": 231407616
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00046962888665997996,
+      "loss": 2.9781,
+      "theoretical_loss": 4.27727246406319,
+      "tokens_seen": 231473152
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00046961885656970914,
+      "loss": 3.153,
+      "theoretical_loss": 4.277124374933917,
+      "tokens_seen": 231538688
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004696088264794383,
+      "loss": 3.3026,
+      "theoretical_loss": 4.276976339447372,
+      "tokens_seen": 231604224
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004695987963891675,
+      "loss": 3.1475,
+      "theoretical_loss": 4.276828357568952,
+      "tokens_seen": 231669760
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004695887662988967,
+      "loss": 2.9757,
+      "theoretical_loss": 4.276680429264088,
+      "tokens_seen": 231735296
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004695787362086259,
+      "loss": 3.2429,
+      "theoretical_loss": 4.276532554498242,
+      "tokens_seen": 231800832
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00046956870611835505,
+      "loss": 2.9495,
+      "theoretical_loss": 4.276384733236909,
+      "tokens_seen": 231866368
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004695586760280843,
+      "loss": 3.2042,
+      "theoretical_loss": 4.276236965445612,
+      "tokens_seen": 231931904
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00046954864593781346,
+      "loss": 3.0181,
+      "theoretical_loss": 4.276089251089914,
+      "tokens_seen": 231997440
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00046953861584754264,
+      "loss": 3.0194,
+      "theoretical_loss": 4.275941590135401,
+      "tokens_seen": 232062976
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004695285857572718,
+      "loss": 3.1368,
+      "theoretical_loss": 4.275793982547698,
+      "tokens_seen": 232128512
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.000469518555667001,
+      "loss": 3.1842,
+      "theoretical_loss": 4.275646428292456,
+      "tokens_seen": 232194048
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004695085255767302,
+      "loss": 3.0394,
+      "theoretical_loss": 4.275498927335362,
+      "tokens_seen": 232259584
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004694984954864594,
+      "loss": 3.2062,
+      "theoretical_loss": 4.275351479642134,
+      "tokens_seen": 232325120
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00046948846539618855,
+      "loss": 3.2058,
+      "theoretical_loss": 4.27520408517852,
+      "tokens_seen": 232390656
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004694784353059178,
+      "loss": 2.9021,
+      "theoretical_loss": 4.275056743910301,
+      "tokens_seen": 232456192
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004694684052156469,
+      "loss": 3.1015,
+      "theoretical_loss": 4.2749094558032885,
+      "tokens_seen": 232521728
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00046945837512537615,
+      "loss": 3.1834,
+      "theoretical_loss": 4.274762220823328,
+      "tokens_seen": 232587264
+    },
+    {
+      "epoch": 0.07,
+      "objective/train/docs_used": 139202,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 3.1069910526275635,
+      "objective/train/theoretical_loss": 4.274615038936293,
+      "objective/train/tokens_used": 253112800,
+      "theoretical_loss": 4.274615038936293,
+      "tokens_seen": 232652800
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004694483450351053,
+      "loss": 3.0744,
+      "theoretical_loss": 4.274615038936293,
+      "tokens_seen": 232652800
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004694383149448345,
+      "loss": 3.0827,
+      "theoretical_loss": 4.274467910108093,
+      "tokens_seen": 232718336
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004694282848545637,
+      "loss": 3.0734,
+      "theoretical_loss": 4.274320834304666,
+      "tokens_seen": 232783872
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00046941825476429287,
+      "loss": 3.1049,
+      "theoretical_loss": 4.27417381149198,
+      "tokens_seen": 232849408
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00046940822467402205,
+      "loss": 3.0458,
+      "theoretical_loss": 4.274026841636037,
+      "tokens_seen": 232914944
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004693981945837513,
+      "loss": 3.2492,
+      "theoretical_loss": 4.273879924702872,
+      "tokens_seen": 232980480
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004693881644934804,
+      "loss": 3.0141,
+      "theoretical_loss": 4.273733060658546,
+      "tokens_seen": 233046016
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00046937813440320965,
+      "loss": 2.9252,
+      "theoretical_loss": 4.273586249469155,
+      "tokens_seen": 233111552
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00046936810431293883,
+      "loss": 3.0968,
+      "theoretical_loss": 4.273439491100828,
+      "tokens_seen": 233177088
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.000469358074222668,
+      "loss": 2.9294,
+      "theoretical_loss": 4.273292785519722,
+      "tokens_seen": 233242624
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004693480441323972,
+      "loss": 2.9721,
+      "theoretical_loss": 4.273146132692023,
+      "tokens_seen": 233308160
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004693380140421264,
+      "loss": 3.2223,
+      "theoretical_loss": 4.272999532583955,
+      "tokens_seen": 233373696
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00046932798395185555,
+      "loss": 3.1393,
+      "theoretical_loss": 4.272852985161768,
+      "tokens_seen": 233439232
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004693179538615848,
+      "loss": 3.2015,
+      "theoretical_loss": 4.272706490391743,
+      "tokens_seen": 233504768
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004693079237713139,
+      "loss": 3.0477,
+      "theoretical_loss": 4.272560048240195,
+      "tokens_seen": 233570304
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00046929789368104315,
+      "loss": 3.0628,
+      "theoretical_loss": 4.272413658673468,
+      "tokens_seen": 233635840
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004692878635907723,
+      "loss": 3.2822,
+      "theoretical_loss": 4.272267321657937,
+      "tokens_seen": 233701376
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004692778335005015,
+      "loss": 3.2682,
+      "theoretical_loss": 4.272121037160009,
+      "tokens_seen": 233766912
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004692678034102307,
+      "loss": 3.1303,
+      "theoretical_loss": 4.271974805146121,
+      "tokens_seen": 233832448
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004692577733199599,
+      "loss": 3.1491,
+      "theoretical_loss": 4.27182862558274,
+      "tokens_seen": 233897984
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00046924774322968906,
+      "loss": 3.1131,
+      "theoretical_loss": 4.271682498436365,
+      "tokens_seen": 233963520
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00046923771313941824,
+      "loss": 3.1811,
+      "theoretical_loss": 4.271536423673528,
+      "tokens_seen": 234029056
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004692276830491474,
+      "loss": 3.2045,
+      "theoretical_loss": 4.271390401260785,
+      "tokens_seen": 234094592
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00046921765295887666,
+      "loss": 3.1978,
+      "theoretical_loss": 4.271244431164731,
+      "tokens_seen": 234160128
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00046920762286860584,
+      "loss": 3.2577,
+      "theoretical_loss": 4.271098513351986,
+      "tokens_seen": 234225664
+    },
+    {
+      "epoch": 0.07,
+      "objective/train/docs_used": 140513,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 3.1093058586120605,
+      "objective/train/theoretical_loss": 4.270952647789202,
+      "objective/train/tokens_used": 254751200,
+      "theoretical_loss": 4.270952647789202,
+      "tokens_seen": 234291200
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.000469197592778335,
+      "loss": 3.3018,
+      "theoretical_loss": 4.270952647789202,
+      "tokens_seen": 234291200
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00046918756268806425,
+      "loss": 3.1679,
+      "theoretical_loss": 4.2708068344430625,
+      "tokens_seen": 234356736
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004691775325977934,
+      "loss": 3.092,
+      "theoretical_loss": 4.270661073280282,
+      "tokens_seen": 234422272
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004691675025075226,
+      "loss": 3.2188,
+      "theoretical_loss": 4.2705153642676015,
+      "tokens_seen": 234487808
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00046915747241725174,
+      "loss": 3.162,
+      "theoretical_loss": 4.270369707371799,
+      "tokens_seen": 234553344
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.000469147442326981,
+      "loss": 3.123,
+      "theoretical_loss": 4.270224102559677,
+      "tokens_seen": 234618880
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00046913741223671016,
+      "loss": 3.2137,
+      "theoretical_loss": 4.270078549798071,
+      "tokens_seen": 234684416
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00046912738214643934,
+      "loss": 3.1062,
+      "theoretical_loss": 4.269933049053847,
+      "tokens_seen": 234749952
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004691173520561685,
+      "loss": 3.2631,
+      "theoretical_loss": 4.269787600293903,
+      "tokens_seen": 234815488
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004691073219658977,
+      "loss": 3.0201,
+      "theoretical_loss": 4.269642203485162,
+      "tokens_seen": 234881024
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004690972918756269,
+      "loss": 2.9759,
+      "theoretical_loss": 4.269496858594582,
+      "tokens_seen": 234946560
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004690872617853561,
+      "loss": 3.1686,
+      "theoretical_loss": 4.269351565589151,
+      "tokens_seen": 235012096
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00046907723169508525,
+      "loss": 3.1832,
+      "theoretical_loss": 4.269206324435885,
+      "tokens_seen": 235077632
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004690672016048145,
+      "loss": 3.1976,
+      "theoretical_loss": 4.26906113510183,
+      "tokens_seen": 235143168
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00046905717151454366,
+      "loss": 3.1711,
+      "theoretical_loss": 4.268915997554065,
+      "tokens_seen": 235208704
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00046904714142427284,
+      "loss": 3.105,
+      "theoretical_loss": 4.268770911759697,
+      "tokens_seen": 235274240
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.000469037111334002,
+      "loss": 2.9095,
+      "theoretical_loss": 4.268625877685863,
+      "tokens_seen": 235339776
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004690270812437312,
+      "loss": 3.0525,
+      "theoretical_loss": 4.268480895299731,
+      "tokens_seen": 235405312
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004690170511534604,
+      "loss": 3.2324,
+      "theoretical_loss": 4.268335964568497,
+      "tokens_seen": 235470848
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004690070210631896,
+      "loss": 3.1499,
+      "theoretical_loss": 4.26819108545939,
+      "tokens_seen": 235536384
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00046899699097291875,
+      "loss": 3.1792,
+      "theoretical_loss": 4.268046257939666,
+      "tokens_seen": 235601920
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.000468986960882648,
+      "loss": 3.2761,
+      "theoretical_loss": 4.2679014819766135,
+      "tokens_seen": 235667456
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004689769307923771,
+      "loss": 3.0686,
+      "theoretical_loss": 4.267756757537548,
+      "tokens_seen": 235732992
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00046896690070210635,
+      "loss": 3.2238,
+      "theoretical_loss": 4.267612084589816,
+      "tokens_seen": 235798528
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00046895687061183553,
+      "loss": 3.346,
+      "theoretical_loss": 4.267467463100796,
+      "tokens_seen": 235864064
+    },
+    {
+      "epoch": 0.07,
+      "objective/train/docs_used": 141147,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 3.236262321472168,
+      "objective/train/theoretical_loss": 4.267322893037893,
+      "objective/train/tokens_used": 256389600,
+      "theoretical_loss": 4.267322893037893,
+      "tokens_seen": 235929600
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004689468405215647,
+      "loss": 3.064,
+      "theoretical_loss": 4.267322893037893,
+      "tokens_seen": 235929600
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004689368104312939,
+      "loss": 3.0627,
+      "theoretical_loss": 4.267178374368543,
+      "tokens_seen": 235995136
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00046892678034102307,
+      "loss": 3.1095,
+      "theoretical_loss": 4.267033907060211,
+      "tokens_seen": 236060672
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00046891675025075225,
+      "loss": 3.0446,
+      "theoretical_loss": 4.266889491080392,
+      "tokens_seen": 236126208
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004689067201604815,
+      "loss": 3.1192,
+      "theoretical_loss": 4.2667451263966125,
+      "tokens_seen": 236191744
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004688966900702106,
+      "loss": 2.9901,
+      "theoretical_loss": 4.266600812976426,
+      "tokens_seen": 236257280
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00046888665997993985,
+      "loss": 3.1156,
+      "theoretical_loss": 4.2664565507874155,
+      "tokens_seen": 236322816
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00046887662988966903,
+      "loss": 3.1053,
+      "theoretical_loss": 4.266312339797195,
+      "tokens_seen": 236388352
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004688665997993982,
+      "loss": 3.0449,
+      "theoretical_loss": 4.266168179973407,
+      "tokens_seen": 236453888
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004688565697091274,
+      "loss": 3.2738,
+      "theoretical_loss": 4.266024071283725,
+      "tokens_seen": 236519424
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004688465396188566,
+      "loss": 2.8719,
+      "theoretical_loss": 4.265880013695849,
+      "tokens_seen": 236584960
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00046883650952858575,
+      "loss": 3.2135,
+      "theoretical_loss": 4.265736007177511,
+      "tokens_seen": 236650496
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.000468826479438315,
+      "loss": 3.0794,
+      "theoretical_loss": 4.265592051696469,
+      "tokens_seen": 236716032
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004688164493480441,
+      "loss": 2.9949,
+      "theoretical_loss": 4.265448147220515,
+      "tokens_seen": 236781568
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00046880641925777335,
+      "loss": 2.8953,
+      "theoretical_loss": 4.265304293717467,
+      "tokens_seen": 236847104
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004687963891675025,
+      "loss": 2.9984,
+      "theoretical_loss": 4.2651604911551715,
+      "tokens_seen": 236912640
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004687863590772317,
+      "loss": 3.054,
+      "theoretical_loss": 4.265016739501508,
+      "tokens_seen": 236978176
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004687763289869609,
+      "loss": 3.052,
+      "theoretical_loss": 4.264873038724381,
+      "tokens_seen": 237043712
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004687662988966901,
+      "loss": 3.0228,
+      "theoretical_loss": 4.264729388791726,
+      "tokens_seen": 237109248
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00046875626880641926,
+      "loss": 3.0213,
+      "theoretical_loss": 4.264585789671508,
+      "tokens_seen": 237174784
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00046874623871614844,
+      "loss": 3.2672,
+      "theoretical_loss": 4.26444224133172,
+      "tokens_seen": 237240320
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004687362086258776,
+      "loss": 3.0529,
+      "theoretical_loss": 4.264298743740384,
+      "tokens_seen": 237305856
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00046872617853560686,
+      "loss": 2.9217,
+      "theoretical_loss": 4.264155296865552,
+      "tokens_seen": 237371392
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.000468716148445336,
+      "loss": 3.0414,
+      "theoretical_loss": 4.264011900675305,
+      "tokens_seen": 237436928
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004687061183550652,
+      "loss": 3.0795,
+      "theoretical_loss": 4.26386855513775,
+      "tokens_seen": 237502464
+    },
+    {
+      "epoch": 0.07,
+      "objective/train/docs_used": 142360,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 2.9394404888153076,
+      "objective/train/theoretical_loss": 4.263725260221027,
+      "objective/train/tokens_used": 258028000,
+      "theoretical_loss": 4.263725260221027,
+      "tokens_seen": 237568000
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004686960882647944,
+      "loss": 3.1755,
+      "theoretical_loss": 4.263725260221027,
+      "tokens_seen": 237568000
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004686860581745236,
+      "loss": 3.2614,
+      "theoretical_loss": 4.263582015893302,
+      "tokens_seen": 237633536
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00046867602808425276,
+      "loss": 3.2817,
+      "theoretical_loss": 4.263438822122771,
+      "tokens_seen": 237699072
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00046866599799398194,
+      "loss": 3.1451,
+      "theoretical_loss": 4.263295678877658,
+      "tokens_seen": 237764608
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004686559679037111,
+      "loss": 3.2993,
+      "theoretical_loss": 4.263152586126217,
+      "tokens_seen": 237830144
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00046864593781344036,
+      "loss": 3.1979,
+      "theoretical_loss": 4.263009543836728,
+      "tokens_seen": 237895680
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004686359077231695,
+      "loss": 3.1446,
+      "theoretical_loss": 4.2628665519775035,
+      "tokens_seen": 237961216
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004686258776328987,
+      "loss": 3.3129,
+      "theoretical_loss": 4.262723610516881,
+      "tokens_seen": 238026752
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00046861584754262785,
+      "loss": 3.0843,
+      "theoretical_loss": 4.26258071942323,
+      "tokens_seen": 238092288
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004686058174523571,
+      "loss": 3.0462,
+      "theoretical_loss": 4.262437878664945,
+      "tokens_seen": 238157824
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00046859578736208626,
+      "loss": 3.1094,
+      "theoretical_loss": 4.262295088210451,
+      "tokens_seen": 238223360
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00046858575727181545,
+      "loss": 3.0836,
+      "theoretical_loss": 4.262152348028202,
+      "tokens_seen": 238288896
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004685757271815446,
+      "loss": 3.2288,
+      "theoretical_loss": 4.26200965808668,
+      "tokens_seen": 238354432
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004685656970912738,
+      "loss": 3.2874,
+      "theoretical_loss": 4.261867018354394,
+      "tokens_seen": 238419968
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.000468555667001003,
+      "loss": 3.0589,
+      "theoretical_loss": 4.261724428799884,
+      "tokens_seen": 238485504
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004685456369107322,
+      "loss": 3.0927,
+      "theoretical_loss": 4.261581889391715,
+      "tokens_seen": 238551040
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00046853560682046135,
+      "loss": 3.2002,
+      "theoretical_loss": 4.2614394000984825,
+      "tokens_seen": 238616576
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004685255767301906,
+      "loss": 3.1648,
+      "theoretical_loss": 4.261296960888811,
+      "tokens_seen": 238682112
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00046851554663991977,
+      "loss": 3.3789,
+      "theoretical_loss": 4.2611545717313515,
+      "tokens_seen": 238747648
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00046850551654964895,
+      "loss": 3.0458,
+      "theoretical_loss": 4.261012232594784,
+      "tokens_seen": 238813184
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00046849548645937813,
+      "loss": 2.9991,
+      "theoretical_loss": 4.260869943447817,
+      "tokens_seen": 238878720
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004684854563691073,
+      "loss": 3.0852,
+      "theoretical_loss": 4.260727704259187,
+      "tokens_seen": 238944256
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004684754262788365,
+      "loss": 3.372,
+      "theoretical_loss": 4.260585514997658,
+      "tokens_seen": 239009792
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00046846539618856573,
+      "loss": 3.0656,
+      "theoretical_loss": 4.260443375632022,
+      "tokens_seen": 239075328
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004684553660982949,
+      "loss": 3.048,
+      "theoretical_loss": 4.260301286131099,
+      "tokens_seen": 239140864
+    },
+    {
+      "epoch": 0.07,
+      "objective/train/docs_used": 143562,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 3.0513010025024414,
+      "objective/train/theoretical_loss": 4.26015924646374,
+      "objective/train/tokens_used": 259666400,
+      "theoretical_loss": 4.26015924646374,
+      "tokens_seen": 239206400
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004684453360080241,
+      "loss": 2.9484,
+      "theoretical_loss": 4.26015924646374,
+      "tokens_seen": 239206400
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00046843530591775327,
+      "loss": 3.4032,
+      "theoretical_loss": 4.26001725659882,
+      "tokens_seen": 239271936
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00046842527582748245,
+      "loss": 3.1223,
+      "theoretical_loss": 4.2598753165052425,
+      "tokens_seen": 239337472
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004684152457372117,
+      "loss": 3.1745,
+      "theoretical_loss": 4.259733426151941,
+      "tokens_seen": 239403008
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004684052156469408,
+      "loss": 3.0693,
+      "theoretical_loss": 4.259591585507876,
+      "tokens_seen": 239468544
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00046839518555667005,
+      "loss": 3.2599,
+      "theoretical_loss": 4.259449794542036,
+      "tokens_seen": 239534080
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00046838515546639923,
+      "loss": 3.051,
+      "theoretical_loss": 4.259308053223437,
+      "tokens_seen": 239599616
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004683751253761284,
+      "loss": 3.2346,
+      "theoretical_loss": 4.259166361521121,
+      "tokens_seen": 239665152
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004683650952858576,
+      "loss": 3.1637,
+      "theoretical_loss": 4.259024719404161,
+      "tokens_seen": 239730688
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004683550651955868,
+      "loss": 3.0852,
+      "theoretical_loss": 4.258883126841656,
+      "tokens_seen": 239796224
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00046834503510531595,
+      "loss": 3.0005,
+      "theoretical_loss": 4.258741583802734,
+      "tokens_seen": 239861760
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004683350050150452,
+      "loss": 3.2479,
+      "theoretical_loss": 4.258600090256548,
+      "tokens_seen": 239927296
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004683249749247743,
+      "loss": 2.7133,
+      "theoretical_loss": 4.258458646172281,
+      "tokens_seen": 239992832
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00046831494483450355,
+      "loss": 3.1386,
+      "theoretical_loss": 4.258317251519143,
+      "tokens_seen": 240058368
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004683049147442327,
+      "loss": 3.2625,
+      "theoretical_loss": 4.258175906266371,
+      "tokens_seen": 240123904
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004682948846539619,
+      "loss": 3.3216,
+      "theoretical_loss": 4.2580346103832305,
+      "tokens_seen": 240189440
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004682848545636911,
+      "loss": 3.1082,
+      "theoretical_loss": 4.257893363839013,
+      "tokens_seen": 240254976
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004682748244734203,
+      "loss": 2.8933,
+      "theoretical_loss": 4.257752166603041,
+      "tokens_seen": 240320512
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00046826479438314946,
+      "loss": 3.1841,
+      "theoretical_loss": 4.257611018644658,
+      "tokens_seen": 240386048
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00046825476429287864,
+      "loss": 2.9544,
+      "theoretical_loss": 4.257469919933241,
+      "tokens_seen": 240451584
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004682447342026078,
+      "loss": 3.0391,
+      "theoretical_loss": 4.257328870438193,
+      "tokens_seen": 240517120
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00046823470411233706,
+      "loss": 2.95,
+      "theoretical_loss": 4.257187870128942,
+      "tokens_seen": 240582656
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004682246740220662,
+      "loss": 3.0321,
+      "theoretical_loss": 4.257046918974945,
+      "tokens_seen": 240648192
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004682146439317954,
+      "loss": 3.1266,
+      "theoretical_loss": 4.2569060169456865,
+      "tokens_seen": 240713728
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004682046138415246,
+      "loss": 3.1535,
+      "theoretical_loss": 4.256765164010678,
+      "tokens_seen": 240779264
+    },
+    {
+      "epoch": 0.07,
+      "objective/train/docs_used": 144045,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 3.4141147136688232,
+      "objective/train/theoretical_loss": 4.256624360139456,
+      "objective/train/tokens_used": 261304800,
+      "theoretical_loss": 4.256624360139456,
+      "tokens_seen": 240844800
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004681945837512538,
+      "loss": 3.314,
+      "theoretical_loss": 4.256624360139456,
+      "tokens_seen": 240844800
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00046818455366098296,
+      "loss": 3.0936,
+      "theoretical_loss": 4.25648360530159,
+      "tokens_seen": 240910336
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00046817452357071214,
+      "loss": 2.9071,
+      "theoretical_loss": 4.25634289946667,
+      "tokens_seen": 240975872
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004681644934804413,
+      "loss": 3.0465,
+      "theoretical_loss": 4.256202242604317,
+      "tokens_seen": 241041408
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00046815446339017056,
+      "loss": 3.2584,
+      "theoretical_loss": 4.256061634684178,
+      "tokens_seen": 241106944
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004681444332998997,
+      "loss": 3.0502,
+      "theoretical_loss": 4.255921075675927,
+      "tokens_seen": 241172480
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004681344032096289,
+      "loss": 3.1567,
+      "theoretical_loss": 4.255780565549264,
+      "tokens_seen": 241238016
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00046812437311935805,
+      "loss": 3.0096,
+      "theoretical_loss": 4.255640104273919,
+      "tokens_seen": 241303552
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004681143430290873,
+      "loss": 2.7541,
+      "theoretical_loss": 4.255499691819647,
+      "tokens_seen": 241369088
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00046810431293881646,
+      "loss": 3.1162,
+      "theoretical_loss": 4.255359328156229,
+      "tokens_seen": 241434624
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00046809428284854565,
+      "loss": 3.1105,
+      "theoretical_loss": 4.255219013253475,
+      "tokens_seen": 241500160
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004680842527582748,
+      "loss": 3.1662,
+      "theoretical_loss": 4.255078747081219,
+      "tokens_seen": 241565696
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.000468074222668004,
+      "loss": 3.0573,
+      "theoretical_loss": 4.254938529609326,
+      "tokens_seen": 241631232
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004680641925777332,
+      "loss": 3.2336,
+      "theoretical_loss": 4.254798360807684,
+      "tokens_seen": 241696768
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004680541624874624,
+      "loss": 3.1564,
+      "theoretical_loss": 4.254658240646211,
+      "tokens_seen": 241762304
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00046804413239719155,
+      "loss": 3.0097,
+      "theoretical_loss": 4.254518169094847,
+      "tokens_seen": 241827840
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004680341023069208,
+      "loss": 3.0337,
+      "theoretical_loss": 4.254378146123563,
+      "tokens_seen": 241893376
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00046802407221664997,
+      "loss": 2.9549,
+      "theoretical_loss": 4.254238171702356,
+      "tokens_seen": 241958912
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00046801404212637915,
+      "loss": 2.976,
+      "theoretical_loss": 4.2540982458012495,
+      "tokens_seen": 242024448
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00046800401203610833,
+      "loss": 3.1121,
+      "theoretical_loss": 4.253958368390292,
+      "tokens_seen": 242089984
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004679939819458375,
+      "loss": 3.1283,
+      "theoretical_loss": 4.2538185394395605,
+      "tokens_seen": 242155520
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004679839518555667,
+      "loss": 3.2828,
+      "theoretical_loss": 4.253678758919156,
+      "tokens_seen": 242221056
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00046797392176529593,
+      "loss": 3.1396,
+      "theoretical_loss": 4.25353902679921,
+      "tokens_seen": 242286592
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00046796389167502505,
+      "loss": 3.2791,
+      "theoretical_loss": 4.253399343049878,
+      "tokens_seen": 242352128
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004679538615847543,
+      "loss": 3.0178,
+      "theoretical_loss": 4.253259707641341,
+      "tokens_seen": 242417664
+    },
+    {
+      "epoch": 0.07,
+      "objective/train/docs_used": 145400,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 2.983632802963257,
+      "objective/train/theoretical_loss": 4.253120120543809,
+      "objective/train/tokens_used": 262943200,
+      "theoretical_loss": 4.253120120543809,
+      "tokens_seen": 242483200
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004679438314944834,
+      "loss": 3.1131,
+      "theoretical_loss": 4.253120120543809,
+      "tokens_seen": 242483200
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00046793380140421265,
+      "loss": 3.1157,
+      "theoretical_loss": 4.252980581727517,
+      "tokens_seen": 242548736
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00046792377131394183,
+      "loss": 3.3695,
+      "theoretical_loss": 4.2528410911627255,
+      "tokens_seen": 242614272
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.000467913741223671,
+      "loss": 3.0132,
+      "theoretical_loss": 4.2527016488197225,
+      "tokens_seen": 242679808
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004679037111334002,
+      "loss": 3.2259,
+      "theoretical_loss": 4.252562254668824,
+      "tokens_seen": 242745344
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00046789368104312943,
+      "loss": 3.3286,
+      "theoretical_loss": 4.2524229086803675,
+      "tokens_seen": 242810880
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00046788365095285856,
+      "loss": 2.9194,
+      "theoretical_loss": 4.252283610824723,
+      "tokens_seen": 242876416
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004678736208625878,
+      "loss": 3.1186,
+      "theoretical_loss": 4.25214436107228,
+      "tokens_seen": 242941952
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004678635907723169,
+      "loss": 3.0458,
+      "theoretical_loss": 4.25200515939346,
+      "tokens_seen": 243007488
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00046785356068204616,
+      "loss": 3.1178,
+      "theoretical_loss": 4.251866005758707,
+      "tokens_seen": 243073024
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00046784353059177534,
+      "loss": 2.9132,
+      "theoretical_loss": 4.251726900138494,
+      "tokens_seen": 243138560
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004678335005015045,
+      "loss": 3.1386,
+      "theoretical_loss": 4.251587842503317,
+      "tokens_seen": 243204096
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004678234704112337,
+      "loss": 3.1516,
+      "theoretical_loss": 4.2514488328237,
+      "tokens_seen": 243269632
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004678134403209629,
+      "loss": 3.0271,
+      "theoretical_loss": 4.251309871070193,
+      "tokens_seen": 243335168
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00046780341023069206,
+      "loss": 3.1534,
+      "theoretical_loss": 4.25117095721337,
+      "tokens_seen": 243400704
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004677933801404213,
+      "loss": 3.2002,
+      "theoretical_loss": 4.251032091223834,
+      "tokens_seen": 243466240
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004677833500501504,
+      "loss": 3.2106,
+      "theoretical_loss": 4.250893273072213,
+      "tokens_seen": 243531776
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00046777331995987966,
+      "loss": 3.0027,
+      "theoretical_loss": 4.250754502729161,
+      "tokens_seen": 243597312
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004677632898696088,
+      "loss": 3.0599,
+      "theoretical_loss": 4.250615780165355,
+      "tokens_seen": 243662848
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.000467753259779338,
+      "loss": 2.7333,
+      "theoretical_loss": 4.250477105351502,
+      "tokens_seen": 243728384
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004677432296890672,
+      "loss": 3.1958,
+      "theoretical_loss": 4.250338478258333,
+      "tokens_seen": 243793920
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004677331995987964,
+      "loss": 3.2165,
+      "theoretical_loss": 4.250199898856604,
+      "tokens_seen": 243859456
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00046772316950852556,
+      "loss": 3.0671,
+      "theoretical_loss": 4.250061367117097,
+      "tokens_seen": 243924992
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004677131394182548,
+      "loss": 3.0396,
+      "theoretical_loss": 4.249922883010623,
+      "tokens_seen": 243990528
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.000467703109327984,
+      "loss": 3.0198,
+      "theoretical_loss": 4.249784446508014,
+      "tokens_seen": 244056064
+    },
+    {
+      "epoch": 0.07,
+      "objective/train/docs_used": 146117,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 3.144657850265503,
+      "objective/train/theoretical_loss": 4.24964605758013,
+      "objective/train/tokens_used": 264581600,
+      "theoretical_loss": 4.24964605758013,
+      "tokens_seen": 244121600
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00046769307923771316,
+      "loss": 3.141,
+      "theoretical_loss": 4.24964605758013,
+      "tokens_seen": 244121600
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00046768304914744234,
+      "loss": 3.0374,
+      "theoretical_loss": 4.249507716197858,
+      "tokens_seen": 244187136
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004676730190571715,
+      "loss": 2.9348,
+      "theoretical_loss": 4.249369422332107,
+      "tokens_seen": 244252672
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00046766298896690076,
+      "loss": 3.2713,
+      "theoretical_loss": 4.249231175953814,
+      "tokens_seen": 244318208
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004676529588766299,
+      "loss": 3.0096,
+      "theoretical_loss": 4.249092977033943,
+      "tokens_seen": 244383744
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004676429287863591,
+      "loss": 3.0521,
+      "theoretical_loss": 4.248954825543479,
+      "tokens_seen": 244449280
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00046763289869608825,
+      "loss": 3.049,
+      "theoretical_loss": 4.248816721453438,
+      "tokens_seen": 244514816
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004676228686058175,
+      "loss": 3.0529,
+      "theoretical_loss": 4.248678664734857,
+      "tokens_seen": 244580352
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00046761283851554666,
+      "loss": 2.8181,
+      "theoretical_loss": 4.248540655358801,
+      "tokens_seen": 244645888
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00046760280842527585,
+      "loss": 2.9386,
+      "theoretical_loss": 4.2484026932963594,
+      "tokens_seen": 244711424
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.000467592778335005,
+      "loss": 3.1262,
+      "theoretical_loss": 4.248264778518647,
+      "tokens_seen": 244776960
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004675827482447342,
+      "loss": 3.0133,
+      "theoretical_loss": 4.248126910996804,
+      "tokens_seen": 244842496
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004675727181544634,
+      "loss": 2.8972,
+      "theoretical_loss": 4.2479890907019975,
+      "tokens_seen": 244908032
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004675626880641926,
+      "loss": 2.9783,
+      "theoretical_loss": 4.2478513176054165,
+      "tokens_seen": 244973568
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00046755265797392175,
+      "loss": 3.045,
+      "theoretical_loss": 4.247713591678278,
+      "tokens_seen": 245039104
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.000467542627883651,
+      "loss": 2.8848,
+      "theoretical_loss": 4.247575912891824,
+      "tokens_seen": 245104640
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00046753259779338017,
+      "loss": 3.0541,
+      "theoretical_loss": 4.247438281217321,
+      "tokens_seen": 245170176
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00046752256770310935,
+      "loss": 3.0006,
+      "theoretical_loss": 4.247300696626062,
+      "tokens_seen": 245235712
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00046751253761283853,
+      "loss": 3.1447,
+      "theoretical_loss": 4.247163159089362,
+      "tokens_seen": 245301248
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004675025075225677,
+      "loss": 3.204,
+      "theoretical_loss": 4.247025668578564,
+      "tokens_seen": 245366784
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004674924774322969,
+      "loss": 3.2769,
+      "theoretical_loss": 4.246888225065036,
+      "tokens_seen": 245432320
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00046748244734202613,
+      "loss": 2.7871,
+      "theoretical_loss": 4.24675082852017,
+      "tokens_seen": 245497856
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00046747241725175525,
+      "loss": 2.8847,
+      "theoretical_loss": 4.246613478915384,
+      "tokens_seen": 245563392
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004674623871614845,
+      "loss": 3.0322,
+      "theoretical_loss": 4.246476176222119,
+      "tokens_seen": 245628928
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004674523570712136,
+      "loss": 3.3512,
+      "theoretical_loss": 4.246338920411843,
+      "tokens_seen": 245694464
+    },
+    {
+      "epoch": 0.07,
+      "objective/train/docs_used": 147054,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 3.2560789585113525,
+      "objective/train/theoretical_loss": 4.24620171145605,
+      "objective/train/tokens_used": 266220000,
+      "theoretical_loss": 4.24620171145605,
+      "tokens_seen": 245760000
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00046744232698094285,
+      "loss": 3.1654,
+      "theoretical_loss": 4.24620171145605,
+      "tokens_seen": 245760000
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00046743229689067203,
+      "loss": 2.945,
+      "theoretical_loss": 4.246064549326255,
+      "tokens_seen": 245825536
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004674222668004012,
+      "loss": 2.9211,
+      "theoretical_loss": 4.245927433994002,
+      "tokens_seen": 245891072
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004674122367101304,
+      "loss": 3.1739,
+      "theoretical_loss": 4.245790365430857,
+      "tokens_seen": 245956608
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00046740220661985963,
+      "loss": 3.2878,
+      "theoretical_loss": 4.245653343608412,
+      "tokens_seen": 246022144
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00046739217652958876,
+      "loss": 2.9528,
+      "theoretical_loss": 4.245516368498285,
+      "tokens_seen": 246087680
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.000467382146439318,
+      "loss": 3.1466,
+      "theoretical_loss": 4.245379440072116,
+      "tokens_seen": 246153216
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004673721163490471,
+      "loss": 2.9806,
+      "theoretical_loss": 4.245242558301572,
+      "tokens_seen": 246218752
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00046736208625877636,
+      "loss": 2.9725,
+      "theoretical_loss": 4.2451057231583444,
+      "tokens_seen": 246284288
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00046735205616850554,
+      "loss": 3.115,
+      "theoretical_loss": 4.24496893461415,
+      "tokens_seen": 246349824
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004673420260782347,
+      "loss": 3.1308,
+      "theoretical_loss": 4.244832192640725,
+      "tokens_seen": 246415360
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004673319959879639,
+      "loss": 3.1996,
+      "theoretical_loss": 4.24469549720984,
+      "tokens_seen": 246480896
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004673219658976931,
+      "loss": 3.2534,
+      "theoretical_loss": 4.244558848293281,
+      "tokens_seen": 246546432
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00046731193580742226,
+      "loss": 2.9928,
+      "theoretical_loss": 4.244422245862863,
+      "tokens_seen": 246611968
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004673019057171515,
+      "loss": 3.2244,
+      "theoretical_loss": 4.244285689890425,
+      "tokens_seen": 246677504
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004672918756268806,
+      "loss": 3.103,
+      "theoretical_loss": 4.244149180347831,
+      "tokens_seen": 246743040
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00046728184553660986,
+      "loss": 3.0084,
+      "theoretical_loss": 4.244012717206969,
+      "tokens_seen": 246808576
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.000467271815446339,
+      "loss": 3.0494,
+      "theoretical_loss": 4.24387630043975,
+      "tokens_seen": 246874112
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004672617853560682,
+      "loss": 3.2686,
+      "theoretical_loss": 4.243739930018112,
+      "tokens_seen": 246939648
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004672517552657974,
+      "loss": 3.0173,
+      "theoretical_loss": 4.243603605914016,
+      "tokens_seen": 247005184
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004672417251755266,
+      "loss": 3.2096,
+      "theoretical_loss": 4.243467328099448,
+      "tokens_seen": 247070720
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00046723169508525576,
+      "loss": 3.3022,
+      "theoretical_loss": 4.2433310965464175,
+      "tokens_seen": 247136256
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.000467221664994985,
+      "loss": 3.2016,
+      "theoretical_loss": 4.24319491122696,
+      "tokens_seen": 247201792
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004672116349047141,
+      "loss": 3.0706,
+      "theoretical_loss": 4.243058772113131,
+      "tokens_seen": 247267328
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00046720160481444336,
+      "loss": 3.0435,
+      "theoretical_loss": 4.242922679177017,
+      "tokens_seen": 247332864
+    },
+    {
+      "epoch": 0.07,
+      "objective/train/docs_used": 147484,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 3.0415401458740234,
+      "objective/train/theoretical_loss": 4.2427866323907235,
+      "objective/train/tokens_used": 267858400,
+      "theoretical_loss": 4.2427866323907235,
+      "tokens_seen": 247398400
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004671915747241725,
+      "loss": 3.1482,
+      "theoretical_loss": 4.2427866323907235,
+      "tokens_seen": 247398400
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004671815446339017,
+      "loss": 3.0442,
+      "theoretical_loss": 4.242650631726383,
+      "tokens_seen": 247463936
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004671715145436309,
+      "loss": 3.1308,
+      "theoretical_loss": 4.24251467715615,
+      "tokens_seen": 247529472
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004671614844533601,
+      "loss": 3.0189,
+      "theoretical_loss": 4.2423787686522045,
+      "tokens_seen": 247595008
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046715145436308927,
+      "loss": 3.2769,
+      "theoretical_loss": 4.2422429061867515,
+      "tokens_seen": 247660544
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046714142427281845,
+      "loss": 3.1076,
+      "theoretical_loss": 4.242107089732018,
+      "tokens_seen": 247726080
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046713139418254763,
+      "loss": 2.9247,
+      "theoretical_loss": 4.241971319260257,
+      "tokens_seen": 247791616
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046712136409227686,
+      "loss": 3.0152,
+      "theoretical_loss": 4.241835594743743,
+      "tokens_seen": 247857152
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.000467111334002006,
+      "loss": 3.1742,
+      "theoretical_loss": 4.241699916154779,
+      "tokens_seen": 247922688
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004671013039117352,
+      "loss": 2.9643,
+      "theoretical_loss": 4.241564283465689,
+      "tokens_seen": 247988224
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046709127382146435,
+      "loss": 3.1792,
+      "theoretical_loss": 4.241428696648818,
+      "tokens_seen": 248053760
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004670812437311936,
+      "loss": 2.9589,
+      "theoretical_loss": 4.241293155676542,
+      "tokens_seen": 248119296
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046707121364092277,
+      "loss": 2.9542,
+      "theoretical_loss": 4.241157660521255,
+      "tokens_seen": 248184832
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046706118355065195,
+      "loss": 3.0795,
+      "theoretical_loss": 4.241022211155379,
+      "tokens_seen": 248250368
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046705115346038113,
+      "loss": 2.8946,
+      "theoretical_loss": 4.240886807551356,
+      "tokens_seen": 248315904
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046704112337011037,
+      "loss": 3.1148,
+      "theoretical_loss": 4.240751449681655,
+      "tokens_seen": 248381440
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004670310932798395,
+      "loss": 3.2406,
+      "theoretical_loss": 4.240616137518767,
+      "tokens_seen": 248446976
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046702106318956873,
+      "loss": 3.0567,
+      "theoretical_loss": 4.240480871035208,
+      "tokens_seen": 248512512
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046701103309929786,
+      "loss": 3.0778,
+      "theoretical_loss": 4.240345650203517,
+      "tokens_seen": 248578048
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004670010030090271,
+      "loss": 3.1317,
+      "theoretical_loss": 4.240210474996257,
+      "tokens_seen": 248643584
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004669909729187563,
+      "loss": 2.8978,
+      "theoretical_loss": 4.240075345386014,
+      "tokens_seen": 248709120
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046698094282848545,
+      "loss": 3.1967,
+      "theoretical_loss": 4.239940261345399,
+      "tokens_seen": 248774656
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046697091273821464,
+      "loss": 3.2585,
+      "theoretical_loss": 4.239805222847047,
+      "tokens_seen": 248840192
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004669608826479438,
+      "loss": 3.2483,
+      "theoretical_loss": 4.239670229863615,
+      "tokens_seen": 248905728
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046695085255767305,
+      "loss": 3.1135,
+      "theoretical_loss": 4.2395352823677825,
+      "tokens_seen": 248971264
+    },
+    {
+      "epoch": 0.08,
+      "objective/train/docs_used": 148704,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 2.5283594131469727,
+      "objective/train/theoretical_loss": 4.239400380332256,
+      "objective/train/tokens_used": 269496800,
+      "theoretical_loss": 4.239400380332256,
+      "tokens_seen": 249036800
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046694082246740223,
+      "loss": 2.9646,
+      "theoretical_loss": 4.239400380332256,
+      "tokens_seen": 249036800
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004669307923771314,
+      "loss": 3.1003,
+      "theoretical_loss": 4.239265523729765,
+      "tokens_seen": 249102336
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004669207622868606,
+      "loss": 3.0558,
+      "theoretical_loss": 4.239130712533059,
+      "tokens_seen": 249167872
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046691073219658983,
+      "loss": 3.2011,
+      "theoretical_loss": 4.2389959467149145,
+      "tokens_seen": 249233408
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046690070210631896,
+      "loss": 3.1159,
+      "theoretical_loss": 4.238861226248131,
+      "tokens_seen": 249298944
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004668906720160482,
+      "loss": 2.9774,
+      "theoretical_loss": 4.238726551105531,
+      "tokens_seen": 249364480
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004668806419257773,
+      "loss": 2.8515,
+      "theoretical_loss": 4.2385919212599585,
+      "tokens_seen": 249430016
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046687061183550656,
+      "loss": 3.0703,
+      "theoretical_loss": 4.238457336684284,
+      "tokens_seen": 249495552
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046686058174523574,
+      "loss": 3.0331,
+      "theoretical_loss": 4.2383227973514,
+      "tokens_seen": 249561088
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004668505516549649,
+      "loss": 2.9647,
+      "theoretical_loss": 4.238188303234223,
+      "tokens_seen": 249626624
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004668405215646941,
+      "loss": 3.195,
+      "theoretical_loss": 4.23805385430569,
+      "tokens_seen": 249692160
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004668304914744233,
+      "loss": 3.1549,
+      "theoretical_loss": 4.237919450538767,
+      "tokens_seen": 249757696
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046682046138415246,
+      "loss": 3.257,
+      "theoretical_loss": 4.237785091906437,
+      "tokens_seen": 249823232
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004668104312938817,
+      "loss": 3.1791,
+      "theoretical_loss": 4.23765077838171,
+      "tokens_seen": 249888768
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004668004012036108,
+      "loss": 3.0966,
+      "theoretical_loss": 4.237516509937617,
+      "tokens_seen": 249954304
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046679037111334006,
+      "loss": 3.0634,
+      "theoretical_loss": 4.2373822865472155,
+      "tokens_seen": 250019840
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004667803410230692,
+      "loss": 3.2033,
+      "theoretical_loss": 4.237248108183583,
+      "tokens_seen": 250085376
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004667703109327984,
+      "loss": 3.0081,
+      "theoretical_loss": 4.237113974819821,
+      "tokens_seen": 250150912
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004667602808425276,
+      "loss": 2.9963,
+      "theoretical_loss": 4.236979886429055,
+      "tokens_seen": 250216448
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004667502507522568,
+      "loss": 3.0911,
+      "theoretical_loss": 4.236845842984433,
+      "tokens_seen": 250281984
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046674022066198596,
+      "loss": 3.1609,
+      "theoretical_loss": 4.236711844459125,
+      "tokens_seen": 250347520
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004667301905717152,
+      "loss": 2.8868,
+      "theoretical_loss": 4.236577890826326,
+      "tokens_seen": 250413056
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004667201604814443,
+      "loss": 2.8415,
+      "theoretical_loss": 4.236443982059253,
+      "tokens_seen": 250478592
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046671013039117356,
+      "loss": 3.1447,
+      "theoretical_loss": 4.236310118131144,
+      "tokens_seen": 250544128
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004667001003009027,
+      "loss": 2.6377,
+      "theoretical_loss": 4.236176299015265,
+      "tokens_seen": 250609664
+    },
+    {
+      "epoch": 0.08,
+      "objective/train/docs_used": 149300,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 3.2010979652404785,
+      "objective/train/theoretical_loss": 4.2360425246849,
+      "objective/train/tokens_used": 271135200,
+      "theoretical_loss": 4.2360425246849,
+      "tokens_seen": 250675200
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004666900702106319,
+      "loss": 3.063,
+      "theoretical_loss": 4.2360425246849,
+      "tokens_seen": 250675200
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004666800401203611,
+      "loss": 3.1872,
+      "theoretical_loss": 4.2359087951133585,
+      "tokens_seen": 250740736
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004666700100300903,
+      "loss": 3.1641,
+      "theoretical_loss": 4.235775110273972,
+      "tokens_seen": 250806272
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046665997993981947,
+      "loss": 3.2843,
+      "theoretical_loss": 4.235641470140094,
+      "tokens_seen": 250871808
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046664994984954865,
+      "loss": 3.0174,
+      "theoretical_loss": 4.235507874685103,
+      "tokens_seen": 250937344
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046663991975927783,
+      "loss": 2.8743,
+      "theoretical_loss": 4.235374323882399,
+      "tokens_seen": 251002880
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046662988966900706,
+      "loss": 3.1391,
+      "theoretical_loss": 4.235240817705406,
+      "tokens_seen": 251068416
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004666198595787362,
+      "loss": 3.0944,
+      "theoretical_loss": 4.235107356127567,
+      "tokens_seen": 251133952
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046660982948846543,
+      "loss": 3.2513,
+      "theoretical_loss": 4.234973939122352,
+      "tokens_seen": 251199488
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046659979939819455,
+      "loss": 2.703,
+      "theoretical_loss": 4.2348405666632525,
+      "tokens_seen": 251265024
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004665897693079238,
+      "loss": 2.8943,
+      "theoretical_loss": 4.234707238723782,
+      "tokens_seen": 251330560
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046657973921765297,
+      "loss": 2.98,
+      "theoretical_loss": 4.234573955277476,
+      "tokens_seen": 251396096
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046656970912738215,
+      "loss": 3.1948,
+      "theoretical_loss": 4.234440716297895,
+      "tokens_seen": 251461632
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046655967903711133,
+      "loss": 3.2021,
+      "theoretical_loss": 4.234307521758619,
+      "tokens_seen": 251527168
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046654964894684057,
+      "loss": 3.0329,
+      "theoretical_loss": 4.234174371633253,
+      "tokens_seen": 251592704
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004665396188565697,
+      "loss": 3.0375,
+      "theoretical_loss": 4.234041265895425,
+      "tokens_seen": 251658240
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046652958876629893,
+      "loss": 2.9757,
+      "theoretical_loss": 4.233908204518783,
+      "tokens_seen": 251723776
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046651955867602806,
+      "loss": 2.956,
+      "theoretical_loss": 4.233775187476998,
+      "tokens_seen": 251789312
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004665095285857573,
+      "loss": 3.0873,
+      "theoretical_loss": 4.233642214743766,
+      "tokens_seen": 251854848
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004664994984954865,
+      "loss": 3.1756,
+      "theoretical_loss": 4.2335092862928025,
+      "tokens_seen": 251920384
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046648946840521565,
+      "loss": 3.047,
+      "theoretical_loss": 4.233376402097846,
+      "tokens_seen": 251985920
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046647943831494484,
+      "loss": 2.7,
+      "theoretical_loss": 4.233243562132661,
+      "tokens_seen": 252051456
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.000466469408224674,
+      "loss": 2.8238,
+      "theoretical_loss": 4.233110766371029,
+      "tokens_seen": 252116992
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004664593781344032,
+      "loss": 3.0103,
+      "theoretical_loss": 4.232978014786756,
+      "tokens_seen": 252182528
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046644934804413243,
+      "loss": 2.9662,
+      "theoretical_loss": 4.2328453073536725,
+      "tokens_seen": 252248064
+    },
+    {
+      "epoch": 0.08,
+      "objective/train/docs_used": 150610,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 2.978238582611084,
+      "objective/train/theoretical_loss": 4.232712644045627,
+      "objective/train/tokens_used": 272773600,
+      "theoretical_loss": 4.232712644045627,
+      "tokens_seen": 252313600
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046643931795386156,
+      "loss": 3.1531,
+      "theoretical_loss": 4.232712644045627,
+      "tokens_seen": 252313600
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004664292878635908,
+      "loss": 3.2135,
+      "theoretical_loss": 4.232580024836494,
+      "tokens_seen": 252379136
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004664192577733199,
+      "loss": 3.0233,
+      "theoretical_loss": 4.232447449700169,
+      "tokens_seen": 252444672
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046640922768304916,
+      "loss": 2.8671,
+      "theoretical_loss": 4.23231491861057,
+      "tokens_seen": 252510208
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046639919759277834,
+      "loss": 3.0974,
+      "theoretical_loss": 4.232182431541636,
+      "tokens_seen": 252575744
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004663891675025075,
+      "loss": 3.0586,
+      "theoretical_loss": 4.232049988467328,
+      "tokens_seen": 252641280
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004663791374122367,
+      "loss": 2.9412,
+      "theoretical_loss": 4.2319175893616325,
+      "tokens_seen": 252706816
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046636910732196594,
+      "loss": 2.9511,
+      "theoretical_loss": 4.231785234198555,
+      "tokens_seen": 252772352
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046635907723169506,
+      "loss": 2.9213,
+      "theoretical_loss": 4.231652922952122,
+      "tokens_seen": 252837888
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004663490471414243,
+      "loss": 2.9723,
+      "theoretical_loss": 4.231520655596388,
+      "tokens_seen": 252903424
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004663390170511534,
+      "loss": 3.0252,
+      "theoretical_loss": 4.231388432105421,
+      "tokens_seen": 252968960
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046632898696088266,
+      "loss": 3.1199,
+      "theoretical_loss": 4.231256252453317,
+      "tokens_seen": 253034496
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046631895687061184,
+      "loss": 3.194,
+      "theoretical_loss": 4.231124116614194,
+      "tokens_seen": 253100032
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.000466308926780341,
+      "loss": 3.1208,
+      "theoretical_loss": 4.230992024562189,
+      "tokens_seen": 253165568
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004662988966900702,
+      "loss": 2.9426,
+      "theoretical_loss": 4.230859976271464,
+      "tokens_seen": 253231104
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004662888665997994,
+      "loss": 3.0458,
+      "theoretical_loss": 4.2307279717162,
+      "tokens_seen": 253296640
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046627883650952857,
+      "loss": 3.193,
+      "theoretical_loss": 4.2305960108706016,
+      "tokens_seen": 253362176
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004662688064192578,
+      "loss": 3.1534,
+      "theoretical_loss": 4.230464093708894,
+      "tokens_seen": 253427712
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046625877632898693,
+      "loss": 3.1943,
+      "theoretical_loss": 4.230332220205328,
+      "tokens_seen": 253493248
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046624874623871616,
+      "loss": 3.0607,
+      "theoretical_loss": 4.230200390334172,
+      "tokens_seen": 253558784
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046623871614844535,
+      "loss": 2.8528,
+      "theoretical_loss": 4.230068604069717,
+      "tokens_seen": 253624320
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004662286860581745,
+      "loss": 2.9088,
+      "theoretical_loss": 4.2299368613862764,
+      "tokens_seen": 253689856
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004662186559679037,
+      "loss": 3.2573,
+      "theoretical_loss": 4.229805162258187,
+      "tokens_seen": 253755392
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004662086258776329,
+      "loss": 2.9836,
+      "theoretical_loss": 4.229673506659805,
+      "tokens_seen": 253820928
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004661985957873621,
+      "loss": 3.1131,
+      "theoretical_loss": 4.229541894565508,
+      "tokens_seen": 253886464
+    },
+    {
+      "epoch": 0.08,
+      "objective/train/docs_used": 151678,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 2.8198091983795166,
+      "objective/train/theoretical_loss": 4.229410325949698,
+      "objective/train/tokens_used": 274412000,
+      "theoretical_loss": 4.229410325949698,
+      "tokens_seen": 253952000
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004661885656970913,
+      "loss": 2.9257,
+      "theoretical_loss": 4.229410325949698,
+      "tokens_seen": 253952000
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004661785356068205,
+      "loss": 3.1713,
+      "theoretical_loss": 4.229278800786796,
+      "tokens_seen": 254017536
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046616850551654967,
+      "loss": 2.9401,
+      "theoretical_loss": 4.229147319051247,
+      "tokens_seen": 254083072
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046615847542627885,
+      "loss": 2.8864,
+      "theoretical_loss": 4.2290158807175136,
+      "tokens_seen": 254148608
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046614844533600803,
+      "loss": 3.1424,
+      "theoretical_loss": 4.228884485760086,
+      "tokens_seen": 254214144
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046613841524573727,
+      "loss": 3.2015,
+      "theoretical_loss": 4.22875313415347,
+      "tokens_seen": 254279680
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004661283851554664,
+      "loss": 3.1367,
+      "theoretical_loss": 4.228621825872198,
+      "tokens_seen": 254345216
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046611835506519563,
+      "loss": 2.9269,
+      "theoretical_loss": 4.228490560890819,
+      "tokens_seen": 254410752
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046610832497492475,
+      "loss": 3.1191,
+      "theoretical_loss": 4.228359339183908,
+      "tokens_seen": 254476288
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.000466098294884654,
+      "loss": 3.2313,
+      "theoretical_loss": 4.228228160726059,
+      "tokens_seen": 254541824
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046608826479438317,
+      "loss": 3.2163,
+      "theoretical_loss": 4.228097025491887,
+      "tokens_seen": 254607360
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046607823470411235,
+      "loss": 3.0566,
+      "theoretical_loss": 4.22796593345603,
+      "tokens_seen": 254672896
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046606820461384153,
+      "loss": 2.9742,
+      "theoretical_loss": 4.227834884593148,
+      "tokens_seen": 254738432
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046605817452357077,
+      "loss": 2.6936,
+      "theoretical_loss": 4.22770387887792,
+      "tokens_seen": 254803968
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004660481444332999,
+      "loss": 2.989,
+      "theoretical_loss": 4.227572916285046,
+      "tokens_seen": 254869504
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046603811434302913,
+      "loss": 3.1684,
+      "theoretical_loss": 4.227441996789253,
+      "tokens_seen": 254935040
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046602808425275826,
+      "loss": 3.0604,
+      "theoretical_loss": 4.227311120365281,
+      "tokens_seen": 255000576
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004660180541624875,
+      "loss": 3.3224,
+      "theoretical_loss": 4.227180286987897,
+      "tokens_seen": 255066112
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004660080240722167,
+      "loss": 3.2097,
+      "theoretical_loss": 4.227049496631889,
+      "tokens_seen": 255131648
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046599799398194586,
+      "loss": 3.1686,
+      "theoretical_loss": 4.226918749272063,
+      "tokens_seen": 255197184
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046598796389167504,
+      "loss": 3.0575,
+      "theoretical_loss": 4.226788044883248,
+      "tokens_seen": 255262720
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004659779338014042,
+      "loss": 3.1225,
+      "theoretical_loss": 4.226657383440297,
+      "tokens_seen": 255328256
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004659679037111334,
+      "loss": 2.9431,
+      "theoretical_loss": 4.226526764918079,
+      "tokens_seen": 255393792
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046595787362086263,
+      "loss": 3.1373,
+      "theoretical_loss": 4.2263961892914885,
+      "tokens_seen": 255459328
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046594784353059176,
+      "loss": 3.2497,
+      "theoretical_loss": 4.226265656535437,
+      "tokens_seen": 255524864
+    },
+    {
+      "epoch": 0.08,
+      "objective/train/docs_used": 152377,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 3.094141960144043,
+      "objective/train/theoretical_loss": 4.226135166624862,
+      "objective/train/tokens_used": 276050400,
+      "theoretical_loss": 4.226135166624862,
+      "tokens_seen": 255590400
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.000465937813440321,
+      "loss": 3.2235,
+      "theoretical_loss": 4.226135166624862,
+      "tokens_seen": 255590400
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004659277833500501,
+      "loss": 3.158,
+      "theoretical_loss": 4.226004719534718,
+      "tokens_seen": 255655936
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046591775325977936,
+      "loss": 2.9932,
+      "theoretical_loss": 4.225874315239982,
+      "tokens_seen": 255721472
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046590772316950854,
+      "loss": 2.9383,
+      "theoretical_loss": 4.225743953715653,
+      "tokens_seen": 255787008
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004658976930792377,
+      "loss": 3.0384,
+      "theoretical_loss": 4.225613634936749,
+      "tokens_seen": 255852544
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004658876629889669,
+      "loss": 3.1105,
+      "theoretical_loss": 4.22548335887831,
+      "tokens_seen": 255918080
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046587763289869614,
+      "loss": 3.0846,
+      "theoretical_loss": 4.225353125515398,
+      "tokens_seen": 255983616
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046586760280842526,
+      "loss": 3.013,
+      "theoretical_loss": 4.225222934823094,
+      "tokens_seen": 256049152
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004658575727181545,
+      "loss": 3.1509,
+      "theoretical_loss": 4.225092786776502,
+      "tokens_seen": 256114688
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004658475426278836,
+      "loss": 2.9779,
+      "theoretical_loss": 4.224962681350746,
+      "tokens_seen": 256180224
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046583751253761286,
+      "loss": 2.778,
+      "theoretical_loss": 4.224832618520969,
+      "tokens_seen": 256245760
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046582748244734204,
+      "loss": 3.263,
+      "theoretical_loss": 4.224702598262336,
+      "tokens_seen": 256311296
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004658174523570712,
+      "loss": 2.9598,
+      "theoretical_loss": 4.2245726205500365,
+      "tokens_seen": 256376832
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004658074222668004,
+      "loss": 3.1954,
+      "theoretical_loss": 4.224442685359275,
+      "tokens_seen": 256442368
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004657973921765296,
+      "loss": 2.9688,
+      "theoretical_loss": 4.22431279266528,
+      "tokens_seen": 256507904
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046578736208625877,
+      "loss": 3.1653,
+      "theoretical_loss": 4.2241829424433,
+      "tokens_seen": 256573440
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.000465777331995988,
+      "loss": 3.0042,
+      "theoretical_loss": 4.224053134668605,
+      "tokens_seen": 256638976
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046576730190571713,
+      "loss": 2.9824,
+      "theoretical_loss": 4.223923369316485,
+      "tokens_seen": 256704512
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046575727181544636,
+      "loss": 2.9607,
+      "theoretical_loss": 4.2237936463622505,
+      "tokens_seen": 256770048
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046574724172517555,
+      "loss": 3.0764,
+      "theoretical_loss": 4.223663965781233,
+      "tokens_seen": 256835584
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004657372116349047,
+      "loss": 3.1061,
+      "theoretical_loss": 4.2235343275487836,
+      "tokens_seen": 256901120
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004657271815446339,
+      "loss": 2.9893,
+      "theoretical_loss": 4.223404731640277,
+      "tokens_seen": 256966656
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004657171514543631,
+      "loss": 2.9472,
+      "theoretical_loss": 4.223275178031105,
+      "tokens_seen": 257032192
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046570712136409227,
+      "loss": 3.0962,
+      "theoretical_loss": 4.223145666696682,
+      "tokens_seen": 257097728
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004656970912738215,
+      "loss": 3.2783,
+      "theoretical_loss": 4.223016197612442,
+      "tokens_seen": 257163264
+    },
+    {
+      "epoch": 0.08,
+      "objective/train/docs_used": 153004,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 2.8444950580596924,
+      "objective/train/theoretical_loss": 4.222886770753841,
+      "objective/train/tokens_used": 277688800,
+      "theoretical_loss": 4.222886770753841,
+      "tokens_seen": 257228800
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046568706118355063,
+      "loss": 2.8758,
+      "theoretical_loss": 4.222886770753841,
+      "tokens_seen": 257228800
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046567703109327987,
+      "loss": 3.2012,
+      "theoretical_loss": 4.222757386096354,
+      "tokens_seen": 257294336
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.000465667001003009,
+      "loss": 3.0649,
+      "theoretical_loss": 4.222628043615476,
+      "tokens_seen": 257359872
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046565697091273823,
+      "loss": 3.1071,
+      "theoretical_loss": 4.2224987432867245,
+      "tokens_seen": 257425408
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004656469408224674,
+      "loss": 3.1137,
+      "theoretical_loss": 4.222369485085636,
+      "tokens_seen": 257490944
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004656369107321966,
+      "loss": 3.1091,
+      "theoretical_loss": 4.222240268987768,
+      "tokens_seen": 257556480
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004656268806419258,
+      "loss": 2.8566,
+      "theoretical_loss": 4.222111094968698,
+      "tokens_seen": 257622016
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046561685055165495,
+      "loss": 3.0556,
+      "theoretical_loss": 4.2219819630040245,
+      "tokens_seen": 257687552
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046560682046138414,
+      "loss": 3.1304,
+      "theoretical_loss": 4.2218528730693645,
+      "tokens_seen": 257753088
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046559679037111337,
+      "loss": 2.937,
+      "theoretical_loss": 4.221723825140359,
+      "tokens_seen": 257818624
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004655867602808425,
+      "loss": 2.9502,
+      "theoretical_loss": 4.221594819192665,
+      "tokens_seen": 257884160
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046557673019057173,
+      "loss": 3.1884,
+      "theoretical_loss": 4.221465855201963,
+      "tokens_seen": 257949696
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004655667001003009,
+      "loss": 3.1707,
+      "theoretical_loss": 4.221336933143952,
+      "tokens_seen": 258015232
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004655566700100301,
+      "loss": 2.9765,
+      "theoretical_loss": 4.221208052994352,
+      "tokens_seen": 258080768
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004655466399197593,
+      "loss": 3.1372,
+      "theoretical_loss": 4.221079214728903,
+      "tokens_seen": 258146304
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046553660982948846,
+      "loss": 3.0284,
+      "theoretical_loss": 4.2209504183233655,
+      "tokens_seen": 258211840
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046552657973921764,
+      "loss": 3.1995,
+      "theoretical_loss": 4.22082166375352,
+      "tokens_seen": 258277376
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004655165496489469,
+      "loss": 2.8978,
+      "theoretical_loss": 4.220692950995167,
+      "tokens_seen": 258342912
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.000465506519558676,
+      "loss": 3.0251,
+      "theoretical_loss": 4.220564280024126,
+      "tokens_seen": 258408448
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046549648946840524,
+      "loss": 2.9952,
+      "theoretical_loss": 4.22043565081624,
+      "tokens_seen": 258473984
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046548645937813436,
+      "loss": 3.1847,
+      "theoretical_loss": 4.220307063347369,
+      "tokens_seen": 258539520
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004654764292878636,
+      "loss": 2.9241,
+      "theoretical_loss": 4.220178517593393,
+      "tokens_seen": 258605056
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004654663991975928,
+      "loss": 2.8651,
+      "theoretical_loss": 4.220050013530215,
+      "tokens_seen": 258670592
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046545636910732196,
+      "loss": 2.9641,
+      "theoretical_loss": 4.219921551133754,
+      "tokens_seen": 258736128
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004654463390170512,
+      "loss": 3.1291,
+      "theoretical_loss": 4.219793130379952,
+      "tokens_seen": 258801664
+    },
+    {
+      "epoch": 0.08,
+      "objective/train/docs_used": 154245,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 3.155519485473633,
+      "objective/train/theoretical_loss": 4.21966475124477,
+      "objective/train/tokens_used": 279327200,
+      "theoretical_loss": 4.21966475124477,
+      "tokens_seen": 258867200
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004654363089267803,
+      "loss": 3.232,
+      "theoretical_loss": 4.21966475124477,
+      "tokens_seen": 258867200
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046542627883650956,
+      "loss": 3.1265,
+      "theoretical_loss": 4.21953641370419,
+      "tokens_seen": 258932736
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046541624874623874,
+      "loss": 2.8509,
+      "theoretical_loss": 4.21940811773421,
+      "tokens_seen": 258998272
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004654062186559679,
+      "loss": 3.0692,
+      "theoretical_loss": 4.219279863310855,
+      "tokens_seen": 259063808
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004653961885656971,
+      "loss": 2.9505,
+      "theoretical_loss": 4.219151650410162,
+      "tokens_seen": 259129344
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046538615847542634,
+      "loss": 3.049,
+      "theoretical_loss": 4.219023479008193,
+      "tokens_seen": 259194880
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046537612838515546,
+      "loss": 3.0422,
+      "theoretical_loss": 4.21889534908103,
+      "tokens_seen": 259260416
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004653660982948847,
+      "loss": 3.2427,
+      "theoretical_loss": 4.218767260604771,
+      "tokens_seen": 259325952
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004653560682046138,
+      "loss": 2.8318,
+      "theoretical_loss": 4.218639213555537,
+      "tokens_seen": 259391488
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046534603811434306,
+      "loss": 2.817,
+      "theoretical_loss": 4.218511207909467,
+      "tokens_seen": 259457024
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046533600802407224,
+      "loss": 2.9972,
+      "theoretical_loss": 4.218383243642723,
+      "tokens_seen": 259522560
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004653259779338014,
+      "loss": 2.8747,
+      "theoretical_loss": 4.2182553207314815,
+      "tokens_seen": 259588096
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004653159478435306,
+      "loss": 3.0103,
+      "theoretical_loss": 4.218127439151944,
+      "tokens_seen": 259653632
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004653059177532598,
+      "loss": 3.1678,
+      "theoretical_loss": 4.217999598880327,
+      "tokens_seen": 259719168
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046529588766298897,
+      "loss": 3.1776,
+      "theoretical_loss": 4.2178717998928725,
+      "tokens_seen": 259784704
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004652858575727182,
+      "loss": 3.0578,
+      "theoretical_loss": 4.217744042165835,
+      "tokens_seen": 259850240
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046527582748244733,
+      "loss": 3.0386,
+      "theoretical_loss": 4.217616325675493,
+      "tokens_seen": 259915776
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046526579739217656,
+      "loss": 2.9572,
+      "theoretical_loss": 4.2174886503981455,
+      "tokens_seen": 259981312
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046525576730190575,
+      "loss": 2.985,
+      "theoretical_loss": 4.217361016310109,
+      "tokens_seen": 260046848
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004652457372116349,
+      "loss": 3.0667,
+      "theoretical_loss": 4.217233423387719,
+      "tokens_seen": 260112384
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004652357071213641,
+      "loss": 3.0605,
+      "theoretical_loss": 4.217105871607333,
+      "tokens_seen": 260177920
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004652256770310933,
+      "loss": 3.354,
+      "theoretical_loss": 4.216978360945325,
+      "tokens_seen": 260243456
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046521564694082247,
+      "loss": 2.9401,
+      "theoretical_loss": 4.216850891378092,
+      "tokens_seen": 260308992
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004652056168505517,
+      "loss": 3.0811,
+      "theoretical_loss": 4.216723462882047,
+      "tokens_seen": 260374528
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046519558676028083,
+      "loss": 3.1081,
+      "theoretical_loss": 4.2165960754336265,
+      "tokens_seen": 260440064
+    },
+    {
+      "epoch": 0.08,
+      "objective/train/docs_used": 155441,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 2.7761213779449463,
+      "objective/train/theoretical_loss": 4.216468729009281,
+      "objective/train/tokens_used": 280965600,
+      "theoretical_loss": 4.216468729009281,
+      "tokens_seen": 260505600
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046518555667001007,
+      "loss": 2.6967,
+      "theoretical_loss": 4.216468729009281,
+      "tokens_seen": 260505600
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004651755265797392,
+      "loss": 3.038,
+      "theoretical_loss": 4.216341423585486,
+      "tokens_seen": 260571136
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046516549648946843,
+      "loss": 3.3396,
+      "theoretical_loss": 4.216214159138732,
+      "tokens_seen": 260636672
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004651554663991976,
+      "loss": 3.0875,
+      "theoretical_loss": 4.216086935645532,
+      "tokens_seen": 260702208
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004651454363089268,
+      "loss": 3.1269,
+      "theoretical_loss": 4.2159597530824175,
+      "tokens_seen": 260767744
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.000465135406218656,
+      "loss": 2.9654,
+      "theoretical_loss": 4.215832611425937,
+      "tokens_seen": 260833280
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046512537612838515,
+      "loss": 2.9985,
+      "theoretical_loss": 4.215705510652662,
+      "tokens_seen": 260898816
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046511534603811434,
+      "loss": 3.0659,
+      "theoretical_loss": 4.215578450739182,
+      "tokens_seen": 260964352
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046510531594784357,
+      "loss": 2.859,
+      "theoretical_loss": 4.215451431662104,
+      "tokens_seen": 261029888
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004650952858575727,
+      "loss": 3.0105,
+      "theoretical_loss": 4.215324453398057,
+      "tokens_seen": 261095424
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046508525576730193,
+      "loss": 3.0586,
+      "theoretical_loss": 4.215197515923687,
+      "tokens_seen": 261160960
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004650752256770311,
+      "loss": 3.0384,
+      "theoretical_loss": 4.21507061921566,
+      "tokens_seen": 261226496
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004650651955867603,
+      "loss": 2.9461,
+      "theoretical_loss": 4.214943763250662,
+      "tokens_seen": 261292032
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004650551654964895,
+      "loss": 3.0812,
+      "theoretical_loss": 4.214816948005396,
+      "tokens_seen": 261357568
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046504513540621866,
+      "loss": 2.9739,
+      "theoretical_loss": 4.214690173456588,
+      "tokens_seen": 261423104
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046503510531594784,
+      "loss": 3.2378,
+      "theoretical_loss": 4.214563439580979,
+      "tokens_seen": 261488640
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004650250752256771,
+      "loss": 2.9873,
+      "theoretical_loss": 4.214436746355331,
+      "tokens_seen": 261554176
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004650150451354062,
+      "loss": 3.0108,
+      "theoretical_loss": 4.214310093756427,
+      "tokens_seen": 261619712
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046500501504513544,
+      "loss": 2.9422,
+      "theoretical_loss": 4.214183481761064,
+      "tokens_seen": 261685248
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046499498495486456,
+      "loss": 3.1189,
+      "theoretical_loss": 4.214056910346063,
+      "tokens_seen": 261750784
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004649849548645938,
+      "loss": 3.1657,
+      "theoretical_loss": 4.2139303794882625,
+      "tokens_seen": 261816320
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.000464974924774323,
+      "loss": 2.7213,
+      "theoretical_loss": 4.213803889164518,
+      "tokens_seen": 261881856
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046496489468405216,
+      "loss": 2.8726,
+      "theoretical_loss": 4.213677439351708,
+      "tokens_seen": 261947392
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046495486459378134,
+      "loss": 2.7336,
+      "theoretical_loss": 4.213551030026725,
+      "tokens_seen": 262012928
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004649448345035105,
+      "loss": 2.9864,
+      "theoretical_loss": 4.213424661166485,
+      "tokens_seen": 262078464
+    },
+    {
+      "epoch": 0.08,
+      "objective/train/docs_used": 156138,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 2.797309398651123,
+      "objective/train/theoretical_loss": 4.21329833274792,
+      "objective/train/tokens_used": 282604000,
+      "theoretical_loss": 4.21329833274792,
+      "tokens_seen": 262144000
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004649348044132397,
+      "loss": 2.5665,
+      "theoretical_loss": 4.21329833274792,
+      "tokens_seen": 262144000
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046492477432296894,
+      "loss": 3.0384,
+      "theoretical_loss": 4.213172044747982,
+      "tokens_seen": 262209536
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046491474423269807,
+      "loss": 2.8508,
+      "theoretical_loss": 4.213045797143643,
+      "tokens_seen": 262275072
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004649047141424273,
+      "loss": 2.933,
+      "theoretical_loss": 4.212919589911891,
+      "tokens_seen": 262340608
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004648946840521565,
+      "loss": 2.8909,
+      "theoretical_loss": 4.212793423029735,
+      "tokens_seen": 262406144
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046488465396188566,
+      "loss": 3.0675,
+      "theoretical_loss": 4.212667296474204,
+      "tokens_seen": 262471680
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046487462387161485,
+      "loss": 3.0812,
+      "theoretical_loss": 4.2125412102223425,
+      "tokens_seen": 262537216
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.000464864593781344,
+      "loss": 3.2516,
+      "theoretical_loss": 4.212415164251215,
+      "tokens_seen": 262602752
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004648545636910732,
+      "loss": 2.9727,
+      "theoretical_loss": 4.212289158537908,
+      "tokens_seen": 262668288
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046484453360080244,
+      "loss": 3.2213,
+      "theoretical_loss": 4.212163193059521,
+      "tokens_seen": 262733824
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046483450351053157,
+      "loss": 3.0169,
+      "theoretical_loss": 4.2120372677931766,
+      "tokens_seen": 262799360
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004648244734202608,
+      "loss": 3.2831,
+      "theoretical_loss": 4.211911382716014,
+      "tokens_seen": 262864896
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046481444332998993,
+      "loss": 2.9297,
+      "theoretical_loss": 4.211785537805193,
+      "tokens_seen": 262930432
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046480441323971917,
+      "loss": 3.0726,
+      "theoretical_loss": 4.211659733037891,
+      "tokens_seen": 262995968
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046479438314944835,
+      "loss": 3.0694,
+      "theoretical_loss": 4.2115339683913025,
+      "tokens_seen": 263061504
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046478435305917753,
+      "loss": 2.8509,
+      "theoretical_loss": 4.211408243842643,
+      "tokens_seen": 263127040
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004647743229689067,
+      "loss": 3.0094,
+      "theoretical_loss": 4.2112825593691445,
+      "tokens_seen": 263192576
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046476429287863595,
+      "loss": 3.1927,
+      "theoretical_loss": 4.21115691494806,
+      "tokens_seen": 263258112
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004647542627883651,
+      "loss": 3.0788,
+      "theoretical_loss": 4.211031310556661,
+      "tokens_seen": 263323648
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004647442326980943,
+      "loss": 2.9797,
+      "theoretical_loss": 4.210905746172235,
+      "tokens_seen": 263389184
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046473420260782344,
+      "loss": 2.7703,
+      "theoretical_loss": 4.210780221772088,
+      "tokens_seen": 263454720
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046472417251755267,
+      "loss": 2.9913,
+      "theoretical_loss": 4.210654737333549,
+      "tokens_seen": 263520256
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004647141424272819,
+      "loss": 3.0574,
+      "theoretical_loss": 4.21052929283396,
+      "tokens_seen": 263585792
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046470411233701103,
+      "loss": 2.9517,
+      "theoretical_loss": 4.210403888250685,
+      "tokens_seen": 263651328
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046469408224674027,
+      "loss": 2.9469,
+      "theoretical_loss": 4.210278523561106,
+      "tokens_seen": 263716864
+    },
+    {
+      "epoch": 0.08,
+      "objective/train/docs_used": 157246,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 2.921288251876831,
+      "objective/train/theoretical_loss": 4.2101531987426215,
+      "objective/train/tokens_used": 284242400,
+      "theoretical_loss": 4.2101531987426215,
+      "tokens_seen": 263782400
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004646840521564694,
+      "loss": 3.1574,
+      "theoretical_loss": 4.2101531987426215,
+      "tokens_seen": 263782400
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046467402206619863,
+      "loss": 2.9865,
+      "theoretical_loss": 4.210027913772651,
+      "tokens_seen": 263847936
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004646639919759278,
+      "loss": 2.9807,
+      "theoretical_loss": 4.209902668628629,
+      "tokens_seen": 263913472
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.000464653961885657,
+      "loss": 3.0513,
+      "theoretical_loss": 4.209777463288012,
+      "tokens_seen": 263979008
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004646439317953862,
+      "loss": 2.8659,
+      "theoretical_loss": 4.209652297728274,
+      "tokens_seen": 264044544
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046463390170511535,
+      "loss": 3.1063,
+      "theoretical_loss": 4.209527171926906,
+      "tokens_seen": 264110080
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046462387161484454,
+      "loss": 2.9362,
+      "theoretical_loss": 4.209402085861418,
+      "tokens_seen": 264175616
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046461384152457377,
+      "loss": 3.1464,
+      "theoretical_loss": 4.209277039509338,
+      "tokens_seen": 264241152
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004646038114343029,
+      "loss": 3.0352,
+      "theoretical_loss": 4.209152032848213,
+      "tokens_seen": 264306688
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046459378134403213,
+      "loss": 2.7659,
+      "theoretical_loss": 4.209027065855607,
+      "tokens_seen": 264372224
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004645837512537613,
+      "loss": 2.8415,
+      "theoretical_loss": 4.208902138509105,
+      "tokens_seen": 264437760
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004645737211634905,
+      "loss": 3.2915,
+      "theoretical_loss": 4.208777250786305,
+      "tokens_seen": 264503296
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004645636910732197,
+      "loss": 3.0093,
+      "theoretical_loss": 4.20865240266483,
+      "tokens_seen": 264568832
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046455366098294886,
+      "loss": 3.0984,
+      "theoretical_loss": 4.208527594122316,
+      "tokens_seen": 264634368
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046454363089267804,
+      "loss": 3.171,
+      "theoretical_loss": 4.208402825136419,
+      "tokens_seen": 264699904
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004645336008024073,
+      "loss": 3.2183,
+      "theoretical_loss": 4.208278095684813,
+      "tokens_seen": 264765440
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004645235707121364,
+      "loss": 3.1366,
+      "theoretical_loss": 4.20815340574519,
+      "tokens_seen": 264830976
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046451354062186564,
+      "loss": 2.9705,
+      "theoretical_loss": 4.20802875529526,
+      "tokens_seen": 264896512
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046450351053159476,
+      "loss": 3.2497,
+      "theoretical_loss": 4.207904144312752,
+      "tokens_seen": 264962048
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.000464493480441324,
+      "loss": 3.1657,
+      "theoretical_loss": 4.2077795727754115,
+      "tokens_seen": 265027584
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004644834503510532,
+      "loss": 2.869,
+      "theoretical_loss": 4.207655040661004,
+      "tokens_seen": 265093120
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046447342026078236,
+      "loss": 2.8442,
+      "theoretical_loss": 4.207530547947311,
+      "tokens_seen": 265158656
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046446339017051154,
+      "loss": 3.1521,
+      "theoretical_loss": 4.207406094612133,
+      "tokens_seen": 265224192
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004644533600802407,
+      "loss": 2.845,
+      "theoretical_loss": 4.207281680633289,
+      "tokens_seen": 265289728
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004644433299899699,
+      "loss": 2.7843,
+      "theoretical_loss": 4.207157305988615,
+      "tokens_seen": 265355264
+    },
+    {
+      "epoch": 0.08,
+      "objective/train/docs_used": 157714,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 2.635972738265991,
+      "objective/train/theoretical_loss": 4.207032970655965,
+      "objective/train/tokens_used": 285880800,
+      "theoretical_loss": 4.207032970655965,
+      "tokens_seen": 265420800
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046443329989969914,
+      "loss": 2.9511,
+      "theoretical_loss": 4.207032970655965,
+      "tokens_seen": 265420800
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046442326980942827,
+      "loss": 3.0488,
+      "theoretical_loss": 4.206908674613213,
+      "tokens_seen": 265486336
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004644132397191575,
+      "loss": 2.7703,
+      "theoretical_loss": 4.2067844178382465,
+      "tokens_seen": 265551872
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004644032096288867,
+      "loss": 2.9723,
+      "theoretical_loss": 4.206660200308975,
+      "tokens_seen": 265617408
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046439317953861586,
+      "loss": 2.896,
+      "theoretical_loss": 4.206536022003325,
+      "tokens_seen": 265682944
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046438314944834505,
+      "loss": 3.1176,
+      "theoretical_loss": 4.206411882899239,
+      "tokens_seen": 265748480
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004643731193580742,
+      "loss": 3.054,
+      "theoretical_loss": 4.20628778297468,
+      "tokens_seen": 265814016
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004643630892678034,
+      "loss": 3.0583,
+      "theoretical_loss": 4.206163722207627,
+      "tokens_seen": 265879552
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046435305917753264,
+      "loss": 2.9522,
+      "theoretical_loss": 4.206039700576078,
+      "tokens_seen": 265945088
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046434302908726177,
+      "loss": 3.1155,
+      "theoretical_loss": 4.205915718058046,
+      "tokens_seen": 266010624
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.000464332998996991,
+      "loss": 3.1129,
+      "theoretical_loss": 4.205791774631566,
+      "tokens_seen": 266076160
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046432296890672013,
+      "loss": 3.1271,
+      "theoretical_loss": 4.205667870274686,
+      "tokens_seen": 266141696
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046431293881644937,
+      "loss": 3.0881,
+      "theoretical_loss": 4.2055440049654775,
+      "tokens_seen": 266207232
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046430290872617855,
+      "loss": 3.1299,
+      "theoretical_loss": 4.205420178682024,
+      "tokens_seen": 266272768
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046429287863590773,
+      "loss": 2.8421,
+      "theoretical_loss": 4.205296391402431,
+      "tokens_seen": 266338304
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004642828485456369,
+      "loss": 2.9059,
+      "theoretical_loss": 4.205172643104818,
+      "tokens_seen": 266403840
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046427281845536615,
+      "loss": 2.8721,
+      "theoretical_loss": 4.205048933767326,
+      "tokens_seen": 266469376
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004642627883650953,
+      "loss": 3.1183,
+      "theoretical_loss": 4.20492526336811,
+      "tokens_seen": 266534912
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004642527582748245,
+      "loss": 2.9405,
+      "theoretical_loss": 4.204801631885346,
+      "tokens_seen": 266600448
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046424272818455364,
+      "loss": 2.9166,
+      "theoretical_loss": 4.204678039297223,
+      "tokens_seen": 266665984
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046423269809428287,
+      "loss": 3.1439,
+      "theoretical_loss": 4.204554485581953,
+      "tokens_seen": 266731520
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046422266800401205,
+      "loss": 3.0142,
+      "theoretical_loss": 4.2044309707177625,
+      "tokens_seen": 266797056
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046421263791374123,
+      "loss": 3.166,
+      "theoretical_loss": 4.204307494682895,
+      "tokens_seen": 266862592
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004642026078234704,
+      "loss": 2.91,
+      "theoretical_loss": 4.204184057455613,
+      "tokens_seen": 266928128
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004641925777331996,
+      "loss": 2.7895,
+      "theoretical_loss": 4.204060659014197,
+      "tokens_seen": 266993664
+    },
+    {
+      "epoch": 0.08,
+      "objective/train/docs_used": 158889,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 2.5656661987304688,
+      "objective/train/theoretical_loss": 4.2039372993369435,
+      "objective/train/tokens_used": 287519200,
+      "theoretical_loss": 4.2039372993369435,
+      "tokens_seen": 267059200
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004641825476429288,
+      "loss": 2.9939,
+      "theoretical_loss": 4.2039372993369435,
+      "tokens_seen": 267059200
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.000464172517552658,
+      "loss": 2.7803,
+      "theoretical_loss": 4.203813978402167,
+      "tokens_seen": 267124736
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046416248746238714,
+      "loss": 3.129,
+      "theoretical_loss": 4.203690696188198,
+      "tokens_seen": 267190272
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004641524573721164,
+      "loss": 3.0649,
+      "theoretical_loss": 4.203567452673388,
+      "tokens_seen": 267255808
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004641424272818455,
+      "loss": 3.0718,
+      "theoretical_loss": 4.203444247836103,
+      "tokens_seen": 267321344
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046413239719157474,
+      "loss": 3.2325,
+      "theoretical_loss": 4.203321081654726,
+      "tokens_seen": 267386880
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004641223671013039,
+      "loss": 3.0568,
+      "theoretical_loss": 4.203197954107661,
+      "tokens_seen": 267452416
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004641123370110331,
+      "loss": 2.8217,
+      "theoretical_loss": 4.203074865173325,
+      "tokens_seen": 267517952
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004641023069207623,
+      "loss": 2.9454,
+      "theoretical_loss": 4.202951814830155,
+      "tokens_seen": 267583488
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004640922768304915,
+      "loss": 2.9256,
+      "theoretical_loss": 4.202828803056605,
+      "tokens_seen": 267649024
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046408224674022064,
+      "loss": 2.9524,
+      "theoretical_loss": 4.202705829831144,
+      "tokens_seen": 267714560
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004640722166499499,
+      "loss": 2.9487,
+      "theoretical_loss": 4.202582895132262,
+      "tokens_seen": 267780096
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.000464062186559679,
+      "loss": 2.6154,
+      "theoretical_loss": 4.202459998938464,
+      "tokens_seen": 267845632
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046405215646940824,
+      "loss": 3.114,
+      "theoretical_loss": 4.202337141228272,
+      "tokens_seen": 267911168
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004640421263791374,
+      "loss": 3.0631,
+      "theoretical_loss": 4.202214321980227,
+      "tokens_seen": 267976704
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004640320962888666,
+      "loss": 2.9271,
+      "theoretical_loss": 4.202091541172885,
+      "tokens_seen": 268042240
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004640220661985958,
+      "loss": 2.9358,
+      "theoretical_loss": 4.201968798784821,
+      "tokens_seen": 268107776
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046401203610832496,
+      "loss": 2.9304,
+      "theoretical_loss": 4.201846094794627,
+      "tokens_seen": 268173312
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046400200601805414,
+      "loss": 3.2132,
+      "theoretical_loss": 4.201723429180911,
+      "tokens_seen": 268238848
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004639919759277834,
+      "loss": 2.8554,
+      "theoretical_loss": 4.2016008019222975,
+      "tokens_seen": 268304384
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004639819458375125,
+      "loss": 3.0379,
+      "theoretical_loss": 4.201478212997431,
+      "tokens_seen": 268369920
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046397191574724174,
+      "loss": 2.8709,
+      "theoretical_loss": 4.201355662384971,
+      "tokens_seen": 268435456
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004639618856569709,
+      "loss": 3.0275,
+      "theoretical_loss": 4.2012331500635955,
+      "tokens_seen": 268500992
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004639518555667001,
+      "loss": 3.1757,
+      "theoretical_loss": 4.201110676011998,
+      "tokens_seen": 268566528
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046394182547642934,
+      "loss": 3.3162,
+      "theoretical_loss": 4.200988240208889,
+      "tokens_seen": 268632064
+    },
+    {
+      "epoch": 0.08,
+      "objective/train/docs_used": 159274,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 3.012082099914551,
+      "objective/train/theoretical_loss": 4.2008658426329974,
+      "objective/train/tokens_used": 289157600,
+      "theoretical_loss": 4.2008658426329974,
+      "tokens_seen": 268697600
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046393179538615847,
+      "loss": 2.954,
+      "theoretical_loss": 4.2008658426329974,
+      "tokens_seen": 268697600
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004639217652958877,
+      "loss": 2.9394,
+      "theoretical_loss": 4.200743483263068,
+      "tokens_seen": 268763136
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004639117352056169,
+      "loss": 2.9931,
+      "theoretical_loss": 4.200621162077864,
+      "tokens_seen": 268828672
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046390170511534606,
+      "loss": 3.1958,
+      "theoretical_loss": 4.200498879056164,
+      "tokens_seen": 268894208
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046389167502507525,
+      "loss": 3.0091,
+      "theoretical_loss": 4.200376634176765,
+      "tokens_seen": 268959744
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004638816449348044,
+      "loss": 2.9129,
+      "theoretical_loss": 4.200254427418478,
+      "tokens_seen": 269025280
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004638716148445336,
+      "loss": 3.0744,
+      "theoretical_loss": 4.200132258760135,
+      "tokens_seen": 269090816
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046386158475426284,
+      "loss": 3.0598,
+      "theoretical_loss": 4.200010128180582,
+      "tokens_seen": 269156352
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046385155466399197,
+      "loss": 3.3544,
+      "theoretical_loss": 4.1998880356586845,
+      "tokens_seen": 269221888
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004638415245737212,
+      "loss": 3.2152,
+      "theoretical_loss": 4.1997659811733214,
+      "tokens_seen": 269287424
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046383149448345033,
+      "loss": 3.3844,
+      "theoretical_loss": 4.199643964703391,
+      "tokens_seen": 269352960
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046382146439317957,
+      "loss": 3.2429,
+      "theoretical_loss": 4.199521986227808,
+      "tokens_seen": 269418496
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046381143430290875,
+      "loss": 3.002,
+      "theoretical_loss": 4.199400045725502,
+      "tokens_seen": 269484032
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046380140421263793,
+      "loss": 2.9083,
+      "theoretical_loss": 4.199278143175423,
+      "tokens_seen": 269549568
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004637913741223671,
+      "loss": 3.2151,
+      "theoretical_loss": 4.1991562785565355,
+      "tokens_seen": 269615104
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046378134403209635,
+      "loss": 2.7854,
+      "theoretical_loss": 4.199034451847821,
+      "tokens_seen": 269680640
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004637713139418255,
+      "loss": 3.1079,
+      "theoretical_loss": 4.198912663028278,
+      "tokens_seen": 269746176
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004637612838515547,
+      "loss": 3.2046,
+      "theoretical_loss": 4.19879091207692,
+      "tokens_seen": 269811712
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046375125376128384,
+      "loss": 2.8742,
+      "theoretical_loss": 4.19866919897278,
+      "tokens_seen": 269877248
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046374122367101307,
+      "loss": 2.8417,
+      "theoretical_loss": 4.1985475236949075,
+      "tokens_seen": 269942784
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046373119358074225,
+      "loss": 3.0719,
+      "theoretical_loss": 4.198425886222367,
+      "tokens_seen": 270008320
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046372116349047143,
+      "loss": 3.1394,
+      "theoretical_loss": 4.198304286534239,
+      "tokens_seen": 270073856
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004637111334002006,
+      "loss": 2.8706,
+      "theoretical_loss": 4.198182724609624,
+      "tokens_seen": 270139392
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004637011033099298,
+      "loss": 2.9891,
+      "theoretical_loss": 4.198061200427635,
+      "tokens_seen": 270204928
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.000463691073219659,
+      "loss": 2.9082,
+      "theoretical_loss": 4.197939713967407,
+      "tokens_seen": 270270464
+    },
+    {
+      "epoch": 0.08,
+      "objective/train/docs_used": 160520,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 3.315969467163086,
+      "objective/train/theoretical_loss": 4.197818265208085,
+      "objective/train/tokens_used": 290796000,
+      "theoretical_loss": 4.197818265208085,
+      "tokens_seen": 270336000
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004636810431293882,
+      "loss": 2.939,
+      "theoretical_loss": 4.197818265208085,
+      "tokens_seen": 270336000
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046367101303911734,
+      "loss": 2.9865,
+      "theoretical_loss": 4.197696854128836,
+      "tokens_seen": 270401536
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004636609829488466,
+      "loss": 2.9617,
+      "theoretical_loss": 4.1975754807088395,
+      "tokens_seen": 270467072
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004636509528585757,
+      "loss": 3.1313,
+      "theoretical_loss": 4.1974541449272955,
+      "tokens_seen": 270532608
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046364092276830494,
+      "loss": 2.8589,
+      "theoretical_loss": 4.197332846763418,
+      "tokens_seen": 270598144
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004636308926780341,
+      "loss": 3.168,
+      "theoretical_loss": 4.197211586196437,
+      "tokens_seen": 270663680
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004636208625877633,
+      "loss": 3.1609,
+      "theoretical_loss": 4.197090363205602,
+      "tokens_seen": 270729216
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004636108324974925,
+      "loss": 2.8238,
+      "theoretical_loss": 4.196969177770175,
+      "tokens_seen": 270794752
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004636008024072217,
+      "loss": 3.0672,
+      "theoretical_loss": 4.196848029869437,
+      "tokens_seen": 270860288
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046359077231695084,
+      "loss": 3.1223,
+      "theoretical_loss": 4.1967269194826855,
+      "tokens_seen": 270925824
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004635807422266801,
+      "loss": 3.1919,
+      "theoretical_loss": 4.196605846589234,
+      "tokens_seen": 270991360
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004635707121364092,
+      "loss": 3.1143,
+      "theoretical_loss": 4.196484811168412,
+      "tokens_seen": 271056896
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046356068204613844,
+      "loss": 2.7825,
+      "theoretical_loss": 4.196363813199565,
+      "tokens_seen": 271122432
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004635506519558676,
+      "loss": 3.1101,
+      "theoretical_loss": 4.196242852662056,
+      "tokens_seen": 271187968
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004635406218655968,
+      "loss": 3.1081,
+      "theoretical_loss": 4.1961219295352645,
+      "tokens_seen": 271253504
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.000463530591775326,
+      "loss": 3.1529,
+      "theoretical_loss": 4.196001043798585,
+      "tokens_seen": 271319040
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046352056168505516,
+      "loss": 3.1475,
+      "theoretical_loss": 4.195880195431428,
+      "tokens_seen": 271384576
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046351053159478434,
+      "loss": 2.9622,
+      "theoretical_loss": 4.195759384413224,
+      "tokens_seen": 271450112
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004635005015045136,
+      "loss": 2.9907,
+      "theoretical_loss": 4.195638610723415,
+      "tokens_seen": 271515648
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004634904714142427,
+      "loss": 2.7124,
+      "theoretical_loss": 4.195517874341462,
+      "tokens_seen": 271581184
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046348044132397194,
+      "loss": 2.9731,
+      "theoretical_loss": 4.195397175246841,
+      "tokens_seen": 271646720
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046347041123370107,
+      "loss": 3.1014,
+      "theoretical_loss": 4.195276513419046,
+      "tokens_seen": 271712256
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004634603811434303,
+      "loss": 2.8613,
+      "theoretical_loss": 4.195155888837585,
+      "tokens_seen": 271777792
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004634503510531595,
+      "loss": 3.0449,
+      "theoretical_loss": 4.195035301481984,
+      "tokens_seen": 271843328
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046344032096288867,
+      "loss": 2.7986,
+      "theoretical_loss": 4.194914751331785,
+      "tokens_seen": 271908864
+    },
+    {
+      "epoch": 0.08,
+      "objective/train/docs_used": 160942,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 3.2946550846099854,
+      "objective/train/theoretical_loss": 4.194794238366544,
+      "objective/train/tokens_used": 292434400,
+      "theoretical_loss": 4.194794238366544,
+      "tokens_seen": 271974400
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046343029087261785,
+      "loss": 3.0722,
+      "theoretical_loss": 4.194794238366544,
+      "tokens_seen": 271974400
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004634202607823471,
+      "loss": 3.0775,
+      "theoretical_loss": 4.194673762565837,
+      "tokens_seen": 272039936
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004634102306920762,
+      "loss": 2.9018,
+      "theoretical_loss": 4.194553323909252,
+      "tokens_seen": 272105472
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046340020060180545,
+      "loss": 3.1252,
+      "theoretical_loss": 4.1944329223763965,
+      "tokens_seen": 272171008
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046339017051153457,
+      "loss": 2.9432,
+      "theoretical_loss": 4.194312557946892,
+      "tokens_seen": 272236544
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004633801404212638,
+      "loss": 3.1044,
+      "theoretical_loss": 4.194192230600376,
+      "tokens_seen": 272302080
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.000463370110330993,
+      "loss": 3.0371,
+      "theoretical_loss": 4.194071940316505,
+      "tokens_seen": 272367616
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046336008024072217,
+      "loss": 3.1066,
+      "theoretical_loss": 4.193951687074948,
+      "tokens_seen": 272433152
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046335005015045135,
+      "loss": 2.9569,
+      "theoretical_loss": 4.193831470855391,
+      "tokens_seen": 272498688
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046334002006018053,
+      "loss": 3.1706,
+      "theoretical_loss": 4.193711291637538,
+      "tokens_seen": 272564224
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004633299899699097,
+      "loss": 3.0875,
+      "theoretical_loss": 4.1935911494011044,
+      "tokens_seen": 272629760
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046331995987963895,
+      "loss": 3.0274,
+      "theoretical_loss": 4.193471044125828,
+      "tokens_seen": 272695296
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004633099297893681,
+      "loss": 3.1833,
+      "theoretical_loss": 4.193350975791458,
+      "tokens_seen": 272760832
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004632998996990973,
+      "loss": 3.0265,
+      "theoretical_loss": 4.193230944377762,
+      "tokens_seen": 272826368
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046328986960882644,
+      "loss": 2.999,
+      "theoretical_loss": 4.1931109498645185,
+      "tokens_seen": 272891904
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004632798395185557,
+      "loss": 2.8691,
+      "theoretical_loss": 4.19299099223153,
+      "tokens_seen": 272957440
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046326980942828485,
+      "loss": 3.1468,
+      "theoretical_loss": 4.192871071458608,
+      "tokens_seen": 273022976
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046325977933801404,
+      "loss": 3.101,
+      "theoretical_loss": 4.192751187525585,
+      "tokens_seen": 273088512
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004632497492477432,
+      "loss": 2.8904,
+      "theoretical_loss": 4.192631340412304,
+      "tokens_seen": 273154048
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046323971915747245,
+      "loss": 2.8277,
+      "theoretical_loss": 4.192511530098629,
+      "tokens_seen": 273219584
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004632296890672016,
+      "loss": 3.0174,
+      "theoretical_loss": 4.192391756564437,
+      "tokens_seen": 273285120
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004632196589769308,
+      "loss": 2.9337,
+      "theoretical_loss": 4.19227201978962,
+      "tokens_seen": 273350656
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046320962888666,
+      "loss": 3.2491,
+      "theoretical_loss": 4.19215231975409,
+      "tokens_seen": 273416192
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004631995987963892,
+      "loss": 3.1248,
+      "theoretical_loss": 4.192032656437769,
+      "tokens_seen": 273481728
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004631895687061184,
+      "loss": 3.1044,
+      "theoretical_loss": 4.191913029820601,
+      "tokens_seen": 273547264
+    },
+    {
+      "epoch": 0.08,
+      "objective/train/docs_used": 162274,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 3.1903703212738037,
+      "objective/train/theoretical_loss": 4.19179343988254,
+      "objective/train/tokens_used": 294072800,
+      "theoretical_loss": 4.19179343988254,
+      "tokens_seen": 273612800
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046317953861584754,
+      "loss": 3.1483,
+      "theoretical_loss": 4.19179343988254,
+      "tokens_seen": 273612800
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004631695085255768,
+      "loss": 2.8653,
+      "theoretical_loss": 4.191673886603559,
+      "tokens_seen": 273678336
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004631594784353059,
+      "loss": 3.0805,
+      "theoretical_loss": 4.191554369963647,
+      "tokens_seen": 273743872
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046314944834503514,
+      "loss": 2.9344,
+      "theoretical_loss": 4.191434889942807,
+      "tokens_seen": 273809408
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004631394182547643,
+      "loss": 3.0085,
+      "theoretical_loss": 4.191315446521058,
+      "tokens_seen": 273874944
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004631293881644935,
+      "loss": 2.9289,
+      "theoretical_loss": 4.191196039678436,
+      "tokens_seen": 273940480
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004631193580742227,
+      "loss": 3.1368,
+      "theoretical_loss": 4.191076669394992,
+      "tokens_seen": 274006016
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004631093279839519,
+      "loss": 3.1613,
+      "theoretical_loss": 4.190957335650791,
+      "tokens_seen": 274071552
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046309929789368104,
+      "loss": 2.8155,
+      "theoretical_loss": 4.190838038425916,
+      "tokens_seen": 274137088
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004630892678034103,
+      "loss": 3.1902,
+      "theoretical_loss": 4.190718777700465,
+      "tokens_seen": 274202624
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004630792377131394,
+      "loss": 3.0513,
+      "theoretical_loss": 4.190599553454551,
+      "tokens_seen": 274268160
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046306920762286864,
+      "loss": 3.0037,
+      "theoretical_loss": 4.190480365668304,
+      "tokens_seen": 274333696
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004630591775325978,
+      "loss": 2.8063,
+      "theoretical_loss": 4.190361214321866,
+      "tokens_seen": 274399232
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.000463049147442327,
+      "loss": 2.9742,
+      "theoretical_loss": 4.190242099395399,
+      "tokens_seen": 274464768
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004630391173520562,
+      "loss": 2.9558,
+      "theoretical_loss": 4.190123020869077,
+      "tokens_seen": 274530304
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046302908726178536,
+      "loss": 2.9187,
+      "theoretical_loss": 4.190003978723093,
+      "tokens_seen": 274595840
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046301905717151455,
+      "loss": 3.0974,
+      "theoretical_loss": 4.189884972937652,
+      "tokens_seen": 274661376
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004630090270812438,
+      "loss": 3.1408,
+      "theoretical_loss": 4.189766003492977,
+      "tokens_seen": 274726912
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004629989969909729,
+      "loss": 3.0551,
+      "theoretical_loss": 4.189647070369304,
+      "tokens_seen": 274792448
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046298896690070214,
+      "loss": 3.1843,
+      "theoretical_loss": 4.189528173546888,
+      "tokens_seen": 274857984
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046297893681043127,
+      "loss": 3.0686,
+      "theoretical_loss": 4.189409313005997,
+      "tokens_seen": 274923520
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004629689067201605,
+      "loss": 3.0667,
+      "theoretical_loss": 4.189290488726913,
+      "tokens_seen": 274989056
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004629588766298897,
+      "loss": 3.2895,
+      "theoretical_loss": 4.189171700689937,
+      "tokens_seen": 275054592
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046294884653961887,
+      "loss": 3.0592,
+      "theoretical_loss": 4.189052948875384,
+      "tokens_seen": 275120128
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046293881644934805,
+      "loss": 2.9131,
+      "theoretical_loss": 4.188934233263582,
+      "tokens_seen": 275185664
+    },
+    {
+      "epoch": 0.08,
+      "objective/train/docs_used": 163440,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 3.128859519958496,
+      "objective/train/theoretical_loss": 4.188815553834879,
+      "objective/train/tokens_used": 295711200,
+      "theoretical_loss": 4.188815553834879,
+      "tokens_seen": 275251200
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004629287863590773,
+      "loss": 3.0631,
+      "theoretical_loss": 4.188815553834879,
+      "tokens_seen": 275251200
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004629187562688064,
+      "loss": 2.8185,
+      "theoretical_loss": 4.188696910569634,
+      "tokens_seen": 275316736
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046290872617853565,
+      "loss": 3.0597,
+      "theoretical_loss": 4.188578303448223,
+      "tokens_seen": 275382272
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046289869608826477,
+      "loss": 3.0644,
+      "theoretical_loss": 4.188459732451038,
+      "tokens_seen": 275447808
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.000462888665997994,
+      "loss": 3.2541,
+      "theoretical_loss": 4.188341197558486,
+      "tokens_seen": 275513344
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004628786359077232,
+      "loss": 2.772,
+      "theoretical_loss": 4.188222698750988,
+      "tokens_seen": 275578880
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046286860581745237,
+      "loss": 2.945,
+      "theoretical_loss": 4.188104236008982,
+      "tokens_seen": 275644416
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046285857572718155,
+      "loss": 2.9857,
+      "theoretical_loss": 4.18798580931292,
+      "tokens_seen": 275709952
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046284854563691073,
+      "loss": 2.8546,
+      "theoretical_loss": 4.18786741864327,
+      "tokens_seen": 275775488
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004628385155466399,
+      "loss": 3.1056,
+      "theoretical_loss": 4.187749063980516,
+      "tokens_seen": 275841024
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046282848545636915,
+      "loss": 2.8899,
+      "theoretical_loss": 4.187630745305154,
+      "tokens_seen": 275906560
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004628184553660983,
+      "loss": 2.9852,
+      "theoretical_loss": 4.187512462597699,
+      "tokens_seen": 275972096
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004628084252758275,
+      "loss": 3.1059,
+      "theoretical_loss": 4.187394215838678,
+      "tokens_seen": 276037632
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046279839518555664,
+      "loss": 2.945,
+      "theoretical_loss": 4.187276005008637,
+      "tokens_seen": 276103168
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004627883650952859,
+      "loss": 3.1584,
+      "theoretical_loss": 4.1871578300881325,
+      "tokens_seen": 276168704
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046277833500501505,
+      "loss": 3.1305,
+      "theoretical_loss": 4.18703969105774,
+      "tokens_seen": 276234240
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046276830491474424,
+      "loss": 2.8816,
+      "theoretical_loss": 4.186921587898048,
+      "tokens_seen": 276299776
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004627582748244734,
+      "loss": 2.988,
+      "theoretical_loss": 4.1868035205896605,
+      "tokens_seen": 276365312
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046274824473420265,
+      "loss": 2.8155,
+      "theoretical_loss": 4.186685489113197,
+      "tokens_seen": 276430848
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004627382146439318,
+      "loss": 3.1242,
+      "theoretical_loss": 4.186567493449292,
+      "tokens_seen": 276496384
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.000462728184553661,
+      "loss": 3.0243,
+      "theoretical_loss": 4.186449533578595,
+      "tokens_seen": 276561920
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046271815446339014,
+      "loss": 2.9596,
+      "theoretical_loss": 4.186331609481769,
+      "tokens_seen": 276627456
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004627081243731194,
+      "loss": 3.3328,
+      "theoretical_loss": 4.186213721139496,
+      "tokens_seen": 276692992
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046269809428284856,
+      "loss": 3.114,
+      "theoretical_loss": 4.186095868532469,
+      "tokens_seen": 276758528
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046268806419257774,
+      "loss": 2.8937,
+      "theoretical_loss": 4.1859780516413965,
+      "tokens_seen": 276824064
+    },
+    {
+      "epoch": 0.08,
+      "objective/train/docs_used": 163970,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 3.389463186264038,
+      "objective/train/theoretical_loss": 4.185860270447004,
+      "objective/train/tokens_used": 297349600,
+      "theoretical_loss": 4.185860270447004,
+      "tokens_seen": 276889600
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004626780341023069,
+      "loss": 3.0341,
+      "theoretical_loss": 4.185860270447004,
+      "tokens_seen": 276889600
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004626680040120361,
+      "loss": 3.1028,
+      "theoretical_loss": 4.185742524930031,
+      "tokens_seen": 276955136
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004626579739217653,
+      "loss": 2.9541,
+      "theoretical_loss": 4.185624815071232,
+      "tokens_seen": 277020672
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004626479438314945,
+      "loss": 2.9788,
+      "theoretical_loss": 4.185507140851375,
+      "tokens_seen": 277086208
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046263791374122364,
+      "loss": 3.0129,
+      "theoretical_loss": 4.185389502251246,
+      "tokens_seen": 277151744
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004626278836509529,
+      "loss": 2.8961,
+      "theoretical_loss": 4.185271899251642,
+      "tokens_seen": 277217280
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.000462617853560682,
+      "loss": 2.8754,
+      "theoretical_loss": 4.185154331833379,
+      "tokens_seen": 277282816
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046260782347041124,
+      "loss": 2.9643,
+      "theoretical_loss": 4.185036799977285,
+      "tokens_seen": 277348352
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004625977933801404,
+      "loss": 3.0866,
+      "theoretical_loss": 4.184919303664203,
+      "tokens_seen": 277413888
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004625877632898696,
+      "loss": 3.1173,
+      "theoretical_loss": 4.184801842874992,
+      "tokens_seen": 277479424
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004625777331995988,
+      "loss": 3.0844,
+      "theoretical_loss": 4.1846844175905265,
+      "tokens_seen": 277544960
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.000462567703109328,
+      "loss": 3.0707,
+      "theoretical_loss": 4.184567027791693,
+      "tokens_seen": 277610496
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046255767301905715,
+      "loss": 2.9221,
+      "theoretical_loss": 4.184449673459395,
+      "tokens_seen": 277676032
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004625476429287864,
+      "loss": 3.0805,
+      "theoretical_loss": 4.1843323545745505,
+      "tokens_seen": 277741568
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004625376128385155,
+      "loss": 2.9624,
+      "theoretical_loss": 4.184215071118093,
+      "tokens_seen": 277807104
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046252758274824475,
+      "loss": 2.9347,
+      "theoretical_loss": 4.184097823070967,
+      "tokens_seen": 277872640
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004625175526579739,
+      "loss": 3.0535,
+      "theoretical_loss": 4.183980610414137,
+      "tokens_seen": 277938176
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004625075225677031,
+      "loss": 2.9498,
+      "theoretical_loss": 4.18386343312858,
+      "tokens_seen": 278003712
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004624974924774323,
+      "loss": 2.9249,
+      "theoretical_loss": 4.183746291195285,
+      "tokens_seen": 278069248
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046248746238716147,
+      "loss": 3.0889,
+      "theoretical_loss": 4.18362918459526,
+      "tokens_seen": 278134784
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046247743229689065,
+      "loss": 2.9025,
+      "theoretical_loss": 4.183512113309525,
+      "tokens_seen": 278200320
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004624674022066199,
+      "loss": 3.1737,
+      "theoretical_loss": 4.1833950773191155,
+      "tokens_seen": 278265856
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046245737211634907,
+      "loss": 2.8199,
+      "theoretical_loss": 4.183278076605083,
+      "tokens_seen": 278331392
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046244734202607825,
+      "loss": 3.1762,
+      "theoretical_loss": 4.18316111114849,
+      "tokens_seen": 278396928
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004624373119358075,
+      "loss": 3.1915,
+      "theoretical_loss": 4.183044180930418,
+      "tokens_seen": 278462464
+    },
+    {
+      "epoch": 0.08,
+      "objective/train/docs_used": 165182,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 3.31074857711792,
+      "objective/train/theoretical_loss": 4.182927285931959,
+      "objective/train/tokens_used": 298988000,
+      "theoretical_loss": 4.182927285931959,
+      "tokens_seen": 278528000
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004624272818455366,
+      "loss": 3.0716,
+      "theoretical_loss": 4.182927285931959,
+      "tokens_seen": 278528000
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046241725175526585,
+      "loss": 2.8782,
+      "theoretical_loss": 4.182810426134223,
+      "tokens_seen": 278593536
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.000462407221664995,
+      "loss": 3.1372,
+      "theoretical_loss": 4.1826936015183325,
+      "tokens_seen": 278659072
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004623971915747242,
+      "loss": 3.0399,
+      "theoretical_loss": 4.182576812065426,
+      "tokens_seen": 278724608
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004623871614844534,
+      "loss": 3.1069,
+      "theoretical_loss": 4.1824600577566535,
+      "tokens_seen": 278790144
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046237713139418257,
+      "loss": 3.003,
+      "theoretical_loss": 4.182343338573185,
+      "tokens_seen": 278855680
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046236710130391175,
+      "loss": 2.981,
+      "theoretical_loss": 4.1822266544961995,
+      "tokens_seen": 278921216
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046235707121364093,
+      "loss": 2.9963,
+      "theoretical_loss": 4.1821100055068925,
+      "tokens_seen": 278986752
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004623470411233701,
+      "loss": 2.8282,
+      "theoretical_loss": 4.181993391586476,
+      "tokens_seen": 279052288
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046233701103309935,
+      "loss": 2.9174,
+      "theoretical_loss": 4.181876812716174,
+      "tokens_seen": 279117824
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004623269809428285,
+      "loss": 3.0528,
+      "theoretical_loss": 4.181760268877226,
+      "tokens_seen": 279183360
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004623169508525577,
+      "loss": 3.0571,
+      "theoretical_loss": 4.181643760050885,
+      "tokens_seen": 279248896
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046230692076228684,
+      "loss": 3.0749,
+      "theoretical_loss": 4.181527286218419,
+      "tokens_seen": 279314432
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004622968906720161,
+      "loss": 3.0104,
+      "theoretical_loss": 4.181410847361111,
+      "tokens_seen": 279379968
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046228686058174525,
+      "loss": 3.1389,
+      "theoretical_loss": 4.181294443460257,
+      "tokens_seen": 279445504
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046227683049147444,
+      "loss": 2.8566,
+      "theoretical_loss": 4.18117807449717,
+      "tokens_seen": 279511040
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004622668004012036,
+      "loss": 3.1739,
+      "theoretical_loss": 4.181061740453174,
+      "tokens_seen": 279576576
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046225677031093285,
+      "loss": 3.1752,
+      "theoretical_loss": 4.18094544130961,
+      "tokens_seen": 279642112
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.000462246740220662,
+      "loss": 2.9331,
+      "theoretical_loss": 4.180829177047832,
+      "tokens_seen": 279707648
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004622367101303912,
+      "loss": 2.8764,
+      "theoretical_loss": 4.180712947649208,
+      "tokens_seen": 279773184
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046222668004012034,
+      "loss": 3.1994,
+      "theoretical_loss": 4.180596753095122,
+      "tokens_seen": 279838720
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004622166499498496,
+      "loss": 3.2665,
+      "theoretical_loss": 4.18048059336697,
+      "tokens_seen": 279904256
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046220661985957876,
+      "loss": 2.9311,
+      "theoretical_loss": 4.180364468446165,
+      "tokens_seen": 279969792
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046219658976930794,
+      "loss": 3.0726,
+      "theoretical_loss": 4.180248378314133,
+      "tokens_seen": 280035328
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004621865596790371,
+      "loss": 3.0656,
+      "theoretical_loss": 4.180132322952312,
+      "tokens_seen": 280100864
+    },
+    {
+      "epoch": 0.08,
+      "objective/train/docs_used": 165816,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 2.640207529067993,
+      "objective/train/theoretical_loss": 4.180016302342159,
+      "objective/train/tokens_used": 300626400,
+      "theoretical_loss": 4.180016302342159,
+      "tokens_seen": 280166400
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004621765295887663,
+      "loss": 2.8359,
+      "theoretical_loss": 4.180016302342159,
+      "tokens_seen": 280166400
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004621664994984955,
+      "loss": 3.0966,
+      "theoretical_loss": 4.179900316465139,
+      "tokens_seen": 280231936
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004621564694082247,
+      "loss": 3.0808,
+      "theoretical_loss": 4.179784365302739,
+      "tokens_seen": 280297472
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046214643931795384,
+      "loss": 2.9101,
+      "theoretical_loss": 4.179668448836454,
+      "tokens_seen": 280363008
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004621364092276831,
+      "loss": 3.1295,
+      "theoretical_loss": 4.179552567047794,
+      "tokens_seen": 280428544
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004621263791374122,
+      "loss": 2.9127,
+      "theoretical_loss": 4.179436719918286,
+      "tokens_seen": 280494080
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.00046211634904714144,
+      "loss": 3.2209,
+      "theoretical_loss": 4.179320907429468,
+      "tokens_seen": 280559616
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004621063189568706,
+      "loss": 3.0106,
+      "theoretical_loss": 4.179205129562895,
+      "tokens_seen": 280625152
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004620962888665998,
+      "loss": 3.0269,
+      "theoretical_loss": 4.179089386300134,
+      "tokens_seen": 280690688
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.000462086258776329,
+      "loss": 3.1179,
+      "theoretical_loss": 4.178973677622768,
+      "tokens_seen": 280756224
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004620762286860582,
+      "loss": 3.278,
+      "theoretical_loss": 4.17885800351239,
+      "tokens_seen": 280821760
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.00046206619859578735,
+      "loss": 2.9086,
+      "theoretical_loss": 4.178742363950614,
+      "tokens_seen": 280887296
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004620561685055166,
+      "loss": 3.1124,
+      "theoretical_loss": 4.178626758919062,
+      "tokens_seen": 280952832
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004620461384152457,
+      "loss": 3.0546,
+      "theoretical_loss": 4.178511188399372,
+      "tokens_seen": 281018368
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.00046203610832497495,
+      "loss": 3.1334,
+      "theoretical_loss": 4.1783956523731955,
+      "tokens_seen": 281083904
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004620260782347041,
+      "loss": 2.9334,
+      "theoretical_loss": 4.178280150822202,
+      "tokens_seen": 281149440
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004620160481444333,
+      "loss": 2.8178,
+      "theoretical_loss": 4.178164683728068,
+      "tokens_seen": 281214976
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004620060180541625,
+      "loss": 2.8805,
+      "theoretical_loss": 4.17804925107249,
+      "tokens_seen": 281280512
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.00046199598796389167,
+      "loss": 2.919,
+      "theoretical_loss": 4.1779338528371746,
+      "tokens_seen": 281346048
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.00046198595787362085,
+      "loss": 3.1013,
+      "theoretical_loss": 4.177818489003846,
+      "tokens_seen": 281411584
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004619759277833501,
+      "loss": 3.1344,
+      "theoretical_loss": 4.177703159554238,
+      "tokens_seen": 281477120
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004619658976930792,
+      "loss": 3.0054,
+      "theoretical_loss": 4.177587864470103,
+      "tokens_seen": 281542656
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.00046195586760280845,
+      "loss": 2.8447,
+      "theoretical_loss": 4.1774726037332055,
+      "tokens_seen": 281608192
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.00046194583751253763,
+      "loss": 3.0365,
+      "theoretical_loss": 4.17735737732532,
+      "tokens_seen": 281673728
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004619358074222668,
+      "loss": 3.0124,
+      "theoretical_loss": 4.177242185228242,
+      "tokens_seen": 281739264
+    },
+    {
+      "epoch": 0.09,
+      "objective/train/docs_used": 166963,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 3.019186019897461,
+      "objective/train/theoretical_loss": 4.177127027423776,
+      "objective/train/tokens_used": 302264800,
+      "theoretical_loss": 4.177127027423776,
+      "tokens_seen": 281804800
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.000461925777331996,
+      "loss": 2.9856,
+      "theoretical_loss": 4.177127027423776,
+      "tokens_seen": 281804800
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004619157472417252,
+      "loss": 3.0215,
+      "theoretical_loss": 4.17701190389374,
+      "tokens_seen": 281870336
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.00046190571715145435,
+      "loss": 2.9971,
+      "theoretical_loss": 4.176896814619969,
+      "tokens_seen": 281935872
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004618956870611836,
+      "loss": 3.0423,
+      "theoretical_loss": 4.176781759584312,
+      "tokens_seen": 282001408
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004618856569709127,
+      "loss": 3.0798,
+      "theoretical_loss": 4.176666738768627,
+      "tokens_seen": 282066944
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.00046187562688064195,
+      "loss": 2.8872,
+      "theoretical_loss": 4.176551752154791,
+      "tokens_seen": 282132480
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004618655967903711,
+      "loss": 3.1228,
+      "theoretical_loss": 4.1764367997246925,
+      "tokens_seen": 282198016
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004618555667001003,
+      "loss": 3.0256,
+      "theoretical_loss": 4.1763218814602325,
+      "tokens_seen": 282263552
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004618455366098295,
+      "loss": 3.1382,
+      "theoretical_loss": 4.17620699734333,
+      "tokens_seen": 282329088
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004618355065195587,
+      "loss": 2.807,
+      "theoretical_loss": 4.176092147355915,
+      "tokens_seen": 282394624
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.00046182547642928786,
+      "loss": 2.9633,
+      "theoretical_loss": 4.175977331479929,
+      "tokens_seen": 282460160
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.00046181544633901704,
+      "loss": 3.0836,
+      "theoretical_loss": 4.175862549697332,
+      "tokens_seen": 282525696
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004618054162487462,
+      "loss": 3.0762,
+      "theoretical_loss": 4.175747801990093,
+      "tokens_seen": 282591232
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.00046179538615847545,
+      "loss": 3.1594,
+      "theoretical_loss": 4.1756330883402,
+      "tokens_seen": 282656768
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004617853560682046,
+      "loss": 3.136,
+      "theoretical_loss": 4.1755184087296495,
+      "tokens_seen": 282722304
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004617753259779338,
+      "loss": 3.0347,
+      "theoretical_loss": 4.175403763140456,
+      "tokens_seen": 282787840
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.000461765295887663,
+      "loss": 2.9652,
+      "theoretical_loss": 4.175289151554643,
+      "tokens_seen": 282853376
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004617552657973922,
+      "loss": 3.0566,
+      "theoretical_loss": 4.175174573954252,
+      "tokens_seen": 282918912
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.00046174523570712136,
+      "loss": 2.8965,
+      "theoretical_loss": 4.175060030321337,
+      "tokens_seen": 282984448
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.00046173520561685054,
+      "loss": 3.0043,
+      "theoretical_loss": 4.174945520637964,
+      "tokens_seen": 283049984
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004617251755265797,
+      "loss": 3.124,
+      "theoretical_loss": 4.174831044886213,
+      "tokens_seen": 283115520
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.00046171514543630896,
+      "loss": 3.0059,
+      "theoretical_loss": 4.17471660304818,
+      "tokens_seen": 283181056
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.00046170511534603814,
+      "loss": 3.1171,
+      "theoretical_loss": 4.174602195105972,
+      "tokens_seen": 283246592
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004616950852557673,
+      "loss": 2.735,
+      "theoretical_loss": 4.174487821041711,
+      "tokens_seen": 283312128
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004616850551654965,
+      "loss": 2.9887,
+      "theoretical_loss": 4.174373480837532,
+      "tokens_seen": 283377664
+    },
+    {
+      "epoch": 0.09,
+      "objective/train/docs_used": 167624,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 3.3214924335479736,
+      "objective/train/theoretical_loss": 4.174259174475583,
+      "objective/train/tokens_used": 303903200,
+      "theoretical_loss": 4.174259174475583,
+      "tokens_seen": 283443200
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004616750250752257,
+      "loss": 3.0628,
+      "theoretical_loss": 4.174259174475583,
+      "tokens_seen": 283443200
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004616649949849549,
+      "loss": 3.1753,
+      "theoretical_loss": 4.174144901938025,
+      "tokens_seen": 283508736
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.00046165496489468404,
+      "loss": 2.9032,
+      "theoretical_loss": 4.174030663207036,
+      "tokens_seen": 283574272
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004616449348044133,
+      "loss": 3.0913,
+      "theoretical_loss": 4.173916458264804,
+      "tokens_seen": 283639808
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004616349047141424,
+      "loss": 3.0559,
+      "theoretical_loss": 4.173802287093531,
+      "tokens_seen": 283705344
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.00046162487462387164,
+      "loss": 2.8405,
+      "theoretical_loss": 4.173688149675435,
+      "tokens_seen": 283770880
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004616148445336008,
+      "loss": 3.1749,
+      "theoretical_loss": 4.173574045992744,
+      "tokens_seen": 283836416
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.00046160481444333,
+      "loss": 2.9452,
+      "theoretical_loss": 4.173459976027702,
+      "tokens_seen": 283901952
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004615947843530592,
+      "loss": 3.1586,
+      "theoretical_loss": 4.173345939762564,
+      "tokens_seen": 283967488
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004615847542627884,
+      "loss": 2.9694,
+      "theoretical_loss": 4.173231937179601,
+      "tokens_seen": 284033024
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.00046157472417251755,
+      "loss": 3.2162,
+      "theoretical_loss": 4.173117968261097,
+      "tokens_seen": 284098560
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004615646940822468,
+      "loss": 3.1405,
+      "theoretical_loss": 4.173004032989347,
+      "tokens_seen": 284164096
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004615546639919759,
+      "loss": 3.0749,
+      "theoretical_loss": 4.172890131346662,
+      "tokens_seen": 284229632
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.00046154463390170515,
+      "loss": 3.1627,
+      "theoretical_loss": 4.172776263315367,
+      "tokens_seen": 284295168
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004615346038114343,
+      "loss": 3.1345,
+      "theoretical_loss": 4.172662428877796,
+      "tokens_seen": 284360704
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004615245737211635,
+      "loss": 2.9964,
+      "theoretical_loss": 4.172548628016302,
+      "tokens_seen": 284426240
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004615145436308927,
+      "loss": 3.0997,
+      "theoretical_loss": 4.172434860713246,
+      "tokens_seen": 284491776
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.00046150451354062187,
+      "loss": 3.1053,
+      "theoretical_loss": 4.172321126951008,
+      "tokens_seen": 284557312
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.00046149448345035105,
+      "loss": 3.108,
+      "theoretical_loss": 4.172207426711974,
+      "tokens_seen": 284622848
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004614844533600803,
+      "loss": 2.8824,
+      "theoretical_loss": 4.1720937599785515,
+      "tokens_seen": 284688384
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004614744232698094,
+      "loss": 3.2381,
+      "theoretical_loss": 4.171980126733155,
+      "tokens_seen": 284753920
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.00046146439317953865,
+      "loss": 3.1941,
+      "theoretical_loss": 4.171866526958215,
+      "tokens_seen": 284819456
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.00046145436308926783,
+      "loss": 2.7743,
+      "theoretical_loss": 4.171752960636176,
+      "tokens_seen": 284884992
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.000461444332998997,
+      "loss": 2.9867,
+      "theoretical_loss": 4.171639427749492,
+      "tokens_seen": 284950528
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004614343029087262,
+      "loss": 3.0296,
+      "theoretical_loss": 4.171525928280635,
+      "tokens_seen": 285016064
+    },
+    {
+      "epoch": 0.09,
+      "objective/train/docs_used": 168853,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 2.6973540782928467,
+      "objective/train/theoretical_loss": 4.171412462212087,
+      "objective/train/tokens_used": 305541600,
+      "theoretical_loss": 4.171412462212087,
+      "tokens_seen": 285081600
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004614242728184554,
+      "loss": 2.8165,
+      "theoretical_loss": 4.171412462212087,
+      "tokens_seen": 285081600
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.00046141424272818455,
+      "loss": 3.1382,
+      "theoretical_loss": 4.171299029526344,
+      "tokens_seen": 285147136
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004614042126379138,
+      "loss": 3.1788,
+      "theoretical_loss": 4.171185630205917,
+      "tokens_seen": 285212672
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004613941825476429,
+      "loss": 2.9897,
+      "theoretical_loss": 4.171072264233327,
+      "tokens_seen": 285278208
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.00046138415245737215,
+      "loss": 3.0584,
+      "theoretical_loss": 4.17095893159111,
+      "tokens_seen": 285343744
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004613741223671013,
+      "loss": 3.0978,
+      "theoretical_loss": 4.170845632261815,
+      "tokens_seen": 285409280
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004613640922768305,
+      "loss": 2.9229,
+      "theoretical_loss": 4.1707323662280045,
+      "tokens_seen": 285474816
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004613540621865597,
+      "loss": 3.1211,
+      "theoretical_loss": 4.170619133472253,
+      "tokens_seen": 285540352
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004613440320962889,
+      "loss": 2.9585,
+      "theoretical_loss": 4.170505933977149,
+      "tokens_seen": 285605888
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.00046133400200601806,
+      "loss": 3.1654,
+      "theoretical_loss": 4.1703927677252945,
+      "tokens_seen": 285671424
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.00046132397191574724,
+      "loss": 3.116,
+      "theoretical_loss": 4.170279634699304,
+      "tokens_seen": 285736960
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004613139418254764,
+      "loss": 3.2018,
+      "theoretical_loss": 4.170166534881803,
+      "tokens_seen": 285802496
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.00046130391173520566,
+      "loss": 3.0549,
+      "theoretical_loss": 4.170053468255435,
+      "tokens_seen": 285868032
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004612938816449348,
+      "loss": 2.795,
+      "theoretical_loss": 4.169940434802851,
+      "tokens_seen": 285933568
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.000461283851554664,
+      "loss": 3.0738,
+      "theoretical_loss": 4.16982743450672,
+      "tokens_seen": 285999104
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004612738214643932,
+      "loss": 3.0826,
+      "theoretical_loss": 4.169714467349721,
+      "tokens_seen": 286064640
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004612637913741224,
+      "loss": 2.9928,
+      "theoretical_loss": 4.1696015333145455,
+      "tokens_seen": 286130176
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.00046125376128385156,
+      "loss": 3.0505,
+      "theoretical_loss": 4.169488632383901,
+      "tokens_seen": 286195712
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.00046124373119358074,
+      "loss": 3.1082,
+      "theoretical_loss": 4.169375764540505,
+      "tokens_seen": 286261248
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004612337011033099,
+      "loss": 2.9721,
+      "theoretical_loss": 4.16926292976709,
+      "tokens_seen": 286326784
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.00046122367101303916,
+      "loss": 3.1107,
+      "theoretical_loss": 4.1691501280464,
+      "tokens_seen": 286392320
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004612136409227683,
+      "loss": 3.0903,
+      "theoretical_loss": 4.1690373593611945,
+      "tokens_seen": 286457856
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004612036108324975,
+      "loss": 2.9085,
+      "theoretical_loss": 4.168924623694241,
+      "tokens_seen": 286523392
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.00046119358074222665,
+      "loss": 2.9324,
+      "theoretical_loss": 4.168811921028325,
+      "tokens_seen": 286588928
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004611835506519559,
+      "loss": 2.9307,
+      "theoretical_loss": 4.168699251346243,
+      "tokens_seen": 286654464
+    },
+    {
+      "epoch": 0.09,
+      "objective/train/docs_used": 169449,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 2.8505656719207764,
+      "objective/train/theoretical_loss": 4.168586614630803,
+      "objective/train/tokens_used": 307180000,
+      "theoretical_loss": 4.168586614630803,
+      "tokens_seen": 286720000
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.00046117352056168506,
+      "loss": 3.006,
+      "theoretical_loss": 4.168586614630803,
+      "tokens_seen": 286720000
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.00046116349047141425,
+      "loss": 2.8379,
+      "theoretical_loss": 4.168474010864829,
+      "tokens_seen": 286785536
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004611534603811434,
+      "loss": 2.8133,
+      "theoretical_loss": 4.168361440031154,
+      "tokens_seen": 286851072
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004611434302908726,
+      "loss": 2.8832,
+      "theoretical_loss": 4.1682489021126266,
+      "tokens_seen": 286916608
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004611334002006018,
+      "loss": 3.0822,
+      "theoretical_loss": 4.168136397092109,
+      "tokens_seen": 286982144
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.000461123370110331,
+      "loss": 3.1401,
+      "theoretical_loss": 4.168023924952473,
+      "tokens_seen": 287047680
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.00046111334002006015,
+      "loss": 3.0236,
+      "theoretical_loss": 4.167911485676606,
+      "tokens_seen": 287113216
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004611033099297894,
+      "loss": 2.9008,
+      "theoretical_loss": 4.1677990792474064,
+      "tokens_seen": 287178752
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.00046109327983951857,
+      "loss": 3.0368,
+      "theoretical_loss": 4.167686705647787,
+      "tokens_seen": 287244288
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.00046108324974924775,
+      "loss": 2.8824,
+      "theoretical_loss": 4.167574364860673,
+      "tokens_seen": 287309824
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.00046107321965897693,
+      "loss": 2.8937,
+      "theoretical_loss": 4.167462056869001,
+      "tokens_seen": 287375360
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004610631895687061,
+      "loss": 3.037,
+      "theoretical_loss": 4.167349781655721,
+      "tokens_seen": 287440896
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004610531594784353,
+      "loss": 3.0067,
+      "theoretical_loss": 4.167237539203798,
+      "tokens_seen": 287506432
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004610431293881645,
+      "loss": 2.8215,
+      "theoretical_loss": 4.167125329496207,
+      "tokens_seen": 287571968
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.00046103309929789365,
+      "loss": 3.092,
+      "theoretical_loss": 4.167013152515935,
+      "tokens_seen": 287637504
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004610230692076229,
+      "loss": 2.834,
+      "theoretical_loss": 4.1669010082459845,
+      "tokens_seen": 287703040
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.000461013039117352,
+      "loss": 3.0378,
+      "theoretical_loss": 4.16678889666937,
+      "tokens_seen": 287768576
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.00046100300902708125,
+      "loss": 2.9538,
+      "theoretical_loss": 4.166676817769118,
+      "tokens_seen": 287834112
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.00046099297893681043,
+      "loss": 2.936,
+      "theoretical_loss": 4.166564771528268,
+      "tokens_seen": 287899648
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004609829488465396,
+      "loss": 3.0104,
+      "theoretical_loss": 4.16645275792987,
+      "tokens_seen": 287965184
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004609729187562688,
+      "loss": 3.0223,
+      "theoretical_loss": 4.166340776956991,
+      "tokens_seen": 288030720
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.00046096288866599803,
+      "loss": 2.9447,
+      "theoretical_loss": 4.166228828592708,
+      "tokens_seen": 288096256
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004609528585757272,
+      "loss": 2.9425,
+      "theoretical_loss": 4.166116912820109,
+      "tokens_seen": 288161792
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004609428284854564,
+      "loss": 3.0139,
+      "theoretical_loss": 4.166005029622298,
+      "tokens_seen": 288227328
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004609327983951856,
+      "loss": 2.7581,
+      "theoretical_loss": 4.16589317898239,
+      "tokens_seen": 288292864
+    },
+    {
+      "epoch": 0.09,
+      "objective/train/docs_used": 170373,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 2.9227163791656494,
+      "objective/train/theoretical_loss": 4.165781360883513,
+      "objective/train/tokens_used": 308818400,
+      "theoretical_loss": 4.165781360883513,
+      "tokens_seen": 288358400
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.00046092276830491475,
+      "loss": 3.0162,
+      "theoretical_loss": 4.165781360883513,
+      "tokens_seen": 288358400
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.000460912738214644,
+      "loss": 2.9972,
+      "theoretical_loss": 4.165669575308807,
+      "tokens_seen": 288423936
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004609027081243731,
+      "loss": 2.9231,
+      "theoretical_loss": 4.165557822241425,
+      "tokens_seen": 288489472
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.00046089267803410235,
+      "loss": 2.9746,
+      "theoretical_loss": 4.165446101664531,
+      "tokens_seen": 288555008
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004608826479438315,
+      "loss": 3.0331,
+      "theoretical_loss": 4.165334413561306,
+      "tokens_seen": 288620544
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004608726178535607,
+      "loss": 3.0425,
+      "theoretical_loss": 4.165222757914936,
+      "tokens_seen": 288686080
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004608625877632899,
+      "loss": 2.9917,
+      "theoretical_loss": 4.165111134708629,
+      "tokens_seen": 288751616
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004608525576730191,
+      "loss": 3.0351,
+      "theoretical_loss": 4.164999543925597,
+      "tokens_seen": 288817152
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.00046084252758274826,
+      "loss": 2.9461,
+      "theoretical_loss": 4.164887985549068,
+      "tokens_seen": 288882688
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.00046083249749247744,
+      "loss": 3.0879,
+      "theoretical_loss": 4.164776459562285,
+      "tokens_seen": 288948224
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004608224674022066,
+      "loss": 2.9828,
+      "theoretical_loss": 4.164664965948499,
+      "tokens_seen": 289013760
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.00046081243731193586,
+      "loss": 3.1005,
+      "theoretical_loss": 4.164553504690976,
+      "tokens_seen": 289079296
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.000460802407221665,
+      "loss": 3.232,
+      "theoretical_loss": 4.164442075772993,
+      "tokens_seen": 289144832
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004607923771313942,
+      "loss": 3.0206,
+      "theoretical_loss": 4.164330679177841,
+      "tokens_seen": 289210368
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004607823470411234,
+      "loss": 2.9927,
+      "theoretical_loss": 4.164219314888823,
+      "tokens_seen": 289275904
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004607723169508526,
+      "loss": 2.8879,
+      "theoretical_loss": 4.164107982889252,
+      "tokens_seen": 289341440
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.00046076228686058176,
+      "loss": 2.8926,
+      "theoretical_loss": 4.163996683162458,
+      "tokens_seen": 289406976
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.00046075225677031094,
+      "loss": 2.955,
+      "theoretical_loss": 4.163885415691779,
+      "tokens_seen": 289472512
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004607422266800401,
+      "loss": 2.9791,
+      "theoretical_loss": 4.163774180460569,
+      "tokens_seen": 289538048
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.00046073219658976936,
+      "loss": 2.9538,
+      "theoretical_loss": 4.163662977452192,
+      "tokens_seen": 289603584
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004607221664994985,
+      "loss": 2.9461,
+      "theoretical_loss": 4.163551806650023,
+      "tokens_seen": 289669120
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004607121364092277,
+      "loss": 2.8711,
+      "theoretical_loss": 4.163440668037453,
+      "tokens_seen": 289734656
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.00046070210631895685,
+      "loss": 3.0029,
+      "theoretical_loss": 4.163329561597885,
+      "tokens_seen": 289800192
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004606920762286861,
+      "loss": 2.892,
+      "theoretical_loss": 4.1632184873147295,
+      "tokens_seen": 289865728
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.00046068204613841526,
+      "loss": 2.936,
+      "theoretical_loss": 4.1631074451714145,
+      "tokens_seen": 289931264
+    },
+    {
+      "epoch": 0.09,
+      "objective/train/docs_used": 171595,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 3.1895124912261963,
+      "objective/train/theoretical_loss": 4.162996435151379,
+      "objective/train/tokens_used": 310456800,
+      "theoretical_loss": 4.162996435151379,
+      "tokens_seen": 289996800
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.00046067201604814445,
+      "loss": 3.0838,
+      "theoretical_loss": 4.162996435151379,
+      "tokens_seen": 289996800
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004606619859578736,
+      "loss": 2.8576,
+      "theoretical_loss": 4.1628854572380725,
+      "tokens_seen": 290062336
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004606519558676028,
+      "loss": 2.9221,
+      "theoretical_loss": 4.162774511414959,
+      "tokens_seen": 290127872
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.000460641925777332,
+      "loss": 2.9766,
+      "theoretical_loss": 4.162663597665513,
+      "tokens_seen": 290193408
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004606318956870612,
+      "loss": 3.1019,
+      "theoretical_loss": 4.162552715973222,
+      "tokens_seen": 290258944
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.00046062186559679035,
+      "loss": 3.0226,
+      "theoretical_loss": 4.162441866321586,
+      "tokens_seen": 290324480
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004606118355065196,
+      "loss": 3.1766,
+      "theoretical_loss": 4.162331048694118,
+      "tokens_seen": 290390016
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.00046060180541624877,
+      "loss": 2.9799,
+      "theoretical_loss": 4.162220263074341,
+      "tokens_seen": 290455552
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.00046059177532597795,
+      "loss": 3.0867,
+      "theoretical_loss": 4.1621095094457905,
+      "tokens_seen": 290521088
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.00046058174523570713,
+      "loss": 3.0066,
+      "theoretical_loss": 4.161998787792016,
+      "tokens_seen": 290586624
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004605717151454363,
+      "loss": 3.0877,
+      "theoretical_loss": 4.161888098096579,
+      "tokens_seen": 290652160
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004605616850551655,
+      "loss": 2.8912,
+      "theoretical_loss": 4.161777440343051,
+      "tokens_seen": 290717696
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004605516549648947,
+      "loss": 3.0279,
+      "theoretical_loss": 4.161666814515017,
+      "tokens_seen": 290783232
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.00046054162487462385,
+      "loss": 2.9034,
+      "theoretical_loss": 4.161556220596075,
+      "tokens_seen": 290848768
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004605315947843531,
+      "loss": 2.9769,
+      "theoretical_loss": 4.161445658569833,
+      "tokens_seen": 290914304
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004605215646940822,
+      "loss": 3.0383,
+      "theoretical_loss": 4.161335128419914,
+      "tokens_seen": 290979840
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.00046051153460381145,
+      "loss": 2.9355,
+      "theoretical_loss": 4.16122463012995,
+      "tokens_seen": 291045376
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.00046050150451354063,
+      "loss": 3.299,
+      "theoretical_loss": 4.161114163683587,
+      "tokens_seen": 291110912
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004604914744232698,
+      "loss": 2.8822,
+      "theoretical_loss": 4.161003729064483,
+      "tokens_seen": 291176448
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.000460481444332999,
+      "loss": 3.0251,
+      "theoretical_loss": 4.160893326256307,
+      "tokens_seen": 291241984
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.00046047141424272823,
+      "loss": 2.8968,
+      "theoretical_loss": 4.160782955242741,
+      "tokens_seen": 291307520
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.00046046138415245736,
+      "loss": 3.0828,
+      "theoretical_loss": 4.160672616007479,
+      "tokens_seen": 291373056
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004604513540621866,
+      "loss": 3.2492,
+      "theoretical_loss": 4.160562308534226,
+      "tokens_seen": 291438592
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004604413239719157,
+      "loss": 2.7536,
+      "theoretical_loss": 4.160452032806701,
+      "tokens_seen": 291504128
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.00046043129388164495,
+      "loss": 3.1393,
+      "theoretical_loss": 4.160341788808633,
+      "tokens_seen": 291569664
+    },
+    {
+      "epoch": 0.09,
+      "objective/train/docs_used": 172214,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 3.0195140838623047,
+      "objective/train/theoretical_loss": 4.160231576523763,
+      "objective/train/tokens_used": 312095200,
+      "theoretical_loss": 4.160231576523763,
+      "tokens_seen": 291635200
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.00046042126379137414,
+      "loss": 2.9725,
+      "theoretical_loss": 4.160231576523763,
+      "tokens_seen": 291635200
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004604112337011033,
+      "loss": 3.0591,
+      "theoretical_loss": 4.160121395935847,
+      "tokens_seen": 291700736
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004604012036108325,
+      "loss": 2.6618,
+      "theoretical_loss": 4.160011247028649,
+      "tokens_seen": 291766272
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004603911735205617,
+      "loss": 2.9955,
+      "theoretical_loss": 4.159901129785947,
+      "tokens_seen": 291831808
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.00046038114343029086,
+      "loss": 2.9759,
+      "theoretical_loss": 4.159791044191532,
+      "tokens_seen": 291897344
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004603711133400201,
+      "loss": 2.9241,
+      "theoretical_loss": 4.1596809902292025,
+      "tokens_seen": 291962880
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004603610832497492,
+      "loss": 3.0356,
+      "theoretical_loss": 4.159570967882775,
+      "tokens_seen": 292028416
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.00046035105315947846,
+      "loss": 2.8435,
+      "theoretical_loss": 4.1594609771360735,
+      "tokens_seen": 292093952
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004603410230692076,
+      "loss": 3.1677,
+      "theoretical_loss": 4.159351017972936,
+      "tokens_seen": 292159488
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004603309929789368,
+      "loss": 2.7557,
+      "theoretical_loss": 4.159241090377211,
+      "tokens_seen": 292225024
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.000460320962888666,
+      "loss": 2.997,
+      "theoretical_loss": 4.15913119433276,
+      "tokens_seen": 292290560
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004603109327983952,
+      "loss": 3.1299,
+      "theoretical_loss": 4.159021329823457,
+      "tokens_seen": 292356096
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.00046030090270812436,
+      "loss": 2.9866,
+      "theoretical_loss": 4.158911496833184,
+      "tokens_seen": 292421632
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004602908726178536,
+      "loss": 2.8174,
+      "theoretical_loss": 4.15880169534584,
+      "tokens_seen": 292487168
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004602808425275827,
+      "loss": 3.0642,
+      "theoretical_loss": 4.158691925345334,
+      "tokens_seen": 292552704
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.00046027081243731196,
+      "loss": 2.9502,
+      "theoretical_loss": 4.158582186815584,
+      "tokens_seen": 292618240
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004602607823470411,
+      "loss": 2.957,
+      "theoretical_loss": 4.158472479740524,
+      "tokens_seen": 292683776
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004602507522567703,
+      "loss": 2.9731,
+      "theoretical_loss": 4.158362804104097,
+      "tokens_seen": 292749312
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004602407221664995,
+      "loss": 3.0535,
+      "theoretical_loss": 4.1582531598902595,
+      "tokens_seen": 292814848
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004602306920762287,
+      "loss": 2.985,
+      "theoretical_loss": 4.158143547082978,
+      "tokens_seen": 292880384
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.00046022066198595787,
+      "loss": 2.9562,
+      "theoretical_loss": 4.158033965666233,
+      "tokens_seen": 292945920
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.00046021063189568705,
+      "loss": 2.8592,
+      "theoretical_loss": 4.157924415624014,
+      "tokens_seen": 293011456
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004602006018054163,
+      "loss": 2.9666,
+      "theoretical_loss": 4.157814896940324,
+      "tokens_seen": 293076992
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.00046019057171514546,
+      "loss": 3.2116,
+      "theoretical_loss": 4.157705409599179,
+      "tokens_seen": 293142528
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.00046018054162487465,
+      "loss": 2.9664,
+      "theoretical_loss": 4.157595953584604,
+      "tokens_seen": 293208064
+    },
+    {
+      "epoch": 0.09,
+      "objective/train/docs_used": 173342,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 2.8450427055358887,
+      "objective/train/theoretical_loss": 4.1574865288806375,
+      "objective/train/tokens_used": 313733600,
+      "theoretical_loss": 4.1574865288806375,
+      "tokens_seen": 293273600
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004601705115346038,
+      "loss": 2.9126,
+      "theoretical_loss": 4.1574865288806375,
+      "tokens_seen": 293273600
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.000460160481444333,
+      "loss": 2.7492,
+      "theoretical_loss": 4.157377135471328,
+      "tokens_seen": 293339136
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004601504513540622,
+      "loss": 2.8848,
+      "theoretical_loss": 4.157267773340738,
+      "tokens_seen": 293404672
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004601404212637914,
+      "loss": 3.0064,
+      "theoretical_loss": 4.15715844247294,
+      "tokens_seen": 293470208
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.00046013039117352055,
+      "loss": 2.9549,
+      "theoretical_loss": 4.157049142852019,
+      "tokens_seen": 293535744
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004601203610832498,
+      "loss": 2.9487,
+      "theoretical_loss": 4.15693987446207,
+      "tokens_seen": 293601280
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.00046011033099297897,
+      "loss": 3.0182,
+      "theoretical_loss": 4.156830637287203,
+      "tokens_seen": 293666816
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.00046010030090270815,
+      "loss": 2.9087,
+      "theoretical_loss": 4.156721431311535,
+      "tokens_seen": 293732352
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.00046009027081243733,
+      "loss": 3.1088,
+      "theoretical_loss": 4.1566122565192,
+      "tokens_seen": 293797888
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004600802407221665,
+      "loss": 3.0616,
+      "theoretical_loss": 4.156503112894339,
+      "tokens_seen": 293863424
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004600702106318957,
+      "loss": 2.9094,
+      "theoretical_loss": 4.156394000421107,
+      "tokens_seen": 293928960
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.00046006018054162493,
+      "loss": 3.0927,
+      "theoretical_loss": 4.15628491908367,
+      "tokens_seen": 293994496
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.00046005015045135405,
+      "loss": 3.1256,
+      "theoretical_loss": 4.156175868866205,
+      "tokens_seen": 294060032
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004600401203610833,
+      "loss": 3.0551,
+      "theoretical_loss": 4.156066849752902,
+      "tokens_seen": 294125568
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004600300902708124,
+      "loss": 3.0325,
+      "theoretical_loss": 4.15595786172796,
+      "tokens_seen": 294191104
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.00046002006018054165,
+      "loss": 3.0107,
+      "theoretical_loss": 4.155848904775594,
+      "tokens_seen": 294256640
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.00046001003009027083,
+      "loss": 2.9292,
+      "theoretical_loss": 4.155739978880025,
+      "tokens_seen": 294322176
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.00046,
+      "loss": 2.831,
+      "theoretical_loss": 4.155631084025489,
+      "tokens_seen": 294387712
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004599899699097292,
+      "loss": 3.1475,
+      "theoretical_loss": 4.155522220196234,
+      "tokens_seen": 294453248
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.00045997993981945843,
+      "loss": 2.9871,
+      "theoretical_loss": 4.155413387376518,
+      "tokens_seen": 294518784
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.00045996990972918756,
+      "loss": 3.1932,
+      "theoretical_loss": 4.155304585550609,
+      "tokens_seen": 294584320
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004599598796389168,
+      "loss": 2.9215,
+      "theoretical_loss": 4.15519581470279,
+      "tokens_seen": 294649856
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004599498495486459,
+      "loss": 2.9558,
+      "theoretical_loss": 4.155087074817353,
+      "tokens_seen": 294715392
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.00045993981945837515,
+      "loss": 3.2962,
+      "theoretical_loss": 4.154978365878603,
+      "tokens_seen": 294780928
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.00045992978936810434,
+      "loss": 3.1586,
+      "theoretical_loss": 4.154869687870854,
+      "tokens_seen": 294846464
+    },
+    {
+      "epoch": 0.09,
+      "objective/train/docs_used": 173989,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 3.6182363033294678,
+      "objective/train/theoretical_loss": 4.1547610407784346,
+      "objective/train/tokens_used": 315372000,
+      "theoretical_loss": 4.1547610407784346,
+      "tokens_seen": 294912000
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004599197592778335,
+      "loss": 3.244,
+      "theoretical_loss": 4.1547610407784346,
+      "tokens_seen": 294912000
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004599097291875627,
+      "loss": 3.042,
+      "theoretical_loss": 4.154652424585683,
+      "tokens_seen": 294977536
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004598996990972919,
+      "loss": 2.722,
+      "theoretical_loss": 4.154543839276948,
+      "tokens_seen": 295043072
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.00045988966900702106,
+      "loss": 3.0362,
+      "theoretical_loss": 4.154435284836591,
+      "tokens_seen": 295108608
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004598796389167503,
+      "loss": 2.8414,
+      "theoretical_loss": 4.154326761248987,
+      "tokens_seen": 295174144
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004598696088264794,
+      "loss": 2.858,
+      "theoretical_loss": 4.154218268498517,
+      "tokens_seen": 295239680
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.00045985957873620866,
+      "loss": 2.9538,
+      "theoretical_loss": 4.154109806569577,
+      "tokens_seen": 295305216
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004598495486459378,
+      "loss": 3.0794,
+      "theoretical_loss": 4.154001375446576,
+      "tokens_seen": 295370752
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.000459839518555667,
+      "loss": 3.0819,
+      "theoretical_loss": 4.15389297511393,
+      "tokens_seen": 295436288
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004598294884653962,
+      "loss": 3.1095,
+      "theoretical_loss": 4.153784605556069,
+      "tokens_seen": 295501824
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004598194583751254,
+      "loss": 3.2757,
+      "theoretical_loss": 4.153676266757434,
+      "tokens_seen": 295567360
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.00045980942828485456,
+      "loss": 2.7567,
+      "theoretical_loss": 4.153567958702476,
+      "tokens_seen": 295632896
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004597993981945838,
+      "loss": 3.0827,
+      "theoretical_loss": 4.153459681375661,
+      "tokens_seen": 295698432
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004597893681043129,
+      "loss": 3.2228,
+      "theoretical_loss": 4.15335143476146,
+      "tokens_seen": 295763968
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.00045977933801404216,
+      "loss": 2.9004,
+      "theoretical_loss": 4.153243218844362,
+      "tokens_seen": 295829504
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004597693079237713,
+      "loss": 2.9718,
+      "theoretical_loss": 4.153135033608863,
+      "tokens_seen": 295895040
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004597592778335005,
+      "loss": 2.9733,
+      "theoretical_loss": 4.153026879039471,
+      "tokens_seen": 295960576
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004597492477432297,
+      "loss": 2.9921,
+      "theoretical_loss": 4.152918755120708,
+      "tokens_seen": 296026112
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004597392176529589,
+      "loss": 2.9436,
+      "theoretical_loss": 4.152810661837102,
+      "tokens_seen": 296091648
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.00045972918756268807,
+      "loss": 3.1248,
+      "theoretical_loss": 4.152702599173197,
+      "tokens_seen": 296157184
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.00045971915747241725,
+      "loss": 3.1309,
+      "theoretical_loss": 4.152594567113546,
+      "tokens_seen": 296222720
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.00045970912738214643,
+      "loss": 2.9432,
+      "theoretical_loss": 4.152486565642714,
+      "tokens_seen": 296288256
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.00045969909729187566,
+      "loss": 3.2128,
+      "theoretical_loss": 4.152378594745276,
+      "tokens_seen": 296353792
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004596890672016048,
+      "loss": 3.153,
+      "theoretical_loss": 4.15227065440582,
+      "tokens_seen": 296419328
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.000459679037111334,
+      "loss": 3.1355,
+      "theoretical_loss": 4.152162744608944,
+      "tokens_seen": 296484864
+    },
+    {
+      "epoch": 0.09,
+      "objective/train/docs_used": 175144,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 3.142443895339966,
+      "objective/train/theoretical_loss": 4.152054865339256,
+      "objective/train/tokens_used": 317010400,
+      "theoretical_loss": 4.152054865339256,
+      "tokens_seen": 296550400
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.00045966900702106315,
+      "loss": 3.1106,
+      "theoretical_loss": 4.152054865339256,
+      "tokens_seen": 296550400
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004596589769307924,
+      "loss": 3.0485,
+      "theoretical_loss": 4.151947016581378,
+      "tokens_seen": 296615936
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.00045964894684052157,
+      "loss": 3.0957,
+      "theoretical_loss": 4.151839198319942,
+      "tokens_seen": 296681472
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.00045963891675025075,
+      "loss": 2.7138,
+      "theoretical_loss": 4.15173141053959,
+      "tokens_seen": 296747008
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.00045962888665997993,
+      "loss": 2.8906,
+      "theoretical_loss": 4.151623653224975,
+      "tokens_seen": 296812544
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.00045961885656970917,
+      "loss": 2.9162,
+      "theoretical_loss": 4.151515926360764,
+      "tokens_seen": 296878080
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004596088264794383,
+      "loss": 2.9349,
+      "theoretical_loss": 4.151408229931631,
+      "tokens_seen": 296943616
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.00045959879638916753,
+      "loss": 3.0348,
+      "theoretical_loss": 4.151300563922266,
+      "tokens_seen": 297009152
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.00045958876629889666,
+      "loss": 3.0585,
+      "theoretical_loss": 4.151192928317365,
+      "tokens_seen": 297074688
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004595787362086259,
+      "loss": 2.8223,
+      "theoretical_loss": 4.151085323101638,
+      "tokens_seen": 297140224
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004595687061183551,
+      "loss": 2.8607,
+      "theoretical_loss": 4.150977748259805,
+      "tokens_seen": 297205760
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.00045955867602808425,
+      "loss": 2.8969,
+      "theoretical_loss": 4.1508702037765985,
+      "tokens_seen": 297271296
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.00045954864593781344,
+      "loss": 2.9414,
+      "theoretical_loss": 4.150762689636761,
+      "tokens_seen": 297336832
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004595386158475426,
+      "loss": 3.0862,
+      "theoretical_loss": 4.150655205825045,
+      "tokens_seen": 297402368
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004595285857572718,
+      "loss": 3.018,
+      "theoretical_loss": 4.150547752326216,
+      "tokens_seen": 297467904
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.00045951855566700103,
+      "loss": 2.9801,
+      "theoretical_loss": 4.15044032912505,
+      "tokens_seen": 297533440
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.00045950852557673016,
+      "loss": 3.0379,
+      "theoretical_loss": 4.150332936206332,
+      "tokens_seen": 297598976
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004594984954864594,
+      "loss": 3.249,
+      "theoretical_loss": 4.150225573554859,
+      "tokens_seen": 297664512
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004594884653961885,
+      "loss": 3.0122,
+      "theoretical_loss": 4.150118241155443,
+      "tokens_seen": 297730048
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.00045947843530591776,
+      "loss": 3.2498,
+      "theoretical_loss": 4.1500109389929,
+      "tokens_seen": 297795584
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.00045946840521564694,
+      "loss": 2.7565,
+      "theoretical_loss": 4.1499036670520635,
+      "tokens_seen": 297861120
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004594583751253761,
+      "loss": 3.0607,
+      "theoretical_loss": 4.149796425317772,
+      "tokens_seen": 297926656
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.00045944834503510536,
+      "loss": 3.1681,
+      "theoretical_loss": 4.149689213774879,
+      "tokens_seen": 297992192
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.00045943831494483454,
+      "loss": 3.1897,
+      "theoretical_loss": 4.149582032408249,
+      "tokens_seen": 298057728
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004594282848545637,
+      "loss": 3.1223,
+      "theoretical_loss": 4.149474881202755,
+      "tokens_seen": 298123264
+    },
+    {
+      "epoch": 0.09,
+      "objective/train/docs_used": 176258,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 3.113511323928833,
+      "objective/train/theoretical_loss": 4.1493677601432815,
+      "objective/train/tokens_used": 318648800,
+      "theoretical_loss": 4.1493677601432815,
+      "tokens_seen": 298188800
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004594182547642929,
+      "loss": 3.0056,
+      "theoretical_loss": 4.1493677601432815,
+      "tokens_seen": 298188800
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004594082246740221,
+      "loss": 3.017,
+      "theoretical_loss": 4.149260669214725,
+      "tokens_seen": 298254336
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.00045939819458375126,
+      "loss": 3.0696,
+      "theoretical_loss": 4.149153608401994,
+      "tokens_seen": 298319872
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004593881644934805,
+      "loss": 2.9312,
+      "theoretical_loss": 4.149046577690003,
+      "tokens_seen": 298385408
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004593781344032096,
+      "loss": 2.979,
+      "theoretical_loss": 4.148939577063683,
+      "tokens_seen": 298450944
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.00045936810431293886,
+      "loss": 2.7,
+      "theoretical_loss": 4.148832606507974,
+      "tokens_seen": 298516480
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.000459358074222668,
+      "loss": 2.9176,
+      "theoretical_loss": 4.148725666007824,
+      "tokens_seen": 298582016
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004593480441323972,
+      "loss": 2.9862,
+      "theoretical_loss": 4.1486187555481955,
+      "tokens_seen": 298647552
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004593380140421264,
+      "loss": 2.7821,
+      "theoretical_loss": 4.14851187511406,
+      "tokens_seen": 298713088
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004593279839518556,
+      "loss": 3.0391,
+      "theoretical_loss": 4.148405024690402,
+      "tokens_seen": 298778624
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.00045931795386158476,
+      "loss": 2.9083,
+      "theoretical_loss": 4.148298204262213,
+      "tokens_seen": 298844160
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.000459307923771314,
+      "loss": 2.8753,
+      "theoretical_loss": 4.148191413814497,
+      "tokens_seen": 298909696
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004592978936810431,
+      "loss": 2.9916,
+      "theoretical_loss": 4.148084653332271,
+      "tokens_seen": 298975232
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.00045928786359077236,
+      "loss": 3.0754,
+      "theoretical_loss": 4.14797792280056,
+      "tokens_seen": 299040768
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004592778335005015,
+      "loss": 3.0386,
+      "theoretical_loss": 4.147871222204401,
+      "tokens_seen": 299106304
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004592678034102307,
+      "loss": 2.9645,
+      "theoretical_loss": 4.147764551528841,
+      "tokens_seen": 299171840
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004592577733199599,
+      "loss": 2.7558,
+      "theoretical_loss": 4.147657910758938,
+      "tokens_seen": 299237376
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004592477432296891,
+      "loss": 2.7373,
+      "theoretical_loss": 4.147551299879762,
+      "tokens_seen": 299302912
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.00045923771313941827,
+      "loss": 2.8164,
+      "theoretical_loss": 4.147444718876391,
+      "tokens_seen": 299368448
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.00045922768304914745,
+      "loss": 2.8048,
+      "theoretical_loss": 4.147338167733917,
+      "tokens_seen": 299433984
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.00045921765295887663,
+      "loss": 3.0629,
+      "theoretical_loss": 4.14723164643744,
+      "tokens_seen": 299499520
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.00045920762286860586,
+      "loss": 2.948,
+      "theoretical_loss": 4.1471251549720725,
+      "tokens_seen": 299565056
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.000459197592778335,
+      "loss": 2.9864,
+      "theoretical_loss": 4.147018693322936,
+      "tokens_seen": 299630592
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004591875626880642,
+      "loss": 2.9198,
+      "theoretical_loss": 4.146912261475165,
+      "tokens_seen": 299696128
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.00045917753259779335,
+      "loss": 3.034,
+      "theoretical_loss": 4.1468058594139015,
+      "tokens_seen": 299761664
+    },
+    {
+      "epoch": 0.09,
+      "objective/train/docs_used": 176682,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 2.946582078933716,
+      "objective/train/theoretical_loss": 4.146699487124301,
+      "objective/train/tokens_used": 320287200,
+      "theoretical_loss": 4.146699487124301,
+      "tokens_seen": 299827200
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004591675025075226,
+      "loss": 3.0244,
+      "theoretical_loss": 4.146699487124301,
+      "tokens_seen": 299827200
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.00045915747241725177,
+      "loss": 3.0344,
+      "theoretical_loss": 4.146593144591527,
+      "tokens_seen": 299892736
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.00045914744232698095,
+      "loss": 3.1026,
+      "theoretical_loss": 4.146486831800758,
+      "tokens_seen": 299958272
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.00045913741223671013,
+      "loss": 3.005,
+      "theoretical_loss": 4.146380548737179,
+      "tokens_seen": 300023808
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.00045912738214643937,
+      "loss": 3.1644,
+      "theoretical_loss": 4.146274295385986,
+      "tokens_seen": 300089344
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004591173520561685,
+      "loss": 2.8718,
+      "theoretical_loss": 4.146168071732388,
+      "tokens_seen": 300154880
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.00045910732196589773,
+      "loss": 3.0563,
+      "theoretical_loss": 4.146061877761602,
+      "tokens_seen": 300220416
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.00045909729187562686,
+      "loss": 3.226,
+      "theoretical_loss": 4.145955713458856,
+      "tokens_seen": 300285952
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004590872617853561,
+      "loss": 3.0286,
+      "theoretical_loss": 4.145849578809392,
+      "tokens_seen": 300351488
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004590772316950853,
+      "loss": 2.9362,
+      "theoretical_loss": 4.145743473798457,
+      "tokens_seen": 300417024
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.00045906720160481445,
+      "loss": 3.3005,
+      "theoretical_loss": 4.145637398411313,
+      "tokens_seen": 300482560
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.00045905717151454364,
+      "loss": 3.1683,
+      "theoretical_loss": 4.145531352633229,
+      "tokens_seen": 300548096
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004590471414242728,
+      "loss": 2.8494,
+      "theoretical_loss": 4.14542533644949,
+      "tokens_seen": 300613632
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.000459037111334002,
+      "loss": 2.7638,
+      "theoretical_loss": 4.145319349845384,
+      "tokens_seen": 300679168
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.00045902708124373123,
+      "loss": 3.1229,
+      "theoretical_loss": 4.145213392806217,
+      "tokens_seen": 300744704
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.00045901705115346036,
+      "loss": 2.8249,
+      "theoretical_loss": 4.145107465317299,
+      "tokens_seen": 300810240
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004590070210631896,
+      "loss": 2.7708,
+      "theoretical_loss": 4.1450015673639555,
+      "tokens_seen": 300875776
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004589969909729187,
+      "loss": 2.9347,
+      "theoretical_loss": 4.144895698931521,
+      "tokens_seen": 300941312
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.00045898696088264796,
+      "loss": 3.1446,
+      "theoretical_loss": 4.144789860005337,
+      "tokens_seen": 301006848
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.00045897693079237714,
+      "loss": 2.7626,
+      "theoretical_loss": 4.14468405057076,
+      "tokens_seen": 301072384
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004589669007021063,
+      "loss": 2.9021,
+      "theoretical_loss": 4.144578270613157,
+      "tokens_seen": 301137920
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004589568706118355,
+      "loss": 2.9562,
+      "theoretical_loss": 4.144472520117901,
+      "tokens_seen": 301203456
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.00045894684052156474,
+      "loss": 3.0293,
+      "theoretical_loss": 4.14436679907038,
+      "tokens_seen": 301268992
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.00045893681043129386,
+      "loss": 2.9459,
+      "theoretical_loss": 4.144261107455991,
+      "tokens_seen": 301334528
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004589267803410231,
+      "loss": 2.9452,
+      "theoretical_loss": 4.1441554452601395,
+      "tokens_seen": 301400064
+    },
+    {
+      "epoch": 0.09,
+      "objective/train/docs_used": 177896,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 2.9508297443389893,
+      "objective/train/theoretical_loss": 4.1440498124682446,
+      "objective/train/tokens_used": 321925600,
+      "theoretical_loss": 4.1440498124682446,
+      "tokens_seen": 301465600
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004589167502507522,
+      "loss": 3.0381,
+      "theoretical_loss": 4.1440498124682446,
+      "tokens_seen": 301465600
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.00045890672016048146,
+      "loss": 3.0254,
+      "theoretical_loss": 4.143944209065733,
+      "tokens_seen": 301531136
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.00045889669007021064,
+      "loss": 2.948,
+      "theoretical_loss": 4.143838635038043,
+      "tokens_seen": 301596672
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004588866599799398,
+      "loss": 3.035,
+      "theoretical_loss": 4.143733090370624,
+      "tokens_seen": 301662208
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.000458876629889669,
+      "loss": 2.9586,
+      "theoretical_loss": 4.143627575048935,
+      "tokens_seen": 301727744
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004588665997993982,
+      "loss": 2.904,
+      "theoretical_loss": 4.143522089058446,
+      "tokens_seen": 301793280
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.00045885656970912737,
+      "loss": 2.9791,
+      "theoretical_loss": 4.143416632384635,
+      "tokens_seen": 301858816
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004588465396188566,
+      "loss": 2.9168,
+      "theoretical_loss": 4.143311205012992,
+      "tokens_seen": 301924352
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.00045883650952858573,
+      "loss": 2.9416,
+      "theoretical_loss": 4.14320580692902,
+      "tokens_seen": 301989888
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.00045882647943831496,
+      "loss": 3.2419,
+      "theoretical_loss": 4.143100438118227,
+      "tokens_seen": 302055424
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.00045881644934804415,
+      "loss": 3.0354,
+      "theoretical_loss": 4.142995098566135,
+      "tokens_seen": 302120960
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004588064192577733,
+      "loss": 2.891,
+      "theoretical_loss": 4.142889788258275,
+      "tokens_seen": 302186496
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004587963891675025,
+      "loss": 3.0476,
+      "theoretical_loss": 4.142784507180189,
+      "tokens_seen": 302252032
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004587863590772317,
+      "loss": 2.9312,
+      "theoretical_loss": 4.14267925531743,
+      "tokens_seen": 302317568
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.00045877632898696087,
+      "loss": 3.0632,
+      "theoretical_loss": 4.142574032655558,
+      "tokens_seen": 302383104
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004587662988966901,
+      "loss": 3.0175,
+      "theoretical_loss": 4.142468839180148,
+      "tokens_seen": 302448640
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.00045875626880641923,
+      "loss": 3.1759,
+      "theoretical_loss": 4.14236367487678,
+      "tokens_seen": 302514176
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.00045874623871614847,
+      "loss": 3.0104,
+      "theoretical_loss": 4.142258539731047,
+      "tokens_seen": 302579712
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004587362086258776,
+      "loss": 2.8409,
+      "theoretical_loss": 4.142153433728556,
+      "tokens_seen": 302645248
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.00045872617853560683,
+      "loss": 3.0343,
+      "theoretical_loss": 4.142048356854915,
+      "tokens_seen": 302710784
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.000458716148445336,
+      "loss": 3.1169,
+      "theoretical_loss": 4.141943309095752,
+      "tokens_seen": 302776320
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004587061183550652,
+      "loss": 2.7164,
+      "theoretical_loss": 4.141838290436699,
+      "tokens_seen": 302841856
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004586960882647944,
+      "loss": 3.1328,
+      "theoretical_loss": 4.1417333008633985,
+      "tokens_seen": 302907392
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.00045868605817452355,
+      "loss": 2.9251,
+      "theoretical_loss": 4.141628340361508,
+      "tokens_seen": 302972928
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004586760280842528,
+      "loss": 3.0941,
+      "theoretical_loss": 4.141523408916689,
+      "tokens_seen": 303038464
+    },
+    {
+      "epoch": 0.09,
+      "objective/train/docs_used": 178251,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 3.023979425430298,
+      "objective/train/theoretical_loss": 4.141418506514618,
+      "objective/train/tokens_used": 323564000,
+      "theoretical_loss": 4.141418506514618,
+      "tokens_seen": 303104000
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.00045866599799398197,
+      "loss": 3.0323,
+      "theoretical_loss": 4.141418506514618,
+      "tokens_seen": 303104000
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.00045865596790371115,
+      "loss": 2.966,
+      "theoretical_loss": 4.141313633140978,
+      "tokens_seen": 303169536
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.00045864593781344033,
+      "loss": 3.0848,
+      "theoretical_loss": 4.141208788781466,
+      "tokens_seen": 303235072
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.00045863590772316957,
+      "loss": 3.0573,
+      "theoretical_loss": 4.141103973421784,
+      "tokens_seen": 303300608
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004586258776328987,
+      "loss": 3.0507,
+      "theoretical_loss": 4.14099918704765,
+      "tokens_seen": 303366144
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.00045861584754262793,
+      "loss": 2.9278,
+      "theoretical_loss": 4.140894429644788,
+      "tokens_seen": 303431680
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.00045860581745235706,
+      "loss": 3.1005,
+      "theoretical_loss": 4.1407897011989325,
+      "tokens_seen": 303497216
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004585957873620863,
+      "loss": 3.0779,
+      "theoretical_loss": 4.1406850016958305,
+      "tokens_seen": 303562752
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004585857572718155,
+      "loss": 2.8597,
+      "theoretical_loss": 4.140580331121236,
+      "tokens_seen": 303628288
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.00045857572718154465,
+      "loss": 2.8306,
+      "theoretical_loss": 4.140475689460916,
+      "tokens_seen": 303693824
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.00045856569709127384,
+      "loss": 3.0461,
+      "theoretical_loss": 4.140371076700646,
+      "tokens_seen": 303759360
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.000458555667001003,
+      "loss": 3.0263,
+      "theoretical_loss": 4.140266492826211,
+      "tokens_seen": 303824896
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004585456369107322,
+      "loss": 3.0801,
+      "theoretical_loss": 4.140161937823407,
+      "tokens_seen": 303890432
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.00045853560682046143,
+      "loss": 2.9155,
+      "theoretical_loss": 4.140057411678041,
+      "tokens_seen": 303955968
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.00045852557673019056,
+      "loss": 2.8495,
+      "theoretical_loss": 4.139952914375927,
+      "tokens_seen": 304021504
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004585155466399198,
+      "loss": 2.9827,
+      "theoretical_loss": 4.139848445902892,
+      "tokens_seen": 304087040
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004585055165496489,
+      "loss": 2.9606,
+      "theoretical_loss": 4.139744006244772,
+      "tokens_seen": 304152576
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.00045849548645937816,
+      "loss": 2.8089,
+      "theoretical_loss": 4.139639595387413,
+      "tokens_seen": 304218112
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.00045848545636910734,
+      "loss": 3.0314,
+      "theoretical_loss": 4.139535213316671,
+      "tokens_seen": 304283648
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004584754262788365,
+      "loss": 2.972,
+      "theoretical_loss": 4.139430860018411,
+      "tokens_seen": 304349184
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004584653961885657,
+      "loss": 3.0738,
+      "theoretical_loss": 4.13932653547851,
+      "tokens_seen": 304414720
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.00045845536609829494,
+      "loss": 2.8192,
+      "theoretical_loss": 4.139222239682853,
+      "tokens_seen": 304480256
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.00045844533600802406,
+      "loss": 2.805,
+      "theoretical_loss": 4.139117972617337,
+      "tokens_seen": 304545792
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004584353059177533,
+      "loss": 2.8678,
+      "theoretical_loss": 4.139013734267866,
+      "tokens_seen": 304611328
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004584252758274824,
+      "loss": 2.9425,
+      "theoretical_loss": 4.138909524620358,
+      "tokens_seen": 304676864
+    },
+    {
+      "epoch": 0.09,
+      "objective/train/docs_used": 179536,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 3.0331430435180664,
+      "objective/train/theoretical_loss": 4.138805343660737,
+      "objective/train/tokens_used": 325202400,
+      "theoretical_loss": 4.138805343660737,
+      "tokens_seen": 304742400
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.00045841524573721166,
+      "loss": 2.9569,
+      "theoretical_loss": 4.138805343660737,
+      "tokens_seen": 304742400
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.00045840521564694084,
+      "loss": 3.0136,
+      "theoretical_loss": 4.13870119137494,
+      "tokens_seen": 304807936
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.00045839518555667,
+      "loss": 2.7669,
+      "theoretical_loss": 4.138597067748911,
+      "tokens_seen": 304873472
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004583851554663992,
+      "loss": 2.9702,
+      "theoretical_loss": 4.138492972768606,
+      "tokens_seen": 304939008
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004583751253761284,
+      "loss": 3.1139,
+      "theoretical_loss": 4.138388906419991,
+      "tokens_seen": 305004544
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.00045836509528585757,
+      "loss": 2.9143,
+      "theoretical_loss": 4.138284868689041,
+      "tokens_seen": 305070080
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004583550651955868,
+      "loss": 2.7962,
+      "theoretical_loss": 4.13818085956174,
+      "tokens_seen": 305135616
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.00045834503510531593,
+      "loss": 3.0717,
+      "theoretical_loss": 4.1380768790240845,
+      "tokens_seen": 305201152
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.00045833500501504516,
+      "loss": 2.8641,
+      "theoretical_loss": 4.137972927062078,
+      "tokens_seen": 305266688
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.00045832497492477435,
+      "loss": 2.9519,
+      "theoretical_loss": 4.137869003661737,
+      "tokens_seen": 305332224
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004583149448345035,
+      "loss": 3.1127,
+      "theoretical_loss": 4.137765108809084,
+      "tokens_seen": 305397760
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004583049147442327,
+      "loss": 2.9791,
+      "theoretical_loss": 4.1376612424901555,
+      "tokens_seen": 305463296
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004582948846539619,
+      "loss": 2.9104,
+      "theoretical_loss": 4.137557404690993,
+      "tokens_seen": 305528832
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.00045828485456369107,
+      "loss": 3.0647,
+      "theoretical_loss": 4.137453595397653,
+      "tokens_seen": 305594368
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004582748244734203,
+      "loss": 2.9847,
+      "theoretical_loss": 4.137349814596198,
+      "tokens_seen": 305659904
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.00045826479438314943,
+      "loss": 3.0683,
+      "theoretical_loss": 4.137246062272702,
+      "tokens_seen": 305725440
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.00045825476429287867,
+      "loss": 2.9846,
+      "theoretical_loss": 4.137142338413248,
+      "tokens_seen": 305790976
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004582447342026078,
+      "loss": 2.7466,
+      "theoretical_loss": 4.137038643003931,
+      "tokens_seen": 305856512
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.00045823470411233703,
+      "loss": 2.9283,
+      "theoretical_loss": 4.136934976030852,
+      "tokens_seen": 305922048
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004582246740220662,
+      "loss": 3.0245,
+      "theoretical_loss": 4.136831337480125,
+      "tokens_seen": 305987584
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004582146439317954,
+      "loss": 2.8523,
+      "theoretical_loss": 4.13672772733787,
+      "tokens_seen": 306053120
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.00045820461384152457,
+      "loss": 2.8644,
+      "theoretical_loss": 4.136624145590222,
+      "tokens_seen": 306118656
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.00045819458375125375,
+      "loss": 3.1607,
+      "theoretical_loss": 4.136520592223322,
+      "tokens_seen": 306184192
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.00045818455366098294,
+      "loss": 2.9101,
+      "theoretical_loss": 4.136417067223322,
+      "tokens_seen": 306249728
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.00045817452357071217,
+      "loss": 3.1469,
+      "theoretical_loss": 4.136313570576383,
+      "tokens_seen": 306315264
+    },
+    {
+      "epoch": 0.09,
+      "objective/train/docs_used": 180132,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 3.187363624572754,
+      "objective/train/theoretical_loss": 4.136210102268676,
+      "objective/train/tokens_used": 326840800,
+      "theoretical_loss": 4.136210102268676,
+      "tokens_seen": 306380800
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004581644934804413,
+      "loss": 3.0491,
+      "theoretical_loss": 4.136210102268676,
+      "tokens_seen": 306380800
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.00045815446339017053,
+      "loss": 3.2312,
+      "theoretical_loss": 4.136106662286383,
+      "tokens_seen": 306446336
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004581444332998997,
+      "loss": 3.1742,
+      "theoretical_loss": 4.136003250615693,
+      "tokens_seen": 306511872
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004581344032096289,
+      "loss": 2.983,
+      "theoretical_loss": 4.135899867242808,
+      "tokens_seen": 306577408
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004581243731193581,
+      "loss": 3.1178,
+      "theoretical_loss": 4.135796512153934,
+      "tokens_seen": 306642944
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.00045811434302908726,
+      "loss": 2.8261,
+      "theoretical_loss": 4.135693185335295,
+      "tokens_seen": 306708480
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.00045810431293881644,
+      "loss": 3.001,
+      "theoretical_loss": 4.135589886773119,
+      "tokens_seen": 306774016
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004580942828485457,
+      "loss": 2.9546,
+      "theoretical_loss": 4.135486616453644,
+      "tokens_seen": 306839552
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004580842527582748,
+      "loss": 3.0585,
+      "theoretical_loss": 4.135383374363119,
+      "tokens_seen": 306905088
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.00045807422266800404,
+      "loss": 3.023,
+      "theoretical_loss": 4.1352801604878024,
+      "tokens_seen": 306970624
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.00045806419257773316,
+      "loss": 2.954,
+      "theoretical_loss": 4.135176974813961,
+      "tokens_seen": 307036160
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004580541624874624,
+      "loss": 3.259,
+      "theoretical_loss": 4.135073817327872,
+      "tokens_seen": 307101696
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004580441323971916,
+      "loss": 3.053,
+      "theoretical_loss": 4.134970688015824,
+      "tokens_seen": 307167232
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.00045803410230692076,
+      "loss": 2.8994,
+      "theoretical_loss": 4.134867586864112,
+      "tokens_seen": 307232768
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.00045802407221664994,
+      "loss": 3.019,
+      "theoretical_loss": 4.134764513859043,
+      "tokens_seen": 307298304
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004580140421263791,
+      "loss": 2.7869,
+      "theoretical_loss": 4.134661468986932,
+      "tokens_seen": 307363840
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004580040120361083,
+      "loss": 2.8573,
+      "theoretical_loss": 4.1345584522341055,
+      "tokens_seen": 307429376
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.00045799398194583754,
+      "loss": 2.8166,
+      "theoretical_loss": 4.134455463586898,
+      "tokens_seen": 307494912
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.00045798395185556667,
+      "loss": 2.6522,
+      "theoretical_loss": 4.134352503031652,
+      "tokens_seen": 307560448
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004579739217652959,
+      "loss": 3.0032,
+      "theoretical_loss": 4.134249570554724,
+      "tokens_seen": 307625984
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004579638916750251,
+      "loss": 2.8976,
+      "theoretical_loss": 4.134146666142476,
+      "tokens_seen": 307691520
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.00045795386158475426,
+      "loss": 2.7396,
+      "theoretical_loss": 4.134043789781281,
+      "tokens_seen": 307757056
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004579438314944835,
+      "loss": 3.0013,
+      "theoretical_loss": 4.1339409414575226,
+      "tokens_seen": 307822592
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004579338014042126,
+      "loss": 2.9292,
+      "theoretical_loss": 4.133838121157592,
+      "tokens_seen": 307888128
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.00045792377131394186,
+      "loss": 3.2311,
+      "theoretical_loss": 4.133735328867892,
+      "tokens_seen": 307953664
+    },
+    {
+      "epoch": 0.09,
+      "objective/train/docs_used": 181325,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 3.002859354019165,
+      "objective/train/theoretical_loss": 4.133632564574831,
+      "objective/train/tokens_used": 328479200,
+      "theoretical_loss": 4.133632564574831,
+      "tokens_seen": 308019200
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.00045791374122367104,
+      "loss": 2.918,
+      "theoretical_loss": 4.133632564574831,
+      "tokens_seen": 308019200
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004579037111334002,
+      "loss": 2.8607,
+      "theoretical_loss": 4.133529828264831,
+      "tokens_seen": 308084736
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004578936810431294,
+      "loss": 2.9817,
+      "theoretical_loss": 4.133427119924323,
+      "tokens_seen": 308150272
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004578836509528586,
+      "loss": 3.1387,
+      "theoretical_loss": 4.133324439539745,
+      "tokens_seen": 308215808
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.00045787362086258777,
+      "loss": 3.2184,
+      "theoretical_loss": 4.133221787097546,
+      "tokens_seen": 308281344
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.000457863590772317,
+      "loss": 2.8897,
+      "theoretical_loss": 4.133119162584185,
+      "tokens_seen": 308346880
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.00045785356068204613,
+      "loss": 3.0767,
+      "theoretical_loss": 4.1330165659861295,
+      "tokens_seen": 308412416
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.00045784353059177536,
+      "loss": 2.9042,
+      "theoretical_loss": 4.132913997289856,
+      "tokens_seen": 308477952
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.00045783350050150455,
+      "loss": 2.9545,
+      "theoretical_loss": 4.1328114564818526,
+      "tokens_seen": 308543488
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004578234704112337,
+      "loss": 2.8544,
+      "theoretical_loss": 4.132708943548614,
+      "tokens_seen": 308609024
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004578134403209629,
+      "loss": 3.1091,
+      "theoretical_loss": 4.132606458476647,
+      "tokens_seen": 308674560
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004578034102306921,
+      "loss": 2.9141,
+      "theoretical_loss": 4.132504001252465,
+      "tokens_seen": 308740096
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.00045779338014042127,
+      "loss": 2.9562,
+      "theoretical_loss": 4.132401571862593,
+      "tokens_seen": 308805632
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004577833500501505,
+      "loss": 2.9368,
+      "theoretical_loss": 4.132299170293564,
+      "tokens_seen": 308871168
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.00045777331995987963,
+      "loss": 2.8221,
+      "theoretical_loss": 4.132196796531922,
+      "tokens_seen": 308936704
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.00045776328986960887,
+      "loss": 2.9433,
+      "theoretical_loss": 4.132094450564218,
+      "tokens_seen": 309002240
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.000457753259779338,
+      "loss": 2.8562,
+      "theoretical_loss": 4.1319921323770155,
+      "tokens_seen": 309067776
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.00045774322968906723,
+      "loss": 3.0209,
+      "theoretical_loss": 4.131889841956884,
+      "tokens_seen": 309133312
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004577331995987964,
+      "loss": 3.009,
+      "theoretical_loss": 4.131787579290405,
+      "tokens_seen": 309198848
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004577231695085256,
+      "loss": 2.7863,
+      "theoretical_loss": 4.1316853443641675,
+      "tokens_seen": 309264384
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004577131394182548,
+      "loss": 3.0576,
+      "theoretical_loss": 4.13158313716477,
+      "tokens_seen": 309329920
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.00045770310932798395,
+      "loss": 2.9071,
+      "theoretical_loss": 4.131480957678823,
+      "tokens_seen": 309395456
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.00045769307923771314,
+      "loss": 2.9949,
+      "theoretical_loss": 4.1313788058929415,
+      "tokens_seen": 309460992
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.00045768304914744237,
+      "loss": 2.8319,
+      "theoretical_loss": 4.131276681793754,
+      "tokens_seen": 309526528
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004576730190571715,
+      "loss": 2.7952,
+      "theoretical_loss": 4.131174585367896,
+      "tokens_seen": 309592064
+    },
+    {
+      "epoch": 0.09,
+      "objective/train/docs_used": 182674,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 2.9335567951202393,
+      "objective/train/theoretical_loss": 4.131072516602014,
+      "objective/train/tokens_used": 330117600,
+      "theoretical_loss": 4.131072516602014,
+      "tokens_seen": 309657600
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.00045766298896690073,
+      "loss": 2.9291,
+      "theoretical_loss": 4.131072516602014,
+      "tokens_seen": 309657600
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004576529588766299,
+      "loss": 2.8759,
+      "theoretical_loss": 4.130970475482761,
+      "tokens_seen": 309723136
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004576429287863591,
+      "loss": 2.7693,
+      "theoretical_loss": 4.130868461996803,
+      "tokens_seen": 309788672
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004576328986960883,
+      "loss": 3.2834,
+      "theoretical_loss": 4.1307664761308125,
+      "tokens_seen": 309854208
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.00045762286860581746,
+      "loss": 2.7814,
+      "theoretical_loss": 4.130664517871471,
+      "tokens_seen": 309919744
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.00045761283851554664,
+      "loss": 3.364,
+      "theoretical_loss": 4.130562587205473,
+      "tokens_seen": 309985280
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004576028084252759,
+      "loss": 2.8317,
+      "theoretical_loss": 4.130460684119516,
+      "tokens_seen": 310050816
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.000457592778335005,
+      "loss": 2.9605,
+      "theoretical_loss": 4.130358808600313,
+      "tokens_seen": 310116352
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.00045758274824473424,
+      "loss": 3.1335,
+      "theoretical_loss": 4.130256960634582,
+      "tokens_seen": 310181888
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.00045757271815446336,
+      "loss": 3.0687,
+      "theoretical_loss": 4.130155140209052,
+      "tokens_seen": 310247424
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004575626880641926,
+      "loss": 2.9797,
+      "theoretical_loss": 4.130053347310461,
+      "tokens_seen": 310312960
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004575526579739218,
+      "loss": 3.294,
+      "theoretical_loss": 4.129951581925555,
+      "tokens_seen": 310378496
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.00045754262788365096,
+      "loss": 2.9568,
+      "theoretical_loss": 4.129849844041091,
+      "tokens_seen": 310444032
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.00045753259779338014,
+      "loss": 2.941,
+      "theoretical_loss": 4.129748133643835,
+      "tokens_seen": 310509568
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004575225677031093,
+      "loss": 3.0219,
+      "theoretical_loss": 4.129646450720561,
+      "tokens_seen": 310575104
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004575125376128385,
+      "loss": 2.9852,
+      "theoretical_loss": 4.129544795258052,
+      "tokens_seen": 310640640
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.00045750250752256774,
+      "loss": 2.8331,
+      "theoretical_loss": 4.129443167243101,
+      "tokens_seen": 310706176
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.00045749247743229687,
+      "loss": 3.061,
+      "theoretical_loss": 4.129341566662511,
+      "tokens_seen": 310771712
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004574824473420261,
+      "loss": 2.967,
+      "theoretical_loss": 4.1292399935030915,
+      "tokens_seen": 310837248
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004574724172517553,
+      "loss": 3.0105,
+      "theoretical_loss": 4.129138447751664,
+      "tokens_seen": 310902784
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.00045746238716148446,
+      "loss": 2.9289,
+      "theoretical_loss": 4.129036929395058,
+      "tokens_seen": 310968320
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.00045745235707121364,
+      "loss": 3.0223,
+      "theoretical_loss": 4.12893543842011,
+      "tokens_seen": 311033856
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004574423269809428,
+      "loss": 2.9751,
+      "theoretical_loss": 4.12883397481367,
+      "tokens_seen": 311099392
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.000457432296890672,
+      "loss": 2.8895,
+      "theoretical_loss": 4.128732538562592,
+      "tokens_seen": 311164928
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.00045742226680040124,
+      "loss": 2.7377,
+      "theoretical_loss": 4.128631129653744,
+      "tokens_seen": 311230464
+    },
+    {
+      "epoch": 0.09,
+      "objective/train/docs_used": 183229,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 3.0134546756744385,
+      "objective/train/theoretical_loss": 4.128529748073999,
+      "objective/train/tokens_used": 331756000,
+      "theoretical_loss": 4.128529748073999,
+      "tokens_seen": 311296000
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.00045741223671013037,
+      "loss": 2.6315,
+      "theoretical_loss": 4.128529748073999,
+      "tokens_seen": 311296000
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004574022066198596,
+      "loss": 3.0727,
+      "theoretical_loss": 4.128428393810242,
+      "tokens_seen": 311361536
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.00045739217652958873,
+      "loss": 3.0927,
+      "theoretical_loss": 4.128327066849365,
+      "tokens_seen": 311427072
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.00045738214643931797,
+      "loss": 2.984,
+      "theoretical_loss": 4.12822576717827,
+      "tokens_seen": 311492608
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.00045737211634904715,
+      "loss": 3.1219,
+      "theoretical_loss": 4.128124494783869,
+      "tokens_seen": 311558144
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.00045736208625877633,
+      "loss": 2.8423,
+      "theoretical_loss": 4.12802324965308,
+      "tokens_seen": 311623680
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004573520561685055,
+      "loss": 3.034,
+      "theoretical_loss": 4.127922031772834,
+      "tokens_seen": 311689216
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.00045734202607823475,
+      "loss": 3.0936,
+      "theoretical_loss": 4.127820841130068,
+      "tokens_seen": 311754752
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.00045733199598796387,
+      "loss": 2.8057,
+      "theoretical_loss": 4.127719677711729,
+      "tokens_seen": 311820288
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004573219658976931,
+      "loss": 2.8598,
+      "theoretical_loss": 4.1276185415047735,
+      "tokens_seen": 311885824
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.00045731193580742223,
+      "loss": 2.8944,
+      "theoretical_loss": 4.127517432496166,
+      "tokens_seen": 311951360
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.00045730190571715147,
+      "loss": 2.9157,
+      "theoretical_loss": 4.127416350672879,
+      "tokens_seen": 312016896
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.00045729187562688065,
+      "loss": 2.938,
+      "theoretical_loss": 4.1273152960219,
+      "tokens_seen": 312082432
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.00045728184553660983,
+      "loss": 3.0826,
+      "theoretical_loss": 4.127214268530216,
+      "tokens_seen": 312147968
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.000457271815446339,
+      "loss": 3.0073,
+      "theoretical_loss": 4.12711326818483,
+      "tokens_seen": 312213504
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004572617853560682,
+      "loss": 3.0567,
+      "theoretical_loss": 4.127012294972753,
+      "tokens_seen": 312279040
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004572517552657974,
+      "loss": 3.169,
+      "theoretical_loss": 4.126911348881,
+      "tokens_seen": 312344576
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004572417251755266,
+      "loss": 2.9469,
+      "theoretical_loss": 4.126810429896603,
+      "tokens_seen": 312410112
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.00045723169508525574,
+      "loss": 2.9813,
+      "theoretical_loss": 4.126709538006596,
+      "tokens_seen": 312475648
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.000457221664994985,
+      "loss": 2.934,
+      "theoretical_loss": 4.1266086731980245,
+      "tokens_seen": 312541184
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004572116349047141,
+      "loss": 3.2086,
+      "theoretical_loss": 4.126507835457945,
+      "tokens_seen": 312606720
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.00045720160481444334,
+      "loss": 2.7577,
+      "theoretical_loss": 4.126407024773418,
+      "tokens_seen": 312672256
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.00045719157472417257,
+      "loss": 2.9729,
+      "theoretical_loss": 4.126306241131519,
+      "tokens_seen": 312737792
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004571815446339017,
+      "loss": 2.9627,
+      "theoretical_loss": 4.126205484519327,
+      "tokens_seen": 312803328
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.00045717151454363093,
+      "loss": 3.0685,
+      "theoretical_loss": 4.126104754923931,
+      "tokens_seen": 312868864
+    },
+    {
+      "epoch": 0.09,
+      "objective/train/docs_used": 184180,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 3.2469050884246826,
+      "objective/train/theoretical_loss": 4.126004052332432,
+      "objective/train/tokens_used": 333394400,
+      "theoretical_loss": 4.126004052332432,
+      "tokens_seen": 312934400
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004571614844533601,
+      "loss": 3.0363,
+      "theoretical_loss": 4.126004052332432,
+      "tokens_seen": 312934400
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004571514543630893,
+      "loss": 2.9679,
+      "theoretical_loss": 4.125903376731936,
+      "tokens_seen": 312999936
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004571414242728185,
+      "loss": 2.8487,
+      "theoretical_loss": 4.125802728109562,
+      "tokens_seen": 313065472
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.00045713139418254766,
+      "loss": 2.8367,
+      "theoretical_loss": 4.125702106452433,
+      "tokens_seen": 313131008
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.00045712136409227684,
+      "loss": 2.9977,
+      "theoretical_loss": 4.125601511747685,
+      "tokens_seen": 313196544
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004571113340020061,
+      "loss": 3.0608,
+      "theoretical_loss": 4.125500943982459,
+      "tokens_seen": 313262080
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004571013039117352,
+      "loss": 2.8297,
+      "theoretical_loss": 4.125400403143908,
+      "tokens_seen": 313327616
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.00045709127382146444,
+      "loss": 3.052,
+      "theoretical_loss": 4.125299889219194,
+      "tokens_seen": 313393152
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.00045708124373119356,
+      "loss": 3.0677,
+      "theoretical_loss": 4.125199402195486,
+      "tokens_seen": 313458688
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0004570712136409228,
+      "loss": 2.9362,
+      "theoretical_loss": 4.12509894205996,
+      "tokens_seen": 313524224
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.000457061183550652,
+      "loss": 2.9709,
+      "theoretical_loss": 4.124998508799806,
+      "tokens_seen": 313589760
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.00045705115346038116,
+      "loss": 3.04,
+      "theoretical_loss": 4.124898102402218,
+      "tokens_seen": 313655296
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.00045704112337011034,
+      "loss": 3.0373,
+      "theoretical_loss": 4.124797722854401,
+      "tokens_seen": 313720832
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0004570310932798395,
+      "loss": 2.8581,
+      "theoretical_loss": 4.12469737014357,
+      "tokens_seen": 313786368
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0004570210631895687,
+      "loss": 3.0926,
+      "theoretical_loss": 4.124597044256945,
+      "tokens_seen": 313851904
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.00045701103309929794,
+      "loss": 3.0979,
+      "theoretical_loss": 4.124496745181759,
+      "tokens_seen": 313917440
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.00045700100300902707,
+      "loss": 2.9877,
+      "theoretical_loss": 4.12439647290525,
+      "tokens_seen": 313982976
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0004569909729187563,
+      "loss": 3.2337,
+      "theoretical_loss": 4.124296227414667,
+      "tokens_seen": 314048512
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0004569809428284855,
+      "loss": 2.9723,
+      "theoretical_loss": 4.124196008697267,
+      "tokens_seen": 314114048
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.00045697091273821466,
+      "loss": 3.2751,
+      "theoretical_loss": 4.124095816740317,
+      "tokens_seen": 314179584
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.00045696088264794384,
+      "loss": 2.9168,
+      "theoretical_loss": 4.123995651531089,
+      "tokens_seen": 314245120
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.000456950852557673,
+      "loss": 2.9351,
+      "theoretical_loss": 4.123895513056869,
+      "tokens_seen": 314310656
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0004569408224674022,
+      "loss": 3.0289,
+      "theoretical_loss": 4.123795401304948,
+      "tokens_seen": 314376192
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.00045693079237713144,
+      "loss": 2.9276,
+      "theoretical_loss": 4.123695316262626,
+      "tokens_seen": 314441728
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.00045692076228686057,
+      "loss": 3.3239,
+      "theoretical_loss": 4.123595257917214,
+      "tokens_seen": 314507264
+    },
+    {
+      "epoch": 0.1,
+      "objective/train/docs_used": 184828,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 2.897738218307495,
+      "objective/train/theoretical_loss": 4.123495226256027,
+      "objective/train/tokens_used": 335032800,
+      "theoretical_loss": 4.123495226256027,
+      "tokens_seen": 314572800
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0004569107321965898,
+      "loss": 2.8664,
+      "theoretical_loss": 4.123495226256027,
+      "tokens_seen": 314572800
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.00045690070210631893,
+      "loss": 2.9238,
+      "theoretical_loss": 4.123395221266396,
+      "tokens_seen": 314638336
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.00045689067201604817,
+      "loss": 3.0188,
+      "theoretical_loss": 4.123295242935653,
+      "tokens_seen": 314703872
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.00045688064192577735,
+      "loss": 2.9396,
+      "theoretical_loss": 4.123195291251142,
+      "tokens_seen": 314769408
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.00045687061183550653,
+      "loss": 3.1508,
+      "theoretical_loss": 4.123095366200217,
+      "tokens_seen": 314834944
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0004568605817452357,
+      "loss": 2.9989,
+      "theoretical_loss": 4.122995467770239,
+      "tokens_seen": 314900480
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.00045685055165496495,
+      "loss": 2.9205,
+      "theoretical_loss": 4.122895595948577,
+      "tokens_seen": 314966016
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.00045684052156469407,
+      "loss": 2.8574,
+      "theoretical_loss": 4.122795750722612,
+      "tokens_seen": 315031552
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0004568304914744233,
+      "loss": 2.7756,
+      "theoretical_loss": 4.122695932079728,
+      "tokens_seen": 315097088
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.00045682046138415243,
+      "loss": 3.1906,
+      "theoretical_loss": 4.122596140007323,
+      "tokens_seen": 315162624
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.00045681043129388167,
+      "loss": 2.9296,
+      "theoretical_loss": 4.122496374492799,
+      "tokens_seen": 315228160
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.00045680040120361085,
+      "loss": 2.6739,
+      "theoretical_loss": 4.122396635523571,
+      "tokens_seen": 315293696
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.00045679037111334003,
+      "loss": 2.8686,
+      "theoretical_loss": 4.12229692308706,
+      "tokens_seen": 315359232
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0004567803410230692,
+      "loss": 3.2873,
+      "theoretical_loss": 4.122197237170695,
+      "tokens_seen": 315424768
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0004567703109327984,
+      "loss": 3.0861,
+      "theoretical_loss": 4.122097577761917,
+      "tokens_seen": 315490304
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0004567602808425276,
+      "loss": 3.1712,
+      "theoretical_loss": 4.121997944848171,
+      "tokens_seen": 315555840
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0004567502507522568,
+      "loss": 3.1188,
+      "theoretical_loss": 4.121898338416912,
+      "tokens_seen": 315621376
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.00045674022066198594,
+      "loss": 2.9156,
+      "theoretical_loss": 4.121798758455608,
+      "tokens_seen": 315686912
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0004567301905717152,
+      "loss": 2.9989,
+      "theoretical_loss": 4.121699204951728,
+      "tokens_seen": 315752448
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0004567201604814443,
+      "loss": 2.6808,
+      "theoretical_loss": 4.121599677892755,
+      "tokens_seen": 315817984
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.00045671013039117354,
+      "loss": 2.9749,
+      "theoretical_loss": 4.121500177266178,
+      "tokens_seen": 315883520
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0004567001003009027,
+      "loss": 2.9981,
+      "theoretical_loss": 4.121400703059497,
+      "tokens_seen": 315949056
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0004566900702106319,
+      "loss": 3.0599,
+      "theoretical_loss": 4.121301255260217,
+      "tokens_seen": 316014592
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0004566800401203611,
+      "loss": 2.9343,
+      "theoretical_loss": 4.121201833855856,
+      "tokens_seen": 316080128
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0004566700100300903,
+      "loss": 3.0488,
+      "theoretical_loss": 4.121102438833935,
+      "tokens_seen": 316145664
+    },
+    {
+      "epoch": 0.1,
+      "objective/train/docs_used": 186091,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 3.028201103210449,
+      "objective/train/theoretical_loss": 4.121003070181987,
+      "objective/train/tokens_used": 336671200,
+      "theoretical_loss": 4.121003070181987,
+      "tokens_seen": 316211200
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.00045665997993981944,
+      "loss": 3.0888,
+      "theoretical_loss": 4.121003070181987,
+      "tokens_seen": 316211200
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0004566499498495487,
+      "loss": 2.8606,
+      "theoretical_loss": 4.120903727887554,
+      "tokens_seen": 316276736
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0004566399197592778,
+      "loss": 2.8707,
+      "theoretical_loss": 4.120804411938185,
+      "tokens_seen": 316342272
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.00045662988966900704,
+      "loss": 3.1801,
+      "theoretical_loss": 4.120705122321437,
+      "tokens_seen": 316407808
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0004566198595787362,
+      "loss": 2.9392,
+      "theoretical_loss": 4.1206058590248755,
+      "tokens_seen": 316473344
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0004566098294884654,
+      "loss": 3.0525,
+      "theoretical_loss": 4.120506622036077,
+      "tokens_seen": 316538880
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0004565997993981946,
+      "loss": 2.8932,
+      "theoretical_loss": 4.120407411342623,
+      "tokens_seen": 316604416
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.00045658976930792376,
+      "loss": 3.0008,
+      "theoretical_loss": 4.1203082269321065,
+      "tokens_seen": 316669952
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.00045657973921765294,
+      "loss": 2.9041,
+      "theoretical_loss": 4.120209068792127,
+      "tokens_seen": 316735488
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0004565697091273822,
+      "loss": 2.8144,
+      "theoretical_loss": 4.120109936910291,
+      "tokens_seen": 316801024
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0004565596790371113,
+      "loss": 2.9752,
+      "theoretical_loss": 4.120010831274218,
+      "tokens_seen": 316866560
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.00045654964894684054,
+      "loss": 2.9861,
+      "theoretical_loss": 4.119911751871531,
+      "tokens_seen": 316932096
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.00045653961885656967,
+      "loss": 3.1201,
+      "theoretical_loss": 4.1198126986898655,
+      "tokens_seen": 316997632
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0004565295887662989,
+      "loss": 3.2564,
+      "theoretical_loss": 4.119713671716862,
+      "tokens_seen": 317063168
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0004565195586760281,
+      "loss": 3.0303,
+      "theoretical_loss": 4.11961467094017,
+      "tokens_seen": 317128704
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.00045650952858575727,
+      "loss": 3.0001,
+      "theoretical_loss": 4.119515696347451,
+      "tokens_seen": 317194240
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.00045649949849548645,
+      "loss": 3.1713,
+      "theoretical_loss": 4.119416747926369,
+      "tokens_seen": 317259776
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0004564894684052157,
+      "loss": 2.9097,
+      "theoretical_loss": 4.119317825664601,
+      "tokens_seen": 317325312
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0004564794383149448,
+      "loss": 3.0256,
+      "theoretical_loss": 4.119218929549831,
+      "tokens_seen": 317390848
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.00045646940822467405,
+      "loss": 2.9957,
+      "theoretical_loss": 4.119120059569751,
+      "tokens_seen": 317456384
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.00045645937813440317,
+      "loss": 3.1022,
+      "theoretical_loss": 4.11902121571206,
+      "tokens_seen": 317521920
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0004564493480441324,
+      "loss": 2.8814,
+      "theoretical_loss": 4.118922397964468,
+      "tokens_seen": 317587456
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.00045643931795386164,
+      "loss": 2.8242,
+      "theoretical_loss": 4.118823606314692,
+      "tokens_seen": 317652992
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.00045642928786359077,
+      "loss": 3.1631,
+      "theoretical_loss": 4.118724840750456,
+      "tokens_seen": 317718528
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.00045641925777332,
+      "loss": 3.0526,
+      "theoretical_loss": 4.118626101259496,
+      "tokens_seen": 317784064
+    },
+    {
+      "epoch": 0.1,
+      "objective/train/docs_used": 187059,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 3.0989761352539062,
+      "objective/train/theoretical_loss": 4.118527387829552,
+      "objective/train/tokens_used": 338309600,
+      "theoretical_loss": 4.118527387829552,
+      "tokens_seen": 317849600
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.00045640922768304913,
+      "loss": 3.1599,
+      "theoretical_loss": 4.118527387829552,
+      "tokens_seen": 317849600
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.00045639919759277837,
+      "loss": 2.9083,
+      "theoretical_loss": 4.118428700448375,
+      "tokens_seen": 317915136
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.00045638916750250755,
+      "loss": 2.7942,
+      "theoretical_loss": 4.118330039103723,
+      "tokens_seen": 317980672
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.00045637913741223673,
+      "loss": 2.8139,
+      "theoretical_loss": 4.118231403783365,
+      "tokens_seen": 318046208
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0004563691073219659,
+      "loss": 3.037,
+      "theoretical_loss": 4.118132794475073,
+      "tokens_seen": 318111744
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.00045635907723169515,
+      "loss": 3.0491,
+      "theoretical_loss": 4.118034211166632,
+      "tokens_seen": 318177280
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.00045634904714142427,
+      "loss": 2.8304,
+      "theoretical_loss": 4.117935653845834,
+      "tokens_seen": 318242816
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0004563390170511535,
+      "loss": 2.9257,
+      "theoretical_loss": 4.117837122500477,
+      "tokens_seen": 318308352
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.00045632898696088263,
+      "loss": 3.1308,
+      "theoretical_loss": 4.117738617118372,
+      "tokens_seen": 318373888
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.00045631895687061187,
+      "loss": 2.8832,
+      "theoretical_loss": 4.1176401376873315,
+      "tokens_seen": 318439424
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.00045630892678034105,
+      "loss": 3.0881,
+      "theoretical_loss": 4.117541684195183,
+      "tokens_seen": 318504960
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.00045629889669007023,
+      "loss": 3.0655,
+      "theoretical_loss": 4.117443256629758,
+      "tokens_seen": 318570496
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0004562888665997994,
+      "loss": 3.2203,
+      "theoretical_loss": 4.1173448549788985,
+      "tokens_seen": 318636032
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0004562788365095286,
+      "loss": 2.8932,
+      "theoretical_loss": 4.117246479230452,
+      "tokens_seen": 318701568
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0004562688064192578,
+      "loss": 3.1329,
+      "theoretical_loss": 4.117148129372277,
+      "tokens_seen": 318767104
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.000456258776328987,
+      "loss": 3.0318,
+      "theoretical_loss": 4.117049805392239,
+      "tokens_seen": 318832640
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.00045624874623871614,
+      "loss": 3.0924,
+      "theoretical_loss": 4.116951507278211,
+      "tokens_seen": 318898176
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0004562387161484454,
+      "loss": 3.2415,
+      "theoretical_loss": 4.116853235018075,
+      "tokens_seen": 318963712
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0004562286860581745,
+      "loss": 2.8784,
+      "theoretical_loss": 4.116754988599722,
+      "tokens_seen": 319029248
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.00045621865596790374,
+      "loss": 2.9519,
+      "theoretical_loss": 4.11665676801105,
+      "tokens_seen": 319094784
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0004562086258776329,
+      "loss": 3.0788,
+      "theoretical_loss": 4.116558573239965,
+      "tokens_seen": 319160320
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0004561985957873621,
+      "loss": 2.9773,
+      "theoretical_loss": 4.116460404274381,
+      "tokens_seen": 319225856
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0004561885656970913,
+      "loss": 2.8958,
+      "theoretical_loss": 4.116362261102221,
+      "tokens_seen": 319291392
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0004561785356068205,
+      "loss": 3.1937,
+      "theoretical_loss": 4.116264143711416,
+      "tokens_seen": 319356928
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.00045616850551654964,
+      "loss": 3.0717,
+      "theoretical_loss": 4.1161660520899055,
+      "tokens_seen": 319422464
+    },
+    {
+      "epoch": 0.1,
+      "objective/train/docs_used": 187800,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 3.174799680709839,
+      "objective/train/theoretical_loss": 4.116067986225635,
+      "objective/train/tokens_used": 339948000,
+      "theoretical_loss": 4.116067986225635,
+      "tokens_seen": 319488000
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0004561584754262789,
+      "loss": 3.0015,
+      "theoretical_loss": 4.116067986225635,
+      "tokens_seen": 319488000
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.000456148445336008,
+      "loss": 2.8217,
+      "theoretical_loss": 4.11596994610656,
+      "tokens_seen": 319553536
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.00045613841524573724,
+      "loss": 3.3545,
+      "theoretical_loss": 4.115871931720645,
+      "tokens_seen": 319619072
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0004561283851554664,
+      "loss": 2.7344,
+      "theoretical_loss": 4.11577394305586,
+      "tokens_seen": 319684608
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0004561183550651956,
+      "loss": 2.8487,
+      "theoretical_loss": 4.115675980100185,
+      "tokens_seen": 319750144
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0004561083249749248,
+      "loss": 2.9135,
+      "theoretical_loss": 4.115578042841607,
+      "tokens_seen": 319815680
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.00045609829488465396,
+      "loss": 2.9709,
+      "theoretical_loss": 4.115480131268122,
+      "tokens_seen": 319881216
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.00045608826479438314,
+      "loss": 2.9644,
+      "theoretical_loss": 4.1153822453677344,
+      "tokens_seen": 319946752
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0004560782347041124,
+      "loss": 2.9821,
+      "theoretical_loss": 4.115284385128454,
+      "tokens_seen": 320012288
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0004560682046138415,
+      "loss": 3.1831,
+      "theoretical_loss": 4.115186550538302,
+      "tokens_seen": 320077824
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.00045605817452357074,
+      "loss": 3.0413,
+      "theoretical_loss": 4.115088741585305,
+      "tokens_seen": 320143360
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.00045604814443329987,
+      "loss": 2.9108,
+      "theoretical_loss": 4.114990958257501,
+      "tokens_seen": 320208896
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0004560381143430291,
+      "loss": 3.2611,
+      "theoretical_loss": 4.11489320054293,
+      "tokens_seen": 320274432
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0004560280842527583,
+      "loss": 2.6443,
+      "theoretical_loss": 4.114795468429648,
+      "tokens_seen": 320339968
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.00045601805416248747,
+      "loss": 3.2413,
+      "theoretical_loss": 4.114697761905712,
+      "tokens_seen": 320405504
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.00045600802407221665,
+      "loss": 3.2062,
+      "theoretical_loss": 4.1146000809591925,
+      "tokens_seen": 320471040
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0004559979939819459,
+      "loss": 3.2569,
+      "theoretical_loss": 4.114502425578163,
+      "tokens_seen": 320536576
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.000455987963891675,
+      "loss": 3.047,
+      "theoretical_loss": 4.1144047957507075,
+      "tokens_seen": 320602112
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.00045597793380140425,
+      "loss": 2.9388,
+      "theoretical_loss": 4.114307191464919,
+      "tokens_seen": 320667648
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.00045596790371113337,
+      "loss": 3.0027,
+      "theoretical_loss": 4.114209612708898,
+      "tokens_seen": 320733184
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0004559578736208626,
+      "loss": 2.8668,
+      "theoretical_loss": 4.1141120594707505,
+      "tokens_seen": 320798720
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0004559478435305918,
+      "loss": 3.1505,
+      "theoretical_loss": 4.114014531738593,
+      "tokens_seen": 320864256
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.00045593781344032097,
+      "loss": 3.0058,
+      "theoretical_loss": 4.113917029500549,
+      "tokens_seen": 320929792
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.00045592778335005015,
+      "loss": 3.057,
+      "theoretical_loss": 4.113819552744752,
+      "tokens_seen": 320995328
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.00045591775325977933,
+      "loss": 2.9634,
+      "theoretical_loss": 4.11372210145934,
+      "tokens_seen": 321060864
+    },
+    {
+      "epoch": 0.1,
+      "objective/train/docs_used": 188718,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 2.97586989402771,
+      "objective/train/theoretical_loss": 4.11362467563246,
+      "objective/train/tokens_used": 341586400,
+      "theoretical_loss": 4.11362467563246,
+      "tokens_seen": 321126400
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0004559077231695085,
+      "loss": 3.0639,
+      "theoretical_loss": 4.11362467563246,
+      "tokens_seen": 321126400
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.00045589769307923775,
+      "loss": 3.2388,
+      "theoretical_loss": 4.1135272752522685,
+      "tokens_seen": 321191936
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0004558876629889669,
+      "loss": 3.1461,
+      "theoretical_loss": 4.1134299003069295,
+      "tokens_seen": 321257472
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0004558776328986961,
+      "loss": 3.0283,
+      "theoretical_loss": 4.113332550784613,
+      "tokens_seen": 321323008
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.00045586760280842524,
+      "loss": 2.9909,
+      "theoretical_loss": 4.113235226673499,
+      "tokens_seen": 321388544
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0004558575727181545,
+      "loss": 3.0179,
+      "theoretical_loss": 4.113137927961776,
+      "tokens_seen": 321454080
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.00045584754262788365,
+      "loss": 2.8515,
+      "theoretical_loss": 4.113040654637637,
+      "tokens_seen": 321519616
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.00045583751253761284,
+      "loss": 2.755,
+      "theoretical_loss": 4.112943406689286,
+      "tokens_seen": 321585152
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.000455827482447342,
+      "loss": 2.9629,
+      "theoretical_loss": 4.112846184104933,
+      "tokens_seen": 321650688
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.00045581745235707125,
+      "loss": 2.9968,
+      "theoretical_loss": 4.112748986872798,
+      "tokens_seen": 321716224
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0004558074222668004,
+      "loss": 3.2506,
+      "theoretical_loss": 4.112651814981108,
+      "tokens_seen": 321781760
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0004557973921765296,
+      "loss": 3.103,
+      "theoretical_loss": 4.112554668418095,
+      "tokens_seen": 321847296
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.00045578736208625874,
+      "loss": 3.1271,
+      "theoretical_loss": 4.112457547172005,
+      "tokens_seen": 321912832
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.000455777331995988,
+      "loss": 3.2188,
+      "theoretical_loss": 4.112360451231083,
+      "tokens_seen": 321978368
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.00045576730190571716,
+      "loss": 3.0657,
+      "theoretical_loss": 4.112263380583593,
+      "tokens_seen": 322043904
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.00045575727181544634,
+      "loss": 3.1376,
+      "theoretical_loss": 4.112166335217797,
+      "tokens_seen": 322109440
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0004557472417251755,
+      "loss": 2.8802,
+      "theoretical_loss": 4.112069315121969,
+      "tokens_seen": 322174976
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0004557372116349047,
+      "loss": 3.0532,
+      "theoretical_loss": 4.111972320284392,
+      "tokens_seen": 322240512
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0004557271815446339,
+      "loss": 2.9202,
+      "theoretical_loss": 4.111875350693354,
+      "tokens_seen": 322306048
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0004557171514543631,
+      "loss": 3.1109,
+      "theoretical_loss": 4.111778406337153,
+      "tokens_seen": 322371584
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.00045570712136409224,
+      "loss": 2.8451,
+      "theoretical_loss": 4.111681487204094,
+      "tokens_seen": 322437120
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0004556970912738215,
+      "loss": 2.957,
+      "theoretical_loss": 4.111584593282489,
+      "tokens_seen": 322502656
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0004556870611835507,
+      "loss": 3.2971,
+      "theoretical_loss": 4.111487724560659,
+      "tokens_seen": 322568192
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.00045567703109327984,
+      "loss": 3.227,
+      "theoretical_loss": 4.111390881026932,
+      "tokens_seen": 322633728
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0004556670010030091,
+      "loss": 3.1005,
+      "theoretical_loss": 4.111294062669645,
+      "tokens_seen": 322699264
+    },
+    {
+      "epoch": 0.1,
+      "objective/train/docs_used": 189140,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 2.8794915676116943,
+      "objective/train/theoretical_loss": 4.111197269477141,
+      "objective/train/tokens_used": 343224800,
+      "theoretical_loss": 4.111197269477141,
+      "tokens_seen": 322764800
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0004556569709127382,
+      "loss": 2.9512,
+      "theoretical_loss": 4.111197269477141,
+      "tokens_seen": 322764800
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.00045564694082246744,
+      "loss": 3.0384,
+      "theoretical_loss": 4.111100501437772,
+      "tokens_seen": 322830336
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0004556369107321966,
+      "loss": 2.8792,
+      "theoretical_loss": 4.111003758539896,
+      "tokens_seen": 322895872
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0004556268806419258,
+      "loss": 3.1337,
+      "theoretical_loss": 4.110907040771883,
+      "tokens_seen": 322961408
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.000455616850551655,
+      "loss": 3.186,
+      "theoretical_loss": 4.110810348122105,
+      "tokens_seen": 323026944
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.00045560682046138416,
+      "loss": 2.9821,
+      "theoretical_loss": 4.110713680578947,
+      "tokens_seen": 323092480
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.00045559679037111334,
+      "loss": 3.2378,
+      "theoretical_loss": 4.110617038130797,
+      "tokens_seen": 323158016
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0004555867602808426,
+      "loss": 2.6982,
+      "theoretical_loss": 4.110520420766054,
+      "tokens_seen": 323223552
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0004555767301905717,
+      "loss": 3.3153,
+      "theoretical_loss": 4.110423828473126,
+      "tokens_seen": 323289088
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.00045556670010030094,
+      "loss": 3.0101,
+      "theoretical_loss": 4.110327261240422,
+      "tokens_seen": 323354624
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.00045555667001003007,
+      "loss": 3.1061,
+      "theoretical_loss": 4.110230719056367,
+      "tokens_seen": 323420160
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0004555466399197593,
+      "loss": 3.1276,
+      "theoretical_loss": 4.110134201909388,
+      "tokens_seen": 323485696
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0004555366098294885,
+      "loss": 3.2466,
+      "theoretical_loss": 4.110037709787923,
+      "tokens_seen": 323551232
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.00045552657973921767,
+      "loss": 3.347,
+      "theoretical_loss": 4.109941242680414,
+      "tokens_seen": 323616768
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.00045551654964894685,
+      "loss": 3.1481,
+      "theoretical_loss": 4.109844800575315,
+      "tokens_seen": 323682304
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0004555065195586761,
+      "loss": 2.9884,
+      "theoretical_loss": 4.109748383461085,
+      "tokens_seen": 323747840
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0004554964894684052,
+      "loss": 3.1188,
+      "theoretical_loss": 4.10965199132619,
+      "tokens_seen": 323813376
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.00045548645937813445,
+      "loss": 3.2839,
+      "theoretical_loss": 4.109555624159107,
+      "tokens_seen": 323878912
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.00045547642928786357,
+      "loss": 3.3218,
+      "theoretical_loss": 4.109459281948317,
+      "tokens_seen": 323944448
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0004554663991975928,
+      "loss": 2.8989,
+      "theoretical_loss": 4.109362964682312,
+      "tokens_seen": 324009984
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.000455456369107322,
+      "loss": 2.9077,
+      "theoretical_loss": 4.1092666723495865,
+      "tokens_seen": 324075520
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.00045544633901705117,
+      "loss": 3.2878,
+      "theoretical_loss": 4.109170404938649,
+      "tokens_seen": 324141056
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.00045543630892678035,
+      "loss": 3.2166,
+      "theoretical_loss": 4.109074162438011,
+      "tokens_seen": 324206592
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.00045542627883650953,
+      "loss": 2.8978,
+      "theoretical_loss": 4.1089779448361945,
+      "tokens_seen": 324272128
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0004554162487462387,
+      "loss": 3.1213,
+      "theoretical_loss": 4.108881752121727,
+      "tokens_seen": 324337664
+    },
+    {
+      "epoch": 0.1,
+      "objective/train/docs_used": 190150,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 3.1001152992248535,
+      "objective/train/theoretical_loss": 4.108785584283144,
+      "objective/train/tokens_used": 344863200,
+      "theoretical_loss": 4.108785584283144,
+      "tokens_seen": 324403200
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.00045540621865596795,
+      "loss": 2.9022,
+      "theoretical_loss": 4.108785584283144,
+      "tokens_seen": 324403200
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0004553961885656971,
+      "loss": 2.9865,
+      "theoretical_loss": 4.108689441308989,
+      "tokens_seen": 324468736
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0004553861584754263,
+      "loss": 3.112,
+      "theoretical_loss": 4.108593323187815,
+      "tokens_seen": 324534272
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.00045537612838515544,
+      "loss": 3.1477,
+      "theoretical_loss": 4.108497229908179,
+      "tokens_seen": 324599808
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0004553660982948847,
+      "loss": 3.0519,
+      "theoretical_loss": 4.108401161458646,
+      "tokens_seen": 324665344
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.00045535606820461385,
+      "loss": 3.0665,
+      "theoretical_loss": 4.1083051178277925,
+      "tokens_seen": 324730880
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.00045534603811434304,
+      "loss": 2.968,
+      "theoretical_loss": 4.108209099004198,
+      "tokens_seen": 324796416
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0004553360080240722,
+      "loss": 3.1983,
+      "theoretical_loss": 4.108113104976453,
+      "tokens_seen": 324861952
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.00045532597793380145,
+      "loss": 2.8336,
+      "theoretical_loss": 4.108017135733152,
+      "tokens_seen": 324927488
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0004553159478435306,
+      "loss": 3.0941,
+      "theoretical_loss": 4.107921191262901,
+      "tokens_seen": 324993024
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0004553059177532598,
+      "loss": 2.9691,
+      "theoretical_loss": 4.10782527155431,
+      "tokens_seen": 325058560
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.00045529588766298894,
+      "loss": 2.9584,
+      "theoretical_loss": 4.107729376595999,
+      "tokens_seen": 325124096
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0004552858575727182,
+      "loss": 3.253,
+      "theoretical_loss": 4.107633506376594,
+      "tokens_seen": 325189632
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.00045527582748244736,
+      "loss": 3.0396,
+      "theoretical_loss": 4.10753766088473,
+      "tokens_seen": 325255168
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.00045526579739217654,
+      "loss": 3.0754,
+      "theoretical_loss": 4.107441840109048,
+      "tokens_seen": 325320704
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0004552557673019057,
+      "loss": 3.0014,
+      "theoretical_loss": 4.107346044038197,
+      "tokens_seen": 325386240
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0004552457372116349,
+      "loss": 3.0876,
+      "theoretical_loss": 4.107250272660834,
+      "tokens_seen": 325451776
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0004552357071213641,
+      "loss": 2.9613,
+      "theoretical_loss": 4.107154525965623,
+      "tokens_seen": 325517312
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0004552256770310933,
+      "loss": 2.9588,
+      "theoretical_loss": 4.107058803941236,
+      "tokens_seen": 325582848
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.00045521564694082244,
+      "loss": 2.9034,
+      "theoretical_loss": 4.106963106576352,
+      "tokens_seen": 325648384
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0004552056168505517,
+      "loss": 2.8674,
+      "theoretical_loss": 4.106867433859656,
+      "tokens_seen": 325713920
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0004551955867602808,
+      "loss": 2.9357,
+      "theoretical_loss": 4.106771785779845,
+      "tokens_seen": 325779456
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.00045518555667001004,
+      "loss": 2.9561,
+      "theoretical_loss": 4.106676162325618,
+      "tokens_seen": 325844992
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0004551755265797392,
+      "loss": 3.0155,
+      "theoretical_loss": 4.106580563485685,
+      "tokens_seen": 325910528
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0004551654964894684,
+      "loss": 3.0204,
+      "theoretical_loss": 4.106484989248762,
+      "tokens_seen": 325976064
+    },
+    {
+      "epoch": 0.1,
+      "objective/train/docs_used": 191328,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 3.0341267585754395,
+      "objective/train/theoretical_loss": 4.106389439603573,
+      "objective/train/tokens_used": 346501600,
+      "theoretical_loss": 4.106389439603573,
+      "tokens_seen": 326041600
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0004551554663991976,
+      "loss": 3.0402,
+      "theoretical_loss": 4.106389439603573,
+      "tokens_seen": 326041600
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0004551454363089268,
+      "loss": 3.1653,
+      "theoretical_loss": 4.10629391453885,
+      "tokens_seen": 326107136
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.00045513540621865595,
+      "loss": 2.9886,
+      "theoretical_loss": 4.10619841404333,
+      "tokens_seen": 326172672
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0004551253761283852,
+      "loss": 3.3446,
+      "theoretical_loss": 4.106102938105762,
+      "tokens_seen": 326238208
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0004551153460381143,
+      "loss": 2.8962,
+      "theoretical_loss": 4.106007486714896,
+      "tokens_seen": 326303744
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.00045510531594784354,
+      "loss": 3.1154,
+      "theoretical_loss": 4.105912059859495,
+      "tokens_seen": 326369280
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0004550952858575727,
+      "loss": 3.048,
+      "theoretical_loss": 4.105816657528327,
+      "tokens_seen": 326434816
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0004550852557673019,
+      "loss": 2.9685,
+      "theoretical_loss": 4.105721279710168,
+      "tokens_seen": 326500352
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0004550752256770311,
+      "loss": 3.1754,
+      "theoretical_loss": 4.105625926393801,
+      "tokens_seen": 326565888
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.00045506519558676027,
+      "loss": 3.2148,
+      "theoretical_loss": 4.105530597568015,
+      "tokens_seen": 326631424
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.00045505516549648945,
+      "loss": 3.1651,
+      "theoretical_loss": 4.1054352932216105,
+      "tokens_seen": 326696960
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0004550451354062187,
+      "loss": 3.0107,
+      "theoretical_loss": 4.10534001334339,
+      "tokens_seen": 326762496
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0004550351053159478,
+      "loss": 3.2755,
+      "theoretical_loss": 4.105244757922169,
+      "tokens_seen": 326828032
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.00045502507522567705,
+      "loss": 2.8913,
+      "theoretical_loss": 4.105149526946766,
+      "tokens_seen": 326893568
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.00045501504513540623,
+      "loss": 2.9583,
+      "theoretical_loss": 4.1050543204060075,
+      "tokens_seen": 326959104
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0004550050150451354,
+      "loss": 3.187,
+      "theoretical_loss": 4.10495913828873,
+      "tokens_seen": 327024640
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0004549949849548646,
+      "loss": 2.9074,
+      "theoretical_loss": 4.104863980583774,
+      "tokens_seen": 327090176
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.00045498495486459377,
+      "loss": 3.144,
+      "theoretical_loss": 4.104768847279988,
+      "tokens_seen": 327155712
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.00045497492477432295,
+      "loss": 3.0299,
+      "theoretical_loss": 4.104673738366232,
+      "tokens_seen": 327221248
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0004549648946840522,
+      "loss": 2.9183,
+      "theoretical_loss": 4.104578653831368,
+      "tokens_seen": 327286784
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.00045495486459378137,
+      "loss": 2.8461,
+      "theoretical_loss": 4.104483593664266,
+      "tokens_seen": 327352320
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.00045494483450351055,
+      "loss": 2.9215,
+      "theoretical_loss": 4.104388557853807,
+      "tokens_seen": 327417856
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.00045493480441323973,
+      "loss": 3.0317,
+      "theoretical_loss": 4.104293546388876,
+      "tokens_seen": 327483392
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0004549247743229689,
+      "loss": 3.0664,
+      "theoretical_loss": 4.104198559258364,
+      "tokens_seen": 327548928
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.00045491474423269815,
+      "loss": 2.9383,
+      "theoretical_loss": 4.104103596451175,
+      "tokens_seen": 327614464
+    },
+    {
+      "debugging/Self-BLEU-5": 0.2643082570774211,
+      "debugging/distinct-1-grams": 0.8218199262737969,
+      "debugging/distinct-2-grams": 0.9963369963369964,
+      "debugging/entropy-1-grams": 4.917654182369999,
+      "debugging/entropy-2-grams": 5.417185152102512,
+      "debugging/length": 514.3333333333334,
+      "debugging/num_segments": 3,
+      "epoch": 0.1,
+      "objective/train/docs_used": 191945,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 3.493633270263672,
+      "objective/train/theoretical_loss": 4.104008657956216,
+      "objective/train/tokens_used": 348140000,
+      "theoretical_loss": 4.104008657956216,
+      "tokens_seen": 327680000
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0004549047141424273,
+      "loss": 3.1165,
+      "theoretical_loss": 4.104008657956216,
+      "tokens_seen": 327680000
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0004548946840521565,
+      "loss": 2.9939,
+      "theoretical_loss": 4.1039137437624,
+      "tokens_seen": 327745536
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.00045488465396188564,
+      "loss": 2.8305,
+      "theoretical_loss": 4.103818853858651,
+      "tokens_seen": 327811072
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0004548746238716149,
+      "loss": 3.0165,
+      "theoretical_loss": 4.103723988233898,
+      "tokens_seen": 327876608
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.00045486459378134405,
+      "loss": 3.1865,
+      "theoretical_loss": 4.103629146877078,
+      "tokens_seen": 327942144
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.00045485456369107324,
+      "loss": 3.0442,
+      "theoretical_loss": 4.103534329777135,
+      "tokens_seen": 328007680
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0004548445336008024,
+      "loss": 2.9175,
+      "theoretical_loss": 4.103439536923021,
+      "tokens_seen": 328073216
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.00045483450351053165,
+      "loss": 2.9148,
+      "theoretical_loss": 4.103344768303694,
+      "tokens_seen": 328138752
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0004548244734202608,
+      "loss": 3.2821,
+      "theoretical_loss": 4.103250023908119,
+      "tokens_seen": 328204288
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.00045481444332999,
+      "loss": 3.0728,
+      "theoretical_loss": 4.103155303725271,
+      "tokens_seen": 328269824
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.00045480441323971914,
+      "loss": 2.9522,
+      "theoretical_loss": 4.103060607744128,
+      "tokens_seen": 328335360
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0004547943831494484,
+      "loss": 3.0337,
+      "theoretical_loss": 4.102965935953679,
+      "tokens_seen": 328400896
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.00045478435305917756,
+      "loss": 2.7991,
+      "theoretical_loss": 4.102871288342918,
+      "tokens_seen": 328466432
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.00045477432296890674,
+      "loss": 2.9165,
+      "theoretical_loss": 4.102776664900848,
+      "tokens_seen": 328531968
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0004547642928786359,
+      "loss": 3.0264,
+      "theoretical_loss": 4.1026820656164755,
+      "tokens_seen": 328597504
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0004547542627883651,
+      "loss": 2.956,
+      "theoretical_loss": 4.102587490478819,
+      "tokens_seen": 328663040
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0004547442326980943,
+      "loss": 3.0012,
+      "theoretical_loss": 4.102492939476901,
+      "tokens_seen": 328728576
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0004547342026078235,
+      "loss": 3.2309,
+      "theoretical_loss": 4.102398412599753,
+      "tokens_seen": 328794112
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.00045472417251755264,
+      "loss": 2.9335,
+      "theoretical_loss": 4.102303909836412,
+      "tokens_seen": 328859648
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0004547141424272819,
+      "loss": 3.0824,
+      "theoretical_loss": 4.102209431175923,
+      "tokens_seen": 328925184
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.000454704112337011,
+      "loss": 3.0037,
+      "theoretical_loss": 4.102114976607338,
+      "tokens_seen": 328990720
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.00045469408224674024,
+      "loss": 2.9785,
+      "theoretical_loss": 4.102020546119716,
+      "tokens_seen": 329056256
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0004546840521564694,
+      "loss": 3.0101,
+      "theoretical_loss": 4.101926139702125,
+      "tokens_seen": 329121792
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0004546740220661986,
+      "loss": 3.1876,
+      "theoretical_loss": 4.101831757343637,
+      "tokens_seen": 329187328
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0004546639919759278,
+      "loss": 3.0554,
+      "theoretical_loss": 4.1017373990333335,
+      "tokens_seen": 329252864
+    },
+    {
+      "epoch": 0.1,
+      "objective/train/docs_used": 193205,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 2.53017520904541,
+      "objective/train/theoretical_loss": 4.101643064760301,
+      "objective/train/tokens_used": 349778400,
+      "theoretical_loss": 4.101643064760301,
+      "tokens_seen": 329318400
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.000454653961885657,
+      "loss": 2.8743,
+      "theoretical_loss": 4.101643064760301,
+      "tokens_seen": 329318400
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.00045464393179538615,
+      "loss": 2.7826,
+      "theoretical_loss": 4.101548754513636,
+      "tokens_seen": 329383936
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0004546339017051154,
+      "loss": 2.8613,
+      "theoretical_loss": 4.101454468282439,
+      "tokens_seen": 329449472
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0004546238716148445,
+      "loss": 2.9591,
+      "theoretical_loss": 4.101360206055821,
+      "tokens_seen": 329515008
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.00045461384152457374,
+      "loss": 3.0603,
+      "theoretical_loss": 4.101265967822897,
+      "tokens_seen": 329580544
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0004546038114343029,
+      "loss": 2.8043,
+      "theoretical_loss": 4.101171753572791,
+      "tokens_seen": 329646080
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0004545937813440321,
+      "loss": 3.0562,
+      "theoretical_loss": 4.101077563294632,
+      "tokens_seen": 329711616
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0004545837512537613,
+      "loss": 3.1101,
+      "theoretical_loss": 4.1009833969775595,
+      "tokens_seen": 329777152
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.00045457372116349047,
+      "loss": 3.162,
+      "theoretical_loss": 4.100889254610717,
+      "tokens_seen": 329842688
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.00045456369107321965,
+      "loss": 2.9801,
+      "theoretical_loss": 4.100795136183256,
+      "tokens_seen": 329908224
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0004545536609829489,
+      "loss": 2.9315,
+      "theoretical_loss": 4.100701041684336,
+      "tokens_seen": 329973760
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.000454543630892678,
+      "loss": 3.097,
+      "theoretical_loss": 4.100606971103122,
+      "tokens_seen": 330039296
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.00045453360080240725,
+      "loss": 3.0641,
+      "theoretical_loss": 4.100512924428788,
+      "tokens_seen": 330104832
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.00045452357071213643,
+      "loss": 2.9796,
+      "theoretical_loss": 4.100418901650513,
+      "tokens_seen": 330170368
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0004545135406218656,
+      "loss": 2.9823,
+      "theoretical_loss": 4.1003249027574835,
+      "tokens_seen": 330235904
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0004545035105315948,
+      "loss": 3.0358,
+      "theoretical_loss": 4.100230927738895,
+      "tokens_seen": 330301440
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.00045449348044132397,
+      "loss": 3.0782,
+      "theoretical_loss": 4.100136976583947,
+      "tokens_seen": 330366976
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.00045448345035105315,
+      "loss": 3.0677,
+      "theoretical_loss": 4.100043049281849,
+      "tokens_seen": 330432512
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0004544734202607824,
+      "loss": 2.9041,
+      "theoretical_loss": 4.099949145821815,
+      "tokens_seen": 330498048
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0004544633901705115,
+      "loss": 3.1047,
+      "theoretical_loss": 4.099855266193067,
+      "tokens_seen": 330563584
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.00045445336008024075,
+      "loss": 3.2029,
+      "theoretical_loss": 4.099761410384835,
+      "tokens_seen": 330629120
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0004544433299899699,
+      "loss": 2.8177,
+      "theoretical_loss": 4.099667578386355,
+      "tokens_seen": 330694656
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0004544332998996991,
+      "loss": 2.8579,
+      "theoretical_loss": 4.099573770186869,
+      "tokens_seen": 330760192
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0004544232698094283,
+      "loss": 2.9416,
+      "theoretical_loss": 4.099479985775629,
+      "tokens_seen": 330825728
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0004544132397191575,
+      "loss": 2.997,
+      "theoretical_loss": 4.09938622514189,
+      "tokens_seen": 330891264
+    },
+    {
+      "epoch": 0.1,
+      "objective/train/docs_used": 193823,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 2.6776926517486572,
+      "objective/train/theoretical_loss": 4.099292488274917,
+      "objective/train/tokens_used": 351416800,
+      "theoretical_loss": 4.099292488274917,
+      "tokens_seen": 330956800
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.00045440320962888666,
+      "loss": 2.9025,
+      "theoretical_loss": 4.099292488274917,
+      "tokens_seen": 330956800
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.00045439317953861584,
+      "loss": 2.9272,
+      "theoretical_loss": 4.0991987751639805,
+      "tokens_seen": 331022336
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.000454383149448345,
+      "loss": 3.0293,
+      "theoretical_loss": 4.099105085798359,
+      "tokens_seen": 331087872
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.00045437311935807425,
+      "loss": 2.9882,
+      "theoretical_loss": 4.099011420167338,
+      "tokens_seen": 331153408
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0004543630892678034,
+      "loss": 2.9665,
+      "theoretical_loss": 4.098917778260208,
+      "tokens_seen": 331218944
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0004543530591775326,
+      "loss": 3.145,
+      "theoretical_loss": 4.098824160066269,
+      "tokens_seen": 331284480
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0004543430290872618,
+      "loss": 3.0804,
+      "theoretical_loss": 4.098730565574826,
+      "tokens_seen": 331350016
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.000454332998996991,
+      "loss": 2.8699,
+      "theoretical_loss": 4.098636994775193,
+      "tokens_seen": 331415552
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.00045432296890672016,
+      "loss": 2.8012,
+      "theoretical_loss": 4.098543447656688,
+      "tokens_seen": 331481088
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.00045431293881644934,
+      "loss": 3.1938,
+      "theoretical_loss": 4.098449924208639,
+      "tokens_seen": 331546624
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0004543029087261785,
+      "loss": 2.8317,
+      "theoretical_loss": 4.098356424420379,
+      "tokens_seen": 331612160
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.00045429287863590776,
+      "loss": 2.9114,
+      "theoretical_loss": 4.098262948281247,
+      "tokens_seen": 331677696
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0004542828485456369,
+      "loss": 2.9306,
+      "theoretical_loss": 4.098169495780592,
+      "tokens_seen": 331743232
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0004542728184553661,
+      "loss": 3.0543,
+      "theoretical_loss": 4.098076066907769,
+      "tokens_seen": 331808768
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.00045426278836509525,
+      "loss": 2.9948,
+      "theoretical_loss": 4.0979826616521375,
+      "tokens_seen": 331874304
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0004542527582748245,
+      "loss": 3.0341,
+      "theoretical_loss": 4.0978892800030655,
+      "tokens_seen": 331939840
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.00045424272818455366,
+      "loss": 2.7333,
+      "theoretical_loss": 4.0977959219499285,
+      "tokens_seen": 332005376
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.00045423269809428284,
+      "loss": 2.7174,
+      "theoretical_loss": 4.097702587482107,
+      "tokens_seen": 332070912
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.000454222668004012,
+      "loss": 2.9369,
+      "theoretical_loss": 4.097609276588993,
+      "tokens_seen": 332136448
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0004542126379137412,
+      "loss": 2.9834,
+      "theoretical_loss": 4.097515989259978,
+      "tokens_seen": 332201984
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.00045420260782347044,
+      "loss": 2.7845,
+      "theoretical_loss": 4.097422725484467,
+      "tokens_seen": 332267520
+    }
+  ],
+  "max_steps": 50354,
+  "num_train_epochs": 9223372036854775807,
+  "total_flos": 1.6956807708672e+17,
+  "trial_name": null,
+  "trial_params": null
+}