diff --git "a/checkpoint-2500/trainer_state.json" "b/checkpoint-2500/trainer_state.json"
new file mode 100644--- /dev/null
+++ "b/checkpoint-2500/trainer_state.json"
@@ -0,0 +1,18623 @@
+{
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 0.04964848870000397,
+  "global_step": 2500,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.0,
+      "learning_rate": 9.92063492063492e-07,
+      "loss": 10.8568,
+      "theoretical_loss": 20.812814784551147,
+      "tokens_seen": 65536
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 1.984126984126984e-06,
+      "loss": 10.8727,
+      "theoretical_loss": 17.56619808733264,
+      "tokens_seen": 131072
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 2.9761904761904763e-06,
+      "loss": 10.7356,
+      "theoretical_loss": 15.939474075840563,
+      "tokens_seen": 196608
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 3.968253968253968e-06,
+      "loss": 10.7113,
+      "theoretical_loss": 14.892313738992565,
+      "tokens_seen": 262144
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 4.96031746031746e-06,
+      "loss": 10.3498,
+      "theoretical_loss": 14.13621392076697,
+      "tokens_seen": 327680
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 5.9523809523809525e-06,
+      "loss": 10.1826,
+      "theoretical_loss": 13.552558455554218,
+      "tokens_seen": 393216
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 6.944444444444444e-06,
+      "loss": 10.0176,
+      "theoretical_loss": 13.081805984405184,
+      "tokens_seen": 458752
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 7.936507936507936e-06,
+      "loss": 9.6515,
+      "theoretical_loss": 12.690126608487319,
+      "tokens_seen": 524288
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 8.928571428571428e-06,
+      "loss": 9.2598,
+      "theoretical_loss": 12.356589446877619,
+      "tokens_seen": 589824
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 9.92063492063492e-06,
+      "loss": 9.1219,
+      "theoretical_loss": 12.067409590039073,
+      "tokens_seen": 655360
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 1.0912698412698412e-05,
+      "loss": 9.2934,
+      "theoretical_loss": 11.813063214105672,
+      "tokens_seen": 720896
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 1.1904761904761905e-05,
+      "loss": 9.0973,
+      "theoretical_loss": 11.586716191710725,
+      "tokens_seen": 786432
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 1.2896825396825396e-05,
+      "loss": 9.4804,
+      "theoretical_loss": 11.383311123190783,
+      "tokens_seen": 851968
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 1.3888888888888888e-05,
+      "loss": 9.3208,
+      "theoretical_loss": 11.199008685115867,
+      "tokens_seen": 917504
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 1.4880952380952381e-05,
+      "loss": 9.1864,
+      "theoretical_loss": 11.030830900981908,
+      "tokens_seen": 983040
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 1.5873015873015872e-05,
+      "loss": 9.398,
+      "theoretical_loss": 10.876425069460945,
+      "tokens_seen": 1048576
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 1.6865079365079364e-05,
+      "loss": 8.9394,
+      "theoretical_loss": 10.73390272306672,
+      "tokens_seen": 1114112
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 1.7857142857142855e-05,
+      "loss": 9.0636,
+      "theoretical_loss": 10.601726859234274,
+      "tokens_seen": 1179648
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 1.884920634920635e-05,
+      "loss": 9.1583,
+      "theoretical_loss": 10.478631155360636,
+      "tokens_seen": 1245184
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 1.984126984126984e-05,
+      "loss": 9.1124,
+      "theoretical_loss": 10.363560926767326,
+      "tokens_seen": 1310720
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 2.0833333333333333e-05,
+      "loss": 9.0087,
+      "theoretical_loss": 10.25562920390074,
+      "tokens_seen": 1376256
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 2.1825396825396824e-05,
+      "loss": 9.0788,
+      "theoretical_loss": 10.154083536274014,
+      "tokens_seen": 1441792
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 2.2817460317460315e-05,
+      "loss": 8.4406,
+      "theoretical_loss": 10.058280544736593,
+      "tokens_seen": 1507328
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 2.380952380952381e-05,
+      "loss": 8.9754,
+      "theoretical_loss": 9.967666161844274,
+      "tokens_seen": 1572864
+    },
+    {
+      "epoch": 0.0,
+      "objective/train/docs_used": 12267,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 8.854658126831055,
+      "objective/train/theoretical_loss": 9.881760109397105,
+      "objective/train/tokens_used": 22098400,
+      "theoretical_loss": 9.881760109397105,
+      "tokens_seen": 1638400
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 2.48015873015873e-05,
+      "loss": 8.9164,
+      "theoretical_loss": 9.881760109397105,
+      "tokens_seen": 1638400
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 2.5793650793650793e-05,
+      "loss": 7.9452,
+      "theoretical_loss": 9.800143574544556,
+      "tokens_seen": 1703936
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 2.6785714285714284e-05,
+      "loss": 8.81,
+      "theoretical_loss": 9.722449329911441,
+      "tokens_seen": 1769472
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 2.7777777777777776e-05,
+      "loss": 9.0741,
+      "theoretical_loss": 9.648353742085542,
+      "tokens_seen": 1835008
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 2.876984126984127e-05,
+      "loss": 8.8256,
+      "theoretical_loss": 9.577570254149634,
+      "tokens_seen": 1900544
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 2.9761904761904762e-05,
+      "loss": 8.3274,
+      "theoretical_loss": 9.509844029768848,
+      "tokens_seen": 1966080
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 3.075396825396825e-05,
+      "loss": 8.55,
+      "theoretical_loss": 9.44494752063593,
+      "tokens_seen": 2031616
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 3.1746031746031745e-05,
+      "loss": 8.0979,
+      "theoretical_loss": 9.382676773914453,
+      "tokens_seen": 2097152
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 3.273809523809524e-05,
+      "loss": 8.3374,
+      "theoretical_loss": 9.322848337237977,
+      "tokens_seen": 2162688
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 3.373015873015873e-05,
+      "loss": 8.5177,
+      "theoretical_loss": 9.26529664966427,
+      "tokens_seen": 2228224
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 3.472222222222222e-05,
+      "loss": 8.6686,
+      "theoretical_loss": 9.209871830448751,
+      "tokens_seen": 2293760
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 3.571428571428571e-05,
+      "loss": 8.675,
+      "theoretical_loss": 9.156437795512288,
+      "tokens_seen": 2359296
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 3.670634920634921e-05,
+      "loss": 8.4694,
+      "theoretical_loss": 9.104870645417346,
+      "tokens_seen": 2424832
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 3.76984126984127e-05,
+      "loss": 8.2407,
+      "theoretical_loss": 9.05505727953773,
+      "tokens_seen": 2490368
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 3.8690476190476195e-05,
+      "loss": 8.5934,
+      "theoretical_loss": 9.006894199647824,
+      "tokens_seen": 2555904
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 3.968253968253968e-05,
+      "loss": 7.8491,
+      "theoretical_loss": 8.960286472913351,
+      "tokens_seen": 2621440
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 4.067460317460318e-05,
+      "loss": 7.8615,
+      "theoretical_loss": 8.915146829644607,
+      "tokens_seen": 2686976
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 4.1666666666666665e-05,
+      "loss": 7.7589,
+      "theoretical_loss": 8.871394875482219,
+      "tokens_seen": 2752512
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 4.265873015873016e-05,
+      "loss": 8.0014,
+      "theoretical_loss": 8.828956401157495,
+      "tokens_seen": 2818048
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 4.365079365079365e-05,
+      "loss": 7.8309,
+      "theoretical_loss": 8.787762775782408,
+      "tokens_seen": 2883584
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 4.464285714285714e-05,
+      "loss": 8.259,
+      "theoretical_loss": 8.747750411915451,
+      "tokens_seen": 2949120
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 4.563492063492063e-05,
+      "loss": 8.1107,
+      "theoretical_loss": 8.708860292524829,
+      "tokens_seen": 3014656
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 4.6626984126984126e-05,
+      "loss": 8.2947,
+      "theoretical_loss": 8.671037551512843,
+      "tokens_seen": 3080192
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 4.761904761904762e-05,
+      "loss": 7.764,
+      "theoretical_loss": 8.63423110073947,
+      "tokens_seen": 3145728
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 4.8611111111111115e-05,
+      "loss": 8.1678,
+      "theoretical_loss": 8.59839329754032,
+      "tokens_seen": 3211264
+    },
+    {
+      "epoch": 0.0,
+      "objective/train/docs_used": 13483,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 8.108365058898926,
+      "objective/train/theoretical_loss": 8.563479647615063,
+      "objective/train/tokens_used": 23736800,
+      "theoretical_loss": 8.563479647615063,
+      "tokens_seen": 3276800
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 4.96031746031746e-05,
+      "loss": 7.5801,
+      "theoretical_loss": 8.563479647615063,
+      "tokens_seen": 3276800
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 5.05952380952381e-05,
+      "loss": 7.8434,
+      "theoretical_loss": 8.52944853889911,
+      "tokens_seen": 3342336
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 5.1587301587301586e-05,
+      "loss": 7.8492,
+      "theoretical_loss": 8.496261002649998,
+      "tokens_seen": 3407872
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 5.257936507936508e-05,
+      "loss": 7.8312,
+      "theoretical_loss": 8.463880498501183,
+      "tokens_seen": 3473408
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 5.357142857142857e-05,
+      "loss": 7.3015,
+      "theoretical_loss": 8.432272720676774,
+      "tokens_seen": 3538944
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 5.4563492063492063e-05,
+      "loss": 7.9474,
+      "theoretical_loss": 8.401405422934712,
+      "tokens_seen": 3604480
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 5.555555555555555e-05,
+      "loss": 7.6148,
+      "theoretical_loss": 8.371248260124204,
+      "tokens_seen": 3670016
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 5.6547619047619046e-05,
+      "loss": 7.5108,
+      "theoretical_loss": 8.341772644515071,
+      "tokens_seen": 3735552
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 5.753968253968254e-05,
+      "loss": 7.6168,
+      "theoretical_loss": 8.312951615289325,
+      "tokens_seen": 3801088
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 5.8531746031746036e-05,
+      "loss": 7.8436,
+      "theoretical_loss": 8.284759719785177,
+      "tokens_seen": 3866624
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 5.9523809523809524e-05,
+      "loss": 7.0651,
+      "theoretical_loss": 8.257172905255858,
+      "tokens_seen": 3932160
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 6.051587301587302e-05,
+      "loss": 7.4859,
+      "theoretical_loss": 8.23016842005411,
+      "tokens_seen": 3997696
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 6.15079365079365e-05,
+      "loss": 7.531,
+      "theoretical_loss": 8.203724723281965,
+      "tokens_seen": 4063232
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 6.25e-05,
+      "loss": 7.5815,
+      "theoretical_loss": 8.177821402057042,
+      "tokens_seen": 4128768
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 6.349206349206349e-05,
+      "loss": 7.2471,
+      "theoretical_loss": 8.152439095643611,
+      "tokens_seen": 4194304
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 6.448412698412699e-05,
+      "loss": 7.4488,
+      "theoretical_loss": 8.127559425781495,
+      "tokens_seen": 4259840
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 6.547619047619048e-05,
+      "loss": 7.4517,
+      "theoretical_loss": 8.103164932619705,
+      "tokens_seen": 4325376
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 6.646825396825397e-05,
+      "loss": 7.1039,
+      "theoretical_loss": 8.079239015726635,
+      "tokens_seen": 4390912
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 6.746031746031745e-05,
+      "loss": 7.1279,
+      "theoretical_loss": 8.055765879705412,
+      "tokens_seen": 4456448
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 6.845238095238096e-05,
+      "loss": 7.3354,
+      "theoretical_loss": 8.032730483993003,
+      "tokens_seen": 4521984
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 6.944444444444444e-05,
+      "loss": 7.1485,
+      "theoretical_loss": 8.01011849646583,
+      "tokens_seen": 4587520
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 7.043650793650793e-05,
+      "loss": 6.9851,
+      "theoretical_loss": 7.987916250513374,
+      "tokens_seen": 4653056
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 7.142857142857142e-05,
+      "loss": 6.9432,
+      "theoretical_loss": 7.966110705275796,
+      "tokens_seen": 4718592
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 7.242063492063492e-05,
+      "loss": 6.9933,
+      "theoretical_loss": 7.944689408771984,
+      "tokens_seen": 4784128
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 7.341269841269842e-05,
+      "loss": 7.0325,
+      "theoretical_loss": 7.923640463671514,
+      "tokens_seen": 4849664
+    },
+    {
+      "epoch": 0.0,
+      "objective/train/docs_used": 14033,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 6.057826519012451,
+      "objective/train/theoretical_loss": 7.902952495488062,
+      "objective/train/tokens_used": 25375200,
+      "theoretical_loss": 7.902952495488062,
+      "tokens_seen": 4915200
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 7.440476190476191e-05,
+      "loss": 6.7791,
+      "theoretical_loss": 7.902952495488062,
+      "tokens_seen": 4915200
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 7.53968253968254e-05,
+      "loss": 6.7978,
+      "theoretical_loss": 7.882614622993199,
+      "tokens_seen": 4980736
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 7.63888888888889e-05,
+      "loss": 6.8558,
+      "theoretical_loss": 7.8626164306686235,
+      "tokens_seen": 5046272
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 7.738095238095239e-05,
+      "loss": 6.9141,
+      "theoretical_loss": 7.842947943031932,
+      "tokens_seen": 5111808
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 7.837301587301588e-05,
+      "loss": 6.9255,
+      "theoretical_loss": 7.8235996006863076,
+      "tokens_seen": 5177344
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 7.936507936507937e-05,
+      "loss": 6.7895,
+      "theoretical_loss": 7.8045622379581605,
+      "tokens_seen": 5242880
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 8.035714285714287e-05,
+      "loss": 6.7059,
+      "theoretical_loss": 7.785827061999068,
+      "tokens_seen": 5308416
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 8.134920634920635e-05,
+      "loss": 6.815,
+      "theoretical_loss": 7.767385633239359,
+      "tokens_seen": 5373952
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 8.234126984126984e-05,
+      "loss": 6.8445,
+      "theoretical_loss": 7.749229847090614,
+      "tokens_seen": 5439488
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 8.333333333333333e-05,
+      "loss": 6.7849,
+      "theoretical_loss": 7.731351916803314,
+      "tokens_seen": 5505024
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 8.432539682539683e-05,
+      "loss": 6.5349,
+      "theoretical_loss": 7.7137443573939155,
+      "tokens_seen": 5570560
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 8.531746031746032e-05,
+      "loss": 6.5103,
+      "theoretical_loss": 7.696399970562929,
+      "tokens_seen": 5636096
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 8.630952380952381e-05,
+      "loss": 6.9307,
+      "theoretical_loss": 7.679311830532177,
+      "tokens_seen": 5701632
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 8.73015873015873e-05,
+      "loss": 6.8115,
+      "theoretical_loss": 7.662473270735324,
+      "tokens_seen": 5767168
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 8.82936507936508e-05,
+      "loss": 6.6601,
+      "theoretical_loss": 7.645877871301275,
+      "tokens_seen": 5832704
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 8.928571428571429e-05,
+      "loss": 6.2238,
+      "theoretical_loss": 7.6295194472748555,
+      "tokens_seen": 5898240
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 9.027777777777777e-05,
+      "loss": 6.6176,
+      "theoretical_loss": 7.613392037523692,
+      "tokens_seen": 5963776
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 9.126984126984126e-05,
+      "loss": 6.5932,
+      "theoretical_loss": 7.597489894284275,
+      "tokens_seen": 6029312
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 9.226190476190476e-05,
+      "loss": 6.51,
+      "theoretical_loss": 7.581807473303883,
+      "tokens_seen": 6094848
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 9.325396825396825e-05,
+      "loss": 6.3768,
+      "theoretical_loss": 7.566339424538366,
+      "tokens_seen": 6160384
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 9.424603174603175e-05,
+      "loss": 6.5465,
+      "theoretical_loss": 7.551080583368945,
+      "tokens_seen": 6225920
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 9.523809523809524e-05,
+      "loss": 6.8931,
+      "theoretical_loss": 7.536025962303915,
+      "tokens_seen": 6291456
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 9.623015873015874e-05,
+      "loss": 6.5266,
+      "theoretical_loss": 7.521170743133757,
+      "tokens_seen": 6356992
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 9.722222222222223e-05,
+      "loss": 6.4856,
+      "theoretical_loss": 7.506510269510493,
+      "tokens_seen": 6422528
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 9.821428571428572e-05,
+      "loss": 6.4763,
+      "theoretical_loss": 7.492040039924245,
+      "tokens_seen": 6488064
+    },
+    {
+      "epoch": 0.0,
+      "objective/train/docs_used": 15357,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 6.699707984924316,
+      "objective/train/theoretical_loss": 7.4777557010520255,
+      "objective/train/tokens_used": 27013600,
+      "theoretical_loss": 7.4777557010520255,
+      "tokens_seen": 6553600
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 9.92063492063492e-05,
+      "loss": 6.6532,
+      "theoretical_loss": 7.4777557010520255,
+      "tokens_seen": 6553600
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00010019841269841271,
+      "loss": 6.4,
+      "theoretical_loss": 7.463653041455457,
+      "tokens_seen": 6619136
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.0001011904761904762,
+      "loss": 6.2462,
+      "theoretical_loss": 7.449727985605911,
+      "tokens_seen": 6684672
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00010218253968253968,
+      "loss": 6.6399,
+      "theoretical_loss": 7.435976588217015,
+      "tokens_seen": 6750208
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00010317460317460317,
+      "loss": 6.1887,
+      "theoretical_loss": 7.4223950288659,
+      "tokens_seen": 6815744
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00010416666666666667,
+      "loss": 6.4129,
+      "theoretical_loss": 7.40897960688587,
+      "tokens_seen": 6881280
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00010515873015873016,
+      "loss": 6.2296,
+      "theoretical_loss": 7.395726736514341,
+      "tokens_seen": 6946816
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00010615079365079365,
+      "loss": 6.3505,
+      "theoretical_loss": 7.3826329422810275,
+      "tokens_seen": 7012352
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00010714285714285714,
+      "loss": 6.5043,
+      "theoretical_loss": 7.369694854622368,
+      "tokens_seen": 7077888
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00010813492063492064,
+      "loss": 6.3689,
+      "theoretical_loss": 7.3569092057090835,
+      "tokens_seen": 7143424
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00010912698412698413,
+      "loss": 6.5583,
+      "theoretical_loss": 7.3442728254746825,
+      "tokens_seen": 7208960
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00011011904761904761,
+      "loss": 6.3757,
+      "theoretical_loss": 7.3317826378335145,
+      "tokens_seen": 7274496
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.0001111111111111111,
+      "loss": 6.3164,
+      "theoretical_loss": 7.319435657077673,
+      "tokens_seen": 7340032
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.0001121031746031746,
+      "loss": 6.205,
+      "theoretical_loss": 7.307228984442819,
+      "tokens_seen": 7405568
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00011309523809523809,
+      "loss": 6.4975,
+      "theoretical_loss": 7.29515980483356,
+      "tokens_seen": 7471104
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00011408730158730158,
+      "loss": 6.087,
+      "theoretical_loss": 7.283225383699648,
+      "tokens_seen": 7536640
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00011507936507936508,
+      "loss": 6.0785,
+      "theoretical_loss": 7.271423064054828,
+      "tokens_seen": 7602176
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00011607142857142858,
+      "loss": 6.4949,
+      "theoretical_loss": 7.259750263630618,
+      "tokens_seen": 7667712
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00011706349206349207,
+      "loss": 6.2155,
+      "theoretical_loss": 7.248204472157866,
+      "tokens_seen": 7733248
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00011805555555555556,
+      "loss": 6.0229,
+      "theoretical_loss": 7.236783248769257,
+      "tokens_seen": 7798784
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00011904761904761905,
+      "loss": 6.6244,
+      "theoretical_loss": 7.225484219516492,
+      "tokens_seen": 7864320
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00012003968253968255,
+      "loss": 5.9387,
+      "theoretical_loss": 7.2143050749961155,
+      "tokens_seen": 7929856
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00012103174603174604,
+      "loss": 6.2842,
+      "theoretical_loss": 7.2032435680783955,
+      "tokens_seen": 7995392
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00012202380952380953,
+      "loss": 6.2047,
+      "theoretical_loss": 7.19229751173401,
+      "tokens_seen": 8060928
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.000123015873015873,
+      "loss": 6.2571,
+      "theoretical_loss": 7.181464776953511,
+      "tokens_seen": 8126464
+    },
+    {
+      "epoch": 0.0,
+      "objective/train/docs_used": 16711,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 6.2001729011535645,
+      "objective/train/theoretical_loss": 7.1707432907549595,
+      "objective/train/tokens_used": 28652000,
+      "theoretical_loss": 7.1707432907549595,
+      "tokens_seen": 8192000
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.0001240079365079365,
+      "loss": 6.0533,
+      "theoretical_loss": 7.1707432907549595,
+      "tokens_seen": 8192000
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.000125,
+      "loss": 6.0526,
+      "theoretical_loss": 7.160131034275267,
+      "tokens_seen": 8257536
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.0001259920634920635,
+      "loss": 6.0532,
+      "theoretical_loss": 7.149626040941133,
+      "tokens_seen": 8323072
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00012698412698412698,
+      "loss": 5.9719,
+      "theoretical_loss": 7.139226394715633,
+      "tokens_seen": 8388608
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00012797619047619048,
+      "loss": 6.0403,
+      "theoretical_loss": 7.12893022841679,
+      "tokens_seen": 8454144
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00012896825396825398,
+      "loss": 5.7847,
+      "theoretical_loss": 7.118735722104612,
+      "tokens_seen": 8519680
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00012996031746031748,
+      "loss": 5.9094,
+      "theoretical_loss": 7.10864110153334,
+      "tokens_seen": 8585216
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00013095238095238096,
+      "loss": 6.0872,
+      "theoretical_loss": 7.098644636665766,
+      "tokens_seen": 8650752
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00013194444444444446,
+      "loss": 6.0185,
+      "theoretical_loss": 7.088744640246688,
+      "tokens_seen": 8716288
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00013293650793650793,
+      "loss": 6.0768,
+      "theoretical_loss": 7.078939466432745,
+      "tokens_seen": 8781824
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00013392857142857144,
+      "loss": 6.3022,
+      "theoretical_loss": 7.069227509475962,
+      "tokens_seen": 8847360
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.0001349206349206349,
+      "loss": 6.0881,
+      "theoretical_loss": 7.059607202458563,
+      "tokens_seen": 8912896
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.0001359126984126984,
+      "loss": 5.7381,
+      "theoretical_loss": 7.0500770160766635,
+      "tokens_seen": 8978432
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.0001369047619047619,
+      "loss": 5.9916,
+      "theoretical_loss": 7.04063545747062,
+      "tokens_seen": 9043968
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00013789682539682541,
+      "loss": 6.0983,
+      "theoretical_loss": 7.031281069099929,
+      "tokens_seen": 9109504
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.0001388888888888889,
+      "loss": 6.0662,
+      "theoretical_loss": 7.022012427660673,
+      "tokens_seen": 9175040
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.0001398809523809524,
+      "loss": 5.7078,
+      "theoretical_loss": 7.012828143043604,
+      "tokens_seen": 9240576
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00014087301587301586,
+      "loss": 6.2455,
+      "theoretical_loss": 7.003726857331067,
+      "tokens_seen": 9306112
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00014186507936507937,
+      "loss": 5.9609,
+      "theoretical_loss": 6.994707243831053,
+      "tokens_seen": 9371648
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00014285714285714284,
+      "loss": 5.9423,
+      "theoretical_loss": 6.985768006146776,
+      "tokens_seen": 9437184
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00014384920634920634,
+      "loss": 6.0329,
+      "theoretical_loss": 6.976907877280185,
+      "tokens_seen": 9502720
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00014484126984126984,
+      "loss": 5.7799,
+      "theoretical_loss": 6.96812561876801,
+      "tokens_seen": 9568256
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00014583333333333335,
+      "loss": 5.9063,
+      "theoretical_loss": 6.95942001984889,
+      "tokens_seen": 9633792
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00014682539682539685,
+      "loss": 5.9317,
+      "theoretical_loss": 6.950789896660304,
+      "tokens_seen": 9699328
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00014781746031746032,
+      "loss": 5.8925,
+      "theoretical_loss": 6.942234091464025,
+      "tokens_seen": 9764864
+    },
+    {
+      "epoch": 0.0,
+      "objective/train/docs_used": 17166,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 6.479184627532959,
+      "objective/train/theoretical_loss": 6.933751471898896,
+      "objective/train/tokens_used": 30290400,
+      "theoretical_loss": 6.933751471898896,
+      "tokens_seen": 9830400
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00014880952380952382,
+      "loss": 6.2539,
+      "theoretical_loss": 6.933751471898896,
+      "tokens_seen": 9830400
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.0001498015873015873,
+      "loss": 5.996,
+      "theoretical_loss": 6.925340930259813,
+      "tokens_seen": 9895936
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.0001507936507936508,
+      "loss": 5.8197,
+      "theoretical_loss": 6.917001382801793,
+      "tokens_seen": 9961472
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00015178571428571427,
+      "loss": 6.0534,
+      "theoretical_loss": 6.908731769068142,
+      "tokens_seen": 10027008
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.0001527777777777778,
+      "loss": 5.7838,
+      "theoretical_loss": 6.900531051241684,
+      "tokens_seen": 10092544
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00015376984126984128,
+      "loss": 5.7799,
+      "theoretical_loss": 6.892398213518156,
+      "tokens_seen": 10158080
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00015476190476190478,
+      "loss": 6.1056,
+      "theoretical_loss": 6.884332261500866,
+      "tokens_seen": 10223616
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00015575396825396825,
+      "loss": 5.5432,
+      "theoretical_loss": 6.876332221615746,
+      "tokens_seen": 10289152
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00015674603174603175,
+      "loss": 5.8572,
+      "theoretical_loss": 6.8683971405459925,
+      "tokens_seen": 10354688
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00015773809523809523,
+      "loss": 6.226,
+      "theoretical_loss": 6.860526084685546,
+      "tokens_seen": 10420224
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00015873015873015873,
+      "loss": 5.7117,
+      "theoretical_loss": 6.852718139610625,
+      "tokens_seen": 10485760
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.0001597222222222222,
+      "loss": 5.6684,
+      "theoretical_loss": 6.844972409568637,
+      "tokens_seen": 10551296
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00016071428571428573,
+      "loss": 5.8166,
+      "theoretical_loss": 6.8372880169837895,
+      "tokens_seen": 10616832
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.0001617063492063492,
+      "loss": 5.971,
+      "theoretical_loss": 6.829664101978745,
+      "tokens_seen": 10682368
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.0001626984126984127,
+      "loss": 5.7343,
+      "theoretical_loss": 6.8220998219117135,
+      "tokens_seen": 10747904
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00016369047619047618,
+      "loss": 5.8061,
+      "theoretical_loss": 6.81459435092839,
+      "tokens_seen": 10813440
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00016468253968253969,
+      "loss": 6.0029,
+      "theoretical_loss": 6.807146879528176,
+      "tokens_seen": 10878976
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00016567460317460316,
+      "loss": 5.5367,
+      "theoretical_loss": 6.79975661414414,
+      "tokens_seen": 10944512
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00016666666666666666,
+      "loss": 5.6256,
+      "theoretical_loss": 6.79242277673622,
+      "tokens_seen": 11010048
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00016765873015873016,
+      "loss": 5.7868,
+      "theoretical_loss": 6.785144604397143,
+      "tokens_seen": 11075584
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00016865079365079366,
+      "loss": 5.6046,
+      "theoretical_loss": 6.777921348970633,
+      "tokens_seen": 11141120
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00016964285714285717,
+      "loss": 5.9753,
+      "theoretical_loss": 6.770752276681419,
+      "tokens_seen": 11206656
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00017063492063492064,
+      "loss": 5.9022,
+      "theoretical_loss": 6.76363666777662,
+      "tokens_seen": 11272192
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00017162698412698414,
+      "loss": 5.8921,
+      "theoretical_loss": 6.756573816178118,
+      "tokens_seen": 11337728
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00017261904761904762,
+      "loss": 5.9722,
+      "theoretical_loss": 6.749563029145481,
+      "tokens_seen": 11403264
+    },
+    {
+      "epoch": 0.0,
+      "objective/train/docs_used": 18369,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 5.672413349151611,
+      "objective/train/theoretical_loss": 6.742603626949085,
+      "objective/train/tokens_used": 31928800,
+      "theoretical_loss": 6.742603626949085,
+      "tokens_seen": 11468800
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00017361111111111112,
+      "loss": 5.4506,
+      "theoretical_loss": 6.742603626949085,
+      "tokens_seen": 11468800
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.0001746031746031746,
+      "loss": 6.0078,
+      "theoretical_loss": 6.735694942553071,
+      "tokens_seen": 11534336
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.0001755952380952381,
+      "loss": 5.8091,
+      "theoretical_loss": 6.728836321307757,
+      "tokens_seen": 11599872
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.0001765873015873016,
+      "loss": 5.6113,
+      "theoretical_loss": 6.722027120651221,
+      "tokens_seen": 11665408
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.0001775793650793651,
+      "loss": 5.6758,
+      "theoretical_loss": 6.715266709819684,
+      "tokens_seen": 11730944
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00017857142857142857,
+      "loss": 5.6378,
+      "theoretical_loss": 6.708554469566408,
+      "tokens_seen": 11796480
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00017956349206349207,
+      "loss": 5.7411,
+      "theoretical_loss": 6.701889791888819,
+      "tokens_seen": 11862016
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00018055555555555555,
+      "loss": 5.8311,
+      "theoretical_loss": 6.695272079763555,
+      "tokens_seen": 11927552
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00018154761904761905,
+      "loss": 5.2846,
+      "theoretical_loss": 6.688700746889175,
+      "tokens_seen": 11993088
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00018253968253968252,
+      "loss": 5.7687,
+      "theoretical_loss": 6.682175217436269,
+      "tokens_seen": 12058624
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00018353174603174602,
+      "loss": 5.7687,
+      "theoretical_loss": 6.675694925804711,
+      "tokens_seen": 12124160
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00018452380952380953,
+      "loss": 5.6209,
+      "theoretical_loss": 6.66925931638781,
+      "tokens_seen": 12189696
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00018551587301587303,
+      "loss": 5.9193,
+      "theoretical_loss": 6.662867843343154,
+      "tokens_seen": 12255232
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.0001865079365079365,
+      "loss": 5.6754,
+      "theoretical_loss": 6.6565199703698745,
+      "tokens_seen": 12320768
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.0001875,
+      "loss": 5.8597,
+      "theoretical_loss": 6.6502151704921655,
+      "tokens_seen": 12386304
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.0001884920634920635,
+      "loss": 5.709,
+      "theoretical_loss": 6.643952925848826,
+      "tokens_seen": 12451840
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00018948412698412698,
+      "loss": 5.7054,
+      "theoretical_loss": 6.637732727488622,
+      "tokens_seen": 12517376
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00019047619047619048,
+      "loss": 5.5972,
+      "theoretical_loss": 6.6315540751713,
+      "tokens_seen": 12582912
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00019146825396825398,
+      "loss": 5.5887,
+      "theoretical_loss": 6.625416477174044,
+      "tokens_seen": 12648448
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00019246031746031748,
+      "loss": 5.7082,
+      "theoretical_loss": 6.619319450103218,
+      "tokens_seen": 12713984
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00019345238095238096,
+      "loss": 5.5405,
+      "theoretical_loss": 6.613262518711206,
+      "tokens_seen": 12779520
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00019444444444444446,
+      "loss": 5.5806,
+      "theoretical_loss": 6.607245215718208,
+      "tokens_seen": 12845056
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00019543650793650793,
+      "loss": 5.4612,
+      "theoretical_loss": 6.601267081638806,
+      "tokens_seen": 12910592
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00019642857142857144,
+      "loss": 5.6405,
+      "theoretical_loss": 6.595327664613176,
+      "tokens_seen": 12976128
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.0001974206349206349,
+      "loss": 5.3968,
+      "theoretical_loss": 6.58942652024278,
+      "tokens_seen": 13041664
+    },
+    {
+      "epoch": 0.0,
+      "objective/train/docs_used": 19017,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 5.398904323577881,
+      "objective/train/theoretical_loss": 6.583563211430409,
+      "objective/train/tokens_used": 33567200,
+      "theoretical_loss": 6.583563211430409,
+      "tokens_seen": 13107200
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.0001984126984126984,
+      "loss": 5.5392,
+      "theoretical_loss": 6.583563211430409,
+      "tokens_seen": 13107200
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00019940476190476191,
+      "loss": 5.3995,
+      "theoretical_loss": 6.577737308224426,
+      "tokens_seen": 13172736
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00020039682539682542,
+      "loss": 5.5665,
+      "theoretical_loss": 6.571948387667094,
+      "tokens_seen": 13238272
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.0002013888888888889,
+      "loss": 5.5745,
+      "theoretical_loss": 6.566196033646859,
+      "tokens_seen": 13303808
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.0002023809523809524,
+      "loss": 5.4756,
+      "theoretical_loss": 6.560479836754459,
+      "tokens_seen": 13369344
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00020337301587301587,
+      "loss": 5.7895,
+      "theoretical_loss": 6.554799394142741,
+      "tokens_seen": 13434880
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00020436507936507937,
+      "loss": 5.3965,
+      "theoretical_loss": 6.549154309390087,
+      "tokens_seen": 13500416
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00020535714285714284,
+      "loss": 5.494,
+      "theoretical_loss": 6.5435441923673245,
+      "tokens_seen": 13565952
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00020634920634920634,
+      "loss": 5.5559,
+      "theoretical_loss": 6.537968659108021,
+      "tokens_seen": 13631488
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00020734126984126985,
+      "loss": 5.9125,
+      "theoretical_loss": 6.532427331682063,
+      "tokens_seen": 13697024
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00020833333333333335,
+      "loss": 5.5235,
+      "theoretical_loss": 6.5269198380724145,
+      "tokens_seen": 13762560
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00020932539682539685,
+      "loss": 5.6307,
+      "theoretical_loss": 6.5214458120549725,
+      "tokens_seen": 13828096
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00021031746031746032,
+      "loss": 5.3661,
+      "theoretical_loss": 6.516004893081412,
+      "tokens_seen": 13893632
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00021130952380952382,
+      "loss": 5.2823,
+      "theoretical_loss": 6.510596726164945,
+      "tokens_seen": 13959168
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.0002123015873015873,
+      "loss": 5.4241,
+      "theoretical_loss": 6.505220961768906,
+      "tokens_seen": 14024704
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.0002132936507936508,
+      "loss": 5.3679,
+      "theoretical_loss": 6.499877255698063,
+      "tokens_seen": 14090240
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00021428571428571427,
+      "loss": 5.2005,
+      "theoretical_loss": 6.494565268992613,
+      "tokens_seen": 14155776
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.0002152777777777778,
+      "loss": 5.6155,
+      "theoretical_loss": 6.489284667824741,
+      "tokens_seen": 14221312
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00021626984126984128,
+      "loss": 5.6027,
+      "theoretical_loss": 6.484035123397694,
+      "tokens_seen": 14286848
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00021726190476190478,
+      "loss": 5.5959,
+      "theoretical_loss": 6.478816311847305,
+      "tokens_seen": 14352384
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00021825396825396825,
+      "loss": 5.7842,
+      "theoretical_loss": 6.473627914145865,
+      "tokens_seen": 14417920
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00021924603174603176,
+      "loss": 5.4174,
+      "theoretical_loss": 6.468469616008303,
+      "tokens_seen": 14483456
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00022023809523809523,
+      "loss": 5.4423,
+      "theoretical_loss": 6.463341107800611,
+      "tokens_seen": 14548992
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00022123015873015873,
+      "loss": 5.2413,
+      "theoretical_loss": 6.458242084450426,
+      "tokens_seen": 14614528
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.0002222222222222222,
+      "loss": 5.4841,
+      "theoretical_loss": 6.453172245359726,
+      "tokens_seen": 14680064
+    },
+    {
+      "epoch": 0.0,
+      "objective/train/docs_used": 20187,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 5.4210638999938965,
+      "objective/train/theoretical_loss": 6.448131294319589,
+      "objective/train/tokens_used": 35205600,
+      "theoretical_loss": 6.448131294319589,
+      "tokens_seen": 14745600
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00022321428571428573,
+      "loss": 5.318,
+      "theoretical_loss": 6.448131294319589,
+      "tokens_seen": 14745600
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.0002242063492063492,
+      "loss": 5.5298,
+      "theoretical_loss": 6.443118939426935,
+      "tokens_seen": 14811136
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.0002251984126984127,
+      "loss": 5.5104,
+      "theoretical_loss": 6.438134893003209,
+      "tokens_seen": 14876672
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00022619047619047618,
+      "loss": 5.4089,
+      "theoretical_loss": 6.43317887151496,
+      "tokens_seen": 14942208
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00022718253968253969,
+      "loss": 5.5783,
+      "theoretical_loss": 6.428250595496234,
+      "tokens_seen": 15007744
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00022817460317460316,
+      "loss": 5.4029,
+      "theoretical_loss": 6.4233497894727805,
+      "tokens_seen": 15073280
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00022916666666666666,
+      "loss": 5.6043,
+      "theoretical_loss": 6.418476181887964,
+      "tokens_seen": 15138816
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00023015873015873016,
+      "loss": 5.5123,
+      "theoretical_loss": 6.413629505030386,
+      "tokens_seen": 15204352
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00023115079365079367,
+      "loss": 5.5084,
+      "theoretical_loss": 6.408809494963135,
+      "tokens_seen": 15269888
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00023214285714285717,
+      "loss": 5.269,
+      "theoretical_loss": 6.404015891454651,
+      "tokens_seen": 15335424
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00023313492063492064,
+      "loss": 5.4379,
+      "theoretical_loss": 6.399248437911128,
+      "tokens_seen": 15400960
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00023412698412698414,
+      "loss": 5.4346,
+      "theoretical_loss": 6.394506881310447,
+      "tokens_seen": 15466496
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00023511904761904762,
+      "loss": 5.7191,
+      "theoretical_loss": 6.38979097213757,
+      "tokens_seen": 15532032
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00023611111111111112,
+      "loss": 5.5484,
+      "theoretical_loss": 6.385100464321383,
+      "tokens_seen": 15597568
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.0002371031746031746,
+      "loss": 5.4379,
+      "theoretical_loss": 6.380435115172919,
+      "tokens_seen": 15663104
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.0002380952380952381,
+      "loss": 5.4785,
+      "theoretical_loss": 6.375794685324962,
+      "tokens_seen": 15728640
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.0002390873015873016,
+      "loss": 5.2088,
+      "theoretical_loss": 6.371178938672962,
+      "tokens_seen": 15794176
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.0002400793650793651,
+      "loss": 5.5076,
+      "theoretical_loss": 6.366587642317243,
+      "tokens_seen": 15859712
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00024107142857142857,
+      "loss": 5.6351,
+      "theoretical_loss": 6.362020566506475,
+      "tokens_seen": 15925248
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00024206349206349207,
+      "loss": 5.6029,
+      "theoretical_loss": 6.3574774845823665,
+      "tokens_seen": 15990784
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00024305555555555555,
+      "loss": 5.1969,
+      "theoretical_loss": 6.352958172925549,
+      "tokens_seen": 16056320
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00024404761904761905,
+      "loss": 5.285,
+      "theoretical_loss": 6.348462410902625,
+      "tokens_seen": 16121856
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00024503968253968255,
+      "loss": 5.6805,
+      "theoretical_loss": 6.343989980814362,
+      "tokens_seen": 16187392
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.000246031746031746,
+      "loss": 5.5855,
+      "theoretical_loss": 6.339540667844965,
+      "tokens_seen": 16252928
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00024702380952380955,
+      "loss": 5.3559,
+      "theoretical_loss": 6.335114260012455,
+      "tokens_seen": 16318464
+    },
+    {
+      "epoch": 0.0,
+      "objective/train/docs_used": 20737,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 5.463630199432373,
+      "objective/train/theoretical_loss": 6.330710548120079,
+      "objective/train/tokens_used": 36844000,
+      "theoretical_loss": 6.330710548120079,
+      "tokens_seen": 16384000
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.000248015873015873,
+      "loss": 5.515,
+      "theoretical_loss": 6.330710548120079,
+      "tokens_seen": 16384000
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.0002490079365079365,
+      "loss": 5.3596,
+      "theoretical_loss": 6.326329325708746,
+      "tokens_seen": 16449536
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00025,
+      "loss": 5.249,
+      "theoretical_loss": 6.321970389010465,
+      "tokens_seen": 16515072
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0002509920634920635,
+      "loss": 5.3675,
+      "theoretical_loss": 6.317633536902759,
+      "tokens_seen": 16580608
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.000251984126984127,
+      "loss": 5.0919,
+      "theoretical_loss": 6.313318570864016,
+      "tokens_seen": 16646144
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00025297619047619046,
+      "loss": 5.1228,
+      "theoretical_loss": 6.30902529492978,
+      "tokens_seen": 16711680
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00025396825396825396,
+      "loss": 5.4573,
+      "theoretical_loss": 6.304753515649935,
+      "tokens_seen": 16777216
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00025496031746031746,
+      "loss": 5.1499,
+      "theoretical_loss": 6.30050304204677,
+      "tokens_seen": 16842752
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00025595238095238096,
+      "loss": 5.6432,
+      "theoretical_loss": 6.296273685573913,
+      "tokens_seen": 16908288
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0002569444444444444,
+      "loss": 5.439,
+      "theoretical_loss": 6.292065260076094,
+      "tokens_seen": 16973824
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00025793650793650796,
+      "loss": 5.2959,
+      "theoretical_loss": 6.287877581749726,
+      "tokens_seen": 17039360
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00025892857142857146,
+      "loss": 5.3502,
+      "theoretical_loss": 6.2837104691042915,
+      "tokens_seen": 17104896
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00025992063492063497,
+      "loss": 5.5134,
+      "theoretical_loss": 6.279563742924502,
+      "tokens_seen": 17170432
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0002609126984126984,
+      "loss": 5.4871,
+      "theoretical_loss": 6.275437226233224,
+      "tokens_seen": 17235968
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0002619047619047619,
+      "loss": 5.3132,
+      "theoretical_loss": 6.271330744255137,
+      "tokens_seen": 17301504
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0002628968253968254,
+      "loss": 5.1796,
+      "theoretical_loss": 6.267244124381133,
+      "tokens_seen": 17367040
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0002638888888888889,
+      "loss": 5.2656,
+      "theoretical_loss": 6.2631771961334035,
+      "tokens_seen": 17432576
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00026488095238095237,
+      "loss": 5.5896,
+      "theoretical_loss": 6.259129791131242,
+      "tokens_seen": 17498112
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00026587301587301587,
+      "loss": 5.14,
+      "theoretical_loss": 6.255101743057493,
+      "tokens_seen": 17563648
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00026686507936507937,
+      "loss": 5.3606,
+      "theoretical_loss": 6.251092887625685,
+      "tokens_seen": 17629184
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00026785714285714287,
+      "loss": 5.487,
+      "theoretical_loss": 6.247103062547796,
+      "tokens_seen": 17694720
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0002688492063492063,
+      "loss": 5.2887,
+      "theoretical_loss": 6.243132107502647,
+      "tokens_seen": 17760256
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0002698412698412698,
+      "loss": 5.4254,
+      "theoretical_loss": 6.239179864104911,
+      "tokens_seen": 17825792
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0002708333333333333,
+      "loss": 5.4668,
+      "theoretical_loss": 6.235246175874727,
+      "tokens_seen": 17891328
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0002718253968253968,
+      "loss": 5.3584,
+      "theoretical_loss": 6.231330888207894,
+      "tokens_seen": 17956864
+    },
+    {
+      "epoch": 0.01,
+      "objective/train/docs_used": 21782,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 5.119491100311279,
+      "objective/train/theoretical_loss": 6.227433848346639,
+      "objective/train/tokens_used": 38482400,
+      "theoretical_loss": 6.227433848346639,
+      "tokens_seen": 18022400
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0002728174603174603,
+      "loss": 5.1069,
+      "theoretical_loss": 6.227433848346639,
+      "tokens_seen": 18022400
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0002738095238095238,
+      "loss": 5.1568,
+      "theoretical_loss": 6.22355490535095,
+      "tokens_seen": 18087936
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0002748015873015873,
+      "loss": 5.4409,
+      "theoretical_loss": 6.219693910070452,
+      "tokens_seen": 18153472
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00027579365079365083,
+      "loss": 5.1939,
+      "theoretical_loss": 6.215850715116817,
+      "tokens_seen": 18219008
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00027678571428571433,
+      "loss": 5.2274,
+      "theoretical_loss": 6.212025174836697,
+      "tokens_seen": 18284544
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0002777777777777778,
+      "loss": 5.5299,
+      "theoretical_loss": 6.208217145285173,
+      "tokens_seen": 18350080
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0002787698412698413,
+      "loss": 5.4441,
+      "theoretical_loss": 6.204426484199696,
+      "tokens_seen": 18415616
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0002797619047619048,
+      "loss": 5.3135,
+      "theoretical_loss": 6.200653050974525,
+      "tokens_seen": 18481152
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0002807539682539683,
+      "loss": 5.189,
+      "theoretical_loss": 6.196896706635635,
+      "tokens_seen": 18546688
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00028174603174603173,
+      "loss": 5.0786,
+      "theoretical_loss": 6.1931573138160925,
+      "tokens_seen": 18612224
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00028273809523809523,
+      "loss": 5.0434,
+      "theoretical_loss": 6.189434736731897,
+      "tokens_seen": 18677760
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00028373015873015873,
+      "loss": 5.0132,
+      "theoretical_loss": 6.185728841158257,
+      "tokens_seen": 18743296
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00028472222222222223,
+      "loss": 5.1159,
+      "theoretical_loss": 6.182039494406309,
+      "tokens_seen": 18808832
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0002857142857142857,
+      "loss": 5.1786,
+      "theoretical_loss": 6.178366565300266,
+      "tokens_seen": 18874368
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0002867063492063492,
+      "loss": 5.1947,
+      "theoretical_loss": 6.174709924154978,
+      "tokens_seen": 18939904
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0002876984126984127,
+      "loss": 5.2396,
+      "theoretical_loss": 6.171069442753909,
+      "tokens_seen": 19005440
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0002886904761904762,
+      "loss": 5.3947,
+      "theoretical_loss": 6.1674449943275045,
+      "tokens_seen": 19070976
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0002896825396825397,
+      "loss": 4.8732,
+      "theoretical_loss": 6.16383645353196,
+      "tokens_seen": 19136512
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0002906746031746032,
+      "loss": 5.3178,
+      "theoretical_loss": 6.160243696428367,
+      "tokens_seen": 19202048
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0002916666666666667,
+      "loss": 5.1441,
+      "theoretical_loss": 6.156666600462238,
+      "tokens_seen": 19267584
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0002926587301587302,
+      "loss": 5.2695,
+      "theoretical_loss": 6.153105044443393,
+      "tokens_seen": 19333120
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0002936507936507937,
+      "loss": 5.3181,
+      "theoretical_loss": 6.149558908526206,
+      "tokens_seen": 19398656
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00029464285714285714,
+      "loss": 5.2188,
+      "theoretical_loss": 6.146028074190217,
+      "tokens_seen": 19464192
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00029563492063492064,
+      "loss": 5.1652,
+      "theoretical_loss": 6.14251242422106,
+      "tokens_seen": 19529728
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00029662698412698414,
+      "loss": 5.3475,
+      "theoretical_loss": 6.139011842691756,
+      "tokens_seen": 19595264
+    },
+    {
+      "epoch": 0.01,
+      "objective/train/docs_used": 22379,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 5.661308765411377,
+      "objective/train/theoretical_loss": 6.135526214944321,
+      "objective/train/tokens_used": 40120800,
+      "theoretical_loss": 6.135526214944321,
+      "tokens_seen": 19660800
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00029761904761904765,
+      "loss": 5.6218,
+      "theoretical_loss": 6.135526214944321,
+      "tokens_seen": 19660800
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0002986111111111111,
+      "loss": 5.4528,
+      "theoretical_loss": 6.1320554275717,
+      "tokens_seen": 19726336
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0002996031746031746,
+      "loss": 5.1971,
+      "theoretical_loss": 6.128599368400017,
+      "tokens_seen": 19791872
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0003005952380952381,
+      "loss": 5.2823,
+      "theoretical_loss": 6.125157926471134,
+      "tokens_seen": 19857408
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0003015873015873016,
+      "loss": 5.1154,
+      "theoretical_loss": 6.121730992025516,
+      "tokens_seen": 19922944
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00030257936507936505,
+      "loss": 4.8293,
+      "theoretical_loss": 6.118318456485394,
+      "tokens_seen": 19988480
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00030357142857142855,
+      "loss": 5.0591,
+      "theoretical_loss": 6.114920212438209,
+      "tokens_seen": 20054016
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00030456349206349205,
+      "loss": 5.2452,
+      "theoretical_loss": 6.111536153620355,
+      "tokens_seen": 20119552
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0003055555555555556,
+      "loss": 5.1158,
+      "theoretical_loss": 6.108166174901191,
+      "tokens_seen": 20185088
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00030654761904761905,
+      "loss": 5.0927,
+      "theoretical_loss": 6.104810172267331,
+      "tokens_seen": 20250624
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00030753968253968255,
+      "loss": 5.1097,
+      "theoretical_loss": 6.101468042807199,
+      "tokens_seen": 20316160
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00030853174603174605,
+      "loss": 5.0109,
+      "theoretical_loss": 6.098139684695851,
+      "tokens_seen": 20381696
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00030952380952380956,
+      "loss": 5.2,
+      "theoretical_loss": 6.094824997180048,
+      "tokens_seen": 20447232
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.000310515873015873,
+      "loss": 4.919,
+      "theoretical_loss": 6.091523880563589,
+      "tokens_seen": 20512768
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0003115079365079365,
+      "loss": 4.7726,
+      "theoretical_loss": 6.088236236192881,
+      "tokens_seen": 20578304
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0003125,
+      "loss": 5.1579,
+      "theoretical_loss": 6.0849619664427586,
+      "tokens_seen": 20643840
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0003134920634920635,
+      "loss": 5.2649,
+      "theoretical_loss": 6.0817009747025415,
+      "tokens_seen": 20709376
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.000314484126984127,
+      "loss": 4.9338,
+      "theoretical_loss": 6.07845316536232,
+      "tokens_seen": 20774912
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00031547619047619046,
+      "loss": 5.2449,
+      "theoretical_loss": 6.075218443799468,
+      "tokens_seen": 20840448
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00031646825396825396,
+      "loss": 5.2222,
+      "theoretical_loss": 6.071996716365382,
+      "tokens_seen": 20905984
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00031746031746031746,
+      "loss": 4.8122,
+      "theoretical_loss": 6.068787890372443,
+      "tokens_seen": 20971520
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00031845238095238096,
+      "loss": 4.983,
+      "theoretical_loss": 6.0655918740811865,
+      "tokens_seen": 21037056
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0003194444444444444,
+      "loss": 4.9428,
+      "theoretical_loss": 6.062408576687682,
+      "tokens_seen": 21102592
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00032043650793650796,
+      "loss": 5.0493,
+      "theoretical_loss": 6.059237908311129,
+      "tokens_seen": 21168128
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00032142857142857147,
+      "loss": 4.8675,
+      "theoretical_loss": 6.056079779981644,
+      "tokens_seen": 21233664
+    },
+    {
+      "epoch": 0.01,
+      "objective/train/docs_used": 23601,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 5.199455738067627,
+      "objective/train/theoretical_loss": 6.052934103628253,
+      "objective/train/tokens_used": 41759200,
+      "theoretical_loss": 6.052934103628253,
+      "tokens_seen": 21299200
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00032242063492063497,
+      "loss": 5.2637,
+      "theoretical_loss": 6.052934103628253,
+      "tokens_seen": 21299200
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0003234126984126984,
+      "loss": 5.4325,
+      "theoretical_loss": 6.049800792067078,
+      "tokens_seen": 21364736
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0003244047619047619,
+      "loss": 4.8274,
+      "theoretical_loss": 6.046679758989709,
+      "tokens_seen": 21430272
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0003253968253968254,
+      "loss": 5.2104,
+      "theoretical_loss": 6.043570918951775,
+      "tokens_seen": 21495808
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0003263888888888889,
+      "loss": 5.1036,
+      "theoretical_loss": 6.040474187361681,
+      "tokens_seen": 21561344
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00032738095238095237,
+      "loss": 4.8811,
+      "theoretical_loss": 6.037389480469548,
+      "tokens_seen": 21626880
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00032837301587301587,
+      "loss": 5.0504,
+      "theoretical_loss": 6.034316715356304,
+      "tokens_seen": 21692416
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00032936507936507937,
+      "loss": 5.0898,
+      "theoretical_loss": 6.031255809922974,
+      "tokens_seen": 21757952
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00033035714285714287,
+      "loss": 5.0121,
+      "theoretical_loss": 6.0282066828801195,
+      "tokens_seen": 21823488
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0003313492063492063,
+      "loss": 5.1363,
+      "theoretical_loss": 6.0251692537374595,
+      "tokens_seen": 21889024
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0003323412698412698,
+      "loss": 5.1503,
+      "theoretical_loss": 6.022143442793643,
+      "tokens_seen": 21954560
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0003333333333333333,
+      "loss": 5.1066,
+      "theoretical_loss": 6.0191291711261945,
+      "tokens_seen": 22020096
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0003343253968253968,
+      "loss": 5.0889,
+      "theoretical_loss": 6.016126360581609,
+      "tokens_seen": 22085632
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0003353174603174603,
+      "loss": 4.9077,
+      "theoretical_loss": 6.013134933765605,
+      "tokens_seen": 22151168
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0003363095238095238,
+      "loss": 5.2813,
+      "theoretical_loss": 6.010154814033528,
+      "tokens_seen": 22216704
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00033730158730158733,
+      "loss": 4.6999,
+      "theoretical_loss": 6.007185925480902,
+      "tokens_seen": 22282240
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00033829365079365083,
+      "loss": 5.0728,
+      "theoretical_loss": 6.0042281929341375,
+      "tokens_seen": 22347776
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00033928571428571433,
+      "loss": 5.1319,
+      "theoretical_loss": 6.001281541941363,
+      "tokens_seen": 22413312
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0003402777777777778,
+      "loss": 5.0915,
+      "theoretical_loss": 5.998345898763421,
+      "tokens_seen": 22478848
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0003412698412698413,
+      "loss": 4.8912,
+      "theoretical_loss": 5.995421190364983,
+      "tokens_seen": 22544384
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0003422619047619048,
+      "loss": 5.0317,
+      "theoretical_loss": 5.992507344405814,
+      "tokens_seen": 22609920
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0003432539682539683,
+      "loss": 5.1603,
+      "theoretical_loss": 5.9896042892321635,
+      "tokens_seen": 22675456
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00034424603174603173,
+      "loss": 4.9822,
+      "theoretical_loss": 5.986711953868287,
+      "tokens_seen": 22740992
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00034523809523809523,
+      "loss": 5.3948,
+      "theoretical_loss": 5.983830268008107,
+      "tokens_seen": 22806528
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00034623015873015873,
+      "loss": 5.222,
+      "theoretical_loss": 5.980959162006979,
+      "tokens_seen": 22872064
+    },
+    {
+      "epoch": 0.01,
+      "objective/train/docs_used": 24859,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 4.439995288848877,
+      "objective/train/theoretical_loss": 5.978098566873603,
+      "objective/train/tokens_used": 43397600,
+      "theoretical_loss": 5.978098566873603,
+      "tokens_seen": 22937600
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00034722222222222224,
+      "loss": 4.9666,
+      "theoretical_loss": 5.978098566873603,
+      "tokens_seen": 22937600
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0003482142857142857,
+      "loss": 4.7873,
+      "theoretical_loss": 5.975248414262053,
+      "tokens_seen": 23003136
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0003492063492063492,
+      "loss": 5.1429,
+      "theoretical_loss": 5.972408636463909,
+      "tokens_seen": 23068672
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0003501984126984127,
+      "loss": 4.8387,
+      "theoretical_loss": 5.969579166400528,
+      "tokens_seen": 23134208
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0003511904761904762,
+      "loss": 5.2741,
+      "theoretical_loss": 5.966759937615427,
+      "tokens_seen": 23199744
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0003521825396825397,
+      "loss": 4.9993,
+      "theoretical_loss": 5.96395088426676,
+      "tokens_seen": 23265280
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0003531746031746032,
+      "loss": 4.838,
+      "theoretical_loss": 5.961151941119932,
+      "tokens_seen": 23330816
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0003541666666666667,
+      "loss": 4.8937,
+      "theoretical_loss": 5.95836304354031,
+      "tokens_seen": 23396352
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0003551587301587302,
+      "loss": 5.1755,
+      "theoretical_loss": 5.9555841274860395,
+      "tokens_seen": 23461888
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0003561507936507937,
+      "loss": 5.0192,
+      "theoretical_loss": 5.952815129500973,
+      "tokens_seen": 23527424
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00035714285714285714,
+      "loss": 5.1528,
+      "theoretical_loss": 5.950055986707699,
+      "tokens_seen": 23592960
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00035813492063492064,
+      "loss": 5.2473,
+      "theoretical_loss": 5.9473066368006755,
+      "tokens_seen": 23658496
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00035912698412698415,
+      "loss": 5.1431,
+      "theoretical_loss": 5.944567018039454,
+      "tokens_seen": 23724032
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00036011904761904765,
+      "loss": 5.2078,
+      "theoretical_loss": 5.941837069242023,
+      "tokens_seen": 23789568
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0003611111111111111,
+      "loss": 5.2074,
+      "theoretical_loss": 5.939116729778224,
+      "tokens_seen": 23855104
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0003621031746031746,
+      "loss": 5.045,
+      "theoretical_loss": 5.936405939563279,
+      "tokens_seen": 23920640
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0003630952380952381,
+      "loss": 5.0272,
+      "theoretical_loss": 5.93370463905141,
+      "tokens_seen": 23986176
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0003640873015873016,
+      "loss": 5.0628,
+      "theoretical_loss": 5.931012769229536,
+      "tokens_seen": 24051712
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00036507936507936505,
+      "loss": 4.6127,
+      "theoretical_loss": 5.928330271611081,
+      "tokens_seen": 24117248
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00036607142857142855,
+      "loss": 4.8408,
+      "theoretical_loss": 5.925657088229862,
+      "tokens_seen": 24182784
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00036706349206349205,
+      "loss": 4.9603,
+      "theoretical_loss": 5.9229931616340545,
+      "tokens_seen": 24248320
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0003680555555555556,
+      "loss": 5.0127,
+      "theoretical_loss": 5.920338434880263,
+      "tokens_seen": 24313856
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00036904761904761905,
+      "loss": 5.3699,
+      "theoretical_loss": 5.9176928515276535,
+      "tokens_seen": 24379392
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00037003968253968255,
+      "loss": 4.9974,
+      "theoretical_loss": 5.915056355632197,
+      "tokens_seen": 24444928
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00037103174603174606,
+      "loss": 4.9348,
+      "theoretical_loss": 5.912428891740967,
+      "tokens_seen": 24510464
+    },
+    {
+      "epoch": 0.01,
+      "objective/train/docs_used": 25411,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 4.414747714996338,
+      "objective/train/theoretical_loss": 5.90981040488653,
+      "objective/train/tokens_used": 45036000,
+      "theoretical_loss": 5.90981040488653,
+      "tokens_seen": 24576000
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00037202380952380956,
+      "loss": 5.0376,
+      "theoretical_loss": 5.90981040488653,
+      "tokens_seen": 24576000
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.000373015873015873,
+      "loss": 5.0724,
+      "theoretical_loss": 5.907200840581417,
+      "tokens_seen": 24641536
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0003740079365079365,
+      "loss": 5.2247,
+      "theoretical_loss": 5.904600144812672,
+      "tokens_seen": 24707072
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.000375,
+      "loss": 5.4811,
+      "theoretical_loss": 5.902008264036468,
+      "tokens_seen": 24772608
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0003759920634920635,
+      "loss": 4.9107,
+      "theoretical_loss": 5.899425145172803,
+      "tokens_seen": 24838144
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.000376984126984127,
+      "loss": 5.0933,
+      "theoretical_loss": 5.896850735600281,
+      "tokens_seen": 24903680
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00037797619047619046,
+      "loss": 5.1802,
+      "theoretical_loss": 5.8942849831509445,
+      "tokens_seen": 24969216
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00037896825396825396,
+      "loss": 5.2521,
+      "theoretical_loss": 5.891727836105194,
+      "tokens_seen": 25034752
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00037996031746031746,
+      "loss": 4.8607,
+      "theoretical_loss": 5.889179243186776,
+      "tokens_seen": 25100288
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00038095238095238096,
+      "loss": 5.0973,
+      "theoretical_loss": 5.886639153557828,
+      "tokens_seen": 25165824
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0003819444444444444,
+      "loss": 5.3397,
+      "theoretical_loss": 5.8841075168140105,
+      "tokens_seen": 25231360
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00038293650793650797,
+      "loss": 4.7777,
+      "theoretical_loss": 5.88158428297969,
+      "tokens_seen": 25296896
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00038392857142857147,
+      "loss": 4.7219,
+      "theoretical_loss": 5.879069402503189,
+      "tokens_seen": 25362432
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00038492063492063497,
+      "loss": 5.0299,
+      "theoretical_loss": 5.876562826252119,
+      "tokens_seen": 25427968
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0003859126984126984,
+      "loss": 4.8369,
+      "theoretical_loss": 5.874064505508748,
+      "tokens_seen": 25493504
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0003869047619047619,
+      "loss": 5.0017,
+      "theoretical_loss": 5.871574391965453,
+      "tokens_seen": 25559040
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0003878968253968254,
+      "loss": 4.9705,
+      "theoretical_loss": 5.8690924377202265,
+      "tokens_seen": 25624576
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0003888888888888889,
+      "loss": 5.034,
+      "theoretical_loss": 5.866618595272241,
+      "tokens_seen": 25690112
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00038988095238095237,
+      "loss": 4.7181,
+      "theoretical_loss": 5.864152817517482,
+      "tokens_seen": 25755648
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00039087301587301587,
+      "loss": 4.9562,
+      "theoretical_loss": 5.8616950577444245,
+      "tokens_seen": 25821184
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00039186507936507937,
+      "loss": 4.9405,
+      "theoretical_loss": 5.859245269629783,
+      "tokens_seen": 25886720
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0003928571428571429,
+      "loss": 5.099,
+      "theoretical_loss": 5.85680340723431,
+      "tokens_seen": 25952256
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0003938492063492063,
+      "loss": 4.6503,
+      "theoretical_loss": 5.85436942499865,
+      "tokens_seen": 26017792
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0003948412698412698,
+      "loss": 4.8672,
+      "theoretical_loss": 5.851943277739253,
+      "tokens_seen": 26083328
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0003958333333333333,
+      "loss": 5.0201,
+      "theoretical_loss": 5.84952492064434,
+      "tokens_seen": 26148864
+    },
+    {
+      "epoch": 0.01,
+      "objective/train/docs_used": 26668,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 5.337137222290039,
+      "objective/train/theoretical_loss": 5.847114309269919,
+      "objective/train/tokens_used": 46674400,
+      "theoretical_loss": 5.847114309269919,
+      "tokens_seen": 26214400
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0003968253968253968,
+      "loss": 5.1138,
+      "theoretical_loss": 5.847114309269919,
+      "tokens_seen": 26214400
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0003978174603174603,
+      "loss": 4.8239,
+      "theoretical_loss": 5.844711399535855,
+      "tokens_seen": 26279936
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00039880952380952383,
+      "loss": 4.7304,
+      "theoretical_loss": 5.842316147722,
+      "tokens_seen": 26345472
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00039980158730158733,
+      "loss": 5.0092,
+      "theoretical_loss": 5.839928510464356,
+      "tokens_seen": 26411008
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00040079365079365083,
+      "loss": 4.4343,
+      "theoretical_loss": 5.837548444751306,
+      "tokens_seen": 26476544
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00040178571428571433,
+      "loss": 4.9309,
+      "theoretical_loss": 5.835175907919885,
+      "tokens_seen": 26542080
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004027777777777778,
+      "loss": 5.0148,
+      "theoretical_loss": 5.832810857652097,
+      "tokens_seen": 26607616
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004037698412698413,
+      "loss": 4.9709,
+      "theoretical_loss": 5.830453251971296,
+      "tokens_seen": 26673152
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004047619047619048,
+      "loss": 5.1225,
+      "theoretical_loss": 5.8281030492385835,
+      "tokens_seen": 26738688
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004057539682539683,
+      "loss": 4.9439,
+      "theoretical_loss": 5.82576020814929,
+      "tokens_seen": 26804224
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00040674603174603173,
+      "loss": 5.154,
+      "theoretical_loss": 5.823424687729469,
+      "tokens_seen": 26869760
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00040773809523809523,
+      "loss": 4.6484,
+      "theoretical_loss": 5.821096447332456,
+      "tokens_seen": 26935296
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00040873015873015874,
+      "loss": 4.6262,
+      "theoretical_loss": 5.818775446635469,
+      "tokens_seen": 27000832
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00040972222222222224,
+      "loss": 5.0236,
+      "theoretical_loss": 5.816461645636238,
+      "tokens_seen": 27066368
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004107142857142857,
+      "loss": 5.1234,
+      "theoretical_loss": 5.814155004649702,
+      "tokens_seen": 27131904
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004117063492063492,
+      "loss": 4.6509,
+      "theoretical_loss": 5.811855484304724,
+      "tokens_seen": 27197440
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004126984126984127,
+      "loss": 4.9814,
+      "theoretical_loss": 5.809563045540864,
+      "tokens_seen": 27262976
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004136904761904762,
+      "loss": 5.0187,
+      "theoretical_loss": 5.8072776496051866,
+      "tokens_seen": 27328512
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004146825396825397,
+      "loss": 4.9255,
+      "theoretical_loss": 5.804999258049106,
+      "tokens_seen": 27394048
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004156746031746032,
+      "loss": 5.0265,
+      "theoretical_loss": 5.802727832725283,
+      "tokens_seen": 27459584
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004166666666666667,
+      "loss": 4.8827,
+      "theoretical_loss": 5.800463335784541,
+      "tokens_seen": 27525120
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004176587301587302,
+      "loss": 5.0962,
+      "theoretical_loss": 5.798205729672842,
+      "tokens_seen": 27590656
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004186507936507937,
+      "loss": 5.0409,
+      "theoretical_loss": 5.795954977128286,
+      "tokens_seen": 27656192
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00041964285714285714,
+      "loss": 4.8332,
+      "theoretical_loss": 5.7937110411781525,
+      "tokens_seen": 27721728
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00042063492063492065,
+      "loss": 4.813,
+      "theoretical_loss": 5.79147388513598,
+      "tokens_seen": 27787264
+    },
+    {
+      "epoch": 0.01,
+      "objective/train/docs_used": 27295,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 4.60736083984375,
+      "objective/train/theoretical_loss": 5.789243472598683,
+      "objective/train/tokens_used": 48312800,
+      "theoretical_loss": 5.789243472598683,
+      "tokens_seen": 27852800
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00042162698412698415,
+      "loss": 4.735,
+      "theoretical_loss": 5.789243472598683,
+      "tokens_seen": 27852800
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00042261904761904765,
+      "loss": 5.0103,
+      "theoretical_loss": 5.787019767443696,
+      "tokens_seen": 27918336
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004236111111111111,
+      "loss": 4.9487,
+      "theoretical_loss": 5.784802733826166,
+      "tokens_seen": 27983872
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004246031746031746,
+      "loss": 5.0393,
+      "theoretical_loss": 5.782592336176171,
+      "tokens_seen": 28049408
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004255952380952381,
+      "loss": 4.8239,
+      "theoretical_loss": 5.780388539195972,
+      "tokens_seen": 28114944
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004265873015873016,
+      "loss": 4.8301,
+      "theoretical_loss": 5.778191307857307,
+      "tokens_seen": 28180480
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00042757936507936505,
+      "loss": 4.8818,
+      "theoretical_loss": 5.776000607398707,
+      "tokens_seen": 28246016
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00042857142857142855,
+      "loss": 5.0365,
+      "theoretical_loss": 5.773816403322854,
+      "tokens_seen": 28311552
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00042956349206349205,
+      "loss": 4.8806,
+      "theoretical_loss": 5.7716386613939665,
+      "tokens_seen": 28377088
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004305555555555556,
+      "loss": 4.8212,
+      "theoretical_loss": 5.769467347635221,
+      "tokens_seen": 28442624
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00043154761904761905,
+      "loss": 4.6856,
+      "theoretical_loss": 5.767302428326196,
+      "tokens_seen": 28508160
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00043253968253968256,
+      "loss": 5.1099,
+      "theoretical_loss": 5.765143870000358,
+      "tokens_seen": 28573696
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00043353174603174606,
+      "loss": 4.9942,
+      "theoretical_loss": 5.762991639442574,
+      "tokens_seen": 28639232
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00043452380952380956,
+      "loss": 4.7269,
+      "theoretical_loss": 5.7608457036866465,
+      "tokens_seen": 28704768
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.000435515873015873,
+      "loss": 4.7079,
+      "theoretical_loss": 5.758706030012889,
+      "tokens_seen": 28770304
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004365079365079365,
+      "loss": 5.0535,
+      "theoretical_loss": 5.756572585945728,
+      "tokens_seen": 28835840
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004375,
+      "loss": 4.9953,
+      "theoretical_loss": 5.754445339251326,
+      "tokens_seen": 28901376
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004384920634920635,
+      "loss": 4.8239,
+      "theoretical_loss": 5.752324257935244,
+      "tokens_seen": 28966912
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.000439484126984127,
+      "loss": 4.8738,
+      "theoretical_loss": 5.750209310240125,
+      "tokens_seen": 29032448
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00044047619047619046,
+      "loss": 4.5774,
+      "theoretical_loss": 5.7481004646434055,
+      "tokens_seen": 29097984
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00044146825396825396,
+      "loss": 4.7821,
+      "theoretical_loss": 5.745997689855058,
+      "tokens_seen": 29163520
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00044246031746031746,
+      "loss": 4.9197,
+      "theoretical_loss": 5.743900954815356,
+      "tokens_seen": 29229056
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00044345238095238096,
+      "loss": 4.9178,
+      "theoretical_loss": 5.741810228692663,
+      "tokens_seen": 29294592
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004444444444444444,
+      "loss": 4.9811,
+      "theoretical_loss": 5.739725480881262,
+      "tokens_seen": 29360128
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00044543650793650797,
+      "loss": 4.595,
+      "theoretical_loss": 5.737646680999193,
+      "tokens_seen": 29425664
+    },
+    {
+      "epoch": 0.01,
+      "objective/train/docs_used": 28553,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 4.193212032318115,
+      "objective/train/theoretical_loss": 5.73557379888612,
+      "objective/train/tokens_used": 49951200,
+      "theoretical_loss": 5.73557379888612,
+      "tokens_seen": 29491200
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00044642857142857147,
+      "loss": 4.6289,
+      "theoretical_loss": 5.73557379888612,
+      "tokens_seen": 29491200
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00044742063492063497,
+      "loss": 4.5175,
+      "theoretical_loss": 5.733506804601236,
+      "tokens_seen": 29556736
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004484126984126984,
+      "loss": 4.97,
+      "theoretical_loss": 5.7314456684211725,
+      "tokens_seen": 29622272
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004494047619047619,
+      "loss": 4.9007,
+      "theoretical_loss": 5.729390360837952,
+      "tokens_seen": 29687808
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004503968253968254,
+      "loss": 5.0718,
+      "theoretical_loss": 5.7273408525569485,
+      "tokens_seen": 29753344
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004513888888888889,
+      "loss": 4.8076,
+      "theoretical_loss": 5.725297114494884,
+      "tokens_seen": 29818880
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00045238095238095237,
+      "loss": 4.9151,
+      "theoretical_loss": 5.7232591177778405,
+      "tokens_seen": 29884416
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00045337301587301587,
+      "loss": 4.7611,
+      "theoretical_loss": 5.7212268337393,
+      "tokens_seen": 29949952
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00045436507936507937,
+      "loss": 4.6533,
+      "theoretical_loss": 5.719200233918203,
+      "tokens_seen": 30015488
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004553571428571429,
+      "loss": 4.7689,
+      "theoretical_loss": 5.717179290057032,
+      "tokens_seen": 30081024
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004563492063492063,
+      "loss": 4.753,
+      "theoretical_loss": 5.715163974099917,
+      "tokens_seen": 30146560
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004573412698412698,
+      "loss": 4.9783,
+      "theoretical_loss": 5.713154258190757,
+      "tokens_seen": 30212096
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004583333333333333,
+      "loss": 4.8919,
+      "theoretical_loss": 5.711150114671375,
+      "tokens_seen": 30277632
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004593253968253968,
+      "loss": 4.6246,
+      "theoretical_loss": 5.709151516079683,
+      "tokens_seen": 30343168
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00046031746031746033,
+      "loss": 4.8047,
+      "theoretical_loss": 5.707158435147875,
+      "tokens_seen": 30408704
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00046130952380952383,
+      "loss": 4.7619,
+      "theoretical_loss": 5.705170844800628,
+      "tokens_seen": 30474240
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00046230158730158733,
+      "loss": 5.0389,
+      "theoretical_loss": 5.703188718153347,
+      "tokens_seen": 30539776
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00046329365079365083,
+      "loss": 4.7978,
+      "theoretical_loss": 5.7012120285104055,
+      "tokens_seen": 30605312
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00046428571428571433,
+      "loss": 4.8182,
+      "theoretical_loss": 5.699240749363417,
+      "tokens_seen": 30670848
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004652777777777778,
+      "loss": 5.1046,
+      "theoretical_loss": 5.69727485438953,
+      "tokens_seen": 30736384
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004662698412698413,
+      "loss": 4.9301,
+      "theoretical_loss": 5.695314317449732,
+      "tokens_seen": 30801920
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004672619047619048,
+      "loss": 4.9513,
+      "theoretical_loss": 5.69335911258718,
+      "tokens_seen": 30867456
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004682539682539683,
+      "loss": 4.7506,
+      "theoretical_loss": 5.691409214025544,
+      "tokens_seen": 30932992
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00046924603174603173,
+      "loss": 4.9203,
+      "theoretical_loss": 5.689464596167383,
+      "tokens_seen": 30998528
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00047023809523809523,
+      "loss": 4.6588,
+      "theoretical_loss": 5.687525233592513,
+      "tokens_seen": 31064064
+    },
+    {
+      "epoch": 0.01,
+      "objective/train/docs_used": 29210,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 4.980744361877441,
+      "objective/train/theoretical_loss": 5.685591101056422,
+      "objective/train/tokens_used": 51589600,
+      "theoretical_loss": 5.685591101056422,
+      "tokens_seen": 31129600
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00047123015873015874,
+      "loss": 4.8131,
+      "theoretical_loss": 5.685591101056422,
+      "tokens_seen": 31129600
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00047222222222222224,
+      "loss": 4.9729,
+      "theoretical_loss": 5.683662173488678,
+      "tokens_seen": 31195136
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004732142857142857,
+      "loss": 4.9988,
+      "theoretical_loss": 5.681738425991377,
+      "tokens_seen": 31260672
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004742063492063492,
+      "loss": 4.9296,
+      "theoretical_loss": 5.679819833837586,
+      "tokens_seen": 31326208
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004751984126984127,
+      "loss": 4.8866,
+      "theoretical_loss": 5.677906372469826,
+      "tokens_seen": 31391744
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004761904761904762,
+      "loss": 4.8619,
+      "theoretical_loss": 5.675998017498549,
+      "tokens_seen": 31457280
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004771825396825397,
+      "loss": 4.4881,
+      "theoretical_loss": 5.674094744700648,
+      "tokens_seen": 31522816
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004781746031746032,
+      "loss": 4.7385,
+      "theoretical_loss": 5.672196530017979,
+      "tokens_seen": 31588352
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004791666666666667,
+      "loss": 4.9446,
+      "theoretical_loss": 5.670303349555893,
+      "tokens_seen": 31653888
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004801587301587302,
+      "loss": 4.5805,
+      "theoretical_loss": 5.668415179581795,
+      "tokens_seen": 31719424
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004811507936507937,
+      "loss": 4.943,
+      "theoretical_loss": 5.666531996523711,
+      "tokens_seen": 31784960
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00048214285714285715,
+      "loss": 4.9994,
+      "theoretical_loss": 5.664653776968867,
+      "tokens_seen": 31850496
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00048313492063492065,
+      "loss": 4.9432,
+      "theoretical_loss": 5.6627804976622995,
+      "tokens_seen": 31916032
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00048412698412698415,
+      "loss": 4.4612,
+      "theoretical_loss": 5.660912135505461,
+      "tokens_seen": 31981568
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00048511904761904765,
+      "loss": 5.0296,
+      "theoretical_loss": 5.659048667554853,
+      "tokens_seen": 32047104
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004861111111111111,
+      "loss": 4.7342,
+      "theoretical_loss": 5.657190071020672,
+      "tokens_seen": 32112640
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004871031746031746,
+      "loss": 5.0271,
+      "theoretical_loss": 5.65533632326547,
+      "tokens_seen": 32178176
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004880952380952381,
+      "loss": 4.5208,
+      "theoretical_loss": 5.6534874018028205,
+      "tokens_seen": 32243712
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004890873015873016,
+      "loss": 4.8883,
+      "theoretical_loss": 5.651643284296018,
+      "tokens_seen": 32309248
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004900793650793651,
+      "loss": 4.5918,
+      "theoretical_loss": 5.6498039485567695,
+      "tokens_seen": 32374784
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004910714285714286,
+      "loss": 4.8362,
+      "theoretical_loss": 5.6479693725439155,
+      "tokens_seen": 32440320
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.000492063492063492,
+      "loss": 4.4816,
+      "theoretical_loss": 5.646139534362161,
+      "tokens_seen": 32505856
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004930555555555556,
+      "loss": 4.7304,
+      "theoretical_loss": 5.644314412260815,
+      "tokens_seen": 32571392
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004940476190476191,
+      "loss": 4.7639,
+      "theoretical_loss": 5.642493984632544,
+      "tokens_seen": 32636928
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004950396825396826,
+      "loss": 4.486,
+      "theoretical_loss": 5.640678230012151,
+      "tokens_seen": 32702464
+    },
+    {
+      "epoch": 0.01,
+      "objective/train/docs_used": 30192,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 4.205135822296143,
+      "objective/train/theoretical_loss": 5.638867127075349,
+      "objective/train/tokens_used": 53228000,
+      "theoretical_loss": 5.638867127075349,
+      "tokens_seen": 32768000
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.000496031746031746,
+      "loss": 4.6383,
+      "theoretical_loss": 5.638867127075349,
+      "tokens_seen": 32768000
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004970238095238095,
+      "loss": 4.7665,
+      "theoretical_loss": 5.63706065463756,
+      "tokens_seen": 32833536
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.000498015873015873,
+      "loss": 4.6651,
+      "theoretical_loss": 5.635258791652723,
+      "tokens_seen": 32899072
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004990079365079365,
+      "loss": 4.6032,
+      "theoretical_loss": 5.633461517212115,
+      "tokens_seen": 32964608
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0005,
+      "loss": 4.8611,
+      "theoretical_loss": 5.631668810543181,
+      "tokens_seen": 33030144
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004999899699097292,
+      "loss": 4.4403,
+      "theoretical_loss": 5.629880651008384,
+      "tokens_seen": 33095680
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004999799398194584,
+      "loss": 4.6342,
+      "theoretical_loss": 5.628097018104057,
+      "tokens_seen": 33161216
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004999699097291876,
+      "loss": 4.789,
+      "theoretical_loss": 5.626317891459275,
+      "tokens_seen": 33226752
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004999598796389167,
+      "loss": 4.4669,
+      "theoretical_loss": 5.6245432508347335,
+      "tokens_seen": 33292288
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.000499949849548646,
+      "loss": 4.5265,
+      "theoretical_loss": 5.622773076121648,
+      "tokens_seen": 33357824
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004999398194583751,
+      "loss": 4.8668,
+      "theoretical_loss": 5.621007347340646,
+      "tokens_seen": 33423360
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004999297893681044,
+      "loss": 4.663,
+      "theoretical_loss": 5.619246044640693,
+      "tokens_seen": 33488896
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004999197592778335,
+      "loss": 4.51,
+      "theoretical_loss": 5.617489148298015,
+      "tokens_seen": 33554432
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004999097291875627,
+      "loss": 4.4827,
+      "theoretical_loss": 5.615736638715033,
+      "tokens_seen": 33619968
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004998996990972919,
+      "loss": 4.6552,
+      "theoretical_loss": 5.613988496419315,
+      "tokens_seen": 33685504
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004998896690070211,
+      "loss": 4.5444,
+      "theoretical_loss": 5.6122447020625374,
+      "tokens_seen": 33751040
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004998796389167503,
+      "loss": 4.7462,
+      "theoretical_loss": 5.610505236419448,
+      "tokens_seen": 33816576
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004998696088264795,
+      "loss": 4.6343,
+      "theoretical_loss": 5.608770080386854,
+      "tokens_seen": 33882112
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004998595787362087,
+      "loss": 4.3194,
+      "theoretical_loss": 5.6070392149826045,
+      "tokens_seen": 33947648
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004998495486459378,
+      "loss": 4.7376,
+      "theoretical_loss": 5.605312621344602,
+      "tokens_seen": 34013184
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.000499839518555667,
+      "loss": 4.6319,
+      "theoretical_loss": 5.603590280729803,
+      "tokens_seen": 34078720
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004998294884653962,
+      "loss": 4.3936,
+      "theoretical_loss": 5.601872174513245,
+      "tokens_seen": 34144256
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004998194583751254,
+      "loss": 4.8045,
+      "theoretical_loss": 5.60015828418708,
+      "tokens_seen": 34209792
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004998094282848546,
+      "loss": 4.8346,
+      "theoretical_loss": 5.59844859135961,
+      "tokens_seen": 34275328
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004997993981945837,
+      "loss": 4.4444,
+      "theoretical_loss": 5.596743077754337,
+      "tokens_seen": 34340864
+    },
+    {
+      "epoch": 0.01,
+      "objective/train/docs_used": 30758,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 4.619051933288574,
+      "objective/train/theoretical_loss": 5.5950417252090325,
+      "objective/train/tokens_used": 54866400,
+      "theoretical_loss": 5.5950417252090325,
+      "tokens_seen": 34406400
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004997893681043129,
+      "loss": 4.5684,
+      "theoretical_loss": 5.5950417252090325,
+      "tokens_seen": 34406400
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004997793380140421,
+      "loss": 4.7403,
+      "theoretical_loss": 5.593344515674795,
+      "tokens_seen": 34471936
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004997693079237714,
+      "loss": 4.7682,
+      "theoretical_loss": 5.591651431215139,
+      "tokens_seen": 34537472
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004997592778335005,
+      "loss": 4.9194,
+      "theoretical_loss": 5.589962454005072,
+      "tokens_seen": 34603008
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004997492477432298,
+      "loss": 4.7926,
+      "theoretical_loss": 5.588277566330207,
+      "tokens_seen": 34668544
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004997392176529588,
+      "loss": 4.5919,
+      "theoretical_loss": 5.586596750585855,
+      "tokens_seen": 34734080
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004997291875626881,
+      "loss": 4.8471,
+      "theoretical_loss": 5.584919989276146,
+      "tokens_seen": 34799616
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004997191574724173,
+      "loss": 4.8823,
+      "theoretical_loss": 5.583247265013155,
+      "tokens_seen": 34865152
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004997091273821465,
+      "loss": 4.7952,
+      "theoretical_loss": 5.581578560516027,
+      "tokens_seen": 34930688
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004996990972918757,
+      "loss": 4.8327,
+      "theoretical_loss": 5.57991385861013,
+      "tokens_seen": 34996224
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004996890672016048,
+      "loss": 4.3745,
+      "theoretical_loss": 5.578253142226191,
+      "tokens_seen": 35061760
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.000499679037111334,
+      "loss": 4.6921,
+      "theoretical_loss": 5.576596394399467,
+      "tokens_seen": 35127296
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004996690070210632,
+      "loss": 4.6698,
+      "theoretical_loss": 5.574943598268902,
+      "tokens_seen": 35192832
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004996589769307924,
+      "loss": 4.9374,
+      "theoretical_loss": 5.573294737076306,
+      "tokens_seen": 35258368
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004996489468405216,
+      "loss": 4.5814,
+      "theoretical_loss": 5.571649794165537,
+      "tokens_seen": 35323904
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004996389167502507,
+      "loss": 4.6064,
+      "theoretical_loss": 5.570008752981688,
+      "tokens_seen": 35389440
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004996288866599799,
+      "loss": 4.7137,
+      "theoretical_loss": 5.5683715970702945,
+      "tokens_seen": 35454976
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004996188565697091,
+      "loss": 4.7919,
+      "theoretical_loss": 5.566738310076531,
+      "tokens_seen": 35520512
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004996088264794383,
+      "loss": 4.5989,
+      "theoretical_loss": 5.5651088757444285,
+      "tokens_seen": 35586048
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004995987963891675,
+      "loss": 4.4273,
+      "theoretical_loss": 5.5634832779161,
+      "tokens_seen": 35651584
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004995887662988968,
+      "loss": 4.6147,
+      "theoretical_loss": 5.561861500530965,
+      "tokens_seen": 35717120
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004995787362086258,
+      "loss": 4.5803,
+      "theoretical_loss": 5.560243527624989,
+      "tokens_seen": 35782656
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004995687061183551,
+      "loss": 4.762,
+      "theoretical_loss": 5.558629343329924,
+      "tokens_seen": 35848192
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004995586760280842,
+      "loss": 4.7117,
+      "theoretical_loss": 5.557018931872566,
+      "tokens_seen": 35913728
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004995486459378135,
+      "loss": 4.7915,
+      "theoretical_loss": 5.555412277574007,
+      "tokens_seen": 35979264
+    },
+    {
+      "epoch": 0.01,
+      "objective/train/docs_used": 31707,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 4.547565460205078,
+      "objective/train/theoretical_loss": 5.553809364848902,
+      "objective/train/tokens_used": 56504800,
+      "theoretical_loss": 5.553809364848902,
+      "tokens_seen": 36044800
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004995386158475427,
+      "loss": 4.6425,
+      "theoretical_loss": 5.553809364848902,
+      "tokens_seen": 36044800
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004995285857572718,
+      "loss": 4.5963,
+      "theoretical_loss": 5.552210178204751,
+      "tokens_seen": 36110336
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.000499518555667001,
+      "loss": 4.6793,
+      "theoretical_loss": 5.550614702241162,
+      "tokens_seen": 36175872
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004995085255767302,
+      "loss": 4.7561,
+      "theoretical_loss": 5.549022921649151,
+      "tokens_seen": 36241408
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004994984954864594,
+      "loss": 4.6607,
+      "theoretical_loss": 5.54743482121043,
+      "tokens_seen": 36306944
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004994884653961886,
+      "loss": 4.6252,
+      "theoretical_loss": 5.545850385796712,
+      "tokens_seen": 36372480
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004994784353059178,
+      "loss": 4.4331,
+      "theoretical_loss": 5.544269600369009,
+      "tokens_seen": 36438016
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004994684052156469,
+      "loss": 4.643,
+      "theoretical_loss": 5.5426924499769505,
+      "tokens_seen": 36503552
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004994583751253761,
+      "loss": 4.7548,
+      "theoretical_loss": 5.541118919758107,
+      "tokens_seen": 36569088
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004994483450351053,
+      "loss": 4.7337,
+      "theoretical_loss": 5.539548994937308,
+      "tokens_seen": 36634624
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004994383149448345,
+      "loss": 4.3641,
+      "theoretical_loss": 5.537982660825981,
+      "tokens_seen": 36700160
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004994282848545637,
+      "loss": 4.5599,
+      "theoretical_loss": 5.536419902821491,
+      "tokens_seen": 36765696
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004994182547642928,
+      "loss": 4.6875,
+      "theoretical_loss": 5.534860706406477,
+      "tokens_seen": 36831232
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004994082246740221,
+      "loss": 4.7352,
+      "theoretical_loss": 5.533305057148215,
+      "tokens_seen": 36896768
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004993981945837512,
+      "loss": 4.6149,
+      "theoretical_loss": 5.531752940697968,
+      "tokens_seen": 36962304
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004993881644934805,
+      "loss": 4.5067,
+      "theoretical_loss": 5.5302043427903484,
+      "tokens_seen": 37027840
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004993781344032096,
+      "loss": 4.9474,
+      "theoretical_loss": 5.528659249242693,
+      "tokens_seen": 37093376
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004993681043129389,
+      "loss": 4.4017,
+      "theoretical_loss": 5.527117645954435,
+      "tokens_seen": 37158912
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.000499358074222668,
+      "loss": 4.5323,
+      "theoretical_loss": 5.525579518906484,
+      "tokens_seen": 37224448
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004993480441323972,
+      "loss": 4.4802,
+      "theoretical_loss": 5.524044854160613,
+      "tokens_seen": 37289984
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004993380140421264,
+      "loss": 4.4531,
+      "theoretical_loss": 5.522513637858855,
+      "tokens_seen": 37355520
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004993279839518556,
+      "loss": 4.6216,
+      "theoretical_loss": 5.520985856222893,
+      "tokens_seen": 37421056
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004993179538615848,
+      "loss": 4.5407,
+      "theoretical_loss": 5.519461495553473,
+      "tokens_seen": 37486592
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004993079237713139,
+      "loss": 4.4097,
+      "theoretical_loss": 5.5179405422298045,
+      "tokens_seen": 37552128
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004992978936810431,
+      "loss": 4.406,
+      "theoretical_loss": 5.516422982708983,
+      "tokens_seen": 37617664
+    },
+    {
+      "epoch": 0.01,
+      "objective/train/docs_used": 32203,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 4.049692630767822,
+      "objective/train/theoretical_loss": 5.514908803525402,
+      "objective/train/tokens_used": 58143200,
+      "theoretical_loss": 5.514908803525402,
+      "tokens_seen": 37683200
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004992878635907723,
+      "loss": 4.3638,
+      "theoretical_loss": 5.514908803525402,
+      "tokens_seen": 37683200
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004992778335005015,
+      "loss": 4.6329,
+      "theoretical_loss": 5.513397991290185,
+      "tokens_seen": 37748736
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004992678034102307,
+      "loss": 4.6538,
+      "theoretical_loss": 5.511890532690611,
+      "tokens_seen": 37814272
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004992577733199598,
+      "loss": 4.7513,
+      "theoretical_loss": 5.510386414489556,
+      "tokens_seen": 37879808
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.000499247743229689,
+      "loss": 4.6591,
+      "theoretical_loss": 5.5088856235249235,
+      "tokens_seen": 37945344
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004992377131394183,
+      "loss": 4.4618,
+      "theoretical_loss": 5.507388146709105,
+      "tokens_seen": 38010880
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004992276830491475,
+      "loss": 4.4231,
+      "theoretical_loss": 5.505893971028418,
+      "tokens_seen": 38076416
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004992176529588767,
+      "loss": 4.2713,
+      "theoretical_loss": 5.504403083542568,
+      "tokens_seen": 38141952
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004992076228686059,
+      "loss": 4.4092,
+      "theoretical_loss": 5.502915471384111,
+      "tokens_seen": 38207488
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.000499197592778335,
+      "loss": 4.7235,
+      "theoretical_loss": 5.501431121757914,
+      "tokens_seen": 38273024
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004991875626880642,
+      "loss": 4.3762,
+      "theoretical_loss": 5.49995002194063,
+      "tokens_seen": 38338560
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004991775325977934,
+      "loss": 4.6923,
+      "theoretical_loss": 5.498472159280171,
+      "tokens_seen": 38404096
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004991675025075226,
+      "loss": 4.8392,
+      "theoretical_loss": 5.4969975211951905,
+      "tokens_seen": 38469632
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004991574724172518,
+      "loss": 4.5345,
+      "theoretical_loss": 5.495526095174563,
+      "tokens_seen": 38535168
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.000499147442326981,
+      "loss": 4.6407,
+      "theoretical_loss": 5.494057868776883,
+      "tokens_seen": 38600704
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004991374122367101,
+      "loss": 4.5119,
+      "theoretical_loss": 5.492592829629946,
+      "tokens_seen": 38666240
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004991273821464393,
+      "loss": 4.6456,
+      "theoretical_loss": 5.491130965430262,
+      "tokens_seen": 38731776
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004991173520561685,
+      "loss": 4.2981,
+      "theoretical_loss": 5.489672263942543,
+      "tokens_seen": 38797312
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004991073219658977,
+      "loss": 4.7488,
+      "theoretical_loss": 5.488216712999222,
+      "tokens_seen": 38862848
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.000499097291875627,
+      "loss": 4.5654,
+      "theoretical_loss": 5.486764300499962,
+      "tokens_seen": 38928384
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.000499087261785356,
+      "loss": 4.3462,
+      "theoretical_loss": 5.4853150144111655,
+      "tokens_seen": 38993920
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004990772316950853,
+      "loss": 4.6389,
+      "theoretical_loss": 5.4838688427655065,
+      "tokens_seen": 39059456
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004990672016048144,
+      "loss": 4.5166,
+      "theoretical_loss": 5.482425773661445,
+      "tokens_seen": 39124992
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004990571715145437,
+      "loss": 4.5858,
+      "theoretical_loss": 5.480985795262759,
+      "tokens_seen": 39190528
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004990471414242729,
+      "loss": 4.6098,
+      "theoretical_loss": 5.479548895798082,
+      "tokens_seen": 39256064
+    },
+    {
+      "epoch": 0.01,
+      "objective/train/docs_used": 33134,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 4.761120319366455,
+      "objective/train/theoretical_loss": 5.478115063560433,
+      "objective/train/tokens_used": 59781600,
+      "theoretical_loss": 5.478115063560433,
+      "tokens_seen": 39321600
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.000499037111334002,
+      "loss": 4.4957,
+      "theoretical_loss": 5.478115063560433,
+      "tokens_seen": 39321600
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004990270812437312,
+      "loss": 4.7046,
+      "theoretical_loss": 5.476684286906764,
+      "tokens_seen": 39387136
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004990170511534604,
+      "loss": 4.2948,
+      "theoretical_loss": 5.475256554257497,
+      "tokens_seen": 39452672
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004990070210631896,
+      "loss": 4.5969,
+      "theoretical_loss": 5.473831854096085,
+      "tokens_seen": 39518208
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004989969909729188,
+      "loss": 4.5511,
+      "theoretical_loss": 5.472410174968553,
+      "tokens_seen": 39583744
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.000498986960882648,
+      "loss": 4.4106,
+      "theoretical_loss": 5.470991505483065,
+      "tokens_seen": 39649280
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004989769307923771,
+      "loss": 4.623,
+      "theoretical_loss": 5.469575834309477,
+      "tokens_seen": 39714816
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004989669007021063,
+      "loss": 4.5077,
+      "theoretical_loss": 5.468163150178907,
+      "tokens_seen": 39780352
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004989568706118355,
+      "loss": 4.475,
+      "theoretical_loss": 5.466753441883302,
+      "tokens_seen": 39845888
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004989468405215647,
+      "loss": 4.6451,
+      "theoretical_loss": 5.465346698275008,
+      "tokens_seen": 39911424
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004989368104312939,
+      "loss": 4.3366,
+      "theoretical_loss": 5.46394290826635,
+      "tokens_seen": 39976960
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.000498926780341023,
+      "loss": 4.4816,
+      "theoretical_loss": 5.462542060829209,
+      "tokens_seen": 40042496
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004989167502507523,
+      "loss": 4.5005,
+      "theoretical_loss": 5.461144144994606,
+      "tokens_seen": 40108032
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004989067201604814,
+      "loss": 4.5075,
+      "theoretical_loss": 5.459749149852287,
+      "tokens_seen": 40173568
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004988966900702107,
+      "loss": 4.555,
+      "theoretical_loss": 5.4583570645503166,
+      "tokens_seen": 40239104
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004988866599799398,
+      "loss": 4.411,
+      "theoretical_loss": 5.45696787829467,
+      "tokens_seen": 40304640
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004988766298896691,
+      "loss": 4.479,
+      "theoretical_loss": 5.455581580348831,
+      "tokens_seen": 40370176
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004988665997993982,
+      "loss": 4.424,
+      "theoretical_loss": 5.454198160033391,
+      "tokens_seen": 40435712
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004988565697091274,
+      "loss": 4.6216,
+      "theoretical_loss": 5.452817606725657,
+      "tokens_seen": 40501248
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004988465396188566,
+      "loss": 4.5915,
+      "theoretical_loss": 5.451439909859255,
+      "tokens_seen": 40566784
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004988365095285858,
+      "loss": 4.4225,
+      "theoretical_loss": 5.450065058923747,
+      "tokens_seen": 40632320
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.000498826479438315,
+      "loss": 4.4659,
+      "theoretical_loss": 5.448693043464235,
+      "tokens_seen": 40697856
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004988164493480441,
+      "loss": 4.7231,
+      "theoretical_loss": 5.447323853080991,
+      "tokens_seen": 40763392
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004988064192577733,
+      "loss": 4.561,
+      "theoretical_loss": 5.445957477429067,
+      "tokens_seen": 40828928
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004987963891675025,
+      "loss": 4.4559,
+      "theoretical_loss": 5.444593906217927,
+      "tokens_seen": 40894464
+    },
+    {
+      "epoch": 0.01,
+      "objective/train/docs_used": 33820,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 4.709114074707031,
+      "objective/train/theoretical_loss": 5.443233129211069,
+      "objective/train/tokens_used": 61420000,
+      "theoretical_loss": 5.443233129211069,
+      "tokens_seen": 40960000
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004987863590772317,
+      "loss": 4.5353,
+      "theoretical_loss": 5.443233129211069,
+      "tokens_seen": 40960000
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004987763289869609,
+      "loss": 4.5888,
+      "theoretical_loss": 5.441875136225658,
+      "tokens_seen": 41025536
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00049876629889669,
+      "loss": 4.6609,
+      "theoretical_loss": 5.440519917132159,
+      "tokens_seen": 41091072
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004987562688064192,
+      "loss": 4.6782,
+      "theoretical_loss": 5.439167461853971,
+      "tokens_seen": 41156608
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004987462387161484,
+      "loss": 4.3218,
+      "theoretical_loss": 5.437817760367073,
+      "tokens_seen": 41222144
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004987362086258777,
+      "loss": 4.6102,
+      "theoretical_loss": 5.436470802699661,
+      "tokens_seen": 41287680
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004987261785356068,
+      "loss": 4.4044,
+      "theoretical_loss": 5.43512657893179,
+      "tokens_seen": 41353216
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004987161484453361,
+      "loss": 4.5902,
+      "theoretical_loss": 5.433785079195035,
+      "tokens_seen": 41418752
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004987061183550651,
+      "loss": 4.4395,
+      "theoretical_loss": 5.432446293672129,
+      "tokens_seen": 41484288
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004986960882647944,
+      "loss": 4.6268,
+      "theoretical_loss": 5.43111021259663,
+      "tokens_seen": 41549824
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004986860581745236,
+      "loss": 4.7247,
+      "theoretical_loss": 5.429776826252565,
+      "tokens_seen": 41615360
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004986760280842528,
+      "loss": 4.7844,
+      "theoretical_loss": 5.428446124974103,
+      "tokens_seen": 41680896
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.000498665997993982,
+      "loss": 4.748,
+      "theoretical_loss": 5.427118099145207,
+      "tokens_seen": 41746432
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004986559679037111,
+      "loss": 4.4521,
+      "theoretical_loss": 5.425792739199306,
+      "tokens_seen": 41811968
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004986459378134403,
+      "loss": 4.5162,
+      "theoretical_loss": 5.424470035618963,
+      "tokens_seen": 41877504
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004986359077231695,
+      "loss": 4.5787,
+      "theoretical_loss": 5.423149978935547,
+      "tokens_seen": 41943040
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004986258776328987,
+      "loss": 4.4447,
+      "theoretical_loss": 5.421832559728902,
+      "tokens_seen": 42008576
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004986158475426279,
+      "loss": 4.5436,
+      "theoretical_loss": 5.420517768627027,
+      "tokens_seen": 42074112
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.000498605817452357,
+      "loss": 4.5508,
+      "theoretical_loss": 5.419205596305758,
+      "tokens_seen": 42139648
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004985957873620862,
+      "loss": 4.3851,
+      "theoretical_loss": 5.417896033488446,
+      "tokens_seen": 42205184
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004985857572718154,
+      "loss": 4.4232,
+      "theoretical_loss": 5.416589070945641,
+      "tokens_seen": 42270720
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004985757271815446,
+      "loss": 4.4672,
+      "theoretical_loss": 5.415284699494782,
+      "tokens_seen": 42336256
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004985656970912738,
+      "loss": 4.3494,
+      "theoretical_loss": 5.413982909999887,
+      "tokens_seen": 42401792
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004985556670010031,
+      "loss": 4.246,
+      "theoretical_loss": 5.412683693371241,
+      "tokens_seen": 42467328
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004985456369107321,
+      "loss": 4.5265,
+      "theoretical_loss": 5.411387040565092,
+      "tokens_seen": 42532864
+    },
+    {
+      "epoch": 0.01,
+      "objective/train/docs_used": 34938,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 3.999885082244873,
+      "objective/train/theoretical_loss": 5.410092942583357,
+      "objective/train/tokens_used": 63058400,
+      "theoretical_loss": 5.410092942583357,
+      "tokens_seen": 42598400
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004985356068204614,
+      "loss": 4.5919,
+      "theoretical_loss": 5.410092942583357,
+      "tokens_seen": 42598400
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004985255767301905,
+      "loss": 4.603,
+      "theoretical_loss": 5.408801390473304,
+      "tokens_seen": 42663936
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004985155466399198,
+      "loss": 4.546,
+      "theoretical_loss": 5.407512375327272,
+      "tokens_seen": 42729472
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.000498505516549649,
+      "loss": 4.5326,
+      "theoretical_loss": 5.406225888282363,
+      "tokens_seen": 42795008
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004984954864593782,
+      "loss": 4.3889,
+      "theoretical_loss": 5.404941920520156,
+      "tokens_seen": 42860544
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004984854563691073,
+      "loss": 4.1785,
+      "theoretical_loss": 5.403660463266413,
+      "tokens_seen": 42926080
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004984754262788365,
+      "loss": 4.5382,
+      "theoretical_loss": 5.402381507790792,
+      "tokens_seen": 42991616
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004984653961885657,
+      "loss": 4.5693,
+      "theoretical_loss": 5.401105045406557,
+      "tokens_seen": 43057152
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004984553660982949,
+      "loss": 4.4403,
+      "theoretical_loss": 5.399831067470302,
+      "tokens_seen": 43122688
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004984453360080241,
+      "loss": 4.7214,
+      "theoretical_loss": 5.398559565381662,
+      "tokens_seen": 43188224
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004984353059177532,
+      "loss": 4.4442,
+      "theoretical_loss": 5.397290530583037,
+      "tokens_seen": 43253760
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004984252758274825,
+      "loss": 4.309,
+      "theoretical_loss": 5.396023954559315,
+      "tokens_seen": 43319296
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004984152457372116,
+      "loss": 4.3474,
+      "theoretical_loss": 5.394759828837596,
+      "tokens_seen": 43384832
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004984052156469409,
+      "loss": 4.5364,
+      "theoretical_loss": 5.393498144986921,
+      "tokens_seen": 43450368
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00049839518555667,
+      "loss": 4.647,
+      "theoretical_loss": 5.392238894618,
+      "tokens_seen": 43515904
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004983851554663993,
+      "loss": 4.4688,
+      "theoretical_loss": 5.390982069382944,
+      "tokens_seen": 43581440
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004983751253761284,
+      "loss": 4.555,
+      "theoretical_loss": 5.389727660974998,
+      "tokens_seen": 43646976
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004983650952858576,
+      "loss": 4.2673,
+      "theoretical_loss": 5.38847566112828,
+      "tokens_seen": 43712512
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004983550651955868,
+      "loss": 4.223,
+      "theoretical_loss": 5.387226061617516,
+      "tokens_seen": 43778048
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.000498345035105316,
+      "loss": 4.5944,
+      "theoretical_loss": 5.38597885425778,
+      "tokens_seen": 43843584
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004983350050150452,
+      "loss": 4.2395,
+      "theoretical_loss": 5.384734030904238,
+      "tokens_seen": 43909120
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004983249749247743,
+      "loss": 4.2047,
+      "theoretical_loss": 5.383491583451892,
+      "tokens_seen": 43974656
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004983149448345035,
+      "loss": 4.6784,
+      "theoretical_loss": 5.382251503835324,
+      "tokens_seen": 44040192
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004983049147442327,
+      "loss": 4.5094,
+      "theoretical_loss": 5.381013784028445,
+      "tokens_seen": 44105728
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004982948846539619,
+      "loss": 4.2719,
+      "theoretical_loss": 5.379778416044247,
+      "tokens_seen": 44171264
+    },
+    {
+      "epoch": 0.01,
+      "objective/train/docs_used": 35534,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 4.2067718505859375,
+      "objective/train/theoretical_loss": 5.378545391934553,
+      "objective/train/tokens_used": 64696800,
+      "theoretical_loss": 5.378545391934553,
+      "tokens_seen": 44236800
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004982848545636911,
+      "loss": 4.3011,
+      "theoretical_loss": 5.378545391934553,
+      "tokens_seen": 44236800
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004982748244734202,
+      "loss": 4.3593,
+      "theoretical_loss": 5.377314703789773,
+      "tokens_seen": 44302336
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004982647943831494,
+      "loss": 4.5546,
+      "theoretical_loss": 5.376086343738656,
+      "tokens_seen": 44367872
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004982547642928786,
+      "loss": 4.2113,
+      "theoretical_loss": 5.3748603039480525,
+      "tokens_seen": 44433408
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004982447342026079,
+      "loss": 4.6548,
+      "theoretical_loss": 5.373636576622671,
+      "tokens_seen": 44498944
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.000498234704112337,
+      "loss": 4.6067,
+      "theoretical_loss": 5.372415154004842,
+      "tokens_seen": 44564480
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004982246740220663,
+      "loss": 4.5434,
+      "theoretical_loss": 5.371196028374278,
+      "tokens_seen": 44630016
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004982146439317953,
+      "loss": 4.219,
+      "theoretical_loss": 5.369979192047846,
+      "tokens_seen": 44695552
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004982046138415246,
+      "loss": 4.4616,
+      "theoretical_loss": 5.368764637379322,
+      "tokens_seen": 44761088
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004981945837512538,
+      "loss": 4.3927,
+      "theoretical_loss": 5.367552356759175,
+      "tokens_seen": 44826624
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.000498184553660983,
+      "loss": 4.3436,
+      "theoretical_loss": 5.366342342614322,
+      "tokens_seen": 44892160
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004981745235707122,
+      "loss": 4.3641,
+      "theoretical_loss": 5.365134587407918,
+      "tokens_seen": 44957696
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004981644934804413,
+      "loss": 4.3878,
+      "theoretical_loss": 5.363929083639112,
+      "tokens_seen": 45023232
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004981544633901705,
+      "loss": 4.252,
+      "theoretical_loss": 5.362725823842838,
+      "tokens_seen": 45088768
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004981444332998997,
+      "loss": 4.1657,
+      "theoretical_loss": 5.361524800589581,
+      "tokens_seen": 45154304
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004981344032096289,
+      "loss": 4.0417,
+      "theoretical_loss": 5.360326006485165,
+      "tokens_seen": 45219840
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004981243731193581,
+      "loss": 4.6023,
+      "theoretical_loss": 5.359129434170529,
+      "tokens_seen": 45285376
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004981143430290873,
+      "loss": 4.3602,
+      "theoretical_loss": 5.357935076321514,
+      "tokens_seen": 45350912
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004981043129388164,
+      "loss": 4.4446,
+      "theoretical_loss": 5.35674292564864,
+      "tokens_seen": 45416448
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004980942828485456,
+      "loss": 4.4122,
+      "theoretical_loss": 5.3555529748969,
+      "tokens_seen": 45481984
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004980842527582748,
+      "loss": 4.465,
+      "theoretical_loss": 5.354365216845546,
+      "tokens_seen": 45547520
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.000498074222668004,
+      "loss": 4.1501,
+      "theoretical_loss": 5.353179644307868,
+      "tokens_seen": 45613056
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004980641925777333,
+      "loss": 4.6183,
+      "theoretical_loss": 5.351996250131005,
+      "tokens_seen": 45678592
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004980541624874623,
+      "loss": 4.4826,
+      "theoretical_loss": 5.350815027195717,
+      "tokens_seen": 45744128
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004980441323971916,
+      "loss": 4.0969,
+      "theoretical_loss": 5.349635968416189,
+      "tokens_seen": 45809664
+    },
+    {
+      "epoch": 0.01,
+      "objective/train/docs_used": 36791,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 3.983624219894409,
+      "objective/train/theoretical_loss": 5.34845906673983,
+      "objective/train/tokens_used": 66335200,
+      "theoretical_loss": 5.34845906673983,
+      "tokens_seen": 45875200
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004980341023069207,
+      "loss": 4.1114,
+      "theoretical_loss": 5.34845906673983,
+      "tokens_seen": 45875200
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00049802407221665,
+      "loss": 4.3748,
+      "theoretical_loss": 5.347284315147059,
+      "tokens_seen": 45940736
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004980140421263792,
+      "loss": 4.3086,
+      "theoretical_loss": 5.346111706651114,
+      "tokens_seen": 46006272
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004980040120361084,
+      "loss": 4.5521,
+      "theoretical_loss": 5.344941234297847,
+      "tokens_seen": 46071808
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004979939819458375,
+      "loss": 4.4492,
+      "theoretical_loss": 5.343772891165527,
+      "tokens_seen": 46137344
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004979839518555667,
+      "loss": 4.4528,
+      "theoretical_loss": 5.34260667036464,
+      "tokens_seen": 46202880
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004979739217652959,
+      "loss": 4.6275,
+      "theoretical_loss": 5.3414425650377,
+      "tokens_seen": 46268416
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004979638916750251,
+      "loss": 4.5955,
+      "theoretical_loss": 5.3402805683590495,
+      "tokens_seen": 46333952
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004979538615847543,
+      "loss": 4.5274,
+      "theoretical_loss": 5.339120673534668,
+      "tokens_seen": 46399488
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004979438314944834,
+      "loss": 4.1984,
+      "theoretical_loss": 5.337962873801984,
+      "tokens_seen": 46465024
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004979338014042126,
+      "loss": 4.4472,
+      "theoretical_loss": 5.336807162429681,
+      "tokens_seen": 46530560
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004979237713139418,
+      "loss": 4.2688,
+      "theoretical_loss": 5.335653532717512,
+      "tokens_seen": 46596096
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.000497913741223671,
+      "loss": 4.5101,
+      "theoretical_loss": 5.33450197799611,
+      "tokens_seen": 46661632
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004979037111334002,
+      "loss": 4.4218,
+      "theoretical_loss": 5.33335249162681,
+      "tokens_seen": 46727168
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004978936810431293,
+      "loss": 4.365,
+      "theoretical_loss": 5.332205067001454,
+      "tokens_seen": 46792704
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004978836509528586,
+      "loss": 4.4012,
+      "theoretical_loss": 5.331059697542216,
+      "tokens_seen": 46858240
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004978736208625877,
+      "loss": 4.178,
+      "theoretical_loss": 5.329916376701417,
+      "tokens_seen": 46923776
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.000497863590772317,
+      "loss": 4.404,
+      "theoretical_loss": 5.328775097961346,
+      "tokens_seen": 46989312
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004978535606820461,
+      "loss": 4.3063,
+      "theoretical_loss": 5.327635854834084,
+      "tokens_seen": 47054848
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004978435305917754,
+      "loss": 4.3593,
+      "theoretical_loss": 5.326498640861321,
+      "tokens_seen": 47120384
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004978335005015045,
+      "loss": 4.458,
+      "theoretical_loss": 5.325363449614185,
+      "tokens_seen": 47185920
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004978234704112337,
+      "loss": 4.4505,
+      "theoretical_loss": 5.324230274693065,
+      "tokens_seen": 47251456
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004978134403209629,
+      "loss": 4.2784,
+      "theoretical_loss": 5.323099109727434,
+      "tokens_seen": 47316992
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004978034102306921,
+      "loss": 4.3645,
+      "theoretical_loss": 5.321969948375687,
+      "tokens_seen": 47382528
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004977933801404213,
+      "loss": 4.3349,
+      "theoretical_loss": 5.320842784324954,
+      "tokens_seen": 47448064
+    },
+    {
+      "epoch": 0.01,
+      "objective/train/docs_used": 38014,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 4.285874366760254,
+      "objective/train/theoretical_loss": 5.319717611290951,
+      "objective/train/tokens_used": 67973600,
+      "theoretical_loss": 5.319717611290951,
+      "tokens_seen": 47513600
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004977833500501504,
+      "loss": 4.2487,
+      "theoretical_loss": 5.319717611290951,
+      "tokens_seen": 47513600
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004977733199598796,
+      "loss": 4.227,
+      "theoretical_loss": 5.318594423017791,
+      "tokens_seen": 47579136
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004977632898696088,
+      "loss": 4.5068,
+      "theoretical_loss": 5.317473213277827,
+      "tokens_seen": 47644672
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.000497753259779338,
+      "loss": 4.4264,
+      "theoretical_loss": 5.316353975871486,
+      "tokens_seen": 47710208
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004977432296890672,
+      "loss": 4.4848,
+      "theoretical_loss": 5.315236704627106,
+      "tokens_seen": 47775744
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004977331995987965,
+      "loss": 4.4288,
+      "theoretical_loss": 5.314121393400763,
+      "tokens_seen": 47841280
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004977231695085255,
+      "loss": 4.0077,
+      "theoretical_loss": 5.313008036076115,
+      "tokens_seen": 47906816
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004977131394182548,
+      "loss": 4.3322,
+      "theoretical_loss": 5.311896626564246,
+      "tokens_seen": 47972352
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.000497703109327984,
+      "loss": 4.1105,
+      "theoretical_loss": 5.3107871588034925,
+      "tokens_seen": 48037888
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004976930792377132,
+      "loss": 4.3437,
+      "theoretical_loss": 5.309679626759296,
+      "tokens_seen": 48103424
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004976830491474424,
+      "loss": 4.292,
+      "theoretical_loss": 5.308574024424041,
+      "tokens_seen": 48168960
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004976730190571715,
+      "loss": 4.3868,
+      "theoretical_loss": 5.3074703458168955,
+      "tokens_seen": 48234496
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004976629889669007,
+      "loss": 4.436,
+      "theoretical_loss": 5.3063685849836615,
+      "tokens_seen": 48300032
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004976529588766299,
+      "loss": 4.368,
+      "theoretical_loss": 5.305268735996615,
+      "tokens_seen": 48365568
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004976429287863591,
+      "loss": 4.2524,
+      "theoretical_loss": 5.304170792954354,
+      "tokens_seen": 48431104
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004976328986960883,
+      "loss": 4.5041,
+      "theoretical_loss": 5.303074749981648,
+      "tokens_seen": 48496640
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004976228686058175,
+      "loss": 4.412,
+      "theoretical_loss": 5.301980601229285,
+      "tokens_seen": 48562176
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004976128385155466,
+      "loss": 4.3484,
+      "theoretical_loss": 5.300888340873924,
+      "tokens_seen": 48627712
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004976028084252758,
+      "loss": 4.4722,
+      "theoretical_loss": 5.299797963117941,
+      "tokens_seen": 48693248
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.000497592778335005,
+      "loss": 4.1945,
+      "theoretical_loss": 5.298709462189283,
+      "tokens_seen": 48758784
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004975827482447342,
+      "loss": 4.2003,
+      "theoretical_loss": 5.297622832341327,
+      "tokens_seen": 48824320
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004975727181544635,
+      "loss": 4.0957,
+      "theoretical_loss": 5.2965380678527225,
+      "tokens_seen": 48889856
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004975626880641925,
+      "loss": 4.2418,
+      "theoretical_loss": 5.295455163027258,
+      "tokens_seen": 48955392
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004975526579739218,
+      "loss": 4.3033,
+      "theoretical_loss": 5.29437411219371,
+      "tokens_seen": 49020928
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004975426278836509,
+      "loss": 4.2827,
+      "theoretical_loss": 5.293294909705702,
+      "tokens_seen": 49086464
+    },
+    {
+      "epoch": 0.01,
+      "objective/train/docs_used": 38689,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 4.5455546379089355,
+      "objective/train/theoretical_loss": 5.292217549941562,
+      "objective/train/tokens_used": 69612000,
+      "theoretical_loss": 5.292217549941562,
+      "tokens_seen": 49152000
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004975325977933802,
+      "loss": 4.2752,
+      "theoretical_loss": 5.292217549941562,
+      "tokens_seen": 49152000
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004975225677031094,
+      "loss": 4.1889,
+      "theoretical_loss": 5.291142027304185,
+      "tokens_seen": 49217536
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004975125376128386,
+      "loss": 3.9788,
+      "theoretical_loss": 5.29006833622089,
+      "tokens_seen": 49283072
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004975025075225677,
+      "loss": 4.4666,
+      "theoretical_loss": 5.28899647114328,
+      "tokens_seen": 49348608
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004974924774322969,
+      "loss": 4.3282,
+      "theoretical_loss": 5.287926426547106,
+      "tokens_seen": 49414144
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004974824473420261,
+      "loss": 4.3775,
+      "theoretical_loss": 5.2868581969321315,
+      "tokens_seen": 49479680
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004974724172517553,
+      "loss": 4.3022,
+      "theoretical_loss": 5.285791776821995,
+      "tokens_seen": 49545216
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004974623871614845,
+      "loss": 4.559,
+      "theoretical_loss": 5.2847271607640725,
+      "tokens_seen": 49610752
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004974523570712136,
+      "loss": 4.5333,
+      "theoretical_loss": 5.283664343329346,
+      "tokens_seen": 49676288
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004974423269809428,
+      "loss": 4.2267,
+      "theoretical_loss": 5.282603319112273,
+      "tokens_seen": 49741824
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.000497432296890672,
+      "loss": 4.3567,
+      "theoretical_loss": 5.281544082730649,
+      "tokens_seen": 49807360
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004974222668004012,
+      "loss": 4.2938,
+      "theoretical_loss": 5.280486628825479,
+      "tokens_seen": 49872896
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004974122367101304,
+      "loss": 4.3754,
+      "theoretical_loss": 5.279430952060843,
+      "tokens_seen": 49938432
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004974022066198595,
+      "loss": 4.3361,
+      "theoretical_loss": 5.278377047123777,
+      "tokens_seen": 50003968
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004973921765295888,
+      "loss": 4.2186,
+      "theoretical_loss": 5.277324908724132,
+      "tokens_seen": 50069504
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004973821464393179,
+      "loss": 4.2978,
+      "theoretical_loss": 5.276274531594453,
+      "tokens_seen": 50135040
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004973721163490472,
+      "loss": 4.3052,
+      "theoretical_loss": 5.27522591048985,
+      "tokens_seen": 50200576
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004973620862587763,
+      "loss": 4.352,
+      "theoretical_loss": 5.274179040187876,
+      "tokens_seen": 50266112
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004973520561685056,
+      "loss": 4.0427,
+      "theoretical_loss": 5.273133915488394,
+      "tokens_seen": 50331648
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004973420260782347,
+      "loss": 4.3078,
+      "theoretical_loss": 5.272090531213462,
+      "tokens_seen": 50397184
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004973319959879639,
+      "loss": 4.0645,
+      "theoretical_loss": 5.271048882207202,
+      "tokens_seen": 50462720
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004973219658976931,
+      "loss": 4.6409,
+      "theoretical_loss": 5.27000896333568,
+      "tokens_seen": 50528256
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004973119358074223,
+      "loss": 4.0496,
+      "theoretical_loss": 5.268970769486789,
+      "tokens_seen": 50593792
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004973019057171515,
+      "loss": 4.4574,
+      "theoretical_loss": 5.267934295570118,
+      "tokens_seen": 50659328
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004972918756268806,
+      "loss": 4.2343,
+      "theoretical_loss": 5.266899536516842,
+      "tokens_seen": 50724864
+    },
+    {
+      "epoch": 0.02,
+      "objective/train/docs_used": 39885,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 4.198443412780762,
+      "objective/train/theoretical_loss": 5.265866487279597,
+      "objective/train/tokens_used": 71250400,
+      "theoretical_loss": 5.265866487279597,
+      "tokens_seen": 50790400
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004972818455366098,
+      "loss": 4.2608,
+      "theoretical_loss": 5.265866487279597,
+      "tokens_seen": 50790400
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.000497271815446339,
+      "loss": 4.2763,
+      "theoretical_loss": 5.264835142832364,
+      "tokens_seen": 50855936
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004972617853560682,
+      "loss": 4.1906,
+      "theoretical_loss": 5.263805498170351,
+      "tokens_seen": 50921472
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004972517552657974,
+      "loss": 4.1761,
+      "theoretical_loss": 5.26277754830987,
+      "tokens_seen": 50987008
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004972417251755266,
+      "loss": 4.2526,
+      "theoretical_loss": 5.261751288288236,
+      "tokens_seen": 51052544
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004972316950852557,
+      "loss": 4.275,
+      "theoretical_loss": 5.260726713163637,
+      "tokens_seen": 51118080
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004972216649949849,
+      "loss": 4.3822,
+      "theoretical_loss": 5.259703818015023,
+      "tokens_seen": 51183616
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004972116349047142,
+      "loss": 4.235,
+      "theoretical_loss": 5.258682597941998,
+      "tokens_seen": 51249152
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004972016048144433,
+      "loss": 4.1933,
+      "theoretical_loss": 5.257663048064704,
+      "tokens_seen": 51314688
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004971915747241726,
+      "loss": 4.0819,
+      "theoretical_loss": 5.256645163523704,
+      "tokens_seen": 51380224
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004971815446339017,
+      "loss": 4.2302,
+      "theoretical_loss": 5.255628939479877,
+      "tokens_seen": 51445760
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004971715145436309,
+      "loss": 4.1741,
+      "theoretical_loss": 5.254614371114306,
+      "tokens_seen": 51511296
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004971614844533601,
+      "loss": 4.0105,
+      "theoretical_loss": 5.253601453628165,
+      "tokens_seen": 51576832
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004971514543630893,
+      "loss": 4.1921,
+      "theoretical_loss": 5.2525901822426135,
+      "tokens_seen": 51642368
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004971414242728185,
+      "loss": 4.1033,
+      "theoretical_loss": 5.251580552198687,
+      "tokens_seen": 51707904
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004971313941825477,
+      "loss": 4.5223,
+      "theoretical_loss": 5.250572558757186,
+      "tokens_seen": 51773440
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004971213640922768,
+      "loss": 4.2685,
+      "theoretical_loss": 5.2495661971985745,
+      "tokens_seen": 51838976
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.000497111334002006,
+      "loss": 4.4627,
+      "theoretical_loss": 5.248561462822871,
+      "tokens_seen": 51904512
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004971013039117352,
+      "loss": 4.2511,
+      "theoretical_loss": 5.247558350949539,
+      "tokens_seen": 51970048
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004970912738214644,
+      "loss": 4.0554,
+      "theoretical_loss": 5.246556856917392,
+      "tokens_seen": 52035584
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004970812437311936,
+      "loss": 4.1646,
+      "theoretical_loss": 5.24555697608448,
+      "tokens_seen": 52101120
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004970712136409227,
+      "loss": 4.2295,
+      "theoretical_loss": 5.244558703827986,
+      "tokens_seen": 52166656
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004970611835506519,
+      "loss": 4.1183,
+      "theoretical_loss": 5.243562035544132,
+      "tokens_seen": 52232192
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004970511534603811,
+      "loss": 4.4799,
+      "theoretical_loss": 5.24256696664807,
+      "tokens_seen": 52297728
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004970411233701103,
+      "loss": 4.1907,
+      "theoretical_loss": 5.24157349257378,
+      "tokens_seen": 52363264
+    },
+    {
+      "epoch": 0.02,
+      "objective/train/docs_used": 40418,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 3.979517936706543,
+      "objective/train/theoretical_loss": 5.240581608773973,
+      "objective/train/tokens_used": 72888800,
+      "theoretical_loss": 5.240581608773973,
+      "tokens_seen": 52428800
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004970310932798396,
+      "loss": 4.0593,
+      "theoretical_loss": 5.240581608773973,
+      "tokens_seen": 52428800
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004970210631895686,
+      "loss": 4.2029,
+      "theoretical_loss": 5.239591310719987,
+      "tokens_seen": 52494336
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004970110330992979,
+      "loss": 4.0965,
+      "theoretical_loss": 5.238602593901694,
+      "tokens_seen": 52559872
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004970010030090271,
+      "loss": 4.0066,
+      "theoretical_loss": 5.23761545382739,
+      "tokens_seen": 52625408
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004969909729187563,
+      "loss": 4.1287,
+      "theoretical_loss": 5.236629886023708,
+      "tokens_seen": 52690944
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004969809428284855,
+      "loss": 4.1441,
+      "theoretical_loss": 5.235645886035517,
+      "tokens_seen": 52756480
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004969709127382147,
+      "loss": 4.2684,
+      "theoretical_loss": 5.2346634494258195,
+      "tokens_seen": 52822016
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004969608826479438,
+      "loss": 4.1129,
+      "theoretical_loss": 5.233682571775664,
+      "tokens_seen": 52887552
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.000496950852557673,
+      "loss": 4.44,
+      "theoretical_loss": 5.232703248684045,
+      "tokens_seen": 52953088
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004969408224674022,
+      "loss": 4.3805,
+      "theoretical_loss": 5.231725475767806,
+      "tokens_seen": 53018624
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004969307923771314,
+      "loss": 4.012,
+      "theoretical_loss": 5.2307492486615494,
+      "tokens_seen": 53084160
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004969207622868606,
+      "loss": 4.3914,
+      "theoretical_loss": 5.229774563017541,
+      "tokens_seen": 53149696
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004969107321965897,
+      "loss": 4.2825,
+      "theoretical_loss": 5.2288014145056145,
+      "tokens_seen": 53215232
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.000496900702106319,
+      "loss": 3.9345,
+      "theoretical_loss": 5.227829798813083,
+      "tokens_seen": 53280768
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004968906720160481,
+      "loss": 4.1139,
+      "theoretical_loss": 5.2268597116446465,
+      "tokens_seen": 53346304
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004968806419257774,
+      "loss": 4.2213,
+      "theoretical_loss": 5.225891148722296,
+      "tokens_seen": 53411840
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004968706118355065,
+      "loss": 4.1851,
+      "theoretical_loss": 5.224924105785227,
+      "tokens_seen": 53477376
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004968605817452358,
+      "loss": 4.1925,
+      "theoretical_loss": 5.22395857858975,
+      "tokens_seen": 53542912
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004968505516549649,
+      "loss": 3.8839,
+      "theoretical_loss": 5.222994562909199,
+      "tokens_seen": 53608448
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004968405215646941,
+      "loss": 4.1607,
+      "theoretical_loss": 5.222032054533841,
+      "tokens_seen": 53673984
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004968304914744233,
+      "loss": 4.2475,
+      "theoretical_loss": 5.2210710492707895,
+      "tokens_seen": 53739520
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004968204613841525,
+      "loss": 3.881,
+      "theoretical_loss": 5.220111542943918,
+      "tokens_seen": 53805056
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004968104312938817,
+      "loss": 4.0171,
+      "theoretical_loss": 5.21915353139377,
+      "tokens_seen": 53870592
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004968004012036108,
+      "loss": 4.0926,
+      "theoretical_loss": 5.218197010477477,
+      "tokens_seen": 53936128
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.00049679037111334,
+      "loss": 4.3292,
+      "theoretical_loss": 5.21724197606866,
+      "tokens_seen": 54001664
+    },
+    {
+      "epoch": 0.02,
+      "objective/train/docs_used": 41741,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 4.073785305023193,
+      "objective/train/theoretical_loss": 5.216288424057362,
+      "objective/train/tokens_used": 74527200,
+      "theoretical_loss": 5.216288424057362,
+      "tokens_seen": 54067200
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004967803410230692,
+      "loss": 4.2491,
+      "theoretical_loss": 5.216288424057362,
+      "tokens_seen": 54067200
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004967703109327984,
+      "loss": 4.2446,
+      "theoretical_loss": 5.21533635034995,
+      "tokens_seen": 54132736
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004967602808425276,
+      "loss": 4.2944,
+      "theoretical_loss": 5.214385750869031,
+      "tokens_seen": 54198272
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004967502507522568,
+      "loss": 4.1296,
+      "theoretical_loss": 5.213436621553377,
+      "tokens_seen": 54263808
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004967402206619859,
+      "loss": 4.17,
+      "theoretical_loss": 5.212488958357831,
+      "tokens_seen": 54329344
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004967301905717151,
+      "loss": 4.0717,
+      "theoretical_loss": 5.211542757253229,
+      "tokens_seen": 54394880
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004967201604814444,
+      "loss": 4.4436,
+      "theoretical_loss": 5.210598014226319,
+      "tokens_seen": 54460416
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004967101303911735,
+      "loss": 4.1394,
+      "theoretical_loss": 5.209654725279678,
+      "tokens_seen": 54525952
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004967001003009028,
+      "loss": 4.4479,
+      "theoretical_loss": 5.208712886431626,
+      "tokens_seen": 54591488
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.000496690070210632,
+      "loss": 3.9774,
+      "theoretical_loss": 5.207772493716154,
+      "tokens_seen": 54657024
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004966800401203611,
+      "loss": 4.0348,
+      "theoretical_loss": 5.206833543182833,
+      "tokens_seen": 54722560
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004966700100300903,
+      "loss": 4.1275,
+      "theoretical_loss": 5.205896030896748,
+      "tokens_seen": 54788096
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004966599799398195,
+      "loss": 4.034,
+      "theoretical_loss": 5.2049599529384025,
+      "tokens_seen": 54853632
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004966499498495487,
+      "loss": 4.2901,
+      "theoretical_loss": 5.204025305403653,
+      "tokens_seen": 54919168
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004966399197592779,
+      "loss": 4.0859,
+      "theoretical_loss": 5.2030920844036235,
+      "tokens_seen": 54984704
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.000496629889669007,
+      "loss": 4.2186,
+      "theoretical_loss": 5.202160286064628,
+      "tokens_seen": 55050240
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004966198595787362,
+      "loss": 4.1753,
+      "theoretical_loss": 5.2012299065281,
+      "tokens_seen": 55115776
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004966098294884654,
+      "loss": 4.2037,
+      "theoretical_loss": 5.2003009419505055,
+      "tokens_seen": 55181312
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004965997993981946,
+      "loss": 4.3058,
+      "theoretical_loss": 5.199373388503272,
+      "tokens_seen": 55246848
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004965897693079238,
+      "loss": 4.3641,
+      "theoretical_loss": 5.198447242372716,
+      "tokens_seen": 55312384
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004965797392176529,
+      "loss": 4.1662,
+      "theoretical_loss": 5.197522499759961,
+      "tokens_seen": 55377920
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004965697091273821,
+      "loss": 4.1908,
+      "theoretical_loss": 5.196599156880863,
+      "tokens_seen": 55443456
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004965596790371113,
+      "loss": 3.9863,
+      "theoretical_loss": 5.195677209965942,
+      "tokens_seen": 55508992
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004965496489468405,
+      "loss": 4.5222,
+      "theoretical_loss": 5.194756655260305,
+      "tokens_seen": 55574528
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004965396188565698,
+      "loss": 3.9698,
+      "theoretical_loss": 5.193837489023565,
+      "tokens_seen": 55640064
+    },
+    {
+      "epoch": 0.02,
+      "objective/train/docs_used": 42409,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 4.305491924285889,
+      "objective/train/theoretical_loss": 5.192919707529784,
+      "objective/train/tokens_used": 76165600,
+      "theoretical_loss": 5.192919707529784,
+      "tokens_seen": 55705600
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004965295887662988,
+      "loss": 4.121,
+      "theoretical_loss": 5.192919707529784,
+      "tokens_seen": 55705600
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004965195586760281,
+      "loss": 4.108,
+      "theoretical_loss": 5.1920033070673846,
+      "tokens_seen": 55771136
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004965095285857573,
+      "loss": 3.9647,
+      "theoretical_loss": 5.191088283939086,
+      "tokens_seen": 55836672
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004964994984954865,
+      "loss": 3.9453,
+      "theoretical_loss": 5.190174634461828,
+      "tokens_seen": 55902208
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004964894684052157,
+      "loss": 3.9193,
+      "theoretical_loss": 5.189262354966708,
+      "tokens_seen": 55967744
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004964794383149449,
+      "loss": 4.0925,
+      "theoretical_loss": 5.188351441798897,
+      "tokens_seen": 56033280
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.000496469408224674,
+      "loss": 4.1099,
+      "theoretical_loss": 5.187441891317581,
+      "tokens_seen": 56098816
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004964593781344032,
+      "loss": 4.0055,
+      "theoretical_loss": 5.186533699895888,
+      "tokens_seen": 56164352
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004964493480441324,
+      "loss": 4.1547,
+      "theoretical_loss": 5.18562686392081,
+      "tokens_seen": 56229888
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004964393179538616,
+      "loss": 4.1385,
+      "theoretical_loss": 5.184721379793145,
+      "tokens_seen": 56295424
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004964292878635908,
+      "loss": 4.1137,
+      "theoretical_loss": 5.183817243927425,
+      "tokens_seen": 56360960
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.00049641925777332,
+      "loss": 3.988,
+      "theoretical_loss": 5.182914452751846,
+      "tokens_seen": 56426496
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004964092276830491,
+      "loss": 3.9087,
+      "theoretical_loss": 5.1820130027081985,
+      "tokens_seen": 56492032
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004963991975927783,
+      "loss": 4.1078,
+      "theoretical_loss": 5.181112890251804,
+      "tokens_seen": 56557568
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004963891675025075,
+      "loss": 3.9657,
+      "theoretical_loss": 5.180214111851447,
+      "tokens_seen": 56623104
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004963791374122367,
+      "loss": 3.8969,
+      "theoretical_loss": 5.179316663989305,
+      "tokens_seen": 56688640
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004963691073219659,
+      "loss": 4.0938,
+      "theoretical_loss": 5.178420543160889,
+      "tokens_seen": 56754176
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004963590772316951,
+      "loss": 4.0717,
+      "theoretical_loss": 5.177525745874968,
+      "tokens_seen": 56819712
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004963490471414242,
+      "loss": 4.0293,
+      "theoretical_loss": 5.176632268653516,
+      "tokens_seen": 56885248
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004963390170511535,
+      "loss": 4.0914,
+      "theoretical_loss": 5.175740108031633,
+      "tokens_seen": 56950784
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004963289869608827,
+      "loss": 4.3386,
+      "theoretical_loss": 5.174849260557494,
+      "tokens_seen": 57016320
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004963189568706119,
+      "loss": 4.1438,
+      "theoretical_loss": 5.173959722792272,
+      "tokens_seen": 57081856
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.000496308926780341,
+      "loss": 4.1502,
+      "theoretical_loss": 5.173071491310086,
+      "tokens_seen": 57147392
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004962988966900702,
+      "loss": 3.9246,
+      "theoretical_loss": 5.172184562697928,
+      "tokens_seen": 57212928
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004962888665997994,
+      "loss": 4.0249,
+      "theoretical_loss": 5.171298933555606,
+      "tokens_seen": 57278464
+    },
+    {
+      "epoch": 0.02,
+      "objective/train/docs_used": 43695,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 3.649152994155884,
+      "objective/train/theoretical_loss": 5.170414600495677,
+      "objective/train/tokens_used": 77804000,
+      "theoretical_loss": 5.170414600495677,
+      "tokens_seen": 57344000
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004962788365095286,
+      "loss": 4.0093,
+      "theoretical_loss": 5.170414600495677,
+      "tokens_seen": 57344000
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004962688064192578,
+      "loss": 4.1786,
+      "theoretical_loss": 5.16953156014339,
+      "tokens_seen": 57409536
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.000496258776328987,
+      "loss": 4.0607,
+      "theoretical_loss": 5.1686498091366175,
+      "tokens_seen": 57475072
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004962487462387161,
+      "loss": 3.6266,
+      "theoretical_loss": 5.1677693441258,
+      "tokens_seen": 57540608
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004962387161484453,
+      "loss": 3.8942,
+      "theoretical_loss": 5.16689016177388,
+      "tokens_seen": 57606144
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004962286860581746,
+      "loss": 3.9167,
+      "theoretical_loss": 5.166012258756243,
+      "tokens_seen": 57671680
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004962186559679037,
+      "loss": 4.2399,
+      "theoretical_loss": 5.16513563176066,
+      "tokens_seen": 57737216
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.000496208625877633,
+      "loss": 3.7377,
+      "theoretical_loss": 5.164260277487221,
+      "tokens_seen": 57802752
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004961985957873621,
+      "loss": 4.0589,
+      "theoretical_loss": 5.163386192648282,
+      "tokens_seen": 57868288
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004961885656970913,
+      "loss": 4.2436,
+      "theoretical_loss": 5.162513373968403,
+      "tokens_seen": 57933824
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004961785356068205,
+      "loss": 4.2688,
+      "theoretical_loss": 5.1616418181842825,
+      "tokens_seen": 57999360
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004961685055165497,
+      "loss": 3.9842,
+      "theoretical_loss": 5.160771522044712,
+      "tokens_seen": 58064896
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004961584754262789,
+      "loss": 4.202,
+      "theoretical_loss": 5.159902482310506,
+      "tokens_seen": 58130432
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004961484453360081,
+      "loss": 4.0252,
+      "theoretical_loss": 5.15903469575445,
+      "tokens_seen": 58195968
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004961384152457372,
+      "loss": 4.0653,
+      "theoretical_loss": 5.1581681591612405,
+      "tokens_seen": 58261504
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004961283851554664,
+      "loss": 4.1853,
+      "theoretical_loss": 5.15730286932743,
+      "tokens_seen": 58327040
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004961183550651956,
+      "loss": 4.0287,
+      "theoretical_loss": 5.156438823061366,
+      "tokens_seen": 58392576
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004961083249749248,
+      "loss": 3.8706,
+      "theoretical_loss": 5.15557601718314,
+      "tokens_seen": 58458112
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.000496098294884654,
+      "loss": 3.4818,
+      "theoretical_loss": 5.154714448524525,
+      "tokens_seen": 58523648
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004960882647943831,
+      "loss": 4.1847,
+      "theoretical_loss": 5.153854113928924,
+      "tokens_seen": 58589184
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004960782347041123,
+      "loss": 4.0123,
+      "theoretical_loss": 5.152995010251313,
+      "tokens_seen": 58654720
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004960682046138415,
+      "loss": 3.8546,
+      "theoretical_loss": 5.1521371343581865,
+      "tokens_seen": 58720256
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004960581745235707,
+      "loss": 3.9651,
+      "theoretical_loss": 5.1512804831275,
+      "tokens_seen": 58785792
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004960481444333,
+      "loss": 3.9367,
+      "theoretical_loss": 5.1504250534486165,
+      "tokens_seen": 58851328
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.000496038114343029,
+      "loss": 4.3063,
+      "theoretical_loss": 5.149570842222257,
+      "tokens_seen": 58916864
+    },
+    {
+      "epoch": 0.02,
+      "objective/train/docs_used": 44395,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 4.063347339630127,
+      "objective/train/theoretical_loss": 5.1487178463604355,
+      "objective/train/tokens_used": 79442400,
+      "theoretical_loss": 5.1487178463604355,
+      "tokens_seen": 58982400
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004960280842527583,
+      "loss": 3.7506,
+      "theoretical_loss": 5.1487178463604355,
+      "tokens_seen": 58982400
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004960180541624875,
+      "loss": 3.9872,
+      "theoretical_loss": 5.147866062786418,
+      "tokens_seen": 59047936
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004960080240722167,
+      "loss": 3.9911,
+      "theoretical_loss": 5.147015488434661,
+      "tokens_seen": 59113472
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004959979939819459,
+      "loss": 3.8278,
+      "theoretical_loss": 5.14616612025076,
+      "tokens_seen": 59179008
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004959879638916751,
+      "loss": 3.9528,
+      "theoretical_loss": 5.145317955191397,
+      "tokens_seen": 59244544
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004959779338014042,
+      "loss": 3.7919,
+      "theoretical_loss": 5.144470990224288,
+      "tokens_seen": 59310080
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004959679037111334,
+      "loss": 3.9634,
+      "theoretical_loss": 5.143625222328134,
+      "tokens_seen": 59375616
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004959578736208626,
+      "loss": 4.096,
+      "theoretical_loss": 5.142780648492563,
+      "tokens_seen": 59441152
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004959478435305918,
+      "loss": 3.9749,
+      "theoretical_loss": 5.141937265718084,
+      "tokens_seen": 59506688
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.000495937813440321,
+      "loss": 4.0214,
+      "theoretical_loss": 5.141095071016032,
+      "tokens_seen": 59572224
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004959277833500501,
+      "loss": 3.9143,
+      "theoretical_loss": 5.14025406140852,
+      "tokens_seen": 59637760
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004959177532597793,
+      "loss": 4.0642,
+      "theoretical_loss": 5.139414233928385,
+      "tokens_seen": 59703296
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004959077231695085,
+      "loss": 3.9695,
+      "theoretical_loss": 5.138575585619142,
+      "tokens_seen": 59768832
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004958976930792377,
+      "loss": 4.1946,
+      "theoretical_loss": 5.13773811353493,
+      "tokens_seen": 59834368
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004958876629889669,
+      "loss": 4.0784,
+      "theoretical_loss": 5.1369018147404635,
+      "tokens_seen": 59899904
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.000495877632898696,
+      "loss": 4.0826,
+      "theoretical_loss": 5.136066686310986,
+      "tokens_seen": 59965440
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004958676028084253,
+      "loss": 4.0869,
+      "theoretical_loss": 5.135232725332212,
+      "tokens_seen": 60030976
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004958575727181544,
+      "loss": 3.9758,
+      "theoretical_loss": 5.134399928900292,
+      "tokens_seen": 60096512
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004958475426278837,
+      "loss": 4.0519,
+      "theoretical_loss": 5.13356829412175,
+      "tokens_seen": 60162048
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004958375125376129,
+      "loss": 4.1545,
+      "theoretical_loss": 5.132737818113444,
+      "tokens_seen": 60227584
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004958274824473421,
+      "loss": 4.2355,
+      "theoretical_loss": 5.131908498002513,
+      "tokens_seen": 60293120
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004958174523570712,
+      "loss": 3.8189,
+      "theoretical_loss": 5.131080330926334,
+      "tokens_seen": 60358656
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004958074222668004,
+      "loss": 4.1869,
+      "theoretical_loss": 5.130253314032469,
+      "tokens_seen": 60424192
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004957973921765296,
+      "loss": 3.728,
+      "theoretical_loss": 5.1294274444786225,
+      "tokens_seen": 60489728
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004957873620862588,
+      "loss": 4.1881,
+      "theoretical_loss": 5.128602719432592,
+      "tokens_seen": 60555264
+    },
+    {
+      "epoch": 0.02,
+      "objective/train/docs_used": 45571,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 4.731624603271484,
+      "objective/train/theoretical_loss": 5.12777913607222,
+      "objective/train/tokens_used": 81080800,
+      "theoretical_loss": 5.12777913607222,
+      "tokens_seen": 60620800
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.000495777331995988,
+      "loss": 4.3579,
+      "theoretical_loss": 5.12777913607222,
+      "tokens_seen": 60620800
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004957673019057172,
+      "loss": 3.8709,
+      "theoretical_loss": 5.126956691585351,
+      "tokens_seen": 60686336
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004957572718154463,
+      "loss": 3.9854,
+      "theoretical_loss": 5.1261353831697845,
+      "tokens_seen": 60751872
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004957472417251755,
+      "loss": 4.2274,
+      "theoretical_loss": 5.125315208033226,
+      "tokens_seen": 60817408
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004957372116349047,
+      "loss": 3.8993,
+      "theoretical_loss": 5.124496163393244,
+      "tokens_seen": 60882944
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004957271815446339,
+      "loss": 3.7806,
+      "theoretical_loss": 5.123678246477226,
+      "tokens_seen": 60948480
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004957171514543631,
+      "loss": 3.977,
+      "theoretical_loss": 5.122861454522329,
+      "tokens_seen": 61014016
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004957071213640923,
+      "loss": 3.9739,
+      "theoretical_loss": 5.122045784775439,
+      "tokens_seen": 61079552
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004956970912738214,
+      "loss": 3.8256,
+      "theoretical_loss": 5.1212312344931235,
+      "tokens_seen": 61145088
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004956870611835507,
+      "loss": 4.0096,
+      "theoretical_loss": 5.120417800941587,
+      "tokens_seen": 61210624
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004956770310932798,
+      "loss": 4.0399,
+      "theoretical_loss": 5.119605481396629,
+      "tokens_seen": 61276160
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004956670010030091,
+      "loss": 3.9718,
+      "theoretical_loss": 5.1187942731436,
+      "tokens_seen": 61341696
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004956569709127383,
+      "loss": 3.9982,
+      "theoretical_loss": 5.117984173477356,
+      "tokens_seen": 61407232
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004956469408224674,
+      "loss": 4.0777,
+      "theoretical_loss": 5.117175179702215,
+      "tokens_seen": 61472768
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004956369107321966,
+      "loss": 3.9865,
+      "theoretical_loss": 5.116367289131917,
+      "tokens_seen": 61538304
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004956268806419258,
+      "loss": 4.1915,
+      "theoretical_loss": 5.115560499089575,
+      "tokens_seen": 61603840
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.000495616850551655,
+      "loss": 3.8813,
+      "theoretical_loss": 5.1147548069076425,
+      "tokens_seen": 61669376
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004956068204613842,
+      "loss": 3.97,
+      "theoretical_loss": 5.113950209927859,
+      "tokens_seen": 61734912
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004955967903711133,
+      "loss": 3.5137,
+      "theoretical_loss": 5.113146705501217,
+      "tokens_seen": 61800448
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004955867602808425,
+      "loss": 3.7782,
+      "theoretical_loss": 5.112344290987915,
+      "tokens_seen": 61865984
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004955767301905717,
+      "loss": 4.1055,
+      "theoretical_loss": 5.1115429637573175,
+      "tokens_seen": 61931520
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004955667001003009,
+      "loss": 3.9588,
+      "theoretical_loss": 5.110742721187915,
+      "tokens_seen": 61997056
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004955566700100301,
+      "loss": 3.8558,
+      "theoretical_loss": 5.109943560667279,
+      "tokens_seen": 62062592
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004955466399197592,
+      "loss": 3.8969,
+      "theoretical_loss": 5.109145479592028,
+      "tokens_seen": 62128128
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004955366098294884,
+      "loss": 4.1078,
+      "theoretical_loss": 5.108348475367774,
+      "tokens_seen": 62193664
+    },
+    {
+      "epoch": 0.02,
+      "objective/train/docs_used": 46598,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 4.215669631958008,
+      "objective/train/theoretical_loss": 5.107552545409097,
+      "objective/train/tokens_used": 82719200,
+      "theoretical_loss": 5.107552545409097,
+      "tokens_seen": 62259200
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004955265797392177,
+      "loss": 4.056,
+      "theoretical_loss": 5.107552545409097,
+      "tokens_seen": 62259200
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004955165496489468,
+      "loss": 3.9227,
+      "theoretical_loss": 5.106757687139494,
+      "tokens_seen": 62324736
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004955065195586761,
+      "loss": 4.2246,
+      "theoretical_loss": 5.1059638979913435,
+      "tokens_seen": 62390272
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004954964894684052,
+      "loss": 4.0438,
+      "theoretical_loss": 5.105171175405864,
+      "tokens_seen": 62455808
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004954864593781344,
+      "loss": 4.0275,
+      "theoretical_loss": 5.104379516833075,
+      "tokens_seen": 62521344
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004954764292878636,
+      "loss": 4.005,
+      "theoretical_loss": 5.103588919731757,
+      "tokens_seen": 62586880
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004954663991975928,
+      "loss": 3.937,
+      "theoretical_loss": 5.1027993815694135,
+      "tokens_seen": 62652416
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.000495456369107322,
+      "loss": 4.1536,
+      "theoretical_loss": 5.10201089982223,
+      "tokens_seen": 62717952
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004954463390170512,
+      "loss": 3.971,
+      "theoretical_loss": 5.101223471975038,
+      "tokens_seen": 62783488
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004954363089267803,
+      "loss": 3.9062,
+      "theoretical_loss": 5.100437095521272,
+      "tokens_seen": 62849024
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004954262788365095,
+      "loss": 3.8866,
+      "theoretical_loss": 5.099651767962936,
+      "tokens_seen": 62914560
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004954162487462387,
+      "loss": 3.8535,
+      "theoretical_loss": 5.098867486810562,
+      "tokens_seen": 62980096
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004954062186559679,
+      "loss": 3.7859,
+      "theoretical_loss": 5.098084249583175,
+      "tokens_seen": 63045632
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004953961885656971,
+      "loss": 3.9093,
+      "theoretical_loss": 5.097302053808251,
+      "tokens_seen": 63111168
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004953861584754263,
+      "loss": 3.5672,
+      "theoretical_loss": 5.0965208970216835,
+      "tokens_seen": 63176704
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004953761283851555,
+      "loss": 3.8846,
+      "theoretical_loss": 5.095740776767743,
+      "tokens_seen": 63242240
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004953660982948846,
+      "loss": 3.9469,
+      "theoretical_loss": 5.0949616905990425,
+      "tokens_seen": 63307776
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004953560682046139,
+      "loss": 4.2765,
+      "theoretical_loss": 5.094183636076501,
+      "tokens_seen": 63373312
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004953460381143431,
+      "loss": 4.015,
+      "theoretical_loss": 5.093406610769302,
+      "tokens_seen": 63438848
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004953360080240723,
+      "loss": 4.2246,
+      "theoretical_loss": 5.092630612254862,
+      "tokens_seen": 63504384
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004953259779338014,
+      "loss": 3.9173,
+      "theoretical_loss": 5.091855638118792,
+      "tokens_seen": 63569920
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004953159478435306,
+      "loss": 4.0249,
+      "theoretical_loss": 5.091081685954863,
+      "tokens_seen": 63635456
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004953059177532598,
+      "loss": 3.7978,
+      "theoretical_loss": 5.090308753364967,
+      "tokens_seen": 63700992
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.000495295887662989,
+      "loss": 3.8605,
+      "theoretical_loss": 5.089536837959084,
+      "tokens_seen": 63766528
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004952858575727182,
+      "loss": 3.9657,
+      "theoretical_loss": 5.0887659373552445,
+      "tokens_seen": 63832064
+    },
+    {
+      "epoch": 0.02,
+      "objective/train/docs_used": 47189,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 3.3733479976654053,
+      "objective/train/theoretical_loss": 5.087996049179497,
+      "objective/train/tokens_used": 84357600,
+      "theoretical_loss": 5.087996049179497,
+      "tokens_seen": 63897600
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004952758274824474,
+      "loss": 3.6824,
+      "theoretical_loss": 5.087996049179497,
+      "tokens_seen": 63897600
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004952657973921765,
+      "loss": 3.7677,
+      "theoretical_loss": 5.087227171065869,
+      "tokens_seen": 63963136
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004952557673019057,
+      "loss": 3.9197,
+      "theoretical_loss": 5.086459300656337,
+      "tokens_seen": 64028672
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004952457372116349,
+      "loss": 3.927,
+      "theoretical_loss": 5.085692435600783,
+      "tokens_seen": 64094208
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004952357071213641,
+      "loss": 3.8821,
+      "theoretical_loss": 5.084926573556972,
+      "tokens_seen": 64159744
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004952256770310933,
+      "loss": 3.7931,
+      "theoretical_loss": 5.084161712190507,
+      "tokens_seen": 64225280
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004952156469408225,
+      "loss": 3.7647,
+      "theoretical_loss": 5.083397849174801,
+      "tokens_seen": 64290816
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004952056168505516,
+      "loss": 3.9431,
+      "theoretical_loss": 5.082634982191042,
+      "tokens_seen": 64356352
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004951955867602809,
+      "loss": 3.9173,
+      "theoretical_loss": 5.081873108928154,
+      "tokens_seen": 64421888
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.00049518555667001,
+      "loss": 4.0204,
+      "theoretical_loss": 5.081112227082771,
+      "tokens_seen": 64487424
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004951755265797393,
+      "loss": 3.816,
+      "theoretical_loss": 5.0803523343591985,
+      "tokens_seen": 64552960
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004951654964894685,
+      "loss": 3.8761,
+      "theoretical_loss": 5.079593428469382,
+      "tokens_seen": 64618496
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004951554663991976,
+      "loss": 4.0756,
+      "theoretical_loss": 5.078835507132873,
+      "tokens_seen": 64684032
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004951454363089268,
+      "loss": 3.8136,
+      "theoretical_loss": 5.078078568076798,
+      "tokens_seen": 64749568
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.000495135406218656,
+      "loss": 4.0581,
+      "theoretical_loss": 5.077322609035821,
+      "tokens_seen": 64815104
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004951253761283852,
+      "loss": 3.8215,
+      "theoretical_loss": 5.076567627752118,
+      "tokens_seen": 64880640
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004951153460381144,
+      "loss": 3.8355,
+      "theoretical_loss": 5.075813621975337,
+      "tokens_seen": 64946176
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004951053159478435,
+      "loss": 3.9719,
+      "theoretical_loss": 5.075060589462572,
+      "tokens_seen": 65011712
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004950952858575727,
+      "loss": 3.7383,
+      "theoretical_loss": 5.074308527978326,
+      "tokens_seen": 65077248
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004950852557673019,
+      "loss": 3.8755,
+      "theoretical_loss": 5.073557435294486,
+      "tokens_seen": 65142784
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004950752256770311,
+      "loss": 3.6252,
+      "theoretical_loss": 5.07280730919028,
+      "tokens_seen": 65208320
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004950651955867603,
+      "loss": 3.7529,
+      "theoretical_loss": 5.072058147452257,
+      "tokens_seen": 65273856
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004950551654964894,
+      "loss": 3.679,
+      "theoretical_loss": 5.071309947874248,
+      "tokens_seen": 65339392
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004950451354062186,
+      "loss": 3.5894,
+      "theoretical_loss": 5.070562708257339,
+      "tokens_seen": 65404928
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004950351053159479,
+      "loss": 3.9951,
+      "theoretical_loss": 5.069816426409837,
+      "tokens_seen": 65470464
+    },
+    {
+      "epoch": 0.02,
+      "objective/train/docs_used": 48433,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 3.8971009254455566,
+      "objective/train/theoretical_loss": 5.069071100147241,
+      "objective/train/tokens_used": 85996000,
+      "theoretical_loss": 5.069071100147241,
+      "tokens_seen": 65536000
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.000495025075225677,
+      "loss": 3.9133,
+      "theoretical_loss": 5.069071100147241,
+      "tokens_seen": 65536000
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004950150451354063,
+      "loss": 4.2034,
+      "theoretical_loss": 5.068326727292211,
+      "tokens_seen": 65601536
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004950050150451354,
+      "loss": 3.7703,
+      "theoretical_loss": 5.067583305674537,
+      "tokens_seen": 65667072
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004949949849548646,
+      "loss": 3.8678,
+      "theoretical_loss": 5.066840833131106,
+      "tokens_seen": 65732608
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004949849548645938,
+      "loss": 3.8256,
+      "theoretical_loss": 5.066099307505878,
+      "tokens_seen": 65798144
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.000494974924774323,
+      "loss": 3.6242,
+      "theoretical_loss": 5.06535872664985,
+      "tokens_seen": 65863680
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004949648946840522,
+      "loss": 3.9033,
+      "theoretical_loss": 5.064619088421029,
+      "tokens_seen": 65929216
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004949548645937814,
+      "loss": 3.5217,
+      "theoretical_loss": 5.0638803906844,
+      "tokens_seen": 65994752
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004949448345035105,
+      "loss": 3.8469,
+      "theoretical_loss": 5.0631426313119,
+      "tokens_seen": 66060288
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004949348044132397,
+      "loss": 3.7472,
+      "theoretical_loss": 5.062405808182383,
+      "tokens_seen": 66125824
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004949247743229689,
+      "loss": 3.4883,
+      "theoretical_loss": 5.061669919181599,
+      "tokens_seen": 66191360
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004949147442326981,
+      "loss": 3.8441,
+      "theoretical_loss": 5.060934962202155,
+      "tokens_seen": 66256896
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004949047141424273,
+      "loss": 3.9786,
+      "theoretical_loss": 5.060200935143492,
+      "tokens_seen": 66322432
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004948946840521565,
+      "loss": 4.0608,
+      "theoretical_loss": 5.059467835911857,
+      "tokens_seen": 66387968
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004948846539618856,
+      "loss": 3.7521,
+      "theoretical_loss": 5.058735662420268,
+      "tokens_seen": 66453504
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004948746238716148,
+      "loss": 3.6167,
+      "theoretical_loss": 5.0580044125884935,
+      "tokens_seen": 66519040
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.000494864593781344,
+      "loss": 3.76,
+      "theoretical_loss": 5.057274084343018,
+      "tokens_seen": 66584576
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004948545636910733,
+      "loss": 3.8083,
+      "theoretical_loss": 5.056544675617017,
+      "tokens_seen": 66650112
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004948445336008024,
+      "loss": 3.7601,
+      "theoretical_loss": 5.055816184350326,
+      "tokens_seen": 66715648
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004948345035105316,
+      "loss": 3.8069,
+      "theoretical_loss": 5.055088608489417,
+      "tokens_seen": 66781184
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004948244734202607,
+      "loss": 3.3711,
+      "theoretical_loss": 5.054361945987363,
+      "tokens_seen": 66846720
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.00049481444332999,
+      "loss": 3.3429,
+      "theoretical_loss": 5.05363619480382,
+      "tokens_seen": 66912256
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004948044132397192,
+      "loss": 4.0226,
+      "theoretical_loss": 5.052911352904992,
+      "tokens_seen": 66977792
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004947943831494484,
+      "loss": 3.7436,
+      "theoretical_loss": 5.052187418263609,
+      "tokens_seen": 67043328
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004947843530591776,
+      "loss": 3.8333,
+      "theoretical_loss": 5.051464388858893,
+      "tokens_seen": 67108864
+    },
+    {
+      "epoch": 0.02,
+      "objective/train/docs_used": 48877,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 3.7897393703460693,
+      "objective/train/theoretical_loss": 5.050742262676536,
+      "objective/train/tokens_used": 87634400,
+      "theoretical_loss": 5.050742262676536,
+      "tokens_seen": 67174400
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004947743229689067,
+      "loss": 3.7583,
+      "theoretical_loss": 5.050742262676536,
+      "tokens_seen": 67174400
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004947642928786359,
+      "loss": 4.0895,
+      "theoretical_loss": 5.050021037708673,
+      "tokens_seen": 67239936
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004947542627883651,
+      "loss": 3.642,
+      "theoretical_loss": 5.049300711953854,
+      "tokens_seen": 67305472
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004947442326980943,
+      "loss": 3.7434,
+      "theoretical_loss": 5.048581283417015,
+      "tokens_seen": 67371008
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004947342026078235,
+      "loss": 3.7182,
+      "theoretical_loss": 5.047862750109456,
+      "tokens_seen": 67436544
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004947241725175527,
+      "loss": 3.9326,
+      "theoretical_loss": 5.047145110048811,
+      "tokens_seen": 67502080
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004947141424272818,
+      "loss": 3.638,
+      "theoretical_loss": 5.046428361259022,
+      "tokens_seen": 67567616
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004947041123370111,
+      "loss": 3.7471,
+      "theoretical_loss": 5.045712501770317,
+      "tokens_seen": 67633152
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004946940822467402,
+      "loss": 3.699,
+      "theoretical_loss": 5.0449975296191765,
+      "tokens_seen": 67698688
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004946840521564695,
+      "loss": 3.8026,
+      "theoretical_loss": 5.0442834428483145,
+      "tokens_seen": 67764224
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004946740220661987,
+      "loss": 3.7185,
+      "theoretical_loss": 5.043570239506648,
+      "tokens_seen": 67829760
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004946639919759278,
+      "loss": 3.8962,
+      "theoretical_loss": 5.042857917649275,
+      "tokens_seen": 67895296
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.000494653961885657,
+      "loss": 3.7692,
+      "theoretical_loss": 5.042146475337448,
+      "tokens_seen": 67960832
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004946439317953862,
+      "loss": 3.5141,
+      "theoretical_loss": 5.041435910638544,
+      "tokens_seen": 68026368
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004946339017051154,
+      "loss": 3.8396,
+      "theoretical_loss": 5.040726221626048,
+      "tokens_seen": 68091904
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004946238716148446,
+      "loss": 3.8633,
+      "theoretical_loss": 5.040017406379521,
+      "tokens_seen": 68157440
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004946138415245737,
+      "loss": 3.6866,
+      "theoretical_loss": 5.0393094629845745,
+      "tokens_seen": 68222976
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004946038114343029,
+      "loss": 3.5706,
+      "theoretical_loss": 5.038602389532853,
+      "tokens_seen": 68288512
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004945937813440321,
+      "loss": 3.4773,
+      "theoretical_loss": 5.037896184122,
+      "tokens_seen": 68354048
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004945837512537613,
+      "loss": 3.7606,
+      "theoretical_loss": 5.037190844855641,
+      "tokens_seen": 68419584
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004945737211634905,
+      "loss": 3.8062,
+      "theoretical_loss": 5.036486369843355,
+      "tokens_seen": 68485120
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004945636910732196,
+      "loss": 3.6164,
+      "theoretical_loss": 5.03578275720065,
+      "tokens_seen": 68550656
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004945536609829488,
+      "loss": 3.7046,
+      "theoretical_loss": 5.03508000504894,
+      "tokens_seen": 68616192
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004945436308926781,
+      "loss": 3.7478,
+      "theoretical_loss": 5.034378111515521,
+      "tokens_seen": 68681728
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004945336008024072,
+      "loss": 3.663,
+      "theoretical_loss": 5.0336770747335455,
+      "tokens_seen": 68747264
+    },
+    {
+      "epoch": 0.02,
+      "objective/train/docs_used": 49922,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 2.9727299213409424,
+      "objective/train/theoretical_loss": 5.032976892842002,
+      "objective/train/tokens_used": 89272800,
+      "theoretical_loss": 5.032976892842002,
+      "tokens_seen": 68812800
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004945235707121365,
+      "loss": 3.6731,
+      "theoretical_loss": 5.032976892842002,
+      "tokens_seen": 68812800
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004945135406218656,
+      "loss": 3.8125,
+      "theoretical_loss": 5.032277563985687,
+      "tokens_seen": 68878336
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004945035105315948,
+      "loss": 3.8008,
+      "theoretical_loss": 5.031579086315183,
+      "tokens_seen": 68943872
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.000494493480441324,
+      "loss": 3.9551,
+      "theoretical_loss": 5.030881457986837,
+      "tokens_seen": 69009408
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004944834503510532,
+      "loss": 3.843,
+      "theoretical_loss": 5.030184677162734,
+      "tokens_seen": 69074944
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004944734202607824,
+      "loss": 3.5448,
+      "theoretical_loss": 5.029488742010676,
+      "tokens_seen": 69140480
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004944633901705116,
+      "loss": 3.8376,
+      "theoretical_loss": 5.028793650704154,
+      "tokens_seen": 69206016
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004944533600802407,
+      "loss": 3.9125,
+      "theoretical_loss": 5.028099401422336,
+      "tokens_seen": 69271552
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004944433299899699,
+      "loss": 3.6831,
+      "theoretical_loss": 5.0274059923500305,
+      "tokens_seen": 69337088
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004944332998996991,
+      "loss": 3.7007,
+      "theoretical_loss": 5.026713421677673,
+      "tokens_seen": 69402624
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004944232698094283,
+      "loss": 3.6241,
+      "theoretical_loss": 5.0260216876013,
+      "tokens_seen": 69468160
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004944132397191575,
+      "loss": 3.7006,
+      "theoretical_loss": 5.025330788322526,
+      "tokens_seen": 69533696
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004944032096288867,
+      "loss": 3.9117,
+      "theoretical_loss": 5.024640722048522,
+      "tokens_seen": 69599232
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004943931795386158,
+      "loss": 3.4086,
+      "theoretical_loss": 5.023951486991994,
+      "tokens_seen": 69664768
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.000494383149448345,
+      "loss": 3.9368,
+      "theoretical_loss": 5.023263081371157,
+      "tokens_seen": 69730304
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004943731193580742,
+      "loss": 3.8784,
+      "theoretical_loss": 5.022575503409717,
+      "tokens_seen": 69795840
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004943630892678035,
+      "loss": 3.941,
+      "theoretical_loss": 5.021888751336848,
+      "tokens_seen": 69861376
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004943530591775326,
+      "loss": 3.8625,
+      "theoretical_loss": 5.02120282338717,
+      "tokens_seen": 69926912
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004943430290872618,
+      "loss": 3.6762,
+      "theoretical_loss": 5.020517717800723,
+      "tokens_seen": 69992448
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004943329989969909,
+      "loss": 3.4889,
+      "theoretical_loss": 5.019833432822953,
+      "tokens_seen": 70057984
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004943229689067202,
+      "loss": 3.5122,
+      "theoretical_loss": 5.019149966704683,
+      "tokens_seen": 70123520
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004943129388164494,
+      "loss": 3.6293,
+      "theoretical_loss": 5.018467317702097,
+      "tokens_seen": 70189056
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004943029087261786,
+      "loss": 3.5085,
+      "theoretical_loss": 5.017785484076715,
+      "tokens_seen": 70254592
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004942928786359078,
+      "loss": 3.6348,
+      "theoretical_loss": 5.017104464095375,
+      "tokens_seen": 70320128
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004942828485456369,
+      "loss": 3.6907,
+      "theoretical_loss": 5.0164242560302075,
+      "tokens_seen": 70385664
+    },
+    {
+      "epoch": 0.02,
+      "objective/train/docs_used": 51260,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 3.9014105796813965,
+      "objective/train/theoretical_loss": 5.015744858158618,
+      "objective/train/tokens_used": 90911200,
+      "theoretical_loss": 5.015744858158618,
+      "tokens_seen": 70451200
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004942728184553661,
+      "loss": 3.8765,
+      "theoretical_loss": 5.015744858158618,
+      "tokens_seen": 70451200
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004942627883650953,
+      "loss": 3.8516,
+      "theoretical_loss": 5.015066268763264,
+      "tokens_seen": 70516736
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004942527582748245,
+      "loss": 3.6519,
+      "theoretical_loss": 5.014388486132036,
+      "tokens_seen": 70582272
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004942427281845537,
+      "loss": 3.9132,
+      "theoretical_loss": 5.0137115085580355,
+      "tokens_seen": 70647808
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004942326980942828,
+      "loss": 3.8834,
+      "theoretical_loss": 5.013035334339554,
+      "tokens_seen": 70713344
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.000494222668004012,
+      "loss": 3.8794,
+      "theoretical_loss": 5.012359961780053,
+      "tokens_seen": 70778880
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004942126379137412,
+      "loss": 3.8483,
+      "theoretical_loss": 5.011685389188142,
+      "tokens_seen": 70844416
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004942026078234704,
+      "loss": 3.4989,
+      "theoretical_loss": 5.011011614877562,
+      "tokens_seen": 70909952
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004941925777331996,
+      "loss": 3.7672,
+      "theoretical_loss": 5.010338637167162,
+      "tokens_seen": 70975488
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004941825476429289,
+      "loss": 3.6359,
+      "theoretical_loss": 5.009666454380877,
+      "tokens_seen": 71041024
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004941725175526579,
+      "loss": 3.8619,
+      "theoretical_loss": 5.008995064847716,
+      "tokens_seen": 71106560
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004941624874623872,
+      "loss": 3.8008,
+      "theoretical_loss": 5.0083244669017315,
+      "tokens_seen": 71172096
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004941524573721163,
+      "loss": 3.9147,
+      "theoretical_loss": 5.0076546588820054,
+      "tokens_seen": 71237632
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004941424272818456,
+      "loss": 3.3797,
+      "theoretical_loss": 5.00698563913263,
+      "tokens_seen": 71303168
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004941323971915748,
+      "loss": 3.617,
+      "theoretical_loss": 5.006317406002687,
+      "tokens_seen": 71368704
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004941223671013039,
+      "loss": 3.5556,
+      "theoretical_loss": 5.0056499578462255,
+      "tokens_seen": 71434240
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004941123370110331,
+      "loss": 3.4481,
+      "theoretical_loss": 5.0049832930222475,
+      "tokens_seen": 71499776
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004941023069207623,
+      "loss": 3.826,
+      "theoretical_loss": 5.004317409894682,
+      "tokens_seen": 71565312
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004940922768304915,
+      "loss": 3.6718,
+      "theoretical_loss": 5.003652306832372,
+      "tokens_seen": 71630848
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004940822467402207,
+      "loss": 3.8578,
+      "theoretical_loss": 5.002987982209052,
+      "tokens_seen": 71696384
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004940722166499498,
+      "loss": 3.758,
+      "theoretical_loss": 5.00232443440333,
+      "tokens_seen": 71761920
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.000494062186559679,
+      "loss": 3.4768,
+      "theoretical_loss": 5.001661661798666,
+      "tokens_seen": 71827456
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004940521564694082,
+      "loss": 3.6442,
+      "theoretical_loss": 5.000999662783359,
+      "tokens_seen": 71892992
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004940421263791374,
+      "loss": 3.7435,
+      "theoretical_loss": 5.0003384357505185,
+      "tokens_seen": 71958528
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004940320962888666,
+      "loss": 3.8745,
+      "theoretical_loss": 4.999677979098056,
+      "tokens_seen": 72024064
+    },
+    {
+      "epoch": 0.02,
+      "objective/train/docs_used": 51878,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 3.4065515995025635,
+      "objective/train/theoretical_loss": 4.999018291228659,
+      "objective/train/tokens_used": 92549600,
+      "theoretical_loss": 4.999018291228659,
+      "tokens_seen": 72089600
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004940220661985958,
+      "loss": 3.5808,
+      "theoretical_loss": 4.999018291228659,
+      "tokens_seen": 72089600
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004940120361083249,
+      "loss": 3.8043,
+      "theoretical_loss": 4.998359370549778,
+      "tokens_seen": 72155136
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004940020060180542,
+      "loss": 3.649,
+      "theoretical_loss": 4.997701215473601,
+      "tokens_seen": 72220672
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004939919759277834,
+      "loss": 3.6107,
+      "theoretical_loss": 4.997043824417044,
+      "tokens_seen": 72286208
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004939819458375126,
+      "loss": 3.6793,
+      "theoretical_loss": 4.996387195801723,
+      "tokens_seen": 72351744
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004939719157472418,
+      "loss": 3.6529,
+      "theoretical_loss": 4.995731328053944,
+      "tokens_seen": 72417280
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.000493961885656971,
+      "loss": 3.7394,
+      "theoretical_loss": 4.995076219604682,
+      "tokens_seen": 72482816
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004939518555667001,
+      "loss": 3.6256,
+      "theoretical_loss": 4.994421868889559,
+      "tokens_seen": 72548352
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004939418254764293,
+      "loss": 3.6209,
+      "theoretical_loss": 4.9937682743488345,
+      "tokens_seen": 72613888
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004939317953861585,
+      "loss": 3.5004,
+      "theoretical_loss": 4.993115434427376,
+      "tokens_seen": 72679424
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004939217652958877,
+      "loss": 3.5103,
+      "theoretical_loss": 4.992463347574654,
+      "tokens_seen": 72744960
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004939117352056169,
+      "loss": 3.7514,
+      "theoretical_loss": 4.991812012244717,
+      "tokens_seen": 72810496
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.000493901705115346,
+      "loss": 3.5319,
+      "theoretical_loss": 4.99116142689617,
+      "tokens_seen": 72876032
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004938916750250752,
+      "loss": 3.8628,
+      "theoretical_loss": 4.990511589992169,
+      "tokens_seen": 72941568
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004938816449348044,
+      "loss": 4.0163,
+      "theoretical_loss": 4.989862500000392,
+      "tokens_seen": 73007104
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004938716148445337,
+      "loss": 3.7264,
+      "theoretical_loss": 4.989214155393025,
+      "tokens_seen": 73072640
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004938615847542628,
+      "loss": 3.9026,
+      "theoretical_loss": 4.988566554646752,
+      "tokens_seen": 73138176
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.000493851554663992,
+      "loss": 3.4925,
+      "theoretical_loss": 4.9879196962427255,
+      "tokens_seen": 73203712
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004938415245737211,
+      "loss": 3.5623,
+      "theoretical_loss": 4.9872735786665565,
+      "tokens_seen": 73269248
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004938314944834504,
+      "loss": 3.415,
+      "theoretical_loss": 4.986628200408299,
+      "tokens_seen": 73334784
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004938214643931796,
+      "loss": 3.5253,
+      "theoretical_loss": 4.985983559962429,
+      "tokens_seen": 73400320
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004938114343029088,
+      "loss": 3.5016,
+      "theoretical_loss": 4.985339655827827,
+      "tokens_seen": 73465856
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.000493801404212638,
+      "loss": 3.8156,
+      "theoretical_loss": 4.984696486507767,
+      "tokens_seen": 73531392
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004937913741223671,
+      "loss": 3.7367,
+      "theoretical_loss": 4.984054050509893,
+      "tokens_seen": 73596928
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004937813440320963,
+      "loss": 3.6885,
+      "theoretical_loss": 4.983412346346206,
+      "tokens_seen": 73662464
+    },
+    {
+      "epoch": 0.02,
+      "objective/train/docs_used": 53017,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 3.889392137527466,
+      "objective/train/theoretical_loss": 4.982771372533048,
+      "objective/train/tokens_used": 94188000,
+      "theoretical_loss": 4.982771372533048,
+      "tokens_seen": 73728000
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004937713139418255,
+      "loss": 3.6721,
+      "theoretical_loss": 4.982771372533048,
+      "tokens_seen": 73728000
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004937612838515547,
+      "loss": 3.9426,
+      "theoretical_loss": 4.982131127591083,
+      "tokens_seen": 73793536
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004937512537612839,
+      "loss": 3.6135,
+      "theoretical_loss": 4.981491610045282,
+      "tokens_seen": 73859072
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.000493741223671013,
+      "loss": 3.6257,
+      "theoretical_loss": 4.980852818424905,
+      "tokens_seen": 73924608
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004937311935807422,
+      "loss": 3.6675,
+      "theoretical_loss": 4.980214751263491,
+      "tokens_seen": 73990144
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004937211634904714,
+      "loss": 3.6453,
+      "theoretical_loss": 4.979577407098831,
+      "tokens_seen": 74055680
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004937111334002006,
+      "loss": 3.7323,
+      "theoretical_loss": 4.978940784472962,
+      "tokens_seen": 74121216
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004937011033099298,
+      "loss": 3.4667,
+      "theoretical_loss": 4.978304881932145,
+      "tokens_seen": 74186752
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004936910732196591,
+      "loss": 3.9063,
+      "theoretical_loss": 4.97766969802685,
+      "tokens_seen": 74252288
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004936810431293881,
+      "loss": 3.3322,
+      "theoretical_loss": 4.977035231311745,
+      "tokens_seen": 74317824
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004936710130391174,
+      "loss": 3.6439,
+      "theoretical_loss": 4.976401480345672,
+      "tokens_seen": 74383360
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004936609829488465,
+      "loss": 3.5552,
+      "theoretical_loss": 4.975768443691636,
+      "tokens_seen": 74448896
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004936509528585758,
+      "loss": 3.5095,
+      "theoretical_loss": 4.97513611991679,
+      "tokens_seen": 74514432
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.000493640922768305,
+      "loss": 3.7738,
+      "theoretical_loss": 4.974504507592419,
+      "tokens_seen": 74579968
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004936308926780341,
+      "loss": 3.8278,
+      "theoretical_loss": 4.973873605293922,
+      "tokens_seen": 74645504
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004936208625877633,
+      "loss": 3.7248,
+      "theoretical_loss": 4.973243411600797,
+      "tokens_seen": 74711040
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004936108324974925,
+      "loss": 3.8762,
+      "theoretical_loss": 4.9726139250966295,
+      "tokens_seen": 74776576
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004936008024072217,
+      "loss": 3.6798,
+      "theoretical_loss": 4.971985144369073,
+      "tokens_seen": 74842112
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004935907723169509,
+      "loss": 3.6282,
+      "theoretical_loss": 4.9713570680098345,
+      "tokens_seen": 74907648
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.00049358074222668,
+      "loss": 3.7215,
+      "theoretical_loss": 4.970729694614662,
+      "tokens_seen": 74973184
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004935707121364092,
+      "loss": 3.5638,
+      "theoretical_loss": 4.970103022783325,
+      "tokens_seen": 75038720
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004935606820461384,
+      "loss": 3.5533,
+      "theoretical_loss": 4.969477051119606,
+      "tokens_seen": 75104256
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004935506519558676,
+      "loss": 3.496,
+      "theoretical_loss": 4.968851778231276,
+      "tokens_seen": 75169792
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004935406218655968,
+      "loss": 3.6459,
+      "theoretical_loss": 4.9682272027300876,
+      "tokens_seen": 75235328
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.000493530591775326,
+      "loss": 3.5601,
+      "theoretical_loss": 4.96760332323176,
+      "tokens_seen": 75300864
+    },
+    {
+      "epoch": 0.02,
+      "objective/train/docs_used": 53591,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 3.8066093921661377,
+      "objective/train/theoretical_loss": 4.966980138355957,
+      "objective/train/tokens_used": 95826400,
+      "theoretical_loss": 4.966980138355957,
+      "tokens_seen": 75366400
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004935205616850551,
+      "loss": 3.7377,
+      "theoretical_loss": 4.966980138355957,
+      "tokens_seen": 75366400
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004935105315947844,
+      "loss": 3.6051,
+      "theoretical_loss": 4.966357646726282,
+      "tokens_seen": 75431936
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004935005015045135,
+      "loss": 3.5847,
+      "theoretical_loss": 4.965735846970255,
+      "tokens_seen": 75497472
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004934904714142428,
+      "loss": 3.7027,
+      "theoretical_loss": 4.965114737719302,
+      "tokens_seen": 75563008
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004934804413239719,
+      "loss": 3.648,
+      "theoretical_loss": 4.964494317608743,
+      "tokens_seen": 75628544
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004934704112337011,
+      "loss": 3.4989,
+      "theoretical_loss": 4.963874585277772,
+      "tokens_seen": 75694080
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004934603811434303,
+      "loss": 3.8889,
+      "theoretical_loss": 4.963255539369445,
+      "tokens_seen": 75759616
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004934503510531595,
+      "loss": 3.5961,
+      "theoretical_loss": 4.9626371785306675,
+      "tokens_seen": 75825152
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004934403209628887,
+      "loss": 3.4261,
+      "theoretical_loss": 4.962019501412178,
+      "tokens_seen": 75890688
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004934302908726179,
+      "loss": 3.7473,
+      "theoretical_loss": 4.961402506668535,
+      "tokens_seen": 75956224
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.000493420260782347,
+      "loss": 3.4055,
+      "theoretical_loss": 4.960786192958103,
+      "tokens_seen": 76021760
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004934102306920762,
+      "loss": 3.6034,
+      "theoretical_loss": 4.960170558943037,
+      "tokens_seen": 76087296
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004934002006018054,
+      "loss": 3.4187,
+      "theoretical_loss": 4.959555603289269,
+      "tokens_seen": 76152832
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004933901705115346,
+      "loss": 3.6193,
+      "theoretical_loss": 4.9589413246664975,
+      "tokens_seen": 76218368
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004933801404212638,
+      "loss": 3.794,
+      "theoretical_loss": 4.958327721748168,
+      "tokens_seen": 76283904
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.000493370110330993,
+      "loss": 3.3982,
+      "theoretical_loss": 4.957714793211462,
+      "tokens_seen": 76349440
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004933600802407221,
+      "loss": 3.3334,
+      "theoretical_loss": 4.957102537737284,
+      "tokens_seen": 76414976
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004933500501504513,
+      "loss": 3.6421,
+      "theoretical_loss": 4.956490954010249,
+      "tokens_seen": 76480512
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004933400200601805,
+      "loss": 3.5119,
+      "theoretical_loss": 4.9558800407186645,
+      "tokens_seen": 76546048
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004933299899699098,
+      "loss": 3.4933,
+      "theoretical_loss": 4.95526979655452,
+      "tokens_seen": 76611584
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004933199598796389,
+      "loss": 3.7028,
+      "theoretical_loss": 4.954660220213473,
+      "tokens_seen": 76677120
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004933099297893682,
+      "loss": 3.7165,
+      "theoretical_loss": 4.954051310394836,
+      "tokens_seen": 76742656
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004932998996990972,
+      "loss": 3.5507,
+      "theoretical_loss": 4.953443065801563,
+      "tokens_seen": 76808192
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004932898696088265,
+      "loss": 3.4901,
+      "theoretical_loss": 4.952835485140236,
+      "tokens_seen": 76873728
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004932798395185557,
+      "loss": 3.2798,
+      "theoretical_loss": 4.952228567121052,
+      "tokens_seen": 76939264
+    },
+    {
+      "epoch": 0.02,
+      "objective/train/docs_used": 54447,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 3.6758010387420654,
+      "objective/train/theoretical_loss": 4.951622310457807,
+      "objective/train/tokens_used": 97464800,
+      "theoretical_loss": 4.951622310457807,
+      "tokens_seen": 77004800
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004932698094282849,
+      "loss": 3.4817,
+      "theoretical_loss": 4.951622310457807,
+      "tokens_seen": 77004800
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004932597793380141,
+      "loss": 3.5561,
+      "theoretical_loss": 4.95101671386789,
+      "tokens_seen": 77070336
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004932497492477432,
+      "loss": 3.6278,
+      "theoretical_loss": 4.950411776072261,
+      "tokens_seen": 77135872
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004932397191574724,
+      "loss": 3.7609,
+      "theoretical_loss": 4.949807495795445,
+      "tokens_seen": 77201408
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004932296890672016,
+      "loss": 3.6186,
+      "theoretical_loss": 4.949203871765516,
+      "tokens_seen": 77266944
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004932196589769308,
+      "loss": 3.6535,
+      "theoretical_loss": 4.948600902714083,
+      "tokens_seen": 77332480
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.00049320962888666,
+      "loss": 3.4356,
+      "theoretical_loss": 4.947998587376282,
+      "tokens_seen": 77398016
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004931995987963893,
+      "loss": 3.7607,
+      "theoretical_loss": 4.947396924490757,
+      "tokens_seen": 77463552
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004931895687061183,
+      "loss": 3.9334,
+      "theoretical_loss": 4.946795912799653,
+      "tokens_seen": 77529088
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004931795386158476,
+      "loss": 3.4391,
+      "theoretical_loss": 4.946195551048597,
+      "tokens_seen": 77594624
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004931695085255767,
+      "loss": 3.4097,
+      "theoretical_loss": 4.945595837986693,
+      "tokens_seen": 77660160
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.000493159478435306,
+      "loss": 3.4903,
+      "theoretical_loss": 4.944996772366503,
+      "tokens_seen": 77725696
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004931494483450352,
+      "loss": 3.5634,
+      "theoretical_loss": 4.944398352944038,
+      "tokens_seen": 77791232
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004931394182547643,
+      "loss": 3.5448,
+      "theoretical_loss": 4.943800578478745,
+      "tokens_seen": 77856768
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004931293881644935,
+      "loss": 3.4788,
+      "theoretical_loss": 4.9432034477334925,
+      "tokens_seen": 77922304
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004931193580742227,
+      "loss": 3.4963,
+      "theoretical_loss": 4.942606959474562,
+      "tokens_seen": 77987840
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004931093279839519,
+      "loss": 3.4853,
+      "theoretical_loss": 4.942011112471633,
+      "tokens_seen": 78053376
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004930992978936811,
+      "loss": 3.3058,
+      "theoretical_loss": 4.94141590549777,
+      "tokens_seen": 78118912
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004930892678034102,
+      "loss": 3.7623,
+      "theoretical_loss": 4.940821337329414,
+      "tokens_seen": 78184448
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004930792377131394,
+      "loss": 3.324,
+      "theoretical_loss": 4.940227406746368,
+      "tokens_seen": 78249984
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004930692076228686,
+      "loss": 3.595,
+      "theoretical_loss": 4.939634112531784,
+      "tokens_seen": 78315520
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004930591775325978,
+      "loss": 3.6571,
+      "theoretical_loss": 4.939041453472152,
+      "tokens_seen": 78381056
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.000493049147442327,
+      "loss": 3.546,
+      "theoretical_loss": 4.9384494283572895,
+      "tokens_seen": 78446592
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004930391173520562,
+      "loss": 3.7395,
+      "theoretical_loss": 4.937858035980327,
+      "tokens_seen": 78512128
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004930290872617853,
+      "loss": 3.5258,
+      "theoretical_loss": 4.9372672751377,
+      "tokens_seen": 78577664
+    },
+    {
+      "epoch": 0.02,
+      "objective/train/docs_used": 54900,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 2.9137701988220215,
+      "objective/train/theoretical_loss": 4.936677144629131,
+      "objective/train/tokens_used": 99103200,
+      "theoretical_loss": 4.936677144629131,
+      "tokens_seen": 78643200
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004930190571715146,
+      "loss": 3.2102,
+      "theoretical_loss": 4.936677144629131,
+      "tokens_seen": 78643200
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004930090270812437,
+      "loss": 3.7118,
+      "theoretical_loss": 4.936087643257625,
+      "tokens_seen": 78708736
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.000492998996990973,
+      "loss": 3.5876,
+      "theoretical_loss": 4.935498769829453,
+      "tokens_seen": 78774272
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004929889669007021,
+      "loss": 3.7919,
+      "theoretical_loss": 4.934910523154139,
+      "tokens_seen": 78839808
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004929789368104313,
+      "loss": 3.8801,
+      "theoretical_loss": 4.934322902044457,
+      "tokens_seen": 78905344
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004929689067201605,
+      "loss": 3.5096,
+      "theoretical_loss": 4.933735905316408,
+      "tokens_seen": 78970880
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004929588766298897,
+      "loss": 3.4217,
+      "theoretical_loss": 4.933149531789217,
+      "tokens_seen": 79036416
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004929488465396189,
+      "loss": 3.6953,
+      "theoretical_loss": 4.932563780285319,
+      "tokens_seen": 79101952
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004929388164493481,
+      "loss": 3.6487,
+      "theoretical_loss": 4.931978649630346,
+      "tokens_seen": 79167488
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004929287863590773,
+      "loss": 3.6641,
+      "theoretical_loss": 4.931394138653117,
+      "tokens_seen": 79233024
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004929187562688064,
+      "loss": 3.8094,
+      "theoretical_loss": 4.930810246185626,
+      "tokens_seen": 79298560
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004929087261785356,
+      "loss": 3.5257,
+      "theoretical_loss": 4.930226971063035,
+      "tokens_seen": 79364096
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004928986960882648,
+      "loss": 3.4985,
+      "theoretical_loss": 4.929644312123655,
+      "tokens_seen": 79429632
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.000492888665997994,
+      "loss": 3.3258,
+      "theoretical_loss": 4.929062268208941,
+      "tokens_seen": 79495168
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004928786359077232,
+      "loss": 3.2647,
+      "theoretical_loss": 4.92848083816348,
+      "tokens_seen": 79560704
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004928686058174523,
+      "loss": 3.8228,
+      "theoretical_loss": 4.927900020834979,
+      "tokens_seen": 79626240
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004928585757271815,
+      "loss": 3.3943,
+      "theoretical_loss": 4.927319815074252,
+      "tokens_seen": 79691776
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004928485456369107,
+      "loss": 3.4187,
+      "theoretical_loss": 4.926740219735214,
+      "tokens_seen": 79757312
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.00049283851554664,
+      "loss": 3.6324,
+      "theoretical_loss": 4.926161233674863,
+      "tokens_seen": 79822848
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004928284854563691,
+      "loss": 3.627,
+      "theoretical_loss": 4.925582855753279,
+      "tokens_seen": 79888384
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004928184553660984,
+      "loss": 3.4882,
+      "theoretical_loss": 4.925005084833603,
+      "tokens_seen": 79953920
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004928084252758274,
+      "loss": 3.8262,
+      "theoretical_loss": 4.924427919782033,
+      "tokens_seen": 80019456
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004927983951855567,
+      "loss": 3.5192,
+      "theoretical_loss": 4.923851359467811,
+      "tokens_seen": 80084992
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004927883650952859,
+      "loss": 3.714,
+      "theoretical_loss": 4.923275402763213,
+      "tokens_seen": 80150528
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004927783350050151,
+      "loss": 3.4187,
+      "theoretical_loss": 4.922700048543536,
+      "tokens_seen": 80216064
+    },
+    {
+      "epoch": 0.02,
+      "objective/train/docs_used": 56090,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 3.1410176753997803,
+      "objective/train/theoretical_loss": 4.922125295687091,
+      "objective/train/tokens_used": 100741600,
+      "theoretical_loss": 4.922125295687091,
+      "tokens_seen": 80281600
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004927683049147443,
+      "loss": 3.5733,
+      "theoretical_loss": 4.922125295687091,
+      "tokens_seen": 80281600
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004927582748244734,
+      "loss": 3.396,
+      "theoretical_loss": 4.92155114307519,
+      "tokens_seen": 80347136
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004927482447342026,
+      "loss": 3.6166,
+      "theoretical_loss": 4.9209775895921375,
+      "tokens_seen": 80412672
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004927382146439318,
+      "loss": 3.4859,
+      "theoretical_loss": 4.920404634125217,
+      "tokens_seen": 80478208
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.000492728184553661,
+      "loss": 3.2846,
+      "theoretical_loss": 4.919832275564684,
+      "tokens_seen": 80543744
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004927181544633902,
+      "loss": 3.653,
+      "theoretical_loss": 4.9192605128037545,
+      "tokens_seen": 80609280
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004927081243731193,
+      "loss": 3.5658,
+      "theoretical_loss": 4.918689344738594,
+      "tokens_seen": 80674816
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004926980942828485,
+      "loss": 3.5587,
+      "theoretical_loss": 4.918118770268305,
+      "tokens_seen": 80740352
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004926880641925777,
+      "loss": 3.3881,
+      "theoretical_loss": 4.917548788294924,
+      "tokens_seen": 80805888
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004926780341023069,
+      "loss": 3.4548,
+      "theoretical_loss": 4.916979397723403,
+      "tokens_seen": 80871424
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004926680040120361,
+      "loss": 3.6868,
+      "theoretical_loss": 4.916410597461606,
+      "tokens_seen": 80936960
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004926579739217654,
+      "loss": 3.6518,
+      "theoretical_loss": 4.9158423864202945,
+      "tokens_seen": 81002496
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004926479438314944,
+      "loss": 3.4168,
+      "theoretical_loss": 4.915274763513119,
+      "tokens_seen": 81068032
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004926379137412237,
+      "loss": 3.353,
+      "theoretical_loss": 4.914707727656609,
+      "tokens_seen": 81133568
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004926278836509528,
+      "loss": 3.3831,
+      "theoretical_loss": 4.914141277770164,
+      "tokens_seen": 81199104
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004926178535606821,
+      "loss": 3.1851,
+      "theoretical_loss": 4.913575412776042,
+      "tokens_seen": 81264640
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004926078234704113,
+      "loss": 3.7187,
+      "theoretical_loss": 4.91301013159935,
+      "tokens_seen": 81330176
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004925977933801404,
+      "loss": 3.5052,
+      "theoretical_loss": 4.912445433168036,
+      "tokens_seen": 81395712
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004925877632898696,
+      "loss": 3.5131,
+      "theoretical_loss": 4.911881316412876,
+      "tokens_seen": 81461248
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004925777331995988,
+      "loss": 3.7694,
+      "theoretical_loss": 4.911317780267467,
+      "tokens_seen": 81526784
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.000492567703109328,
+      "loss": 3.3832,
+      "theoretical_loss": 4.910754823668215,
+      "tokens_seen": 81592320
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004925576730190572,
+      "loss": 3.8546,
+      "theoretical_loss": 4.910192445554329,
+      "tokens_seen": 81657856
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004925476429287864,
+      "loss": 3.6127,
+      "theoretical_loss": 4.909630644867807,
+      "tokens_seen": 81723392
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004925376128385155,
+      "loss": 3.3545,
+      "theoretical_loss": 4.909069420553429,
+      "tokens_seen": 81788928
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004925275827482447,
+      "loss": 3.5589,
+      "theoretical_loss": 4.908508771558748,
+      "tokens_seen": 81854464
+    },
+    {
+      "epoch": 0.02,
+      "objective/train/docs_used": 57069,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 3.6827940940856934,
+      "objective/train/theoretical_loss": 4.907948696834077,
+      "objective/train/tokens_used": 102380000,
+      "theoretical_loss": 4.907948696834077,
+      "tokens_seen": 81920000
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004925175526579739,
+      "loss": 3.7195,
+      "theoretical_loss": 4.907948696834077,
+      "tokens_seen": 81920000
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004925075225677031,
+      "loss": 3.5548,
+      "theoretical_loss": 4.907389195332484,
+      "tokens_seen": 81985536
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004924974924774323,
+      "loss": 3.807,
+      "theoretical_loss": 4.906830266009781,
+      "tokens_seen": 82051072
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004924874623871615,
+      "loss": 3.4511,
+      "theoretical_loss": 4.90627190782451,
+      "tokens_seen": 82116608
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004924774322968907,
+      "loss": 3.7151,
+      "theoretical_loss": 4.905714119737945,
+      "tokens_seen": 82182144
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004924674022066199,
+      "loss": 3.5237,
+      "theoretical_loss": 4.9051569007140685,
+      "tokens_seen": 82247680
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004924573721163491,
+      "loss": 3.332,
+      "theoretical_loss": 4.904600249719573,
+      "tokens_seen": 82313216
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004924473420260783,
+      "loss": 3.504,
+      "theoretical_loss": 4.90404416572385,
+      "tokens_seen": 82378752
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004924373119358075,
+      "loss": 3.7408,
+      "theoretical_loss": 4.903488647698972,
+      "tokens_seen": 82444288
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004924272818455366,
+      "loss": 3.334,
+      "theoretical_loss": 4.902933694619698,
+      "tokens_seen": 82509824
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004924172517552658,
+      "loss": 3.2703,
+      "theoretical_loss": 4.902379305463451,
+      "tokens_seen": 82575360
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.000492407221664995,
+      "loss": 3.5645,
+      "theoretical_loss": 4.901825479210317,
+      "tokens_seen": 82640896
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004923971915747242,
+      "loss": 3.3278,
+      "theoretical_loss": 4.9012722148430345,
+      "tokens_seen": 82706432
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004923871614844534,
+      "loss": 3.6277,
+      "theoretical_loss": 4.900719511346983,
+      "tokens_seen": 82771968
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004923771313941825,
+      "loss": 3.6241,
+      "theoretical_loss": 4.900167367710176,
+      "tokens_seen": 82837504
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004923671013039117,
+      "loss": 3.5454,
+      "theoretical_loss": 4.899615782923251,
+      "tokens_seen": 82903040
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004923570712136409,
+      "loss": 3.6864,
+      "theoretical_loss": 4.899064755979464,
+      "tokens_seen": 82968576
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004923470411233702,
+      "loss": 3.3855,
+      "theoretical_loss": 4.898514285874674,
+      "tokens_seen": 83034112
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004923370110330993,
+      "loss": 3.4924,
+      "theoretical_loss": 4.897964371607341,
+      "tokens_seen": 83099648
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004923269809428286,
+      "loss": 3.4934,
+      "theoretical_loss": 4.897415012178515,
+      "tokens_seen": 83165184
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004923169508525576,
+      "loss": 3.3992,
+      "theoretical_loss": 4.896866206591823,
+      "tokens_seen": 83230720
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004923069207622869,
+      "loss": 3.5205,
+      "theoretical_loss": 4.896317953853467,
+      "tokens_seen": 83296256
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004922968906720161,
+      "loss": 3.624,
+      "theoretical_loss": 4.8957702529722145,
+      "tokens_seen": 83361792
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004922868605817453,
+      "loss": 3.5903,
+      "theoretical_loss": 4.895223102959381,
+      "tokens_seen": 83427328
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004922768304914745,
+      "loss": 3.7871,
+      "theoretical_loss": 4.894676502828836,
+      "tokens_seen": 83492864
+    },
+    {
+      "epoch": 0.03,
+      "objective/train/docs_used": 57750,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 3.804002046585083,
+      "objective/train/theoretical_loss": 4.894130451596979,
+      "objective/train/tokens_used": 104018400,
+      "theoretical_loss": 4.894130451596979,
+      "tokens_seen": 83558400
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004922668004012036,
+      "loss": 3.4882,
+      "theoretical_loss": 4.894130451596979,
+      "tokens_seen": 83558400
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004922567703109328,
+      "loss": 3.2586,
+      "theoretical_loss": 4.893584948282745,
+      "tokens_seen": 83623936
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.000492246740220662,
+      "loss": 3.5408,
+      "theoretical_loss": 4.893039991907587,
+      "tokens_seen": 83689472
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004922367101303912,
+      "loss": 3.6106,
+      "theoretical_loss": 4.892495581495468,
+      "tokens_seen": 83755008
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004922266800401204,
+      "loss": 3.2746,
+      "theoretical_loss": 4.8919517160728585,
+      "tokens_seen": 83820544
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004922166499498495,
+      "loss": 3.5637,
+      "theoretical_loss": 4.891408394668723,
+      "tokens_seen": 83886080
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004922066198595787,
+      "loss": 3.5945,
+      "theoretical_loss": 4.89086561631451,
+      "tokens_seen": 83951616
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004921965897693079,
+      "loss": 3.6297,
+      "theoretical_loss": 4.890323380044153,
+      "tokens_seen": 84017152
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004921865596790371,
+      "loss": 3.6036,
+      "theoretical_loss": 4.889781684894051,
+      "tokens_seen": 84082688
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004921765295887663,
+      "loss": 3.7009,
+      "theoretical_loss": 4.889240529903066,
+      "tokens_seen": 84148224
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004921664994984956,
+      "loss": 3.4759,
+      "theoretical_loss": 4.888699914112516,
+      "tokens_seen": 84213760
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004921564694082246,
+      "loss": 3.4833,
+      "theoretical_loss": 4.888159836566162,
+      "tokens_seen": 84279296
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004921464393179539,
+      "loss": 3.7335,
+      "theoretical_loss": 4.887620296310205,
+      "tokens_seen": 84344832
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.000492136409227683,
+      "loss": 3.8101,
+      "theoretical_loss": 4.887081292393276,
+      "tokens_seen": 84410368
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004921263791374123,
+      "loss": 3.6913,
+      "theoretical_loss": 4.886542823866424,
+      "tokens_seen": 84475904
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004921163490471415,
+      "loss": 3.5273,
+      "theoretical_loss": 4.886004889783116,
+      "tokens_seen": 84541440
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004921063189568706,
+      "loss": 3.5718,
+      "theoretical_loss": 4.885467489199222,
+      "tokens_seen": 84606976
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004920962888665998,
+      "loss": 3.6503,
+      "theoretical_loss": 4.884930621173009,
+      "tokens_seen": 84672512
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.000492086258776329,
+      "loss": 3.5557,
+      "theoretical_loss": 4.884394284765135,
+      "tokens_seen": 84738048
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004920762286860582,
+      "loss": 3.5114,
+      "theoretical_loss": 4.88385847903864,
+      "tokens_seen": 84803584
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004920661985957874,
+      "loss": 3.8069,
+      "theoretical_loss": 4.883323203058936,
+      "tokens_seen": 84869120
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004920561685055166,
+      "loss": 3.529,
+      "theoretical_loss": 4.882788455893802,
+      "tokens_seen": 84934656
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004920461384152457,
+      "loss": 3.5584,
+      "theoretical_loss": 4.882254236613376,
+      "tokens_seen": 85000192
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004920361083249749,
+      "loss": 3.7302,
+      "theoretical_loss": 4.881720544290144,
+      "tokens_seen": 85065728
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004920260782347041,
+      "loss": 3.4352,
+      "theoretical_loss": 4.881187377998938,
+      "tokens_seen": 85131264
+    },
+    {
+      "epoch": 0.03,
+      "objective/train/docs_used": 58904,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 3.6649792194366455,
+      "objective/train/theoretical_loss": 4.880654736816922,
+      "objective/train/tokens_used": 105656800,
+      "theoretical_loss": 4.880654736816922,
+      "tokens_seen": 85196800
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004920160481444333,
+      "loss": 3.7428,
+      "theoretical_loss": 4.880654736816922,
+      "tokens_seen": 85196800
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004920060180541625,
+      "loss": 3.3637,
+      "theoretical_loss": 4.880122619823589,
+      "tokens_seen": 85262336
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004919959879638916,
+      "loss": 3.4179,
+      "theoretical_loss": 4.87959102610075,
+      "tokens_seen": 85327872
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004919859578736209,
+      "loss": 3.3888,
+      "theoretical_loss": 4.879059954732529,
+      "tokens_seen": 85393408
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.00049197592778335,
+      "loss": 3.7427,
+      "theoretical_loss": 4.878529404805356,
+      "tokens_seen": 85458944
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004919658976930793,
+      "loss": 3.7709,
+      "theoretical_loss": 4.8779993754079545,
+      "tokens_seen": 85524480
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004919558676028084,
+      "loss": 3.5494,
+      "theoretical_loss": 4.877469865631338,
+      "tokens_seen": 85590016
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004919458375125377,
+      "loss": 3.5295,
+      "theoretical_loss": 4.876940874568804,
+      "tokens_seen": 85655552
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004919358074222668,
+      "loss": 3.5878,
+      "theoretical_loss": 4.876412401315923,
+      "tokens_seen": 85721088
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.000491925777331996,
+      "loss": 3.57,
+      "theoretical_loss": 4.8758844449705325,
+      "tokens_seen": 85786624
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004919157472417252,
+      "loss": 3.5246,
+      "theoretical_loss": 4.875357004632728,
+      "tokens_seen": 85852160
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004919057171514544,
+      "loss": 3.3963,
+      "theoretical_loss": 4.87483007940486,
+      "tokens_seen": 85917696
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004918956870611836,
+      "loss": 3.6586,
+      "theoretical_loss": 4.874303668391521,
+      "tokens_seen": 85983232
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004918856569709127,
+      "loss": 3.6073,
+      "theoretical_loss": 4.873777770699542,
+      "tokens_seen": 86048768
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004918756268806419,
+      "loss": 3.5692,
+      "theoretical_loss": 4.873252385437985,
+      "tokens_seen": 86114304
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004918655967903711,
+      "loss": 3.5077,
+      "theoretical_loss": 4.872727511718134,
+      "tokens_seen": 86179840
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004918555667001003,
+      "loss": 3.6584,
+      "theoretical_loss": 4.8722031486534885,
+      "tokens_seen": 86245376
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004918455366098295,
+      "loss": 3.4986,
+      "theoretical_loss": 4.8716792953597565,
+      "tokens_seen": 86310912
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004918355065195586,
+      "loss": 3.7466,
+      "theoretical_loss": 4.871155950954847,
+      "tokens_seen": 86376448
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004918254764292878,
+      "loss": 3.627,
+      "theoretical_loss": 4.870633114558865,
+      "tokens_seen": 86441984
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.000491815446339017,
+      "loss": 3.5282,
+      "theoretical_loss": 4.8701107852941,
+      "tokens_seen": 86507520
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004918054162487463,
+      "loss": 3.1841,
+      "theoretical_loss": 4.869588962285023,
+      "tokens_seen": 86573056
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004917953861584754,
+      "loss": 3.6871,
+      "theoretical_loss": 4.869067644658276,
+      "tokens_seen": 86638592
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004917853560682047,
+      "loss": 3.6656,
+      "theoretical_loss": 4.86854683154267,
+      "tokens_seen": 86704128
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004917753259779337,
+      "loss": 3.5156,
+      "theoretical_loss": 4.868026522069172,
+      "tokens_seen": 86769664
+    },
+    {
+      "epoch": 0.03,
+      "objective/train/docs_used": 59520,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 3.652179718017578,
+      "objective/train/theoretical_loss": 4.867506715370903,
+      "objective/train/tokens_used": 107295200,
+      "theoretical_loss": 4.867506715370903,
+      "tokens_seen": 86835200
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.000491765295887663,
+      "loss": 3.4928,
+      "theoretical_loss": 4.867506715370903,
+      "tokens_seen": 86835200
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004917552657973922,
+      "loss": 3.4802,
+      "theoretical_loss": 4.866987410583125,
+      "tokens_seen": 86900736
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004917452357071214,
+      "loss": 3.5752,
+      "theoretical_loss": 4.8664686068432434,
+      "tokens_seen": 86966272
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004917352056168506,
+      "loss": 3.5258,
+      "theoretical_loss": 4.8659503032907905,
+      "tokens_seen": 87031808
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004917251755265797,
+      "loss": 3.7263,
+      "theoretical_loss": 4.865432499067425,
+      "tokens_seen": 87097344
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004917151454363089,
+      "loss": 3.7067,
+      "theoretical_loss": 4.864915193316922,
+      "tokens_seen": 87162880
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004917051153460381,
+      "loss": 3.4247,
+      "theoretical_loss": 4.864398385185169,
+      "tokens_seen": 87228416
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004916950852557673,
+      "loss": 3.5719,
+      "theoretical_loss": 4.863882073820154,
+      "tokens_seen": 87293952
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004916850551654965,
+      "loss": 3.5349,
+      "theoretical_loss": 4.863366258371963,
+      "tokens_seen": 87359488
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004916750250752258,
+      "loss": 3.5815,
+      "theoretical_loss": 4.862850937992777,
+      "tokens_seen": 87425024
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004916649949849548,
+      "loss": 3.5233,
+      "theoretical_loss": 4.862336111836852,
+      "tokens_seen": 87490560
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004916549648946841,
+      "loss": 3.5181,
+      "theoretical_loss": 4.861821779060529,
+      "tokens_seen": 87556096
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004916449348044132,
+      "loss": 3.369,
+      "theoretical_loss": 4.861307938822215,
+      "tokens_seen": 87621632
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004916349047141425,
+      "loss": 3.3532,
+      "theoretical_loss": 4.86079459028238,
+      "tokens_seen": 87687168
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004916248746238717,
+      "loss": 3.4634,
+      "theoretical_loss": 4.860281732603555,
+      "tokens_seen": 87752704
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004916148445336008,
+      "loss": 3.64,
+      "theoretical_loss": 4.859769364950319,
+      "tokens_seen": 87818240
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.00049160481444333,
+      "loss": 3.5164,
+      "theoretical_loss": 4.859257486489293,
+      "tokens_seen": 87883776
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004915947843530592,
+      "loss": 3.5639,
+      "theoretical_loss": 4.858746096389139,
+      "tokens_seen": 87949312
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004915847542627884,
+      "loss": 3.4632,
+      "theoretical_loss": 4.8582351938205495,
+      "tokens_seen": 88014848
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004915747241725176,
+      "loss": 3.2569,
+      "theoretical_loss": 4.857724777956237,
+      "tokens_seen": 88080384
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004915646940822468,
+      "loss": 3.4019,
+      "theoretical_loss": 4.857214847970939,
+      "tokens_seen": 88145920
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004915546639919759,
+      "loss": 3.266,
+      "theoretical_loss": 4.856705403041397,
+      "tokens_seen": 88211456
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004915446339017051,
+      "loss": 3.7935,
+      "theoretical_loss": 4.8561964423463655,
+      "tokens_seen": 88276992
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004915346038114343,
+      "loss": 3.7385,
+      "theoretical_loss": 4.8556879650665925,
+      "tokens_seen": 88342528
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004915245737211635,
+      "loss": 3.3908,
+      "theoretical_loss": 4.855179970384819,
+      "tokens_seen": 88408064
+    },
+    {
+      "epoch": 0.03,
+      "objective/train/docs_used": 60543,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 3.110377788543701,
+      "objective/train/theoretical_loss": 4.8546724574857745,
+      "objective/train/tokens_used": 108933600,
+      "theoretical_loss": 4.8546724574857745,
+      "tokens_seen": 88473600
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004915145436308927,
+      "loss": 3.4761,
+      "theoretical_loss": 4.8546724574857745,
+      "tokens_seen": 88473600
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004915045135406218,
+      "loss": 3.442,
+      "theoretical_loss": 4.854165425556166,
+      "tokens_seen": 88539136
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004914944834503511,
+      "loss": 3.613,
+      "theoretical_loss": 4.853658873784675,
+      "tokens_seen": 88604672
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004914844533600802,
+      "loss": 3.7501,
+      "theoretical_loss": 4.853152801361952,
+      "tokens_seen": 88670208
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004914744232698095,
+      "loss": 3.4899,
+      "theoretical_loss": 4.852647207480604,
+      "tokens_seen": 88735744
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004914643931795386,
+      "loss": 3.6797,
+      "theoretical_loss": 4.8521420913352,
+      "tokens_seen": 88801280
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004914543630892679,
+      "loss": 3.2736,
+      "theoretical_loss": 4.851637452122251,
+      "tokens_seen": 88866816
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.000491444332998997,
+      "loss": 3.5697,
+      "theoretical_loss": 4.851133289040214,
+      "tokens_seen": 88932352
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004914343029087262,
+      "loss": 3.6628,
+      "theoretical_loss": 4.850629601289482,
+      "tokens_seen": 88997888
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004914242728184554,
+      "loss": 3.4603,
+      "theoretical_loss": 4.850126388072379,
+      "tokens_seen": 89063424
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004914142427281846,
+      "loss": 3.5286,
+      "theoretical_loss": 4.849623648593151,
+      "tokens_seen": 89128960
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004914042126379138,
+      "loss": 3.4864,
+      "theoretical_loss": 4.849121382057964,
+      "tokens_seen": 89194496
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004913941825476429,
+      "loss": 3.5269,
+      "theoretical_loss": 4.848619587674898,
+      "tokens_seen": 89260032
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004913841524573721,
+      "loss": 3.4857,
+      "theoretical_loss": 4.8481182646539365,
+      "tokens_seen": 89325568
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004913741223671013,
+      "loss": 3.6437,
+      "theoretical_loss": 4.847617412206963,
+      "tokens_seen": 89391104
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004913640922768305,
+      "loss": 3.4963,
+      "theoretical_loss": 4.847117029547758,
+      "tokens_seen": 89456640
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004913540621865597,
+      "loss": 3.5452,
+      "theoretical_loss": 4.846617115891988,
+      "tokens_seen": 89522176
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004913440320962888,
+      "loss": 3.2749,
+      "theoretical_loss": 4.8461176704572,
+      "tokens_seen": 89587712
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.000491334002006018,
+      "loss": 3.319,
+      "theoretical_loss": 4.845618692462826,
+      "tokens_seen": 89653248
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004913239719157472,
+      "loss": 3.4855,
+      "theoretical_loss": 4.845120181130158,
+      "tokens_seen": 89718784
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004913139418254765,
+      "loss": 3.7812,
+      "theoretical_loss": 4.844622135682359,
+      "tokens_seen": 89784320
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004913039117352056,
+      "loss": 3.1217,
+      "theoretical_loss": 4.84412455534445,
+      "tokens_seen": 89849856
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004912938816449349,
+      "loss": 3.3833,
+      "theoretical_loss": 4.843627439343306,
+      "tokens_seen": 89915392
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004912838515546639,
+      "loss": 3.5363,
+      "theoretical_loss": 4.843130786907647,
+      "tokens_seen": 89980928
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004912738214643932,
+      "loss": 3.4395,
+      "theoretical_loss": 4.842634597268036,
+      "tokens_seen": 90046464
+    },
+    {
+      "epoch": 0.03,
+      "objective/train/docs_used": 60961,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 3.3933932781219482,
+      "objective/train/theoretical_loss": 4.842138869656871,
+      "objective/train/tokens_used": 110572000,
+      "theoretical_loss": 4.842138869656871,
+      "tokens_seen": 90112000
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004912637913741224,
+      "loss": 3.5361,
+      "theoretical_loss": 4.842138869656871,
+      "tokens_seen": 90112000
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004912537612838516,
+      "loss": 3.1717,
+      "theoretical_loss": 4.841643603308384,
+      "tokens_seen": 90177536
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004912437311935808,
+      "loss": 3.6274,
+      "theoretical_loss": 4.841148797458627,
+      "tokens_seen": 90243072
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.00049123370110331,
+      "loss": 3.5958,
+      "theoretical_loss": 4.840654451345472,
+      "tokens_seen": 90308608
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004912236710130391,
+      "loss": 3.6702,
+      "theoretical_loss": 4.840160564208606,
+      "tokens_seen": 90374144
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004912136409227683,
+      "loss": 3.4133,
+      "theoretical_loss": 4.839667135289522,
+      "tokens_seen": 90439680
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004912036108324975,
+      "loss": 3.3868,
+      "theoretical_loss": 4.839174163831515,
+      "tokens_seen": 90505216
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004911935807422267,
+      "loss": 3.4971,
+      "theoretical_loss": 4.838681649079677,
+      "tokens_seen": 90570752
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004911835506519559,
+      "loss": 3.581,
+      "theoretical_loss": 4.838189590280891,
+      "tokens_seen": 90636288
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.000491173520561685,
+      "loss": 3.5547,
+      "theoretical_loss": 4.837697986683825,
+      "tokens_seen": 90701824
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004911634904714142,
+      "loss": 3.6512,
+      "theoretical_loss": 4.837206837538925,
+      "tokens_seen": 90767360
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004911534603811434,
+      "loss": 3.189,
+      "theoretical_loss": 4.836716142098417,
+      "tokens_seen": 90832896
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004911434302908726,
+      "loss": 3.4003,
+      "theoretical_loss": 4.836225899616288,
+      "tokens_seen": 90898432
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004911334002006019,
+      "loss": 3.5343,
+      "theoretical_loss": 4.835736109348293,
+      "tokens_seen": 90963968
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004911233701103309,
+      "loss": 3.491,
+      "theoretical_loss": 4.8352467705519455,
+      "tokens_seen": 91029504
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004911133400200602,
+      "loss": 3.543,
+      "theoretical_loss": 4.83475788248651,
+      "tokens_seen": 91095040
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004911033099297893,
+      "loss": 3.5829,
+      "theoretical_loss": 4.834269444412996,
+      "tokens_seen": 91160576
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004910932798395186,
+      "loss": 3.5038,
+      "theoretical_loss": 4.83378145559416,
+      "tokens_seen": 91226112
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004910832497492478,
+      "loss": 3.6003,
+      "theoretical_loss": 4.83329391529449,
+      "tokens_seen": 91291648
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.000491073219658977,
+      "loss": 3.5657,
+      "theoretical_loss": 4.832806822780208,
+      "tokens_seen": 91357184
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004910631895687061,
+      "loss": 3.3872,
+      "theoretical_loss": 4.832320177319261,
+      "tokens_seen": 91422720
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004910531594784353,
+      "loss": 3.4167,
+      "theoretical_loss": 4.831833978181314,
+      "tokens_seen": 91488256
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004910431293881645,
+      "loss": 3.2479,
+      "theoretical_loss": 4.831348224637752,
+      "tokens_seen": 91553792
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004910330992978937,
+      "loss": 3.5555,
+      "theoretical_loss": 4.830862915961665,
+      "tokens_seen": 91619328
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004910230692076229,
+      "loss": 3.4961,
+      "theoretical_loss": 4.830378051427852,
+      "tokens_seen": 91684864
+    },
+    {
+      "epoch": 0.03,
+      "objective/train/docs_used": 62148,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 3.0938072204589844,
+      "objective/train/theoretical_loss": 4.829893630312808,
+      "objective/train/tokens_used": 112210400,
+      "theoretical_loss": 4.829893630312808,
+      "tokens_seen": 91750400
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.000491013039117352,
+      "loss": 3.1482,
+      "theoretical_loss": 4.829893630312808,
+      "tokens_seen": 91750400
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004910030090270812,
+      "loss": 3.4885,
+      "theoretical_loss": 4.829409651894725,
+      "tokens_seen": 91815936
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004909929789368104,
+      "loss": 3.5385,
+      "theoretical_loss": 4.8289261154534815,
+      "tokens_seen": 91881472
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004909829488465397,
+      "loss": 3.2172,
+      "theoretical_loss": 4.828443020270642,
+      "tokens_seen": 91947008
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004909729187562688,
+      "loss": 3.351,
+      "theoretical_loss": 4.827960365629449,
+      "tokens_seen": 92012544
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004909628886659981,
+      "loss": 3.6262,
+      "theoretical_loss": 4.827478150814821,
+      "tokens_seen": 92078080
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004909528585757272,
+      "loss": 3.4536,
+      "theoretical_loss": 4.826996375113339,
+      "tokens_seen": 92143616
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004909428284854564,
+      "loss": 3.4569,
+      "theoretical_loss": 4.826515037813255,
+      "tokens_seen": 92209152
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004909327983951856,
+      "loss": 3.7922,
+      "theoretical_loss": 4.826034138204474,
+      "tokens_seen": 92274688
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004909227683049148,
+      "loss": 3.4368,
+      "theoretical_loss": 4.825553675578557,
+      "tokens_seen": 92340224
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.000490912738214644,
+      "loss": 3.3922,
+      "theoretical_loss": 4.825073649228713,
+      "tokens_seen": 92405760
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004909027081243731,
+      "loss": 3.5078,
+      "theoretical_loss": 4.824594058449794,
+      "tokens_seen": 92471296
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004908926780341023,
+      "loss": 3.3106,
+      "theoretical_loss": 4.824114902538292,
+      "tokens_seen": 92536832
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004908826479438315,
+      "loss": 3.136,
+      "theoretical_loss": 4.823636180792329,
+      "tokens_seen": 92602368
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004908726178535607,
+      "loss": 3.3286,
+      "theoretical_loss": 4.823157892511661,
+      "tokens_seen": 92667904
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004908625877632899,
+      "loss": 3.6266,
+      "theoretical_loss": 4.82268003699766,
+      "tokens_seen": 92733440
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.000490852557673019,
+      "loss": 3.5695,
+      "theoretical_loss": 4.8222026135533245,
+      "tokens_seen": 92798976
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004908425275827482,
+      "loss": 3.487,
+      "theoretical_loss": 4.821725621483262,
+      "tokens_seen": 92864512
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004908324974924774,
+      "loss": 3.503,
+      "theoretical_loss": 4.821249060093692,
+      "tokens_seen": 92930048
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004908224674022067,
+      "loss": 3.3758,
+      "theoretical_loss": 4.820772928692433,
+      "tokens_seen": 92995584
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004908124373119358,
+      "loss": 3.4497,
+      "theoretical_loss": 4.820297226588908,
+      "tokens_seen": 93061120
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004908024072216651,
+      "loss": 3.6942,
+      "theoretical_loss": 4.819821953094133,
+      "tokens_seen": 93126656
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004907923771313941,
+      "loss": 3.6533,
+      "theoretical_loss": 4.819347107520713,
+      "tokens_seen": 93192192
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004907823470411234,
+      "loss": 3.3973,
+      "theoretical_loss": 4.818872689182836,
+      "tokens_seen": 93257728
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004907723169508526,
+      "loss": 3.2205,
+      "theoretical_loss": 4.818398697396274,
+      "tokens_seen": 93323264
+    },
+    {
+      "epoch": 0.03,
+      "objective/train/docs_used": 62815,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 3.5069005489349365,
+      "objective/train/theoretical_loss": 4.817925131478374,
+      "objective/train/tokens_used": 113848800,
+      "theoretical_loss": 4.817925131478374,
+      "tokens_seen": 93388800
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004907622868605818,
+      "loss": 3.5578,
+      "theoretical_loss": 4.817925131478374,
+      "tokens_seen": 93388800
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.000490752256770311,
+      "loss": 3.5471,
+      "theoretical_loss": 4.817451990748047,
+      "tokens_seen": 93454336
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004907422266800401,
+      "loss": 3.6923,
+      "theoretical_loss": 4.81697927452578,
+      "tokens_seen": 93519872
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004907321965897693,
+      "loss": 3.5227,
+      "theoretical_loss": 4.816506982133613,
+      "tokens_seen": 93585408
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004907221664994985,
+      "loss": 3.2295,
+      "theoretical_loss": 4.816035112895147,
+      "tokens_seen": 93650944
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004907121364092277,
+      "loss": 3.1312,
+      "theoretical_loss": 4.815563666135532,
+      "tokens_seen": 93716480
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004907021063189569,
+      "loss": 3.6798,
+      "theoretical_loss": 4.815092641181467,
+      "tokens_seen": 93782016
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.000490692076228686,
+      "loss": 3.7669,
+      "theoretical_loss": 4.814622037361195,
+      "tokens_seen": 93847552
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004906820461384152,
+      "loss": 3.6417,
+      "theoretical_loss": 4.814151854004493,
+      "tokens_seen": 93913088
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004906720160481444,
+      "loss": 3.2937,
+      "theoretical_loss": 4.813682090442674,
+      "tokens_seen": 93978624
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004906619859578736,
+      "loss": 3.2833,
+      "theoretical_loss": 4.81321274600858,
+      "tokens_seen": 94044160
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004906519558676028,
+      "loss": 3.2788,
+      "theoretical_loss": 4.812743820036577,
+      "tokens_seen": 94109696
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004906419257773321,
+      "loss": 3.5618,
+      "theoretical_loss": 4.81227531186255,
+      "tokens_seen": 94175232
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004906318956870611,
+      "loss": 3.5329,
+      "theoretical_loss": 4.811807220823899,
+      "tokens_seen": 94240768
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004906218655967904,
+      "loss": 3.6156,
+      "theoretical_loss": 4.811339546259536,
+      "tokens_seen": 94306304
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004906118355065195,
+      "loss": 3.3777,
+      "theoretical_loss": 4.810872287509877,
+      "tokens_seen": 94371840
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004906018054162488,
+      "loss": 3.5883,
+      "theoretical_loss": 4.810405443916842,
+      "tokens_seen": 94437376
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.000490591775325978,
+      "loss": 3.6907,
+      "theoretical_loss": 4.809939014823849,
+      "tokens_seen": 94502912
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004905817452357072,
+      "loss": 3.6602,
+      "theoretical_loss": 4.809472999575805,
+      "tokens_seen": 94568448
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004905717151454363,
+      "loss": 3.6322,
+      "theoretical_loss": 4.809007397519108,
+      "tokens_seen": 94633984
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004905616850551655,
+      "loss": 3.4204,
+      "theoretical_loss": 4.80854220800164,
+      "tokens_seen": 94699520
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004905516549648947,
+      "loss": 3.5405,
+      "theoretical_loss": 4.808077430372762,
+      "tokens_seen": 94765056
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004905416248746239,
+      "loss": 3.4647,
+      "theoretical_loss": 4.807613063983311,
+      "tokens_seen": 94830592
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004905315947843531,
+      "loss": 3.7424,
+      "theoretical_loss": 4.807149108185593,
+      "tokens_seen": 94896128
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004905215646940822,
+      "loss": 3.3372,
+      "theoretical_loss": 4.806685562333383,
+      "tokens_seen": 94961664
+    },
+    {
+      "epoch": 0.03,
+      "objective/train/docs_used": 64162,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 3.0013229846954346,
+      "objective/train/theoretical_loss": 4.806222425781916,
+      "objective/train/tokens_used": 115487200,
+      "theoretical_loss": 4.806222425781916,
+      "tokens_seen": 95027200
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004905115346038114,
+      "loss": 3.4672,
+      "theoretical_loss": 4.806222425781916,
+      "tokens_seen": 95027200
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004905015045135406,
+      "loss": 3.4124,
+      "theoretical_loss": 4.805759697887886,
+      "tokens_seen": 95092736
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004904914744232698,
+      "loss": 3.5337,
+      "theoretical_loss": 4.80529737800944,
+      "tokens_seen": 95158272
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.000490481444332999,
+      "loss": 3.1987,
+      "theoretical_loss": 4.804835465506176,
+      "tokens_seen": 95223808
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004904714142427281,
+      "loss": 3.3264,
+      "theoretical_loss": 4.8043739597391335,
+      "tokens_seen": 95289344
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004904613841524574,
+      "loss": 3.421,
+      "theoretical_loss": 4.803912860070795,
+      "tokens_seen": 95354880
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004904513540621865,
+      "loss": 3.5238,
+      "theoretical_loss": 4.803452165865082,
+      "tokens_seen": 95420416
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004904413239719158,
+      "loss": 3.4263,
+      "theoretical_loss": 4.802991876487344,
+      "tokens_seen": 95485952
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004904312938816449,
+      "loss": 3.3975,
+      "theoretical_loss": 4.802531991304359,
+      "tokens_seen": 95551488
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004904212637913742,
+      "loss": 3.5391,
+      "theoretical_loss": 4.802072509684331,
+      "tokens_seen": 95617024
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004904112337011033,
+      "loss": 3.4524,
+      "theoretical_loss": 4.801613430996884,
+      "tokens_seen": 95682560
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004904012036108325,
+      "loss": 3.5944,
+      "theoretical_loss": 4.801154754613056,
+      "tokens_seen": 95748096
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004903911735205617,
+      "loss": 3.2372,
+      "theoretical_loss": 4.800696479905296,
+      "tokens_seen": 95813632
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004903811434302909,
+      "loss": 3.1815,
+      "theoretical_loss": 4.800238606247462,
+      "tokens_seen": 95879168
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004903711133400201,
+      "loss": 3.4315,
+      "theoretical_loss": 4.799781133014815,
+      "tokens_seen": 95944704
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004903610832497492,
+      "loss": 3.2851,
+      "theoretical_loss": 4.799324059584013,
+      "tokens_seen": 96010240
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004903510531594784,
+      "loss": 3.6757,
+      "theoretical_loss": 4.798867385333111,
+      "tokens_seen": 96075776
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004903410230692076,
+      "loss": 3.3841,
+      "theoretical_loss": 4.798411109641553,
+      "tokens_seen": 96141312
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004903309929789368,
+      "loss": 3.6343,
+      "theoretical_loss": 4.797955231890175,
+      "tokens_seen": 96206848
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.000490320962888666,
+      "loss": 3.3026,
+      "theoretical_loss": 4.797499751461188,
+      "tokens_seen": 96272384
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004903109327983952,
+      "loss": 3.3258,
+      "theoretical_loss": 4.797044667738188,
+      "tokens_seen": 96337920
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004903009027081243,
+      "loss": 3.4414,
+      "theoretical_loss": 4.796589980106143,
+      "tokens_seen": 96403456
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004902908726178535,
+      "loss": 3.2512,
+      "theoretical_loss": 4.796135687951393,
+      "tokens_seen": 96468992
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004902808425275828,
+      "loss": 3.4072,
+      "theoretical_loss": 4.795681790661644,
+      "tokens_seen": 96534528
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004902708124373119,
+      "loss": 3.4574,
+      "theoretical_loss": 4.795228287625964,
+      "tokens_seen": 96600064
+    },
+    {
+      "epoch": 0.03,
+      "objective/train/docs_used": 64775,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 3.715846538543701,
+      "objective/train/theoretical_loss": 4.794775178234782,
+      "objective/train/tokens_used": 117125600,
+      "theoretical_loss": 4.794775178234782,
+      "tokens_seen": 96665600
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004902607823470412,
+      "loss": 3.622,
+      "theoretical_loss": 4.794775178234782,
+      "tokens_seen": 96665600
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004902507522567703,
+      "loss": 3.3135,
+      "theoretical_loss": 4.794322461879881,
+      "tokens_seen": 96731136
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004902407221664995,
+      "loss": 3.3367,
+      "theoretical_loss": 4.793870137954395,
+      "tokens_seen": 96796672
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004902306920762287,
+      "loss": 3.3923,
+      "theoretical_loss": 4.793418205852804,
+      "tokens_seen": 96862208
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004902206619859579,
+      "loss": 3.2944,
+      "theoretical_loss": 4.7929666649709315,
+      "tokens_seen": 96927744
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004902106318956871,
+      "loss": 3.4677,
+      "theoretical_loss": 4.7925155147059435,
+      "tokens_seen": 96993280
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004902006018054163,
+      "loss": 3.5354,
+      "theoretical_loss": 4.792064754456337,
+      "tokens_seen": 97058816
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004901905717151454,
+      "loss": 3.4739,
+      "theoretical_loss": 4.791614383621943,
+      "tokens_seen": 97124352
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004901805416248746,
+      "loss": 3.4712,
+      "theoretical_loss": 4.79116440160392,
+      "tokens_seen": 97189888
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004901705115346038,
+      "loss": 3.6806,
+      "theoretical_loss": 4.790714807804751,
+      "tokens_seen": 97255424
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.000490160481444333,
+      "loss": 3.2497,
+      "theoretical_loss": 4.790265601628235,
+      "tokens_seen": 97320960
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004901504513540623,
+      "loss": 3.3692,
+      "theoretical_loss": 4.789816782479494,
+      "tokens_seen": 97386496
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004901404212637913,
+      "loss": 3.576,
+      "theoretical_loss": 4.789368349764956,
+      "tokens_seen": 97452032
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004901303911735206,
+      "loss": 3.4743,
+      "theoretical_loss": 4.788920302892365,
+      "tokens_seen": 97517568
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004901203610832497,
+      "loss": 3.1291,
+      "theoretical_loss": 4.7884726412707606,
+      "tokens_seen": 97583104
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.000490110330992979,
+      "loss": 3.5709,
+      "theoretical_loss": 4.788025364310492,
+      "tokens_seen": 97648640
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004901003009027082,
+      "loss": 3.3566,
+      "theoretical_loss": 4.787578471423203,
+      "tokens_seen": 97714176
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004900902708124374,
+      "loss": 3.6399,
+      "theoretical_loss": 4.787131962021827,
+      "tokens_seen": 97779712
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004900802407221665,
+      "loss": 3.577,
+      "theoretical_loss": 4.786685835520595,
+      "tokens_seen": 97845248
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004900702106318957,
+      "loss": 3.4237,
+      "theoretical_loss": 4.786240091335019,
+      "tokens_seen": 97910784
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004900601805416249,
+      "loss": 3.3724,
+      "theoretical_loss": 4.785794728881895,
+      "tokens_seen": 97976320
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004900501504513541,
+      "loss": 3.3342,
+      "theoretical_loss": 4.785349747579299,
+      "tokens_seen": 98041856
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004900401203610833,
+      "loss": 3.5862,
+      "theoretical_loss": 4.784905146846581,
+      "tokens_seen": 98107392
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004900300902708124,
+      "loss": 3.3465,
+      "theoretical_loss": 4.784460926104362,
+      "tokens_seen": 98172928
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004900200601805416,
+      "loss": 3.3709,
+      "theoretical_loss": 4.7840170847745345,
+      "tokens_seen": 98238464
+    },
+    {
+      "epoch": 0.03,
+      "objective/train/docs_used": 65599,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 3.469925880432129,
+      "objective/train/theoretical_loss": 4.783573622280253,
+      "objective/train/tokens_used": 118764000,
+      "theoretical_loss": 4.783573622280253,
+      "tokens_seen": 98304000
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004900100300902708,
+      "loss": 3.5123,
+      "theoretical_loss": 4.783573622280253,
+      "tokens_seen": 98304000
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.00049,
+      "loss": 3.1796,
+      "theoretical_loss": 4.7831305380459295,
+      "tokens_seen": 98369536
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004899899699097292,
+      "loss": 3.5071,
+      "theoretical_loss": 4.78268783149724,
+      "tokens_seen": 98435072
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004899799398194583,
+      "loss": 3.1618,
+      "theoretical_loss": 4.782245502061111,
+      "tokens_seen": 98500608
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004899699097291876,
+      "loss": 3.5861,
+      "theoretical_loss": 4.781803549165718,
+      "tokens_seen": 98566144
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004899598796389167,
+      "loss": 3.7046,
+      "theoretical_loss": 4.781361972240484,
+      "tokens_seen": 98631680
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.000489949849548646,
+      "loss": 3.4213,
+      "theoretical_loss": 4.780920770716076,
+      "tokens_seen": 98697216
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004899398194583751,
+      "loss": 3.5335,
+      "theoretical_loss": 4.780479944024398,
+      "tokens_seen": 98762752
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004899297893681044,
+      "loss": 3.3783,
+      "theoretical_loss": 4.7800394915985915,
+      "tokens_seen": 98828288
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004899197592778335,
+      "loss": 3.5404,
+      "theoretical_loss": 4.779599412873031,
+      "tokens_seen": 98893824
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004899097291875627,
+      "loss": 3.4117,
+      "theoretical_loss": 4.77915970728332,
+      "tokens_seen": 98959360
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004898996990972919,
+      "loss": 3.1311,
+      "theoretical_loss": 4.7787203742662845,
+      "tokens_seen": 99024896
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004898896690070211,
+      "loss": 3.3778,
+      "theoretical_loss": 4.778281413259977,
+      "tokens_seen": 99090432
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004898796389167503,
+      "loss": 3.3097,
+      "theoretical_loss": 4.777842823703665,
+      "tokens_seen": 99155968
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004898696088264794,
+      "loss": 3.4696,
+      "theoretical_loss": 4.777404605037833,
+      "tokens_seen": 99221504
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004898595787362086,
+      "loss": 3.5204,
+      "theoretical_loss": 4.7769667567041765,
+      "tokens_seen": 99287040
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004898495486459378,
+      "loss": 3.5602,
+      "theoretical_loss": 4.776529278145597,
+      "tokens_seen": 99352576
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.000489839518555667,
+      "loss": 3.4991,
+      "theoretical_loss": 4.776092168806207,
+      "tokens_seen": 99418112
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004898294884653962,
+      "loss": 3.6413,
+      "theoretical_loss": 4.775655428131313,
+      "tokens_seen": 99483648
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004898194583751254,
+      "loss": 3.6154,
+      "theoretical_loss": 4.775219055567424,
+      "tokens_seen": 99549184
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004898094282848545,
+      "loss": 3.4266,
+      "theoretical_loss": 4.7747830505622435,
+      "tokens_seen": 99614720
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004897993981945837,
+      "loss": 3.2347,
+      "theoretical_loss": 4.774347412564666,
+      "tokens_seen": 99680256
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.000489789368104313,
+      "loss": 3.5374,
+      "theoretical_loss": 4.773912141024771,
+      "tokens_seen": 99745792
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004897793380140421,
+      "loss": 3.3582,
+      "theoretical_loss": 4.773477235393826,
+      "tokens_seen": 99811328
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004897693079237714,
+      "loss": 3.4578,
+      "theoretical_loss": 4.773042695124279,
+      "tokens_seen": 99876864
+    },
+    {
+      "epoch": 0.03,
+      "objective/train/docs_used": 66410,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 4.062344551086426,
+      "objective/train/theoretical_loss": 4.772608519669756,
+      "objective/train/tokens_used": 120402400,
+      "theoretical_loss": 4.772608519669756,
+      "tokens_seen": 99942400
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004897592778335005,
+      "loss": 3.5402,
+      "theoretical_loss": 4.772608519669756,
+      "tokens_seen": 99942400
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004897492477432297,
+      "loss": 3.5992,
+      "theoretical_loss": 4.772174708485057,
+      "tokens_seen": 100007936
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004897392176529589,
+      "loss": 3.4125,
+      "theoretical_loss": 4.771741261026155,
+      "tokens_seen": 100073472
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004897291875626881,
+      "loss": 3.3972,
+      "theoretical_loss": 4.771308176750187,
+      "tokens_seen": 100139008
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004897191574724173,
+      "loss": 3.7219,
+      "theoretical_loss": 4.77087545511546,
+      "tokens_seen": 100204544
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004897091273821465,
+      "loss": 3.5976,
+      "theoretical_loss": 4.77044309558144,
+      "tokens_seen": 100270080
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004896990972918756,
+      "loss": 3.5781,
+      "theoretical_loss": 4.7700110976087515,
+      "tokens_seen": 100335616
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004896890672016048,
+      "loss": 3.4885,
+      "theoretical_loss": 4.769579460659173,
+      "tokens_seen": 100401152
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.000489679037111334,
+      "loss": 3.6787,
+      "theoretical_loss": 4.769148184195636,
+      "tokens_seen": 100466688
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004896690070210632,
+      "loss": 3.5478,
+      "theoretical_loss": 4.768717267682223,
+      "tokens_seen": 100532224
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004896589769307924,
+      "loss": 3.4238,
+      "theoretical_loss": 4.768286710584157,
+      "tokens_seen": 100597760
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004896489468405215,
+      "loss": 3.3217,
+      "theoretical_loss": 4.767856512367805,
+      "tokens_seen": 100663296
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004896389167502507,
+      "loss": 3.3783,
+      "theoretical_loss": 4.767426672500677,
+      "tokens_seen": 100728832
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004896288866599799,
+      "loss": 3.2931,
+      "theoretical_loss": 4.766997190451413,
+      "tokens_seen": 100794368
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004896188565697091,
+      "loss": 3.6434,
+      "theoretical_loss": 4.766568065689789,
+      "tokens_seen": 100859904
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004896088264794384,
+      "loss": 3.5874,
+      "theoretical_loss": 4.766139297686712,
+      "tokens_seen": 100925440
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004895987963891674,
+      "loss": 3.4533,
+      "theoretical_loss": 4.76571088591421,
+      "tokens_seen": 100990976
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004895887662988967,
+      "loss": 3.6816,
+      "theoretical_loss": 4.7652828298454395,
+      "tokens_seen": 101056512
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004895787362086259,
+      "loss": 3.3644,
+      "theoretical_loss": 4.764855128954675,
+      "tokens_seen": 101122048
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004895687061183551,
+      "loss": 3.2458,
+      "theoretical_loss": 4.764427782717309,
+      "tokens_seen": 101187584
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004895586760280843,
+      "loss": 3.3975,
+      "theoretical_loss": 4.764000790609848,
+      "tokens_seen": 101253120
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004895486459378135,
+      "loss": 3.3918,
+      "theoretical_loss": 4.763574152109907,
+      "tokens_seen": 101318656
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004895386158475426,
+      "loss": 3.4413,
+      "theoretical_loss": 4.763147866696213,
+      "tokens_seen": 101384192
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004895285857572718,
+      "loss": 3.154,
+      "theoretical_loss": 4.762721933848593,
+      "tokens_seen": 101449728
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.000489518555667001,
+      "loss": 3.367,
+      "theoretical_loss": 4.762296353047979,
+      "tokens_seen": 101515264
+    },
+    {
+      "epoch": 0.03,
+      "objective/train/docs_used": 66745,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 3.258596897125244,
+      "objective/train/theoretical_loss": 4.7618711237764035,
+      "objective/train/tokens_used": 122040800,
+      "theoretical_loss": 4.7618711237764035,
+      "tokens_seen": 101580800
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004895085255767302,
+      "loss": 3.3147,
+      "theoretical_loss": 4.7618711237764035,
+      "tokens_seen": 101580800
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004894984954864594,
+      "loss": 3.2477,
+      "theoretical_loss": 4.761446245516988,
+      "tokens_seen": 101646336
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004894884653961885,
+      "loss": 3.5315,
+      "theoretical_loss": 4.7610217177539536,
+      "tokens_seen": 101711872
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004894784353059178,
+      "loss": 3.3191,
+      "theoretical_loss": 4.760597539972606,
+      "tokens_seen": 101777408
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004894684052156469,
+      "loss": 3.0442,
+      "theoretical_loss": 4.760173711659341,
+      "tokens_seen": 101842944
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004894583751253762,
+      "loss": 3.4036,
+      "theoretical_loss": 4.759750232301635,
+      "tokens_seen": 101908480
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004894483450351053,
+      "loss": 3.3851,
+      "theoretical_loss": 4.759327101388048,
+      "tokens_seen": 101974016
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004894383149448346,
+      "loss": 3.5575,
+      "theoretical_loss": 4.7589043184082165,
+      "tokens_seen": 102039552
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004894282848545637,
+      "loss": 3.3678,
+      "theoretical_loss": 4.75848188285285,
+      "tokens_seen": 102105088
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004894182547642929,
+      "loss": 3.5807,
+      "theoretical_loss": 4.7580597942137315,
+      "tokens_seen": 102170624
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004894082246740221,
+      "loss": 3.4755,
+      "theoretical_loss": 4.757638051983713,
+      "tokens_seen": 102236160
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004893981945837513,
+      "loss": 3.308,
+      "theoretical_loss": 4.757216655656713,
+      "tokens_seen": 102301696
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004893881644934805,
+      "loss": 3.4312,
+      "theoretical_loss": 4.756795604727708,
+      "tokens_seen": 102367232
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004893781344032096,
+      "loss": 3.4421,
+      "theoretical_loss": 4.7563748986927425,
+      "tokens_seen": 102432768
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004893681043129388,
+      "loss": 3.3071,
+      "theoretical_loss": 4.755954537048913,
+      "tokens_seen": 102498304
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.000489358074222668,
+      "loss": 3.7184,
+      "theoretical_loss": 4.755534519294368,
+      "tokens_seen": 102563840
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004893480441323972,
+      "loss": 3.6644,
+      "theoretical_loss": 4.7551148449283165,
+      "tokens_seen": 102629376
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004893380140421264,
+      "loss": 3.3233,
+      "theoretical_loss": 4.754695513451004,
+      "tokens_seen": 102694912
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004893279839518556,
+      "loss": 3.6128,
+      "theoretical_loss": 4.754276524363733,
+      "tokens_seen": 102760448
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004893179538615847,
+      "loss": 3.4243,
+      "theoretical_loss": 4.75385787716884,
+      "tokens_seen": 102825984
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004893079237713139,
+      "loss": 3.6836,
+      "theoretical_loss": 4.7534395713697055,
+      "tokens_seen": 102891520
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004892978936810432,
+      "loss": 3.6634,
+      "theoretical_loss": 4.753021606470748,
+      "tokens_seen": 102957056
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004892878635907723,
+      "loss": 3.3886,
+      "theoretical_loss": 4.752603981977417,
+      "tokens_seen": 103022592
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004892778335005016,
+      "loss": 3.4605,
+      "theoretical_loss": 4.752186697396198,
+      "tokens_seen": 103088128
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004892678034102307,
+      "loss": 3.3429,
+      "theoretical_loss": 4.7517697522345985,
+      "tokens_seen": 103153664
+    },
+    {
+      "epoch": 0.03,
+      "objective/train/docs_used": 68050,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 3.391291379928589,
+      "objective/train/theoretical_loss": 4.7513531460011595,
+      "objective/train/tokens_used": 123679200,
+      "theoretical_loss": 4.7513531460011595,
+      "tokens_seen": 103219200
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004892577733199599,
+      "loss": 3.488,
+      "theoretical_loss": 4.7513531460011595,
+      "tokens_seen": 103219200
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004892477432296891,
+      "loss": 3.4412,
+      "theoretical_loss": 4.750936878205438,
+      "tokens_seen": 103284736
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004892377131394183,
+      "loss": 3.1551,
+      "theoretical_loss": 4.750520948358019,
+      "tokens_seen": 103350272
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004892276830491475,
+      "loss": 3.3288,
+      "theoretical_loss": 4.750105355970497,
+      "tokens_seen": 103415808
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004892176529588767,
+      "loss": 3.3249,
+      "theoretical_loss": 4.749690100555487,
+      "tokens_seen": 103481344
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004892076228686058,
+      "loss": 3.4856,
+      "theoretical_loss": 4.749275181626612,
+      "tokens_seen": 103546880
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.000489197592778335,
+      "loss": 3.447,
+      "theoretical_loss": 4.748860598698509,
+      "tokens_seen": 103612416
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004891875626880642,
+      "loss": 3.3712,
+      "theoretical_loss": 4.748446351286817,
+      "tokens_seen": 103677952
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004891775325977934,
+      "loss": 3.3355,
+      "theoretical_loss": 4.748032438908181,
+      "tokens_seen": 103743488
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004891675025075226,
+      "loss": 3.6823,
+      "theoretical_loss": 4.747618861080247,
+      "tokens_seen": 103809024
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004891574724172517,
+      "loss": 3.5451,
+      "theoretical_loss": 4.74720561732166,
+      "tokens_seen": 103874560
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004891474423269809,
+      "loss": 3.3377,
+      "theoretical_loss": 4.746792707152057,
+      "tokens_seen": 103940096
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004891374122367101,
+      "loss": 3.2533,
+      "theoretical_loss": 4.746380130092074,
+      "tokens_seen": 104005632
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004891273821464393,
+      "loss": 3.5114,
+      "theoretical_loss": 4.745967885663333,
+      "tokens_seen": 104071168
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004891173520561686,
+      "loss": 3.5891,
+      "theoretical_loss": 4.745555973388447,
+      "tokens_seen": 104136704
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004891073219658976,
+      "loss": 3.5146,
+      "theoretical_loss": 4.745144392791008,
+      "tokens_seen": 104202240
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004890972918756269,
+      "loss": 3.477,
+      "theoretical_loss": 4.744733143395598,
+      "tokens_seen": 104267776
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004890872617853561,
+      "loss": 3.5138,
+      "theoretical_loss": 4.744322224727773,
+      "tokens_seen": 104333312
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004890772316950853,
+      "loss": 3.4684,
+      "theoretical_loss": 4.743911636314069,
+      "tokens_seen": 104398848
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004890672016048145,
+      "loss": 3.3884,
+      "theoretical_loss": 4.743501377681995,
+      "tokens_seen": 104464384
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004890571715145437,
+      "loss": 3.2946,
+      "theoretical_loss": 4.743091448360034,
+      "tokens_seen": 104529920
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004890471414242728,
+      "loss": 3.362,
+      "theoretical_loss": 4.742681847877636,
+      "tokens_seen": 104595456
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.000489037111334002,
+      "loss": 3.1456,
+      "theoretical_loss": 4.742272575765218,
+      "tokens_seen": 104660992
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004890270812437312,
+      "loss": 3.6056,
+      "theoretical_loss": 4.741863631554163,
+      "tokens_seen": 104726528
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004890170511534604,
+      "loss": 3.5804,
+      "theoretical_loss": 4.741455014776813,
+      "tokens_seen": 104792064
+    },
+    {
+      "epoch": 0.03,
+      "objective/train/docs_used": 68417,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 3.796355724334717,
+      "objective/train/theoretical_loss": 4.741046724966468,
+      "objective/train/tokens_used": 125317600,
+      "theoretical_loss": 4.741046724966468,
+      "tokens_seen": 104857600
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004890070210631896,
+      "loss": 3.5708,
+      "theoretical_loss": 4.741046724966468,
+      "tokens_seen": 104857600
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004889969909729187,
+      "loss": 3.1131,
+      "theoretical_loss": 4.74063876165739,
+      "tokens_seen": 104923136
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004889869608826479,
+      "loss": 3.4213,
+      "theoretical_loss": 4.740231124384789,
+      "tokens_seen": 104988672
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004889769307923771,
+      "loss": 3.3977,
+      "theoretical_loss": 4.739823812684829,
+      "tokens_seen": 105054208
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004889669007021063,
+      "loss": 3.345,
+      "theoretical_loss": 4.739416826094621,
+      "tokens_seen": 105119744
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004889568706118355,
+      "loss": 3.4305,
+      "theoretical_loss": 4.739010164152224,
+      "tokens_seen": 105185280
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004889468405215647,
+      "loss": 3.393,
+      "theoretical_loss": 4.7386038263966395,
+      "tokens_seen": 105250816
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004889368104312939,
+      "loss": 3.4701,
+      "theoretical_loss": 4.73819781236781,
+      "tokens_seen": 105316352
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.000488926780341023,
+      "loss": 3.2626,
+      "theoretical_loss": 4.737792121606619,
+      "tokens_seen": 105381888
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004889167502507523,
+      "loss": 3.364,
+      "theoretical_loss": 4.737386753654883,
+      "tokens_seen": 105447424
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004889067201604815,
+      "loss": 3.4215,
+      "theoretical_loss": 4.736981708055353,
+      "tokens_seen": 105512960
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004888966900702107,
+      "loss": 3.2655,
+      "theoretical_loss": 4.7365769843517125,
+      "tokens_seen": 105578496
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004888866599799398,
+      "loss": 3.2771,
+      "theoretical_loss": 4.736172582088571,
+      "tokens_seen": 105644032
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.000488876629889669,
+      "loss": 3.3345,
+      "theoretical_loss": 4.735768500811469,
+      "tokens_seen": 105709568
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004888665997993982,
+      "loss": 3.4764,
+      "theoretical_loss": 4.735364740066865,
+      "tokens_seen": 105775104
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004888565697091274,
+      "loss": 3.7094,
+      "theoretical_loss": 4.734961299402143,
+      "tokens_seen": 105840640
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004888465396188566,
+      "loss": 3.4389,
+      "theoretical_loss": 4.734558178365605,
+      "tokens_seen": 105906176
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004888365095285858,
+      "loss": 3.5066,
+      "theoretical_loss": 4.734155376506466,
+      "tokens_seen": 105971712
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004888264794383149,
+      "loss": 3.2004,
+      "theoretical_loss": 4.733752893374863,
+      "tokens_seen": 106037248
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004888164493480441,
+      "loss": 3.9504,
+      "theoretical_loss": 4.733350728521835,
+      "tokens_seen": 106102784
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004888064192577733,
+      "loss": 3.3078,
+      "theoretical_loss": 4.732948881499336,
+      "tokens_seen": 106168320
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004887963891675025,
+      "loss": 3.258,
+      "theoretical_loss": 4.7325473518602275,
+      "tokens_seen": 106233856
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004887863590772317,
+      "loss": 3.4263,
+      "theoretical_loss": 4.732146139158271,
+      "tokens_seen": 106299392
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.000488776328986961,
+      "loss": 3.2168,
+      "theoretical_loss": 4.731745242948134,
+      "tokens_seen": 106364928
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.00048876629889669,
+      "loss": 3.7064,
+      "theoretical_loss": 4.731344662785381,
+      "tokens_seen": 106430464
+    },
+    {
+      "epoch": 0.03,
+      "objective/train/docs_used": 69666,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 3.1660373210906982,
+      "objective/train/theoretical_loss": 4.730944398226477,
+      "objective/train/tokens_used": 126956000,
+      "theoretical_loss": 4.730944398226477,
+      "tokens_seen": 106496000
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004887562688064193,
+      "loss": 3.4624,
+      "theoretical_loss": 4.730944398226477,
+      "tokens_seen": 106496000
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004887462387161484,
+      "loss": 3.2648,
+      "theoretical_loss": 4.730544448828777,
+      "tokens_seen": 106561536
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004887362086258777,
+      "loss": 3.3377,
+      "theoretical_loss": 4.730144814150533,
+      "tokens_seen": 106627072
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004887261785356069,
+      "loss": 3.182,
+      "theoretical_loss": 4.729745493750885,
+      "tokens_seen": 106692608
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.000488716148445336,
+      "loss": 3.5011,
+      "theoretical_loss": 4.729346487189863,
+      "tokens_seen": 106758144
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004887061183550652,
+      "loss": 3.4765,
+      "theoretical_loss": 4.728947794028379,
+      "tokens_seen": 106823680
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004886960882647944,
+      "loss": 3.5698,
+      "theoretical_loss": 4.728549413828231,
+      "tokens_seen": 106889216
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004886860581745236,
+      "loss": 3.5244,
+      "theoretical_loss": 4.728151346152098,
+      "tokens_seen": 106954752
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004886760280842528,
+      "loss": 3.2724,
+      "theoretical_loss": 4.727753590563535,
+      "tokens_seen": 107020288
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004886659979939819,
+      "loss": 3.2698,
+      "theoretical_loss": 4.727356146626976,
+      "tokens_seen": 107085824
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004886559679037111,
+      "loss": 3.3792,
+      "theoretical_loss": 4.726959013907727,
+      "tokens_seen": 107151360
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004886459378134403,
+      "loss": 3.0486,
+      "theoretical_loss": 4.7265621919719685,
+      "tokens_seen": 107216896
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004886359077231695,
+      "loss": 3.2127,
+      "theoretical_loss": 4.726165680386746,
+      "tokens_seen": 107282432
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004886258776328988,
+      "loss": 3.4488,
+      "theoretical_loss": 4.725769478719978,
+      "tokens_seen": 107347968
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004886158475426278,
+      "loss": 3.6296,
+      "theoretical_loss": 4.725373586540442,
+      "tokens_seen": 107413504
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004886058174523571,
+      "loss": 3.1497,
+      "theoretical_loss": 4.724978003417782,
+      "tokens_seen": 107479040
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004885957873620863,
+      "loss": 3.0499,
+      "theoretical_loss": 4.724582728922501,
+      "tokens_seen": 107544576
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004885857572718155,
+      "loss": 3.3971,
+      "theoretical_loss": 4.724187762625959,
+      "tokens_seen": 107610112
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004885757271815447,
+      "loss": 3.4614,
+      "theoretical_loss": 4.723793104100376,
+      "tokens_seen": 107675648
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004885656970912739,
+      "loss": 3.2867,
+      "theoretical_loss": 4.72339875291882,
+      "tokens_seen": 107741184
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.000488555667001003,
+      "loss": 3.5243,
+      "theoretical_loss": 4.723004708655214,
+      "tokens_seen": 107806720
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004885456369107322,
+      "loss": 3.471,
+      "theoretical_loss": 4.722610970884331,
+      "tokens_seen": 107872256
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004885356068204614,
+      "loss": 3.4896,
+      "theoretical_loss": 4.722217539181788,
+      "tokens_seen": 107937792
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004885255767301906,
+      "loss": 3.5163,
+      "theoretical_loss": 4.7218244131240485,
+      "tokens_seen": 108003328
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004885155466399198,
+      "loss": 3.1836,
+      "theoretical_loss": 4.721431592288419,
+      "tokens_seen": 108068864
+    },
+    {
+      "epoch": 0.03,
+      "objective/train/docs_used": 70233,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 3.18623423576355,
+      "objective/train/theoretical_loss": 4.721039076253046,
+      "objective/train/tokens_used": 128594400,
+      "theoretical_loss": 4.721039076253046,
+      "tokens_seen": 108134400
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.000488505516549649,
+      "loss": 3.2707,
+      "theoretical_loss": 4.721039076253046,
+      "tokens_seen": 108134400
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004884954864593781,
+      "loss": 3.3744,
+      "theoretical_loss": 4.720646864596914,
+      "tokens_seen": 108199936
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004884854563691073,
+      "loss": 3.2403,
+      "theoretical_loss": 4.720254956899844,
+      "tokens_seen": 108265472
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004884754262788365,
+      "loss": 3.4192,
+      "theoretical_loss": 4.719863352742491,
+      "tokens_seen": 108331008
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004884653961885657,
+      "loss": 3.2221,
+      "theoretical_loss": 4.719472051706341,
+      "tokens_seen": 108396544
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004884553660982949,
+      "loss": 3.4325,
+      "theoretical_loss": 4.71908105337371,
+      "tokens_seen": 108462080
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004884453360080241,
+      "loss": 3.3533,
+      "theoretical_loss": 4.718690357327743,
+      "tokens_seen": 108527616
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004884353059177532,
+      "loss": 3.5112,
+      "theoretical_loss": 4.718299963152407,
+      "tokens_seen": 108593152
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004884252758274825,
+      "loss": 3.2922,
+      "theoretical_loss": 4.7179098704324955,
+      "tokens_seen": 108658688
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004884152457372117,
+      "loss": 3.4903,
+      "theoretical_loss": 4.717520078753621,
+      "tokens_seen": 108724224
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004884052156469409,
+      "loss": 3.5257,
+      "theoretical_loss": 4.717130587702217,
+      "tokens_seen": 108789760
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.00048839518555667,
+      "loss": 3.371,
+      "theoretical_loss": 4.716741396865531,
+      "tokens_seen": 108855296
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004883851554663992,
+      "loss": 3.388,
+      "theoretical_loss": 4.716352505831628,
+      "tokens_seen": 108920832
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004883751253761284,
+      "loss": 3.5684,
+      "theoretical_loss": 4.715963914189384,
+      "tokens_seen": 108986368
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004883650952858576,
+      "loss": 3.3996,
+      "theoretical_loss": 4.715575621528487,
+      "tokens_seen": 109051904
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004883550651955868,
+      "loss": 3.1784,
+      "theoretical_loss": 4.71518762743943,
+      "tokens_seen": 109117440
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.000488345035105316,
+      "loss": 3.2919,
+      "theoretical_loss": 4.714799931513517,
+      "tokens_seen": 109182976
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004883350050150451,
+      "loss": 3.1558,
+      "theoretical_loss": 4.714412533342855,
+      "tokens_seen": 109248512
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004883249749247743,
+      "loss": 3.5333,
+      "theoretical_loss": 4.7140254325203506,
+      "tokens_seen": 109314048
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004883149448345035,
+      "loss": 3.6226,
+      "theoretical_loss": 4.713638628639713,
+      "tokens_seen": 109379584
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004883049147442327,
+      "loss": 3.2969,
+      "theoretical_loss": 4.713252121295449,
+      "tokens_seen": 109445120
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004882948846539619,
+      "loss": 3.6223,
+      "theoretical_loss": 4.712865910082863,
+      "tokens_seen": 109510656
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004882848545636911,
+      "loss": 3.4854,
+      "theoretical_loss": 4.712479994598052,
+      "tokens_seen": 109576192
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004882748244734203,
+      "loss": 3.4307,
+      "theoretical_loss": 4.7120943744379025,
+      "tokens_seen": 109641728
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.00048826479438314946,
+      "loss": 3.3526,
+      "theoretical_loss": 4.7117090492000955,
+      "tokens_seen": 109707264
+    },
+    {
+      "epoch": 0.03,
+      "objective/train/docs_used": 70991,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 3.7265214920043945,
+      "objective/train/theoretical_loss": 4.711324018483099,
+      "objective/train/tokens_used": 130232800,
+      "theoretical_loss": 4.711324018483099,
+      "tokens_seen": 109772800
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.00048825476429287864,
+      "loss": 3.4221,
+      "theoretical_loss": 4.711324018483099,
+      "tokens_seen": 109772800
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004882447342026078,
+      "loss": 3.156,
+      "theoretical_loss": 4.710939281886164,
+      "tokens_seen": 109838336
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.000488234704112337,
+      "loss": 3.5109,
+      "theoretical_loss": 4.71055483900933,
+      "tokens_seen": 109903872
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.00048822467402206624,
+      "loss": 3.6234,
+      "theoretical_loss": 4.710170689453415,
+      "tokens_seen": 109969408
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.00048821464393179536,
+      "loss": 3.3112,
+      "theoretical_loss": 4.709786832820017,
+      "tokens_seen": 110034944
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004882046138415246,
+      "loss": 3.3367,
+      "theoretical_loss": 4.7094032687115135,
+      "tokens_seen": 110100480
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004881945837512537,
+      "loss": 3.3711,
+      "theoretical_loss": 4.709019996731058,
+      "tokens_seen": 110166016
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.00048818455366098296,
+      "loss": 3.1866,
+      "theoretical_loss": 4.708637016482579,
+      "tokens_seen": 110231552
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.00048817452357071214,
+      "loss": 3.3009,
+      "theoretical_loss": 4.708254327570774,
+      "tokens_seen": 110297088
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004881644934804413,
+      "loss": 3.6798,
+      "theoretical_loss": 4.7078719296011124,
+      "tokens_seen": 110362624
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004881544633901705,
+      "loss": 3.5188,
+      "theoretical_loss": 4.7074898221798325,
+      "tokens_seen": 110428160
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.00048814443329989974,
+      "loss": 3.1,
+      "theoretical_loss": 4.707108004913936,
+      "tokens_seen": 110493696
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.00048813440320962887,
+      "loss": 3.5579,
+      "theoretical_loss": 4.706726477411193,
+      "tokens_seen": 110559232
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004881243731193581,
+      "loss": 3.3834,
+      "theoretical_loss": 4.706345239280133,
+      "tokens_seen": 110624768
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.00048811434302908723,
+      "loss": 3.3456,
+      "theoretical_loss": 4.705964290130046,
+      "tokens_seen": 110690304
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.00048810431293881646,
+      "loss": 3.3642,
+      "theoretical_loss": 4.705583629570983,
+      "tokens_seen": 110755840
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.00048809428284854564,
+      "loss": 3.2147,
+      "theoretical_loss": 4.705203257213746,
+      "tokens_seen": 110821376
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004880842527582748,
+      "loss": 3.4359,
+      "theoretical_loss": 4.7048231726699,
+      "tokens_seen": 110886912
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.000488074222668004,
+      "loss": 3.163,
+      "theoretical_loss": 4.7044433755517545,
+      "tokens_seen": 110952448
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004880641925777332,
+      "loss": 3.2166,
+      "theoretical_loss": 4.704063865472374,
+      "tokens_seen": 111017984
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.00048805416248746237,
+      "loss": 3.2815,
+      "theoretical_loss": 4.703684642045571,
+      "tokens_seen": 111083520
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004880441323971916,
+      "loss": 3.3855,
+      "theoretical_loss": 4.703305704885906,
+      "tokens_seen": 111149056
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.00048803410230692073,
+      "loss": 3.476,
+      "theoretical_loss": 4.7029270536086845,
+      "tokens_seen": 111214592
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.00048802407221664997,
+      "loss": 3.1811,
+      "theoretical_loss": 4.702548687829953,
+      "tokens_seen": 111280128
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.00048801404212637915,
+      "loss": 3.212,
+      "theoretical_loss": 4.702170607166503,
+      "tokens_seen": 111345664
+    },
+    {
+      "epoch": 0.03,
+      "objective/train/docs_used": 71566,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 3.1423094272613525,
+      "objective/train/theoretical_loss": 4.701792811235861,
+      "objective/train/tokens_used": 131871200,
+      "theoretical_loss": 4.701792811235861,
+      "tokens_seen": 111411200
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.00048800401203610833,
+      "loss": 3.3106,
+      "theoretical_loss": 4.701792811235861,
+      "tokens_seen": 111411200
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004879939819458375,
+      "loss": 3.3804,
+      "theoretical_loss": 4.701415299656295,
+      "tokens_seen": 111476736
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004879839518555667,
+      "loss": 3.3137,
+      "theoretical_loss": 4.701038072046808,
+      "tokens_seen": 111542272
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004879739217652959,
+      "loss": 3.4184,
+      "theoretical_loss": 4.7006611280271375,
+      "tokens_seen": 111607808
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004879638916750251,
+      "loss": 3.5134,
+      "theoretical_loss": 4.700284467217749,
+      "tokens_seen": 111673344
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004879538615847543,
+      "loss": 3.515,
+      "theoretical_loss": 4.699908089239845,
+      "tokens_seen": 111738880
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.00048794383149448347,
+      "loss": 3.4391,
+      "theoretical_loss": 4.6995319937153495,
+      "tokens_seen": 111804416
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.00048793380140421265,
+      "loss": 3.4928,
+      "theoretical_loss": 4.699156180266918,
+      "tokens_seen": 111869952
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.00048792377131394183,
+      "loss": 3.4204,
+      "theoretical_loss": 4.69878064851793,
+      "tokens_seen": 111935488
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.00048791374122367107,
+      "loss": 3.2864,
+      "theoretical_loss": 4.698405398092486,
+      "tokens_seen": 112001024
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004879037111334002,
+      "loss": 3.52,
+      "theoretical_loss": 4.698030428615411,
+      "tokens_seen": 112066560
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.00048789368104312943,
+      "loss": 3.2593,
+      "theoretical_loss": 4.6976557397122445,
+      "tokens_seen": 112132096
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.00048788365095285856,
+      "loss": 3.3313,
+      "theoretical_loss": 4.697281331009249,
+      "tokens_seen": 112197632
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004878736208625878,
+      "loss": 3.1676,
+      "theoretical_loss": 4.6969072021333975,
+      "tokens_seen": 112263168
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.00048786359077231697,
+      "loss": 3.4483,
+      "theoretical_loss": 4.696533352712381,
+      "tokens_seen": 112328704
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.00048785356068204615,
+      "loss": 3.3785,
+      "theoretical_loss": 4.696159782374602,
+      "tokens_seen": 112394240
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.00048784353059177533,
+      "loss": 3.0219,
+      "theoretical_loss": 4.695786490749171,
+      "tokens_seen": 112459776
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.00048783350050150457,
+      "loss": 3.3436,
+      "theoretical_loss": 4.6954134774659115,
+      "tokens_seen": 112525312
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004878234704112337,
+      "loss": 3.3403,
+      "theoretical_loss": 4.695040742155349,
+      "tokens_seen": 112590848
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.00048781344032096293,
+      "loss": 3.4199,
+      "theoretical_loss": 4.694668284448717,
+      "tokens_seen": 112656384
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.00048780341023069206,
+      "loss": 3.0947,
+      "theoretical_loss": 4.694296103977953,
+      "tokens_seen": 112721920
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004877933801404213,
+      "loss": 2.9975,
+      "theoretical_loss": 4.693924200375694,
+      "tokens_seen": 112787456
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004877833500501505,
+      "loss": 3.2263,
+      "theoretical_loss": 4.693552573275277,
+      "tokens_seen": 112852992
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.00048777331995987966,
+      "loss": 3.3911,
+      "theoretical_loss": 4.69318122231074,
+      "tokens_seen": 112918528
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.00048776328986960884,
+      "loss": 3.3579,
+      "theoretical_loss": 4.692810147116814,
+      "tokens_seen": 112984064
+    },
+    {
+      "epoch": 0.03,
+      "objective/train/docs_used": 72808,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 3.543510913848877,
+      "objective/train/theoretical_loss": 4.692439347328926,
+      "objective/train/tokens_used": 133509600,
+      "theoretical_loss": 4.692439347328926,
+      "tokens_seen": 113049600
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.000487753259779338,
+      "loss": 3.5704,
+      "theoretical_loss": 4.692439347328926,
+      "tokens_seen": 113049600
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004877432296890672,
+      "loss": 3.2623,
+      "theoretical_loss": 4.692068822583197,
+      "tokens_seen": 113115136
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.00048773319959879644,
+      "loss": 3.7086,
+      "theoretical_loss": 4.6916985725164375,
+      "tokens_seen": 113180672
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.00048772316950852556,
+      "loss": 3.2168,
+      "theoretical_loss": 4.691328596766148,
+      "tokens_seen": 113246208
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004877131394182548,
+      "loss": 3.5721,
+      "theoretical_loss": 4.690958894970518,
+      "tokens_seen": 113311744
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004877031093279839,
+      "loss": 3.5405,
+      "theoretical_loss": 4.690589466768422,
+      "tokens_seen": 113377280
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.00048769307923771316,
+      "loss": 3.4867,
+      "theoretical_loss": 4.69022031179942,
+      "tokens_seen": 113442816
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.00048768304914744234,
+      "loss": 3.3823,
+      "theoretical_loss": 4.689851429703753,
+      "tokens_seen": 113508352
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004876730190571715,
+      "loss": 3.242,
+      "theoretical_loss": 4.689482820122342,
+      "tokens_seen": 113573888
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004876629889669007,
+      "loss": 3.6015,
+      "theoretical_loss": 4.689114482696793,
+      "tokens_seen": 113639424
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.00048765295887662994,
+      "loss": 3.4168,
+      "theoretical_loss": 4.6887464170693836,
+      "tokens_seen": 113704960
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.00048764292878635907,
+      "loss": 3.4562,
+      "theoretical_loss": 4.6883786228830715,
+      "tokens_seen": 113770496
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004876328986960883,
+      "loss": 3.1948,
+      "theoretical_loss": 4.688011099781486,
+      "tokens_seen": 113836032
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.00048762286860581743,
+      "loss": 3.2999,
+      "theoretical_loss": 4.687643847408929,
+      "tokens_seen": 113901568
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.00048761283851554666,
+      "loss": 3.342,
+      "theoretical_loss": 4.687276865410377,
+      "tokens_seen": 113967104
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.00048760280842527584,
+      "loss": 3.4757,
+      "theoretical_loss": 4.686910153431472,
+      "tokens_seen": 114032640
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.000487592778335005,
+      "loss": 3.5307,
+      "theoretical_loss": 4.686543711118526,
+      "tokens_seen": 114098176
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004875827482447342,
+      "loss": 3.2597,
+      "theoretical_loss": 4.686177538118516,
+      "tokens_seen": 114163712
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004875727181544634,
+      "loss": 3.4569,
+      "theoretical_loss": 4.685811634079084,
+      "tokens_seen": 114229248
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.00048756268806419257,
+      "loss": 3.3531,
+      "theoretical_loss": 4.685445998648532,
+      "tokens_seen": 114294784
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004875526579739218,
+      "loss": 3.4034,
+      "theoretical_loss": 4.685080631475831,
+      "tokens_seen": 114360320
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.00048754262788365093,
+      "loss": 3.0558,
+      "theoretical_loss": 4.6847155322106016,
+      "tokens_seen": 114425856
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.00048753259779338017,
+      "loss": 3.4359,
+      "theoretical_loss": 4.6843507005031295,
+      "tokens_seen": 114491392
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.00048752256770310935,
+      "loss": 3.4293,
+      "theoretical_loss": 4.683986136004355,
+      "tokens_seen": 114556928
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.00048751253761283853,
+      "loss": 3.1354,
+      "theoretical_loss": 4.6836218383658705,
+      "tokens_seen": 114622464
+    },
+    {
+      "epoch": 0.03,
+      "objective/train/docs_used": 73349,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 2.8527109622955322,
+      "objective/train/theoretical_loss": 4.683257807239925,
+      "objective/train/tokens_used": 135148000,
+      "theoretical_loss": 4.683257807239925,
+      "tokens_seen": 114688000
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004875025075225677,
+      "loss": 3.1073,
+      "theoretical_loss": 4.683257807239925,
+      "tokens_seen": 114688000
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004874924774322969,
+      "loss": 3.096,
+      "theoretical_loss": 4.682894042279417,
+      "tokens_seen": 114753536
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.00048748244734202607,
+      "loss": 3.2853,
+      "theoretical_loss": 4.682530543137897,
+      "tokens_seen": 114819072
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004874724172517553,
+      "loss": 3.6298,
+      "theoretical_loss": 4.68216730946956,
+      "tokens_seen": 114884608
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.00048746238716148443,
+      "loss": 3.305,
+      "theoretical_loss": 4.6818043409292525,
+      "tokens_seen": 114950144
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.00048745235707121367,
+      "loss": 3.4946,
+      "theoretical_loss": 4.681441637172463,
+      "tokens_seen": 115015680
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004874423269809428,
+      "loss": 3.3972,
+      "theoretical_loss": 4.681079197855325,
+      "tokens_seen": 115081216
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.00048743229689067203,
+      "loss": 3.1421,
+      "theoretical_loss": 4.680717022634612,
+      "tokens_seen": 115146752
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004874222668004012,
+      "loss": 3.6661,
+      "theoretical_loss": 4.680355111167742,
+      "tokens_seen": 115212288
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004874122367101304,
+      "loss": 3.2117,
+      "theoretical_loss": 4.679993463112769,
+      "tokens_seen": 115277824
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004874022066198596,
+      "loss": 3.4083,
+      "theoretical_loss": 4.679632078128385,
+      "tokens_seen": 115343360
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.00048739217652958876,
+      "loss": 3.2653,
+      "theoretical_loss": 4.679270955873918,
+      "tokens_seen": 115408896
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.00048738214643931794,
+      "loss": 3.4833,
+      "theoretical_loss": 4.678910096009329,
+      "tokens_seen": 115474432
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048737211634904717,
+      "loss": 3.3117,
+      "theoretical_loss": 4.678549498195213,
+      "tokens_seen": 115539968
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004873620862587763,
+      "loss": 3.3255,
+      "theoretical_loss": 4.678189162092798,
+      "tokens_seen": 115605504
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048735205616850553,
+      "loss": 3.2552,
+      "theoretical_loss": 4.677829087363937,
+      "tokens_seen": 115671040
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004873420260782347,
+      "loss": 3.2912,
+      "theoretical_loss": 4.677469273671118,
+      "tokens_seen": 115736576
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004873319959879639,
+      "loss": 3.1695,
+      "theoretical_loss": 4.677109720677448,
+      "tokens_seen": 115802112
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004873219658976931,
+      "loss": 3.2226,
+      "theoretical_loss": 4.676750428046665,
+      "tokens_seen": 115867648
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048731193580742226,
+      "loss": 3.2803,
+      "theoretical_loss": 4.676391395443128,
+      "tokens_seen": 115933184
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048730190571715144,
+      "loss": 3.5568,
+      "theoretical_loss": 4.6760326225318165,
+      "tokens_seen": 115998720
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004872918756268807,
+      "loss": 3.339,
+      "theoretical_loss": 4.675674108978335,
+      "tokens_seen": 116064256
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004872818455366098,
+      "loss": 3.1442,
+      "theoretical_loss": 4.675315854448903,
+      "tokens_seen": 116129792
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048727181544633904,
+      "loss": 3.1936,
+      "theoretical_loss": 4.674957858610361,
+      "tokens_seen": 116195328
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048726178535606816,
+      "loss": 3.2333,
+      "theoretical_loss": 4.674600121130163,
+      "tokens_seen": 116260864
+    },
+    {
+      "epoch": 0.04,
+      "objective/train/docs_used": 74658,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 2.9736719131469727,
+      "objective/train/theoretical_loss": 4.674242641676377,
+      "objective/train/tokens_used": 136786400,
+      "theoretical_loss": 4.674242641676377,
+      "tokens_seen": 116326400
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004872517552657974,
+      "loss": 3.3693,
+      "theoretical_loss": 4.674242641676377,
+      "tokens_seen": 116326400
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004872417251755266,
+      "loss": 3.3527,
+      "theoretical_loss": 4.673885419917689,
+      "tokens_seen": 116391936
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048723169508525576,
+      "loss": 3.5978,
+      "theoretical_loss": 4.673528455523392,
+      "tokens_seen": 116457472
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.000487221664994985,
+      "loss": 3.416,
+      "theoretical_loss": 4.673171748163389,
+      "tokens_seen": 116523008
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004872116349047141,
+      "loss": 3.3723,
+      "theoretical_loss": 4.672815297508194,
+      "tokens_seen": 116588544
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048720160481444336,
+      "loss": 3.1412,
+      "theoretical_loss": 4.672459103228929,
+      "tokens_seen": 116654080
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048719157472417254,
+      "loss": 3.6399,
+      "theoretical_loss": 4.672103164997319,
+      "tokens_seen": 116719616
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004871815446339017,
+      "loss": 3.3651,
+      "theoretical_loss": 4.671747482485696,
+      "tokens_seen": 116785152
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004871715145436309,
+      "loss": 3.3461,
+      "theoretical_loss": 4.671392055366992,
+      "tokens_seen": 116850688
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048716148445336014,
+      "loss": 3.2057,
+      "theoretical_loss": 4.671036883314743,
+      "tokens_seen": 116916224
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048715145436308927,
+      "loss": 3.1564,
+      "theoretical_loss": 4.670681966003084,
+      "tokens_seen": 116981760
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004871414242728185,
+      "loss": 3.3224,
+      "theoretical_loss": 4.670327303106748,
+      "tokens_seen": 117047296
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048713139418254763,
+      "loss": 3.5285,
+      "theoretical_loss": 4.6699728943010665,
+      "tokens_seen": 117112832
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048712136409227686,
+      "loss": 3.2677,
+      "theoretical_loss": 4.669618739261967,
+      "tokens_seen": 117178368
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048711133400200604,
+      "loss": 3.1055,
+      "theoretical_loss": 4.669264837665969,
+      "tokens_seen": 117243904
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004871013039117352,
+      "loss": 3.3293,
+      "theoretical_loss": 4.668911189190185,
+      "tokens_seen": 117309440
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004870912738214644,
+      "loss": 3.3489,
+      "theoretical_loss": 4.668557793512322,
+      "tokens_seen": 117374976
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004870812437311936,
+      "loss": 3.1125,
+      "theoretical_loss": 4.668204650310676,
+      "tokens_seen": 117440512
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048707121364092277,
+      "loss": 3.2515,
+      "theoretical_loss": 4.667851759264128,
+      "tokens_seen": 117506048
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.000487061183550652,
+      "loss": 3.3586,
+      "theoretical_loss": 4.667499120052151,
+      "tokens_seen": 117571584
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048705115346038113,
+      "loss": 3.2803,
+      "theoretical_loss": 4.6671467323548,
+      "tokens_seen": 117637120
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048704112337011037,
+      "loss": 3.3058,
+      "theoretical_loss": 4.666794595852718,
+      "tokens_seen": 117702656
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048703109327983955,
+      "loss": 3.2752,
+      "theoretical_loss": 4.666442710227129,
+      "tokens_seen": 117768192
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048702106318956873,
+      "loss": 3.3295,
+      "theoretical_loss": 4.666091075159839,
+      "tokens_seen": 117833728
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004870110330992979,
+      "loss": 3.5682,
+      "theoretical_loss": 4.665739690333233,
+      "tokens_seen": 117899264
+    },
+    {
+      "epoch": 0.04,
+      "objective/train/docs_used": 75891,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 2.9445178508758545,
+      "objective/train/theoretical_loss": 4.665388555430277,
+      "objective/train/tokens_used": 138424800,
+      "theoretical_loss": 4.665388555430277,
+      "tokens_seen": 117964800
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004870010030090271,
+      "loss": 3.4408,
+      "theoretical_loss": 4.665388555430277,
+      "tokens_seen": 117964800
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048699097291875627,
+      "loss": 3.42,
+      "theoretical_loss": 4.665037670134513,
+      "tokens_seen": 118030336
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004869809428284855,
+      "loss": 3.4118,
+      "theoretical_loss": 4.66468703413006,
+      "tokens_seen": 118095872
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048697091273821463,
+      "loss": 2.8531,
+      "theoretical_loss": 4.664336647101612,
+      "tokens_seen": 118161408
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048696088264794387,
+      "loss": 3.5486,
+      "theoretical_loss": 4.663986508734435,
+      "tokens_seen": 118226944
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.000486950852557673,
+      "loss": 3.3137,
+      "theoretical_loss": 4.663636618714368,
+      "tokens_seen": 118292480
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048694082246740223,
+      "loss": 3.2493,
+      "theoretical_loss": 4.663286976727821,
+      "tokens_seen": 118358016
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004869307923771314,
+      "loss": 3.4941,
+      "theoretical_loss": 4.6629375824617725,
+      "tokens_seen": 118423552
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004869207622868606,
+      "loss": 3.1022,
+      "theoretical_loss": 4.6625884356037695,
+      "tokens_seen": 118489088
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004869107321965898,
+      "loss": 3.2497,
+      "theoretical_loss": 4.662239535841925,
+      "tokens_seen": 118554624
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048690070210631896,
+      "loss": 3.465,
+      "theoretical_loss": 4.661890882864919,
+      "tokens_seen": 118620160
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048689067201604814,
+      "loss": 3.3506,
+      "theoretical_loss": 4.661542476361992,
+      "tokens_seen": 118685696
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048688064192577737,
+      "loss": 3.2192,
+      "theoretical_loss": 4.661194316022952,
+      "tokens_seen": 118751232
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004868706118355065,
+      "loss": 3.274,
+      "theoretical_loss": 4.660846401538165,
+      "tokens_seen": 118816768
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048686058174523573,
+      "loss": 3.1325,
+      "theoretical_loss": 4.6604987325985565,
+      "tokens_seen": 118882304
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004868505516549649,
+      "loss": 3.241,
+      "theoretical_loss": 4.660151308895614,
+      "tokens_seen": 118947840
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004868405215646941,
+      "loss": 3.457,
+      "theoretical_loss": 4.659804130121378,
+      "tokens_seen": 119013376
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004868304914744233,
+      "loss": 3.2732,
+      "theoretical_loss": 4.659457195968448,
+      "tokens_seen": 119078912
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048682046138415246,
+      "loss": 3.1738,
+      "theoretical_loss": 4.659110506129977,
+      "tokens_seen": 119144448
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048681043129388164,
+      "loss": 3.2882,
+      "theoretical_loss": 4.6587640602996725,
+      "tokens_seen": 119209984
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004868004012036109,
+      "loss": 3.4992,
+      "theoretical_loss": 4.658417858171794,
+      "tokens_seen": 119275520
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048679037111334,
+      "loss": 3.2217,
+      "theoretical_loss": 4.6580718994411505,
+      "tokens_seen": 119341056
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048678034102306924,
+      "loss": 3.3294,
+      "theoretical_loss": 4.657726183803101,
+      "tokens_seen": 119406592
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048677031093279836,
+      "loss": 3.0235,
+      "theoretical_loss": 4.657380710953554,
+      "tokens_seen": 119472128
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004867602808425276,
+      "loss": 3.2772,
+      "theoretical_loss": 4.657035480588962,
+      "tokens_seen": 119537664
+    },
+    {
+      "epoch": 0.04,
+      "objective/train/docs_used": 76546,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 3.836437225341797,
+      "objective/train/theoretical_loss": 4.656690492406327,
+      "objective/train/tokens_used": 140063200,
+      "theoretical_loss": 4.656690492406327,
+      "tokens_seen": 119603200
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004867502507522568,
+      "loss": 3.395,
+      "theoretical_loss": 4.656690492406327,
+      "tokens_seen": 119603200
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048674022066198596,
+      "loss": 3.2332,
+      "theoretical_loss": 4.6563457461031925,
+      "tokens_seen": 119668736
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048673019057171514,
+      "loss": 3.3067,
+      "theoretical_loss": 4.656001241377647,
+      "tokens_seen": 119734272
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004867201604814443,
+      "loss": 3.4375,
+      "theoretical_loss": 4.655656977928318,
+      "tokens_seen": 119799808
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004867101303911735,
+      "loss": 3.3728,
+      "theoretical_loss": 4.655312955454378,
+      "tokens_seen": 119865344
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048670010030090274,
+      "loss": 3.4504,
+      "theoretical_loss": 4.654969173655536,
+      "tokens_seen": 119930880
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048669007021063187,
+      "loss": 3.168,
+      "theoretical_loss": 4.654625632232037,
+      "tokens_seen": 119996416
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004866800401203611,
+      "loss": 3.1419,
+      "theoretical_loss": 4.654282330884667,
+      "tokens_seen": 120061952
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004866700100300903,
+      "loss": 3.483,
+      "theoretical_loss": 4.653939269314744,
+      "tokens_seen": 120127488
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048665997993981947,
+      "loss": 3.1995,
+      "theoretical_loss": 4.653596447224125,
+      "tokens_seen": 120193024
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048664994984954865,
+      "loss": 3.0174,
+      "theoretical_loss": 4.653253864315193,
+      "tokens_seen": 120258560
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048663991975927783,
+      "loss": 3.2561,
+      "theoretical_loss": 4.6529115202908695,
+      "tokens_seen": 120324096
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.000486629889669007,
+      "loss": 3.3328,
+      "theoretical_loss": 4.652569414854604,
+      "tokens_seen": 120389632
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048661985957873624,
+      "loss": 3.2924,
+      "theoretical_loss": 4.652227547710373,
+      "tokens_seen": 120455168
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048660982948846537,
+      "loss": 3.2271,
+      "theoretical_loss": 4.651885918562684,
+      "tokens_seen": 120520704
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004865997993981946,
+      "loss": 3.2564,
+      "theoretical_loss": 4.65154452711657,
+      "tokens_seen": 120586240
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048658976930792373,
+      "loss": 3.2625,
+      "theoretical_loss": 4.651203373077593,
+      "tokens_seen": 120651776
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048657973921765297,
+      "loss": 3.1227,
+      "theoretical_loss": 4.650862456151833,
+      "tokens_seen": 120717312
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048656970912738215,
+      "loss": 3.2546,
+      "theoretical_loss": 4.650521776045898,
+      "tokens_seen": 120782848
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048655967903711133,
+      "loss": 3.2324,
+      "theoretical_loss": 4.650181332466918,
+      "tokens_seen": 120848384
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004865496489468405,
+      "loss": 3.3792,
+      "theoretical_loss": 4.64984112512254,
+      "tokens_seen": 120913920
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048653961885656975,
+      "loss": 3.4118,
+      "theoretical_loss": 4.6495011537209345,
+      "tokens_seen": 120979456
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004865295887662989,
+      "loss": 3.3581,
+      "theoretical_loss": 4.649161417970789,
+      "tokens_seen": 121044992
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004865195586760281,
+      "loss": 3.4152,
+      "theoretical_loss": 4.648821917581308,
+      "tokens_seen": 121110528
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048650952858575724,
+      "loss": 3.4244,
+      "theoretical_loss": 4.648482652262211,
+      "tokens_seen": 121176064
+    },
+    {
+      "epoch": 0.04,
+      "objective/train/docs_used": 77942,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 2.9632604122161865,
+      "objective/train/theoretical_loss": 4.648143621723734,
+      "objective/train/tokens_used": 141701600,
+      "theoretical_loss": 4.648143621723734,
+      "tokens_seen": 121241600
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048649949849548647,
+      "loss": 3.2822,
+      "theoretical_loss": 4.648143621723734,
+      "tokens_seen": 121241600
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048648946840521565,
+      "loss": 3.0832,
+      "theoretical_loss": 4.6478048256766265,
+      "tokens_seen": 121307136
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048647943831494483,
+      "loss": 3.3079,
+      "theoretical_loss": 4.647466263832148,
+      "tokens_seen": 121372672
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048646940822467407,
+      "loss": 3.2723,
+      "theoretical_loss": 4.647127935902072,
+      "tokens_seen": 121438208
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004864593781344032,
+      "loss": 3.5748,
+      "theoretical_loss": 4.646789841598681,
+      "tokens_seen": 121503744
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048644934804413243,
+      "loss": 3.1642,
+      "theoretical_loss": 4.646451980634765,
+      "tokens_seen": 121569280
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004864393179538616,
+      "loss": 3.1145,
+      "theoretical_loss": 4.646114352723625,
+      "tokens_seen": 121634816
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004864292878635908,
+      "loss": 3.5071,
+      "theoretical_loss": 4.645776957579065,
+      "tokens_seen": 121700352
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048641925777332,
+      "loss": 3.4015,
+      "theoretical_loss": 4.645439794915395,
+      "tokens_seen": 121765888
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048640922768304916,
+      "loss": 3.2914,
+      "theoretical_loss": 4.6451028644474315,
+      "tokens_seen": 121831424
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048639919759277834,
+      "loss": 3.2374,
+      "theoretical_loss": 4.644766165890491,
+      "tokens_seen": 121896960
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048638916750250757,
+      "loss": 3.2349,
+      "theoretical_loss": 4.644429698960394,
+      "tokens_seen": 121962496
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004863791374122367,
+      "loss": 3.4859,
+      "theoretical_loss": 4.644093463373462,
+      "tokens_seen": 122028032
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048636910732196593,
+      "loss": 3.2108,
+      "theoretical_loss": 4.643757458846514,
+      "tokens_seen": 122093568
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004863590772316951,
+      "loss": 3.2694,
+      "theoretical_loss": 4.643421685096866,
+      "tokens_seen": 122159104
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004863490471414243,
+      "loss": 3.525,
+      "theoretical_loss": 4.643086141842337,
+      "tokens_seen": 122224640
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004863390170511535,
+      "loss": 3.4927,
+      "theoretical_loss": 4.642750828801238,
+      "tokens_seen": 122290176
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048632898696088266,
+      "loss": 3.4461,
+      "theoretical_loss": 4.642415745692375,
+      "tokens_seen": 122355712
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048631895687061184,
+      "loss": 3.3788,
+      "theoretical_loss": 4.6420808922350485,
+      "tokens_seen": 122421248
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004863089267803411,
+      "loss": 3.3053,
+      "theoretical_loss": 4.641746268149052,
+      "tokens_seen": 122486784
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004862988966900702,
+      "loss": 3.2416,
+      "theoretical_loss": 4.64141187315467,
+      "tokens_seen": 122552320
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048628886659979944,
+      "loss": 3.0169,
+      "theoretical_loss": 4.641077706972679,
+      "tokens_seen": 122617856
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048627883650952857,
+      "loss": 3.3138,
+      "theoretical_loss": 4.6407437693243425,
+      "tokens_seen": 122683392
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004862688064192578,
+      "loss": 3.1577,
+      "theoretical_loss": 4.640410059931414,
+      "tokens_seen": 122748928
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.000486258776328987,
+      "loss": 3.3863,
+      "theoretical_loss": 4.640076578516132,
+      "tokens_seen": 122814464
+    },
+    {
+      "epoch": 0.04,
+      "objective/train/docs_used": 78541,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 2.7341437339782715,
+      "objective/train/theoretical_loss": 4.639743324801225,
+      "objective/train/tokens_used": 143340000,
+      "theoretical_loss": 4.639743324801225,
+      "tokens_seen": 122880000
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048624874623871616,
+      "loss": 3.1381,
+      "theoretical_loss": 4.639743324801225,
+      "tokens_seen": 122880000
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048623871614844534,
+      "loss": 3.3316,
+      "theoretical_loss": 4.639410298509901,
+      "tokens_seen": 122945536
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004862286860581745,
+      "loss": 3.2356,
+      "theoretical_loss": 4.639077499365856,
+      "tokens_seen": 123011072
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004862186559679037,
+      "loss": 3.355,
+      "theoretical_loss": 4.638744927093268,
+      "tokens_seen": 123076608
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048620862587763294,
+      "loss": 3.3971,
+      "theoretical_loss": 4.6384125814167945,
+      "tokens_seen": 123142144
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048619859578736207,
+      "loss": 3.0734,
+      "theoretical_loss": 4.638080462061575,
+      "tokens_seen": 123207680
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004861885656970913,
+      "loss": 3.1079,
+      "theoretical_loss": 4.63774856875323,
+      "tokens_seen": 123273216
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004861785356068205,
+      "loss": 3.3234,
+      "theoretical_loss": 4.637416901217852,
+      "tokens_seen": 123338752
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048616850551654967,
+      "loss": 3.3152,
+      "theoretical_loss": 4.63708545918202,
+      "tokens_seen": 123404288
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048615847542627885,
+      "loss": 3.345,
+      "theoretical_loss": 4.636754242372782,
+      "tokens_seen": 123469824
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048614844533600803,
+      "loss": 3.3265,
+      "theoretical_loss": 4.636423250517664,
+      "tokens_seen": 123535360
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004861384152457372,
+      "loss": 3.3053,
+      "theoretical_loss": 4.636092483344665,
+      "tokens_seen": 123600896
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048612838515546644,
+      "loss": 3.3878,
+      "theoretical_loss": 4.635761940582256,
+      "tokens_seen": 123666432
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048611835506519557,
+      "loss": 3.3957,
+      "theoretical_loss": 4.635431621959384,
+      "tokens_seen": 123731968
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004861083249749248,
+      "loss": 3.4768,
+      "theoretical_loss": 4.63510152720546,
+      "tokens_seen": 123797504
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048609829488465393,
+      "loss": 3.2736,
+      "theoretical_loss": 4.634771656050372,
+      "tokens_seen": 123863040
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048608826479438317,
+      "loss": 3.1239,
+      "theoretical_loss": 4.63444200822447,
+      "tokens_seen": 123928576
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048607823470411235,
+      "loss": 3.2963,
+      "theoretical_loss": 4.6341125834585775,
+      "tokens_seen": 123994112
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048606820461384153,
+      "loss": 3.4747,
+      "theoretical_loss": 4.633783381483978,
+      "tokens_seen": 124059648
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004860581745235707,
+      "loss": 3.374,
+      "theoretical_loss": 4.633454402032427,
+      "tokens_seen": 124125184
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048604814443329995,
+      "loss": 3.4227,
+      "theoretical_loss": 4.63312564483614,
+      "tokens_seen": 124190720
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004860381143430291,
+      "loss": 3.6141,
+      "theoretical_loss": 4.632797109627798,
+      "tokens_seen": 124256256
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004860280842527583,
+      "loss": 3.2804,
+      "theoretical_loss": 4.632468796140543,
+      "tokens_seen": 124321792
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048601805416248744,
+      "loss": 3.4935,
+      "theoretical_loss": 4.632140704107979,
+      "tokens_seen": 124387328
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048600802407221667,
+      "loss": 3.3112,
+      "theoretical_loss": 4.631812833264169,
+      "tokens_seen": 124452864
+    },
+    {
+      "epoch": 0.04,
+      "objective/train/docs_used": 79709,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 3.385240316390991,
+      "objective/train/theoretical_loss": 4.631485183343639,
+      "objective/train/tokens_used": 144978400,
+      "theoretical_loss": 4.631485183343639,
+      "tokens_seen": 124518400
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048599799398194585,
+      "loss": 3.2349,
+      "theoretical_loss": 4.631485183343639,
+      "tokens_seen": 124518400
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048598796389167503,
+      "loss": 3.3546,
+      "theoretical_loss": 4.631157754081366,
+      "tokens_seen": 124583936
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004859779338014042,
+      "loss": 3.088,
+      "theoretical_loss": 4.630830545212792,
+      "tokens_seen": 124649472
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004859679037111334,
+      "loss": 3.4382,
+      "theoretical_loss": 4.63050355647381,
+      "tokens_seen": 124715008
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004859578736208626,
+      "loss": 3.3753,
+      "theoretical_loss": 4.630176787600771,
+      "tokens_seen": 124780544
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004859478435305918,
+      "loss": 3.1749,
+      "theoretical_loss": 4.6298502383304765,
+      "tokens_seen": 124846080
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048593781344032094,
+      "loss": 3.1883,
+      "theoretical_loss": 4.629523908400184,
+      "tokens_seen": 124911616
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004859277833500502,
+      "loss": 3.4012,
+      "theoretical_loss": 4.6291977975476035,
+      "tokens_seen": 124977152
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004859177532597793,
+      "loss": 3.1556,
+      "theoretical_loss": 4.628871905510893,
+      "tokens_seen": 125042688
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048590772316950854,
+      "loss": 3.5815,
+      "theoretical_loss": 4.628546232028661,
+      "tokens_seen": 125108224
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004858976930792377,
+      "loss": 3.2682,
+      "theoretical_loss": 4.62822077683997,
+      "tokens_seen": 125173760
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004858876629889669,
+      "loss": 3.685,
+      "theoretical_loss": 4.627895539684323,
+      "tokens_seen": 125239296
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004858776328986961,
+      "loss": 3.2897,
+      "theoretical_loss": 4.627570520301673,
+      "tokens_seen": 125304832
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004858676028084253,
+      "loss": 3.1734,
+      "theoretical_loss": 4.627245718432423,
+      "tokens_seen": 125370368
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048585757271815444,
+      "loss": 3.0723,
+      "theoretical_loss": 4.626921133817413,
+      "tokens_seen": 125435904
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004858475426278837,
+      "loss": 3.1019,
+      "theoretical_loss": 4.626596766197936,
+      "tokens_seen": 125501440
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004858375125376128,
+      "loss": 3.21,
+      "theoretical_loss": 4.62627261531572,
+      "tokens_seen": 125566976
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048582748244734204,
+      "loss": 3.1586,
+      "theoretical_loss": 4.625948680912939,
+      "tokens_seen": 125632512
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004858174523570712,
+      "loss": 3.1464,
+      "theoretical_loss": 4.625624962732207,
+      "tokens_seen": 125698048
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004858074222668004,
+      "loss": 3.5956,
+      "theoretical_loss": 4.62530146051658,
+      "tokens_seen": 125763584
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004857973921765296,
+      "loss": 3.2982,
+      "theoretical_loss": 4.624978174009549,
+      "tokens_seen": 125829120
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048578736208625877,
+      "loss": 2.8431,
+      "theoretical_loss": 4.624655102955048,
+      "tokens_seen": 125894656
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048577733199598795,
+      "loss": 3.2039,
+      "theoretical_loss": 4.624332247097442,
+      "tokens_seen": 125960192
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004857673019057172,
+      "loss": 3.255,
+      "theoretical_loss": 4.624009606181539,
+      "tokens_seen": 126025728
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004857572718154463,
+      "loss": 3.3476,
+      "theoretical_loss": 4.623687179952577,
+      "tokens_seen": 126091264
+    },
+    {
+      "epoch": 0.04,
+      "objective/train/docs_used": 80984,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 3.0424399375915527,
+      "objective/train/theoretical_loss": 4.623364968156229,
+      "objective/train/tokens_used": 146616800,
+      "theoretical_loss": 4.623364968156229,
+      "tokens_seen": 126156800
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048574724172517554,
+      "loss": 3.1198,
+      "theoretical_loss": 4.623364968156229,
+      "tokens_seen": 126156800
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048573721163490467,
+      "loss": 3.2627,
+      "theoretical_loss": 4.623042970538604,
+      "tokens_seen": 126222336
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004857271815446339,
+      "loss": 3.2619,
+      "theoretical_loss": 4.622721186846241,
+      "tokens_seen": 126287872
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048571715145436314,
+      "loss": 3.077,
+      "theoretical_loss": 4.62239961682611,
+      "tokens_seen": 126353408
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048570712136409227,
+      "loss": 3.4283,
+      "theoretical_loss": 4.622078260225611,
+      "tokens_seen": 126418944
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004856970912738215,
+      "loss": 3.1504,
+      "theoretical_loss": 4.621757116792576,
+      "tokens_seen": 126484480
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004856870611835507,
+      "loss": 3.2759,
+      "theoretical_loss": 4.621436186275262,
+      "tokens_seen": 126550016
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048567703109327987,
+      "loss": 3.2942,
+      "theoretical_loss": 4.6211154684223565,
+      "tokens_seen": 126615552
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048566700100300905,
+      "loss": 3.462,
+      "theoretical_loss": 4.6207949629829725,
+      "tokens_seen": 126681088
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048565697091273823,
+      "loss": 2.9686,
+      "theoretical_loss": 4.620474669706647,
+      "tokens_seen": 126746624
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004856469408224674,
+      "loss": 3.3596,
+      "theoretical_loss": 4.620154588343343,
+      "tokens_seen": 126812160
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048563691073219664,
+      "loss": 3.3793,
+      "theoretical_loss": 4.619834718643448,
+      "tokens_seen": 126877696
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048562688064192577,
+      "loss": 3.327,
+      "theoretical_loss": 4.619515060357771,
+      "tokens_seen": 126943232
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.000485616850551655,
+      "loss": 3.5938,
+      "theoretical_loss": 4.619195613237542,
+      "tokens_seen": 127008768
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048560682046138413,
+      "loss": 3.1359,
+      "theoretical_loss": 4.6188763770344154,
+      "tokens_seen": 127074304
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048559679037111337,
+      "loss": 3.6939,
+      "theoretical_loss": 4.618557351500461,
+      "tokens_seen": 127139840
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048558676028084255,
+      "loss": 3.3604,
+      "theoretical_loss": 4.618238536388171,
+      "tokens_seen": 127205376
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048557673019057173,
+      "loss": 3.4903,
+      "theoretical_loss": 4.617919931450454,
+      "tokens_seen": 127270912
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004855667001003009,
+      "loss": 3.3289,
+      "theoretical_loss": 4.617601536440637,
+      "tokens_seen": 127336448
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048555667001003015,
+      "loss": 3.1247,
+      "theoretical_loss": 4.617283351112462,
+      "tokens_seen": 127401984
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004855466399197593,
+      "loss": 3.5764,
+      "theoretical_loss": 4.616965375220088,
+      "tokens_seen": 127467520
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004855366098294885,
+      "loss": 3.419,
+      "theoretical_loss": 4.616647608518086,
+      "tokens_seen": 127533056
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048552657973921764,
+      "loss": 3.2447,
+      "theoretical_loss": 4.616330050761444,
+      "tokens_seen": 127598592
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048551654964894687,
+      "loss": 3.0136,
+      "theoretical_loss": 4.616012701705559,
+      "tokens_seen": 127664128
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048550651955867605,
+      "loss": 3.1339,
+      "theoretical_loss": 4.615695561106241,
+      "tokens_seen": 127729664
+    },
+    {
+      "epoch": 0.04,
+      "objective/train/docs_used": 81703,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 3.6864616870880127,
+      "objective/train/theoretical_loss": 4.6153786287197125,
+      "objective/train/tokens_used": 148255200,
+      "theoretical_loss": 4.6153786287197125,
+      "tokens_seen": 127795200
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048549648946840523,
+      "loss": 3.2948,
+      "theoretical_loss": 4.6153786287197125,
+      "tokens_seen": 127795200
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004854864593781344,
+      "loss": 3.0792,
+      "theoretical_loss": 4.615061904302603,
+      "tokens_seen": 127860736
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004854764292878636,
+      "loss": 3.1573,
+      "theoretical_loss": 4.614745387611955,
+      "tokens_seen": 127926272
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004854663991975928,
+      "loss": 3.4941,
+      "theoretical_loss": 4.614429078405214,
+      "tokens_seen": 127991808
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.000485456369107322,
+      "loss": 3.2639,
+      "theoretical_loss": 4.6141129764402375,
+      "tokens_seen": 128057344
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048544633901705114,
+      "loss": 3.2058,
+      "theoretical_loss": 4.613797081475287,
+      "tokens_seen": 128122880
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004854363089267804,
+      "loss": 3.4029,
+      "theoretical_loss": 4.613481393269028,
+      "tokens_seen": 128188416
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004854262788365095,
+      "loss": 3.5993,
+      "theoretical_loss": 4.613165911580533,
+      "tokens_seen": 128253952
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048541624874623874,
+      "loss": 3.1374,
+      "theoretical_loss": 4.612850636169279,
+      "tokens_seen": 128319488
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004854062186559679,
+      "loss": 3.3721,
+      "theoretical_loss": 4.612535566795142,
+      "tokens_seen": 128385024
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004853961885656971,
+      "loss": 3.1656,
+      "theoretical_loss": 4.612220703218403,
+      "tokens_seen": 128450560
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004853861584754263,
+      "loss": 3.4822,
+      "theoretical_loss": 4.611906045199744,
+      "tokens_seen": 128516096
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004853761283851555,
+      "loss": 3.1366,
+      "theoretical_loss": 4.611591592500245,
+      "tokens_seen": 128581632
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048536609829488464,
+      "loss": 3.3858,
+      "theoretical_loss": 4.611277344881389,
+      "tokens_seen": 128647168
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004853560682046139,
+      "loss": 3.1729,
+      "theoretical_loss": 4.610963302105051,
+      "tokens_seen": 128712704
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.000485346038114343,
+      "loss": 3.2684,
+      "theoretical_loss": 4.610649463933511,
+      "tokens_seen": 128778240
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048533600802407224,
+      "loss": 3.1187,
+      "theoretical_loss": 4.61033583012944,
+      "tokens_seen": 128843776
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004853259779338014,
+      "loss": 3.2702,
+      "theoretical_loss": 4.610022400455909,
+      "tokens_seen": 128909312
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004853159478435306,
+      "loss": 3.1401,
+      "theoretical_loss": 4.6097091746763805,
+      "tokens_seen": 128974848
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004853059177532598,
+      "loss": 3.0625,
+      "theoretical_loss": 4.609396152554714,
+      "tokens_seen": 129040384
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048529588766298897,
+      "loss": 3.4469,
+      "theoretical_loss": 4.60908333385516,
+      "tokens_seen": 129105920
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048528585757271815,
+      "loss": 3.2416,
+      "theoretical_loss": 4.608770718342361,
+      "tokens_seen": 129171456
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004852758274824474,
+      "loss": 3.2241,
+      "theoretical_loss": 4.6084583057813555,
+      "tokens_seen": 129236992
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004852657973921765,
+      "loss": 3.2384,
+      "theoretical_loss": 4.608146095937567,
+      "tokens_seen": 129302528
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048525576730190574,
+      "loss": 3.4009,
+      "theoretical_loss": 4.607834088576811,
+      "tokens_seen": 129368064
+    },
+    {
+      "epoch": 0.04,
+      "objective/train/docs_used": 82831,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 3.1748886108398438,
+      "objective/train/theoretical_loss": 4.607522283465294,
+      "objective/train/tokens_used": 149893600,
+      "theoretical_loss": 4.607522283465294,
+      "tokens_seen": 129433600
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048524573721163487,
+      "loss": 3.2513,
+      "theoretical_loss": 4.607522283465294,
+      "tokens_seen": 129433600
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004852357071213641,
+      "loss": 3.3024,
+      "theoretical_loss": 4.607210680369608,
+      "tokens_seen": 129499136
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004852256770310933,
+      "loss": 3.5175,
+      "theoretical_loss": 4.606899279056734,
+      "tokens_seen": 129564672
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048521564694082247,
+      "loss": 3.1854,
+      "theoretical_loss": 4.606588079294038,
+      "tokens_seen": 129630208
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048520561685055165,
+      "loss": 3.1448,
+      "theoretical_loss": 4.606277080849273,
+      "tokens_seen": 129695744
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004851955867602809,
+      "loss": 3.2784,
+      "theoretical_loss": 4.605966283490575,
+      "tokens_seen": 129761280
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048518555667001,
+      "loss": 3.055,
+      "theoretical_loss": 4.6056556869864655,
+      "tokens_seen": 129826816
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048517552657973925,
+      "loss": 3.4625,
+      "theoretical_loss": 4.60534529110585,
+      "tokens_seen": 129892352
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004851654964894684,
+      "loss": 3.2055,
+      "theoretical_loss": 4.605035095618014,
+      "tokens_seen": 129957888
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004851554663991976,
+      "loss": 3.3952,
+      "theoretical_loss": 4.604725100292626,
+      "tokens_seen": 130023424
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004851454363089268,
+      "loss": 3.193,
+      "theoretical_loss": 4.604415304899735,
+      "tokens_seen": 130088960
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048513540621865597,
+      "loss": 3.3349,
+      "theoretical_loss": 4.604105709209769,
+      "tokens_seen": 130154496
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048512537612838515,
+      "loss": 3.4078,
+      "theoretical_loss": 4.603796312993536,
+      "tokens_seen": 130220032
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048511534603811433,
+      "loss": 3.3374,
+      "theoretical_loss": 4.603487116022222,
+      "tokens_seen": 130285568
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004851053159478435,
+      "loss": 3.2205,
+      "theoretical_loss": 4.60317811806739,
+      "tokens_seen": 130351104
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048509528585757275,
+      "loss": 3.3053,
+      "theoretical_loss": 4.60286931890098,
+      "tokens_seen": 130416640
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004850852557673019,
+      "loss": 3.3346,
+      "theoretical_loss": 4.602560718295308,
+      "tokens_seen": 130482176
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004850752256770311,
+      "loss": 3.3864,
+      "theoretical_loss": 4.602252316023064,
+      "tokens_seen": 130547712
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048506519558676024,
+      "loss": 3.4327,
+      "theoretical_loss": 4.601944111857312,
+      "tokens_seen": 130613248
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004850551654964895,
+      "loss": 3.2583,
+      "theoretical_loss": 4.601636105571492,
+      "tokens_seen": 130678784
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048504513540621866,
+      "loss": 3.4605,
+      "theoretical_loss": 4.601328296939414,
+      "tokens_seen": 130744320
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048503510531594784,
+      "loss": 3.2544,
+      "theoretical_loss": 4.601020685735259,
+      "tokens_seen": 130809856
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.000485025075225677,
+      "loss": 3.2836,
+      "theoretical_loss": 4.600713271733582,
+      "tokens_seen": 130875392
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048501504513540625,
+      "loss": 3.4586,
+      "theoretical_loss": 4.600406054709307,
+      "tokens_seen": 130940928
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004850050150451354,
+      "loss": 3.3115,
+      "theoretical_loss": 4.600099034437726,
+      "tokens_seen": 131006464
+    },
+    {
+      "epoch": 0.04,
+      "objective/train/docs_used": 83204,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 3.2750132083892822,
+      "objective/train/theoretical_loss": 4.5997922106945,
+      "objective/train/tokens_used": 151532000,
+      "theoretical_loss": 4.5997922106945,
+      "tokens_seen": 131072000
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004849949849548646,
+      "loss": 3.3318,
+      "theoretical_loss": 4.5997922106945,
+      "tokens_seen": 131072000
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004849849548645938,
+      "loss": 3.5144,
+      "theoretical_loss": 4.5994855832556585,
+      "tokens_seen": 131137536
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.000484974924774323,
+      "loss": 3.4244,
+      "theoretical_loss": 4.5991791518976,
+      "tokens_seen": 131203072
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004849648946840522,
+      "loss": 3.2389,
+      "theoretical_loss": 4.5988729163970845,
+      "tokens_seen": 131268608
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048495486459378134,
+      "loss": 3.3639,
+      "theoretical_loss": 4.59856687653124,
+      "tokens_seen": 131334144
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004849448345035106,
+      "loss": 3.2156,
+      "theoretical_loss": 4.598261032077561,
+      "tokens_seen": 131399680
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004849348044132397,
+      "loss": 3.5008,
+      "theoretical_loss": 4.597955382813903,
+      "tokens_seen": 131465216
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048492477432296894,
+      "loss": 3.4594,
+      "theoretical_loss": 4.597649928518486,
+      "tokens_seen": 131530752
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004849147442326981,
+      "loss": 3.3812,
+      "theoretical_loss": 4.597344668969892,
+      "tokens_seen": 131596288
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004849047141424273,
+      "loss": 3.3369,
+      "theoretical_loss": 4.597039603947064,
+      "tokens_seen": 131661824
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004848946840521565,
+      "loss": 3.2768,
+      "theoretical_loss": 4.596734733229306,
+      "tokens_seen": 131727360
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004848846539618857,
+      "loss": 2.9459,
+      "theoretical_loss": 4.596430056596285,
+      "tokens_seen": 131792896
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048487462387161484,
+      "loss": 3.0683,
+      "theoretical_loss": 4.596125573828021,
+      "tokens_seen": 131858432
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004848645937813441,
+      "loss": 3.3636,
+      "theoretical_loss": 4.595821284704899,
+      "tokens_seen": 131923968
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004848545636910732,
+      "loss": 3.2567,
+      "theoretical_loss": 4.5955171890076585,
+      "tokens_seen": 131989504
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048484453360080244,
+      "loss": 3.5107,
+      "theoretical_loss": 4.595213286517394,
+      "tokens_seen": 132055040
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004848345035105316,
+      "loss": 3.3222,
+      "theoretical_loss": 4.594909577015562,
+      "tokens_seen": 132120576
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004848244734202608,
+      "loss": 3.4088,
+      "theoretical_loss": 4.594606060283969,
+      "tokens_seen": 132186112
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048481444332999,
+      "loss": 3.3058,
+      "theoretical_loss": 4.594302736104778,
+      "tokens_seen": 132251648
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048480441323971917,
+      "loss": 3.3761,
+      "theoretical_loss": 4.593999604260507,
+      "tokens_seen": 132317184
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048479438314944835,
+      "loss": 3.115,
+      "theoretical_loss": 4.593696664534026,
+      "tokens_seen": 132382720
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004847843530591776,
+      "loss": 3.0734,
+      "theoretical_loss": 4.593393916708557,
+      "tokens_seen": 132448256
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004847743229689067,
+      "loss": 3.4007,
+      "theoretical_loss": 4.593091360567676,
+      "tokens_seen": 132513792
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048476429287863594,
+      "loss": 3.4221,
+      "theoretical_loss": 4.59278899589531,
+      "tokens_seen": 132579328
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048475426278836507,
+      "loss": 3.3316,
+      "theoretical_loss": 4.592486822475731,
+      "tokens_seen": 132644864
+    },
+    {
+      "epoch": 0.04,
+      "objective/train/docs_used": 84183,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 3.4309723377227783,
+      "objective/train/theoretical_loss": 4.592184840093566,
+      "objective/train/tokens_used": 153170400,
+      "theoretical_loss": 4.592184840093566,
+      "tokens_seen": 132710400
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004847442326980943,
+      "loss": 3.1609,
+      "theoretical_loss": 4.592184840093566,
+      "tokens_seen": 132710400
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004847342026078235,
+      "loss": 3.3487,
+      "theoretical_loss": 4.59188304853379,
+      "tokens_seen": 132775936
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048472417251755267,
+      "loss": 3.2624,
+      "theoretical_loss": 4.591581447581723,
+      "tokens_seen": 132841472
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048471414242728185,
+      "loss": 3.0568,
+      "theoretical_loss": 4.591280037023036,
+      "tokens_seen": 132907008
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004847041123370111,
+      "loss": 3.3944,
+      "theoretical_loss": 4.590978816643744,
+      "tokens_seen": 132972544
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004846940822467402,
+      "loss": 3.375,
+      "theoretical_loss": 4.590677786230208,
+      "tokens_seen": 133038080
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048468405215646945,
+      "loss": 3.1915,
+      "theoretical_loss": 4.590376945569136,
+      "tokens_seen": 133103616
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004846740220661986,
+      "loss": 3.2734,
+      "theoretical_loss": 4.590076294447578,
+      "tokens_seen": 133169152
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004846639919759278,
+      "loss": 3.1604,
+      "theoretical_loss": 4.589775832652929,
+      "tokens_seen": 133234688
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.000484653961885657,
+      "loss": 3.1837,
+      "theoretical_loss": 4.589475559972927,
+      "tokens_seen": 133300224
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048464393179538617,
+      "loss": 3.3031,
+      "theoretical_loss": 4.589175476195651,
+      "tokens_seen": 133365760
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048463390170511535,
+      "loss": 3.1795,
+      "theoretical_loss": 4.588875581109523,
+      "tokens_seen": 133431296
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048462387161484453,
+      "loss": 3.2039,
+      "theoretical_loss": 4.588575874503304,
+      "tokens_seen": 133496832
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004846138415245737,
+      "loss": 3.4852,
+      "theoretical_loss": 4.588276356166096,
+      "tokens_seen": 133562368
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048460381143430295,
+      "loss": 3.3087,
+      "theoretical_loss": 4.587977025887342,
+      "tokens_seen": 133627904
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004845937813440321,
+      "loss": 3.1137,
+      "theoretical_loss": 4.58767788345682,
+      "tokens_seen": 133693440
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004845837512537613,
+      "loss": 3.2587,
+      "theoretical_loss": 4.5873789286646485,
+      "tokens_seen": 133758976
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048457372116349044,
+      "loss": 3.3583,
+      "theoretical_loss": 4.587080161301283,
+      "tokens_seen": 133824512
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004845636910732197,
+      "loss": 3.2645,
+      "theoretical_loss": 4.586781581157513,
+      "tokens_seen": 133890048
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048455366098294886,
+      "loss": 3.289,
+      "theoretical_loss": 4.58648318802447,
+      "tokens_seen": 133955584
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048454363089267804,
+      "loss": 2.9687,
+      "theoretical_loss": 4.586184981693611,
+      "tokens_seen": 134021120
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004845336008024072,
+      "loss": 3.1314,
+      "theoretical_loss": 4.585886961956737,
+      "tokens_seen": 134086656
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048452357071213645,
+      "loss": 3.6532,
+      "theoretical_loss": 4.585589128605976,
+      "tokens_seen": 134152192
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004845135406218656,
+      "loss": 3.1767,
+      "theoretical_loss": 4.585291481433792,
+      "tokens_seen": 134217728
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004845035105315948,
+      "loss": 3.1699,
+      "theoretical_loss": 4.584994020232981,
+      "tokens_seen": 134283264
+    },
+    {
+      "epoch": 0.04,
+      "objective/train/docs_used": 84861,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 3.49385142326355,
+      "objective/train/theoretical_loss": 4.58469674479667,
+      "objective/train/tokens_used": 154808800,
+      "theoretical_loss": 4.58469674479667,
+      "tokens_seen": 134348800
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048449348044132394,
+      "loss": 3.3437,
+      "theoretical_loss": 4.58469674479667,
+      "tokens_seen": 134348800
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004844834503510532,
+      "loss": 3.3897,
+      "theoretical_loss": 4.584399654918316,
+      "tokens_seen": 134414336
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048447342026078236,
+      "loss": 3.2379,
+      "theoretical_loss": 4.584102750391709,
+      "tokens_seen": 134479872
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048446339017051154,
+      "loss": 3.4963,
+      "theoretical_loss": 4.583806031010965,
+      "tokens_seen": 134545408
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004844533600802407,
+      "loss": 3.4515,
+      "theoretical_loss": 4.5835094965705325,
+      "tokens_seen": 134610944
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004844433299899699,
+      "loss": 3.323,
+      "theoretical_loss": 4.5832131468651855,
+      "tokens_seen": 134676480
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004844332998996991,
+      "loss": 3.0751,
+      "theoretical_loss": 4.582916981690026,
+      "tokens_seen": 134742016
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004844232698094283,
+      "loss": 3.2973,
+      "theoretical_loss": 4.582621000840484,
+      "tokens_seen": 134807552
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048441323971915745,
+      "loss": 3.3443,
+      "theoretical_loss": 4.582325204112314,
+      "tokens_seen": 134873088
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004844032096288867,
+      "loss": 3.2763,
+      "theoretical_loss": 4.582029591301595,
+      "tokens_seen": 134938624
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048439317953861586,
+      "loss": 3.3043,
+      "theoretical_loss": 4.581734162204734,
+      "tokens_seen": 135004160
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048438314944834504,
+      "loss": 3.2092,
+      "theoretical_loss": 4.5814389166184615,
+      "tokens_seen": 135069696
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004843731193580742,
+      "loss": 3.2379,
+      "theoretical_loss": 4.581143854339828,
+      "tokens_seen": 135135232
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004843630892678034,
+      "loss": 3.2659,
+      "theoretical_loss": 4.580848975166209,
+      "tokens_seen": 135200768
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004843530591775326,
+      "loss": 3.0079,
+      "theoretical_loss": 4.580554278895304,
+      "tokens_seen": 135266304
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004843430290872618,
+      "loss": 3.2396,
+      "theoretical_loss": 4.58025976532513,
+      "tokens_seen": 135331840
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048433299899699095,
+      "loss": 3.2323,
+      "theoretical_loss": 4.579965434254028,
+      "tokens_seen": 135397376
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004843229689067202,
+      "loss": 3.0602,
+      "theoretical_loss": 4.579671285480657,
+      "tokens_seen": 135462912
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004843129388164493,
+      "loss": 3.2404,
+      "theoretical_loss": 4.579377318803996,
+      "tokens_seen": 135528448
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048430290872617855,
+      "loss": 3.0519,
+      "theoretical_loss": 4.579083534023344,
+      "tokens_seen": 135593984
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048429287863590773,
+      "loss": 3.3365,
+      "theoretical_loss": 4.578789930938315,
+      "tokens_seen": 135659520
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004842828485456369,
+      "loss": 3.2243,
+      "theoretical_loss": 4.5784965093488434,
+      "tokens_seen": 135725056
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004842728184553661,
+      "loss": 3.1942,
+      "theoretical_loss": 4.578203269055179,
+      "tokens_seen": 135790592
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048426278836509527,
+      "loss": 3.4092,
+      "theoretical_loss": 4.577910209857889,
+      "tokens_seen": 135856128
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048425275827482445,
+      "loss": 3.1487,
+      "theoretical_loss": 4.577617331557854,
+      "tokens_seen": 135921664
+    },
+    {
+      "epoch": 0.04,
+      "objective/train/docs_used": 86086,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 3.4546592235565186,
+      "objective/train/theoretical_loss": 4.577324633956271,
+      "objective/train/tokens_used": 156447200,
+      "theoretical_loss": 4.577324633956271,
+      "tokens_seen": 135987200
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004842427281845537,
+      "loss": 3.3413,
+      "theoretical_loss": 4.577324633956271,
+      "tokens_seen": 135987200
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048423269809428287,
+      "loss": 3.4106,
+      "theoretical_loss": 4.5770321168546495,
+      "tokens_seen": 136052736
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048422266800401205,
+      "loss": 3.4241,
+      "theoretical_loss": 4.576739780054815,
+      "tokens_seen": 136118272
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004842126379137413,
+      "loss": 3.254,
+      "theoretical_loss": 4.576447623358905,
+      "tokens_seen": 136183808
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004842026078234704,
+      "loss": 3.0941,
+      "theoretical_loss": 4.576155646569367,
+      "tokens_seen": 136249344
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048419257773319965,
+      "loss": 3.3943,
+      "theoretical_loss": 4.575863849488963,
+      "tokens_seen": 136314880
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004841825476429288,
+      "loss": 3.2206,
+      "theoretical_loss": 4.575572231920763,
+      "tokens_seen": 136380416
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.000484172517552658,
+      "loss": 3.1229,
+      "theoretical_loss": 4.575280793668151,
+      "tokens_seen": 136445952
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004841624874623872,
+      "loss": 2.9849,
+      "theoretical_loss": 4.574989534534817,
+      "tokens_seen": 136511488
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048415245737211637,
+      "loss": 3.2635,
+      "theoretical_loss": 4.574698454324764,
+      "tokens_seen": 136577024
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048414242728184555,
+      "loss": 3.1675,
+      "theoretical_loss": 4.574407552842299,
+      "tokens_seen": 136642560
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048413239719157473,
+      "loss": 3.5906,
+      "theoretical_loss": 4.57411682989204,
+      "tokens_seen": 136708096
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004841223671013039,
+      "loss": 3.2567,
+      "theoretical_loss": 4.57382628527891,
+      "tokens_seen": 136773632
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048411233701103315,
+      "loss": 3.5297,
+      "theoretical_loss": 4.573535918808141,
+      "tokens_seen": 136839168
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004841023069207623,
+      "loss": 3.3214,
+      "theoretical_loss": 4.573245730285268,
+      "tokens_seen": 136904704
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004840922768304915,
+      "loss": 3.2527,
+      "theoretical_loss": 4.572955719516132,
+      "tokens_seen": 136970240
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048408224674022064,
+      "loss": 3.0917,
+      "theoretical_loss": 4.572665886306881,
+      "tokens_seen": 137035776
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004840722166499499,
+      "loss": 3.0382,
+      "theoretical_loss": 4.572376230463965,
+      "tokens_seen": 137101312
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048406218655967906,
+      "loss": 3.3253,
+      "theoretical_loss": 4.572086751794135,
+      "tokens_seen": 137166848
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048405215646940824,
+      "loss": 3.4228,
+      "theoretical_loss": 4.571797450104451,
+      "tokens_seen": 137232384
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004840421263791374,
+      "loss": 3.4935,
+      "theoretical_loss": 4.571508325202268,
+      "tokens_seen": 137297920
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048403209628886665,
+      "loss": 3.2935,
+      "theoretical_loss": 4.5712193768952485,
+      "tokens_seen": 137363456
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004840220661985958,
+      "loss": 3.0949,
+      "theoretical_loss": 4.570930604991352,
+      "tokens_seen": 137428992
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.000484012036108325,
+      "loss": 3.0731,
+      "theoretical_loss": 4.57064200929884,
+      "tokens_seen": 137494528
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048400200601805414,
+      "loss": 3.2526,
+      "theoretical_loss": 4.570353589626273,
+      "tokens_seen": 137560064
+    },
+    {
+      "epoch": 0.04,
+      "objective/train/docs_used": 86654,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 3.495647668838501,
+      "objective/train/theoretical_loss": 4.570065345782512,
+      "objective/train/tokens_used": 158085600,
+      "theoretical_loss": 4.570065345782512,
+      "tokens_seen": 137625600
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004839919759277834,
+      "loss": 3.3585,
+      "theoretical_loss": 4.570065345782512,
+      "tokens_seen": 137625600
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048398194583751256,
+      "loss": 3.331,
+      "theoretical_loss": 4.569777277576713,
+      "tokens_seen": 137691136
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048397191574724174,
+      "loss": 3.2636,
+      "theoretical_loss": 4.569489384818334,
+      "tokens_seen": 137756672
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004839618856569709,
+      "loss": 3.3772,
+      "theoretical_loss": 4.569201667317128,
+      "tokens_seen": 137822208
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004839518555667001,
+      "loss": 3.1082,
+      "theoretical_loss": 4.568914124883145,
+      "tokens_seen": 137887744
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004839418254764293,
+      "loss": 3.567,
+      "theoretical_loss": 4.568626757326731,
+      "tokens_seen": 137953280
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004839317953861585,
+      "loss": 3.3668,
+      "theoretical_loss": 4.568339564458528,
+      "tokens_seen": 138018816
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048392176529588765,
+      "loss": 3.2898,
+      "theoretical_loss": 4.568052546089472,
+      "tokens_seen": 138084352
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004839117352056169,
+      "loss": 3.2154,
+      "theoretical_loss": 4.567765702030792,
+      "tokens_seen": 138149888
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048390170511534606,
+      "loss": 3.3787,
+      "theoretical_loss": 4.567479032094015,
+      "tokens_seen": 138215424
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048389167502507524,
+      "loss": 3.2051,
+      "theoretical_loss": 4.567192536090956,
+      "tokens_seen": 138280960
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004838816449348044,
+      "loss": 3.2675,
+      "theoretical_loss": 4.566906213833725,
+      "tokens_seen": 138346496
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004838716148445336,
+      "loss": 3.4467,
+      "theoretical_loss": 4.566620065134725,
+      "tokens_seen": 138412032
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004838615847542628,
+      "loss": 3.3219,
+      "theoretical_loss": 4.566334089806647,
+      "tokens_seen": 138477568
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.000483851554663992,
+      "loss": 3.3479,
+      "theoretical_loss": 4.566048287662475,
+      "tokens_seen": 138543104
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048384152457372115,
+      "loss": 3.3473,
+      "theoretical_loss": 4.565762658515482,
+      "tokens_seen": 138608640
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004838314944834504,
+      "loss": 3.3177,
+      "theoretical_loss": 4.565477202179232,
+      "tokens_seen": 138674176
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004838214643931795,
+      "loss": 3.2072,
+      "theoretical_loss": 4.565191918467576,
+      "tokens_seen": 138739712
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048381143430290875,
+      "loss": 3.4187,
+      "theoretical_loss": 4.564906807194655,
+      "tokens_seen": 138805248
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048380140421263793,
+      "loss": 2.8605,
+      "theoretical_loss": 4.564621868174897,
+      "tokens_seen": 138870784
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004837913741223671,
+      "loss": 3.0615,
+      "theoretical_loss": 4.564337101223017,
+      "tokens_seen": 138936320
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004837813440320963,
+      "loss": 3.3648,
+      "theoretical_loss": 4.564052506154015,
+      "tokens_seen": 139001856
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048377131394182547,
+      "loss": 3.051,
+      "theoretical_loss": 4.563768082783182,
+      "tokens_seen": 139067392
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048376128385155465,
+      "loss": 3.1675,
+      "theoretical_loss": 4.563483830926089,
+      "tokens_seen": 139132928
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004837512537612839,
+      "loss": 3.1113,
+      "theoretical_loss": 4.563199750398592,
+      "tokens_seen": 139198464
+    },
+    {
+      "epoch": 0.04,
+      "objective/train/docs_used": 87912,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 3.470696210861206,
+      "objective/train/theoretical_loss": 4.562915841016838,
+      "objective/train/tokens_used": 159724000,
+      "theoretical_loss": 4.562915841016838,
+      "tokens_seen": 139264000
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.000483741223671013,
+      "loss": 3.5703,
+      "theoretical_loss": 4.562915841016838,
+      "tokens_seen": 139264000
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048373119358074225,
+      "loss": 3.2538,
+      "theoretical_loss": 4.5626321025972505,
+      "tokens_seen": 139329536
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048372116349047143,
+      "loss": 3.2385,
+      "theoretical_loss": 4.562348534956538,
+      "tokens_seen": 139395072
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004837111334002006,
+      "loss": 3.4436,
+      "theoretical_loss": 4.562065137911691,
+      "tokens_seen": 139460608
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004837011033099298,
+      "loss": 3.1403,
+      "theoretical_loss": 4.561781911279987,
+      "tokens_seen": 139526144
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.000483691073219659,
+      "loss": 3.1669,
+      "theoretical_loss": 4.5614988548789785,
+      "tokens_seen": 139591680
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048368104312938816,
+      "loss": 3.1656,
+      "theoretical_loss": 4.561215968526502,
+      "tokens_seen": 139657216
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004836710130391174,
+      "loss": 3.2068,
+      "theoretical_loss": 4.560933252040675,
+      "tokens_seen": 139722752
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004836609829488465,
+      "loss": 3.2876,
+      "theoretical_loss": 4.56065070523989,
+      "tokens_seen": 139788288
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048365095285857575,
+      "loss": 3.7268,
+      "theoretical_loss": 4.560368327942827,
+      "tokens_seen": 139853824
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004836409227683049,
+      "loss": 3.0936,
+      "theoretical_loss": 4.560086119968435,
+      "tokens_seen": 139919360
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004836308926780341,
+      "loss": 3.362,
+      "theoretical_loss": 4.559804081135949,
+      "tokens_seen": 139984896
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004836208625877633,
+      "loss": 3.4023,
+      "theoretical_loss": 4.559522211264877,
+      "tokens_seen": 140050432
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004836108324974925,
+      "loss": 3.299,
+      "theoretical_loss": 4.559240510175006,
+      "tokens_seen": 140115968
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048360080240722166,
+      "loss": 3.5194,
+      "theoretical_loss": 4.558958977686398,
+      "tokens_seen": 140181504
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048359077231695084,
+      "loss": 3.1333,
+      "theoretical_loss": 4.558677613619393,
+      "tokens_seen": 140247040
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048358074222668,
+      "loss": 3.1792,
+      "theoretical_loss": 4.558396417794603,
+      "tokens_seen": 140312576
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048357071213640926,
+      "loss": 3.2967,
+      "theoretical_loss": 4.558115390032916,
+      "tokens_seen": 140378112
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004835606820461384,
+      "loss": 3.2453,
+      "theoretical_loss": 4.557834530155497,
+      "tokens_seen": 140443648
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004835506519558676,
+      "loss": 3.2309,
+      "theoretical_loss": 4.5575538379837806,
+      "tokens_seen": 140509184
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004835406218655968,
+      "loss": 3.1676,
+      "theoretical_loss": 4.557273313339477,
+      "tokens_seen": 140574720
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.000483530591775326,
+      "loss": 3.3045,
+      "theoretical_loss": 4.556992956044569,
+      "tokens_seen": 140640256
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048352056168505516,
+      "loss": 3.4098,
+      "theoretical_loss": 4.55671276592131,
+      "tokens_seen": 140705792
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048351053159478434,
+      "loss": 3.2187,
+      "theoretical_loss": 4.556432742792225,
+      "tokens_seen": 140771328
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004835005015045135,
+      "loss": 3.2732,
+      "theoretical_loss": 4.55615288648011,
+      "tokens_seen": 140836864
+    },
+    {
+      "epoch": 0.04,
+      "objective/train/docs_used": 88907,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 3.168332576751709,
+      "objective/train/theoretical_loss": 4.555873196808033,
+      "objective/train/tokens_used": 161362400,
+      "theoretical_loss": 4.555873196808033,
+      "tokens_seen": 140902400
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048349047141424276,
+      "loss": 3.2827,
+      "theoretical_loss": 4.555873196808033,
+      "tokens_seen": 140902400
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048348044132397194,
+      "loss": 3.0851,
+      "theoretical_loss": 4.555593673599329,
+      "tokens_seen": 140967936
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004834704112337011,
+      "loss": 3.2459,
+      "theoretical_loss": 4.555314316677606,
+      "tokens_seen": 141033472
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004834603811434303,
+      "loss": 3.1295,
+      "theoretical_loss": 4.555035125866738,
+      "tokens_seen": 141099008
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004834503510531595,
+      "loss": 3.2718,
+      "theoretical_loss": 4.5547561009908675,
+      "tokens_seen": 141164544
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004834403209628887,
+      "loss": 3.4012,
+      "theoretical_loss": 4.554477241874405,
+      "tokens_seen": 141230080
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048343029087261785,
+      "loss": 3.2815,
+      "theoretical_loss": 4.554198548342029,
+      "tokens_seen": 141295616
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004834202607823471,
+      "loss": 3.3752,
+      "theoretical_loss": 4.553920020218683,
+      "tokens_seen": 141361152
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048341023069207626,
+      "loss": 3.4304,
+      "theoretical_loss": 4.55364165732958,
+      "tokens_seen": 141426688
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048340020060180544,
+      "loss": 3.06,
+      "theoretical_loss": 4.553363459500193,
+      "tokens_seen": 141492224
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004833901705115346,
+      "loss": 3.4717,
+      "theoretical_loss": 4.553085426556265,
+      "tokens_seen": 141557760
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004833801404212638,
+      "loss": 3.2863,
+      "theoretical_loss": 4.552807558323801,
+      "tokens_seen": 141623296
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.000483370110330993,
+      "loss": 3.5702,
+      "theoretical_loss": 4.552529854629072,
+      "tokens_seen": 141688832
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004833600802407222,
+      "loss": 3.3102,
+      "theoretical_loss": 4.552252315298611,
+      "tokens_seen": 141754368
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048335005015045135,
+      "loss": 3.2317,
+      "theoretical_loss": 4.551974940159212,
+      "tokens_seen": 141819904
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004833400200601806,
+      "loss": 3.3219,
+      "theoretical_loss": 4.551697729037938,
+      "tokens_seen": 141885440
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004833299899699097,
+      "loss": 3.4492,
+      "theoretical_loss": 4.551420681762107,
+      "tokens_seen": 141950976
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048331995987963895,
+      "loss": 3.2021,
+      "theoretical_loss": 4.551143798159299,
+      "tokens_seen": 142016512
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048330992978936813,
+      "loss": 3.2159,
+      "theoretical_loss": 4.550867078057361,
+      "tokens_seen": 142082048
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004832998996990973,
+      "loss": 3.2044,
+      "theoretical_loss": 4.550590521284393,
+      "tokens_seen": 142147584
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004832898696088265,
+      "loss": 3.1647,
+      "theoretical_loss": 4.550314127668761,
+      "tokens_seen": 142213120
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048327983951855567,
+      "loss": 3.4173,
+      "theoretical_loss": 4.550037897039086,
+      "tokens_seen": 142278656
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048326980942828485,
+      "loss": 3.0888,
+      "theoretical_loss": 4.54976182922425,
+      "tokens_seen": 142344192
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004832597793380141,
+      "loss": 3.0527,
+      "theoretical_loss": 4.549485924053392,
+      "tokens_seen": 142409728
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004832497492477432,
+      "loss": 3.3659,
+      "theoretical_loss": 4.54921018135591,
+      "tokens_seen": 142475264
+    },
+    {
+      "epoch": 0.04,
+      "objective/train/docs_used": 89401,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 3.601848602294922,
+      "objective/train/theoretical_loss": 4.548934600961459,
+      "objective/train/tokens_used": 163000800,
+      "theoretical_loss": 4.548934600961459,
+      "tokens_seen": 142540800
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048323971915747245,
+      "loss": 3.4687,
+      "theoretical_loss": 4.548934600961459,
+      "tokens_seen": 142540800
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048322968906720163,
+      "loss": 3.2753,
+      "theoretical_loss": 4.54865918269995,
+      "tokens_seen": 142606336
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004832196589769308,
+      "loss": 3.1228,
+      "theoretical_loss": 4.548383926401552,
+      "tokens_seen": 142671872
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048320962888666,
+      "loss": 3.0876,
+      "theoretical_loss": 4.548108831896688,
+      "tokens_seen": 142737408
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004831995987963892,
+      "loss": 3.44,
+      "theoretical_loss": 4.547833899016037,
+      "tokens_seen": 142802944
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048318956870611836,
+      "loss": 3.0777,
+      "theoretical_loss": 4.547559127590535,
+      "tokens_seen": 142868480
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004831795386158476,
+      "loss": 3.5091,
+      "theoretical_loss": 4.547284517451367,
+      "tokens_seen": 142934016
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004831695085255767,
+      "loss": 3.3175,
+      "theoretical_loss": 4.547010068429978,
+      "tokens_seen": 142999552
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048315947843530595,
+      "loss": 3.1291,
+      "theoretical_loss": 4.546735780358061,
+      "tokens_seen": 143065088
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004831494483450351,
+      "loss": 3.1034,
+      "theoretical_loss": 4.546461653067565,
+      "tokens_seen": 143130624
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004831394182547643,
+      "loss": 3.0849,
+      "theoretical_loss": 4.54618768639069,
+      "tokens_seen": 143196160
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004831293881644935,
+      "loss": 3.0665,
+      "theoretical_loss": 4.545913880159889,
+      "tokens_seen": 143261696
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004831193580742227,
+      "loss": 3.2806,
+      "theoretical_loss": 4.545640234207866,
+      "tokens_seen": 143327232
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048310932798395186,
+      "loss": 3.3249,
+      "theoretical_loss": 4.545366748367574,
+      "tokens_seen": 143392768
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048309929789368104,
+      "loss": 3.4583,
+      "theoretical_loss": 4.545093422472218,
+      "tokens_seen": 143458304
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004830892678034102,
+      "loss": 3.5278,
+      "theoretical_loss": 4.544820256355253,
+      "tokens_seen": 143523840
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048307923771313946,
+      "loss": 3.0959,
+      "theoretical_loss": 4.544547249850384,
+      "tokens_seen": 143589376
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004830692076228686,
+      "loss": 3.4636,
+      "theoretical_loss": 4.544274402791562,
+      "tokens_seen": 143654912
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004830591775325978,
+      "loss": 3.1348,
+      "theoretical_loss": 4.544001715012989,
+      "tokens_seen": 143720448
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.000483049147442327,
+      "loss": 3.3713,
+      "theoretical_loss": 4.543729186349115,
+      "tokens_seen": 143785984
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004830391173520562,
+      "loss": 3.334,
+      "theoretical_loss": 4.5434568166346345,
+      "tokens_seen": 143851520
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048302908726178536,
+      "loss": 3.2556,
+      "theoretical_loss": 4.543184605704493,
+      "tokens_seen": 143917056
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048301905717151454,
+      "loss": 3.24,
+      "theoretical_loss": 4.542912553393879,
+      "tokens_seen": 143982592
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004830090270812437,
+      "loss": 3.0543,
+      "theoretical_loss": 4.542640659538229,
+      "tokens_seen": 144048128
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048299899699097296,
+      "loss": 3.3306,
+      "theoretical_loss": 4.542368923973226,
+      "tokens_seen": 144113664
+    },
+    {
+      "epoch": 0.04,
+      "objective/train/docs_used": 90537,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 3.076249361038208,
+      "objective/train/theoretical_loss": 4.542097346534794,
+      "objective/train/tokens_used": 164639200,
+      "theoretical_loss": 4.542097346534794,
+      "tokens_seen": 144179200
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004829889669007021,
+      "loss": 3.2985,
+      "theoretical_loss": 4.542097346534794,
+      "tokens_seen": 144179200
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004829789368104313,
+      "loss": 3.2073,
+      "theoretical_loss": 4.541825927059106,
+      "tokens_seen": 144244736
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048296890672016045,
+      "loss": 3.3592,
+      "theoretical_loss": 4.541554665382575,
+      "tokens_seen": 144310272
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004829588766298897,
+      "loss": 3.5993,
+      "theoretical_loss": 4.541283561341862,
+      "tokens_seen": 144375808
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048294884653961887,
+      "loss": 3.0916,
+      "theoretical_loss": 4.541012614773868,
+      "tokens_seen": 144441344
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048293881644934805,
+      "loss": 3.2114,
+      "theoretical_loss": 4.540741825515736,
+      "tokens_seen": 144506880
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048292878635907723,
+      "loss": 3.4082,
+      "theoretical_loss": 4.540471193404855,
+      "tokens_seen": 144572416
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048291875626880646,
+      "loss": 3.1695,
+      "theoretical_loss": 4.54020071827885,
+      "tokens_seen": 144637952
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004829087261785356,
+      "loss": 3.4008,
+      "theoretical_loss": 4.539930399975596,
+      "tokens_seen": 144703488
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004828986960882648,
+      "loss": 3.1431,
+      "theoretical_loss": 4.539660238333198,
+      "tokens_seen": 144769024
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048288866599799395,
+      "loss": 3.3694,
+      "theoretical_loss": 4.53939023319001,
+      "tokens_seen": 144834560
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004828786359077232,
+      "loss": 3.3767,
+      "theoretical_loss": 4.539120384384621,
+      "tokens_seen": 144900096
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048286860581745237,
+      "loss": 3.5769,
+      "theoretical_loss": 4.538850691755861,
+      "tokens_seen": 144965632
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048285857572718155,
+      "loss": 3.3304,
+      "theoretical_loss": 4.538581155142799,
+      "tokens_seen": 145031168
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048284854563691073,
+      "loss": 3.1185,
+      "theoretical_loss": 4.538311774384743,
+      "tokens_seen": 145096704
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004828385155466399,
+      "loss": 3.4686,
+      "theoretical_loss": 4.538042549321238,
+      "tokens_seen": 145162240
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004828284854563691,
+      "loss": 3.1808,
+      "theoretical_loss": 4.537773479792067,
+      "tokens_seen": 145227776
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048281845536609833,
+      "loss": 3.2259,
+      "theoretical_loss": 4.537504565637248,
+      "tokens_seen": 145293312
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048280842527582746,
+      "loss": 3.3492,
+      "theoretical_loss": 4.537235806697041,
+      "tokens_seen": 145358848
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004827983951855567,
+      "loss": 3.4474,
+      "theoretical_loss": 4.536967202811935,
+      "tokens_seen": 145424384
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004827883650952858,
+      "loss": 3.2921,
+      "theoretical_loss": 4.536698753822661,
+      "tokens_seen": 145489920
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048277833500501505,
+      "loss": 3.4976,
+      "theoretical_loss": 4.53643045957018,
+      "tokens_seen": 145555456
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048276830491474423,
+      "loss": 3.4497,
+      "theoretical_loss": 4.536162319895695,
+      "tokens_seen": 145620992
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004827582748244734,
+      "loss": 3.4119,
+      "theoretical_loss": 4.535894334640634,
+      "tokens_seen": 145686528
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004827482447342026,
+      "loss": 3.0995,
+      "theoretical_loss": 4.535626503646666,
+      "tokens_seen": 145752064
+    },
+    {
+      "epoch": 0.04,
+      "objective/train/docs_used": 91176,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 2.9223389625549316,
+      "objective/train/theoretical_loss": 4.535358826755691,
+      "objective/train/tokens_used": 166277600,
+      "theoretical_loss": 4.535358826755691,
+      "tokens_seen": 145817600
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048273821464393183,
+      "loss": 3.2178,
+      "theoretical_loss": 4.535358826755691,
+      "tokens_seen": 145817600
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.000482728184553661,
+      "loss": 3.4704,
+      "theoretical_loss": 4.535091303809842,
+      "tokens_seen": 145883136
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004827181544633902,
+      "loss": 3.2936,
+      "theoretical_loss": 4.534823934651484,
+      "tokens_seen": 145948672
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004827081243731194,
+      "loss": 3.1424,
+      "theoretical_loss": 4.5345567191232155,
+      "tokens_seen": 146014208
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048269809428284856,
+      "loss": 3.3168,
+      "theoretical_loss": 4.534289657067865,
+      "tokens_seen": 146079744
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004826880641925778,
+      "loss": 2.9011,
+      "theoretical_loss": 4.5340227483284945,
+      "tokens_seen": 146145280
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004826780341023069,
+      "loss": 3.2725,
+      "theoretical_loss": 4.533755992748395,
+      "tokens_seen": 146210816
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048266800401203615,
+      "loss": 3.2624,
+      "theoretical_loss": 4.533489390171088,
+      "tokens_seen": 146276352
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004826579739217653,
+      "loss": 3.4202,
+      "theoretical_loss": 4.533222940440325,
+      "tokens_seen": 146341888
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004826479438314945,
+      "loss": 3.1386,
+      "theoretical_loss": 4.532956643400087,
+      "tokens_seen": 146407424
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004826379137412237,
+      "loss": 3.3923,
+      "theoretical_loss": 4.532690498894585,
+      "tokens_seen": 146472960
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004826278836509529,
+      "loss": 3.4023,
+      "theoretical_loss": 4.5324245067682565,
+      "tokens_seen": 146538496
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048261785356068206,
+      "loss": 3.1156,
+      "theoretical_loss": 4.532158666865769,
+      "tokens_seen": 146604032
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048260782347041124,
+      "loss": 3.1851,
+      "theoretical_loss": 4.5318929790320155,
+      "tokens_seen": 146669568
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004825977933801404,
+      "loss": 3.0943,
+      "theoretical_loss": 4.53162744311212,
+      "tokens_seen": 146735104
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048258776328986966,
+      "loss": 3.392,
+      "theoretical_loss": 4.53136205895143,
+      "tokens_seen": 146800640
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004825777331995988,
+      "loss": 3.4047,
+      "theoretical_loss": 4.531096826395519,
+      "tokens_seen": 146866176
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.000482567703109328,
+      "loss": 3.2556,
+      "theoretical_loss": 4.5308317452901905,
+      "tokens_seen": 146931712
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004825576730190572,
+      "loss": 3.1769,
+      "theoretical_loss": 4.530566815481469,
+      "tokens_seen": 146997248
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004825476429287864,
+      "loss": 3.2533,
+      "theoretical_loss": 4.530302036815607,
+      "tokens_seen": 147062784
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048253761283851556,
+      "loss": 3.3405,
+      "theoretical_loss": 4.53003740913908,
+      "tokens_seen": 147128320
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048252758274824474,
+      "loss": 3.3032,
+      "theoretical_loss": 4.529772932298588,
+      "tokens_seen": 147193856
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004825175526579739,
+      "loss": 3.4738,
+      "theoretical_loss": 4.529508606141057,
+      "tokens_seen": 147259392
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048250752256770316,
+      "loss": 3.2304,
+      "theoretical_loss": 4.529244430513632,
+      "tokens_seen": 147324928
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004824974924774323,
+      "loss": 3.2967,
+      "theoretical_loss": 4.528980405263686,
+      "tokens_seen": 147390464
+    },
+    {
+      "epoch": 0.04,
+      "objective/train/docs_used": 92532,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 3.4305601119995117,
+      "objective/train/theoretical_loss": 4.528716530238812,
+      "objective/train/tokens_used": 167916000,
+      "theoretical_loss": 4.528716530238812,
+      "tokens_seen": 147456000
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004824874623871615,
+      "loss": 3.4659,
+      "theoretical_loss": 4.528716530238812,
+      "tokens_seen": 147456000
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048247743229689065,
+      "loss": 3.2645,
+      "theoretical_loss": 4.528452805286824,
+      "tokens_seen": 147521536
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004824674022066199,
+      "loss": 3.2269,
+      "theoretical_loss": 4.528189230255759,
+      "tokens_seen": 147587072
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048245737211634907,
+      "loss": 3.2869,
+      "theoretical_loss": 4.527925804993875,
+      "tokens_seen": 147652608
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048244734202607825,
+      "loss": 3.3874,
+      "theoretical_loss": 4.5276625293496515,
+      "tokens_seen": 147718144
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048243731193580743,
+      "loss": 3.1961,
+      "theoretical_loss": 4.527399403171788,
+      "tokens_seen": 147783680
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048242728184553666,
+      "loss": 3.3541,
+      "theoretical_loss": 4.5271364263092035,
+      "tokens_seen": 147849216
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004824172517552658,
+      "loss": 3.2284,
+      "theoretical_loss": 4.526873598611038,
+      "tokens_seen": 147914752
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.000482407221664995,
+      "loss": 3.1859,
+      "theoretical_loss": 4.52661091992665,
+      "tokens_seen": 147980288
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048239719157472415,
+      "loss": 3.3306,
+      "theoretical_loss": 4.526348390105614,
+      "tokens_seen": 148045824
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004823871614844534,
+      "loss": 3.0245,
+      "theoretical_loss": 4.526086008997728,
+      "tokens_seen": 148111360
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048237713139418257,
+      "loss": 3.1419,
+      "theoretical_loss": 4.525823776453004,
+      "tokens_seen": 148176896
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048236710130391175,
+      "loss": 3.0672,
+      "theoretical_loss": 4.525561692321673,
+      "tokens_seen": 148242432
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048235707121364093,
+      "loss": 3.259,
+      "theoretical_loss": 4.525299756454182,
+      "tokens_seen": 148307968
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004823470411233701,
+      "loss": 3.3995,
+      "theoretical_loss": 4.525037968701198,
+      "tokens_seen": 148373504
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004823370110330993,
+      "loss": 3.1846,
+      "theoretical_loss": 4.5247763289136,
+      "tokens_seen": 148439040
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00048232698094282853,
+      "loss": 3.3233,
+      "theoretical_loss": 4.524514836942485,
+      "tokens_seen": 148504576
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00048231695085255766,
+      "loss": 3.1244,
+      "theoretical_loss": 4.524253492639165,
+      "tokens_seen": 148570112
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004823069207622869,
+      "loss": 3.1294,
+      "theoretical_loss": 4.523992295855169,
+      "tokens_seen": 148635648
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.000482296890672016,
+      "loss": 3.4438,
+      "theoretical_loss": 4.523731246442237,
+      "tokens_seen": 148701184
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00048228686058174525,
+      "loss": 3.2735,
+      "theoretical_loss": 4.523470344252326,
+      "tokens_seen": 148766720
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00048227683049147443,
+      "loss": 3.2357,
+      "theoretical_loss": 4.523209589137606,
+      "tokens_seen": 148832256
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004822668004012036,
+      "loss": 3.3941,
+      "theoretical_loss": 4.522948980950461,
+      "tokens_seen": 148897792
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004822567703109328,
+      "loss": 3.0532,
+      "theoretical_loss": 4.522688519543487,
+      "tokens_seen": 148963328
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00048224674022066203,
+      "loss": 3.1432,
+      "theoretical_loss": 4.5224282047694935,
+      "tokens_seen": 149028864
+    },
+    {
+      "epoch": 0.05,
+      "objective/train/docs_used": 93022,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 3.5269510746002197,
+      "objective/train/theoretical_loss": 4.522168036481502,
+      "objective/train/tokens_used": 169554400,
+      "theoretical_loss": 4.522168036481502,
+      "tokens_seen": 149094400
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00048223671013039116,
+      "loss": 3.5101,
+      "theoretical_loss": 4.522168036481502,
+      "tokens_seen": 149094400
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004822266800401204,
+      "loss": 3.2561,
+      "theoretical_loss": 4.521908014532746,
+      "tokens_seen": 149159936
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004822166499498495,
+      "loss": 3.3344,
+      "theoretical_loss": 4.52164813877667,
+      "tokens_seen": 149225472
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00048220661985957876,
+      "loss": 3.2209,
+      "theoretical_loss": 4.52138840906693,
+      "tokens_seen": 149291008
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00048219658976930794,
+      "loss": 3.1369,
+      "theoretical_loss": 4.521128825257392,
+      "tokens_seen": 149356544
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004821865596790371,
+      "loss": 3.1751,
+      "theoretical_loss": 4.520869387202134,
+      "tokens_seen": 149422080
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004821765295887663,
+      "loss": 2.9797,
+      "theoretical_loss": 4.52061009475544,
+      "tokens_seen": 149487616
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004821664994984955,
+      "loss": 3.196,
+      "theoretical_loss": 4.52035094777181,
+      "tokens_seen": 149553152
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00048215646940822466,
+      "loss": 3.3068,
+      "theoretical_loss": 4.520091946105946,
+      "tokens_seen": 149618688
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004821464393179539,
+      "loss": 3.4288,
+      "theoretical_loss": 4.519833089612764,
+      "tokens_seen": 149684224
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.000482136409227683,
+      "loss": 3.2192,
+      "theoretical_loss": 4.519574378147384,
+      "tokens_seen": 149749760
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00048212637913741226,
+      "loss": 3.291,
+      "theoretical_loss": 4.519315811565137,
+      "tokens_seen": 149815296
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004821163490471414,
+      "loss": 3.3752,
+      "theoretical_loss": 4.51905738972156,
+      "tokens_seen": 149880832
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004821063189568706,
+      "loss": 3.6035,
+      "theoretical_loss": 4.518799112472401,
+      "tokens_seen": 149946368
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004820962888665998,
+      "loss": 3.5239,
+      "theoretical_loss": 4.518540979673606,
+      "tokens_seen": 150011904
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.000482086258776329,
+      "loss": 3.1651,
+      "theoretical_loss": 4.518282991181337,
+      "tokens_seen": 150077440
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00048207622868605816,
+      "loss": 3.1406,
+      "theoretical_loss": 4.518025146851956,
+      "tokens_seen": 150142976
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004820661985957874,
+      "loss": 3.2538,
+      "theoretical_loss": 4.517767446542033,
+      "tokens_seen": 150208512
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00048205616850551653,
+      "loss": 3.1976,
+      "theoretical_loss": 4.517509890108343,
+      "tokens_seen": 150274048
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00048204613841524576,
+      "loss": 3.1097,
+      "theoretical_loss": 4.5172524774078635,
+      "tokens_seen": 150339584
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004820361083249749,
+      "loss": 3.3081,
+      "theoretical_loss": 4.5169952082977805,
+      "tokens_seen": 150405120
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004820260782347041,
+      "loss": 3.2403,
+      "theoretical_loss": 4.51673808263548,
+      "tokens_seen": 150470656
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004820160481444333,
+      "loss": 3.0648,
+      "theoretical_loss": 4.516481100278555,
+      "tokens_seen": 150536192
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004820060180541625,
+      "loss": 3.1256,
+      "theoretical_loss": 4.516224261084799,
+      "tokens_seen": 150601728
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00048199598796389167,
+      "loss": 3.0304,
+      "theoretical_loss": 4.515967564912211,
+      "tokens_seen": 150667264
+    },
+    {
+      "epoch": 0.05,
+      "objective/train/docs_used": 94173,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 3.2401325702667236,
+      "objective/train/theoretical_loss": 4.515711011618992,
+      "objective/train/tokens_used": 171192800,
+      "theoretical_loss": 4.515711011618992,
+      "tokens_seen": 150732800
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00048198595787362085,
+      "loss": 3.2269,
+      "theoretical_loss": 4.515711011618992,
+      "tokens_seen": 150732800
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004819759277833501,
+      "loss": 3.1091,
+      "theoretical_loss": 4.515454601063542,
+      "tokens_seen": 150798336
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00048196589769307927,
+      "loss": 3.2182,
+      "theoretical_loss": 4.5151983331044665,
+      "tokens_seen": 150863872
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00048195586760280845,
+      "loss": 3.2107,
+      "theoretical_loss": 4.514942207600572,
+      "tokens_seen": 150929408
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00048194583751253763,
+      "loss": 3.1827,
+      "theoretical_loss": 4.514686224410863,
+      "tokens_seen": 150994944
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00048193580742226686,
+      "loss": 3.3203,
+      "theoretical_loss": 4.514430383394549,
+      "tokens_seen": 151060480
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.000481925777331996,
+      "loss": 3.0874,
+      "theoretical_loss": 4.514174684411037,
+      "tokens_seen": 151126016
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004819157472417252,
+      "loss": 3.2352,
+      "theoretical_loss": 4.513919127319935,
+      "tokens_seen": 151191552
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00048190571715145435,
+      "loss": 3.3179,
+      "theoretical_loss": 4.513663711981049,
+      "tokens_seen": 151257088
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004818956870611836,
+      "loss": 3.266,
+      "theoretical_loss": 4.513408438254386,
+      "tokens_seen": 151322624
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00048188565697091277,
+      "loss": 3.1682,
+      "theoretical_loss": 4.5131533060001505,
+      "tokens_seen": 151388160
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00048187562688064195,
+      "loss": 3.2016,
+      "theoretical_loss": 4.512898315078747,
+      "tokens_seen": 151453696
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00048186559679037113,
+      "loss": 3.2221,
+      "theoretical_loss": 4.512643465350775,
+      "tokens_seen": 151519232
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004818555667001003,
+      "loss": 3.394,
+      "theoretical_loss": 4.512388756677037,
+      "tokens_seen": 151584768
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004818455366098295,
+      "loss": 3.3002,
+      "theoretical_loss": 4.5121341889185285,
+      "tokens_seen": 151650304
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00048183550651955873,
+      "loss": 3.131,
+      "theoretical_loss": 4.511879761936442,
+      "tokens_seen": 151715840
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00048182547642928786,
+      "loss": 3.3703,
+      "theoretical_loss": 4.511625475592169,
+      "tokens_seen": 151781376
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004818154463390171,
+      "loss": 3.2669,
+      "theoretical_loss": 4.511371329747295,
+      "tokens_seen": 151846912
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004818054162487462,
+      "loss": 3.0424,
+      "theoretical_loss": 4.511117324263603,
+      "tokens_seen": 151912448
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00048179538615847545,
+      "loss": 3.6565,
+      "theoretical_loss": 4.5108634590030725,
+      "tokens_seen": 151977984
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00048178535606820463,
+      "loss": 3.411,
+      "theoretical_loss": 4.510609733827873,
+      "tokens_seen": 152043520
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004817753259779338,
+      "loss": 3.1894,
+      "theoretical_loss": 4.5103561486003745,
+      "tokens_seen": 152109056
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.000481765295887663,
+      "loss": 3.053,
+      "theoretical_loss": 4.51010270318314,
+      "tokens_seen": 152174592
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00048175526579739223,
+      "loss": 3.1504,
+      "theoretical_loss": 4.509849397438925,
+      "tokens_seen": 152240128
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00048174523570712136,
+      "loss": 3.2262,
+      "theoretical_loss": 4.509596231230679,
+      "tokens_seen": 152305664
+    },
+    {
+      "epoch": 0.05,
+      "objective/train/docs_used": 94867,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 3.429643392562866,
+      "objective/train/theoretical_loss": 4.509343204421548,
+      "objective/train/tokens_used": 172831200,
+      "theoretical_loss": 4.509343204421548,
+      "tokens_seen": 152371200
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004817352056168506,
+      "loss": 3.4209,
+      "theoretical_loss": 4.509343204421548,
+      "tokens_seen": 152371200
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004817251755265797,
+      "loss": 3.2597,
+      "theoretical_loss": 4.509090316874865,
+      "tokens_seen": 152436736
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00048171514543630896,
+      "loss": 3.2857,
+      "theoretical_loss": 4.508837568454162,
+      "tokens_seen": 152502272
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00048170511534603814,
+      "loss": 3.5512,
+      "theoretical_loss": 4.508584959023159,
+      "tokens_seen": 152567808
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004816950852557673,
+      "loss": 3.3878,
+      "theoretical_loss": 4.50833248844577,
+      "tokens_seen": 152633344
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004816850551654965,
+      "loss": 3.3808,
+      "theoretical_loss": 4.5080801565861,
+      "tokens_seen": 152698880
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004816750250752257,
+      "loss": 3.1163,
+      "theoretical_loss": 4.507827963308447,
+      "tokens_seen": 152764416
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00048166499498495486,
+      "loss": 3.2056,
+      "theoretical_loss": 4.507575908477296,
+      "tokens_seen": 152829952
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004816549648946841,
+      "loss": 3.1235,
+      "theoretical_loss": 4.507323991957325,
+      "tokens_seen": 152895488
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004816449348044132,
+      "loss": 3.439,
+      "theoretical_loss": 4.507072213613402,
+      "tokens_seen": 152961024
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00048163490471414246,
+      "loss": 3.2998,
+      "theoretical_loss": 4.506820573310586,
+      "tokens_seen": 153026560
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004816248746238716,
+      "loss": 3.5354,
+      "theoretical_loss": 4.506569070914123,
+      "tokens_seen": 153092096
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004816148445336008,
+      "loss": 3.1693,
+      "theoretical_loss": 4.506317706289451,
+      "tokens_seen": 153157632
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00048160481444333,
+      "loss": 3.3531,
+      "theoretical_loss": 4.506066479302193,
+      "tokens_seen": 153223168
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004815947843530592,
+      "loss": 3.3859,
+      "theoretical_loss": 4.505815389818165,
+      "tokens_seen": 153288704
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00048158475426278837,
+      "loss": 3.2548,
+      "theoretical_loss": 4.505564437703368,
+      "tokens_seen": 153354240
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004815747241725176,
+      "loss": 3.1131,
+      "theoretical_loss": 4.5053136228239925,
+      "tokens_seen": 153419776
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00048156469408224673,
+      "loss": 3.3206,
+      "theoretical_loss": 4.505062945046413,
+      "tokens_seen": 153485312
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00048155466399197596,
+      "loss": 3.3988,
+      "theoretical_loss": 4.504812404237197,
+      "tokens_seen": 153550848
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004815446339017051,
+      "loss": 3.1752,
+      "theoretical_loss": 4.504562000263093,
+      "tokens_seen": 153616384
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004815346038114343,
+      "loss": 3.1504,
+      "theoretical_loss": 4.504311732991039,
+      "tokens_seen": 153681920
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004815245737211635,
+      "loss": 3.0018,
+      "theoretical_loss": 4.504061602288159,
+      "tokens_seen": 153747456
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004815145436308927,
+      "loss": 2.9879,
+      "theoretical_loss": 4.503811608021761,
+      "tokens_seen": 153812992
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00048150451354062187,
+      "loss": 3.1938,
+      "theoretical_loss": 4.50356175005934,
+      "tokens_seen": 153878528
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00048149448345035105,
+      "loss": 3.5199,
+      "theoretical_loss": 4.503312028268576,
+      "tokens_seen": 153944064
+    },
+    {
+      "epoch": 0.05,
+      "objective/train/docs_used": 95853,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 3.367169141769409,
+      "objective/train/theoretical_loss": 4.503062442517334,
+      "objective/train/tokens_used": 174469600,
+      "theoretical_loss": 4.503062442517334,
+      "tokens_seen": 154009600
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00048148445336008023,
+      "loss": 3.257,
+      "theoretical_loss": 4.503062442517334,
+      "tokens_seen": 154009600
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00048147442326980947,
+      "loss": 3.4149,
+      "theoretical_loss": 4.502812992673661,
+      "tokens_seen": 154075136
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004814643931795386,
+      "loss": 3.3831,
+      "theoretical_loss": 4.50256367860579,
+      "tokens_seen": 154140672
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00048145436308926783,
+      "loss": 3.2127,
+      "theoretical_loss": 4.502314500182138,
+      "tokens_seen": 154206208
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00048144433299899696,
+      "loss": 3.0253,
+      "theoretical_loss": 4.502065457271304,
+      "tokens_seen": 154271744
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004814343029087262,
+      "loss": 3.522,
+      "theoretical_loss": 4.501816549742072,
+      "tokens_seen": 154337280
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00048142427281845537,
+      "loss": 3.031,
+      "theoretical_loss": 4.501567777463406,
+      "tokens_seen": 154402816
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00048141424272818455,
+      "loss": 3.33,
+      "theoretical_loss": 4.501319140304456,
+      "tokens_seen": 154468352
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00048140421263791373,
+      "loss": 3.0943,
+      "theoretical_loss": 4.50107063813455,
+      "tokens_seen": 154533888
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00048139418254764297,
+      "loss": 2.9423,
+      "theoretical_loss": 4.5008222708232,
+      "tokens_seen": 154599424
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004813841524573721,
+      "loss": 3.2762,
+      "theoretical_loss": 4.500574038240099,
+      "tokens_seen": 154664960
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00048137412236710133,
+      "loss": 3.1341,
+      "theoretical_loss": 4.500325940255123,
+      "tokens_seen": 154730496
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00048136409227683046,
+      "loss": 3.323,
+      "theoretical_loss": 4.500077976738324,
+      "tokens_seen": 154796032
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004813540621865597,
+      "loss": 3.2968,
+      "theoretical_loss": 4.499830147559939,
+      "tokens_seen": 154861568
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004813440320962889,
+      "loss": 3.3287,
+      "theoretical_loss": 4.499582452590383,
+      "tokens_seen": 154927104
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00048133400200601806,
+      "loss": 3.4478,
+      "theoretical_loss": 4.499334891700251,
+      "tokens_seen": 154992640
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00048132397191574724,
+      "loss": 3.1801,
+      "theoretical_loss": 4.499087464760317,
+      "tokens_seen": 155058176
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004813139418254764,
+      "loss": 3.0457,
+      "theoretical_loss": 4.498840171641534,
+      "tokens_seen": 155123712
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004813039117352056,
+      "loss": 2.9473,
+      "theoretical_loss": 4.498593012215036,
+      "tokens_seen": 155189248
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00048129388164493483,
+      "loss": 3.1367,
+      "theoretical_loss": 4.498345986352133,
+      "tokens_seen": 155254784
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00048128385155466396,
+      "loss": 3.2789,
+      "theoretical_loss": 4.4980990939243135,
+      "tokens_seen": 155320320
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004812738214643932,
+      "loss": 3.265,
+      "theoretical_loss": 4.497852334803245,
+      "tokens_seen": 155385856
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004812637913741223,
+      "loss": 3.2818,
+      "theoretical_loss": 4.497605708860771,
+      "tokens_seen": 155451392
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00048125376128385156,
+      "loss": 3.148,
+      "theoretical_loss": 4.497359215968914,
+      "tokens_seen": 155516928
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00048124373119358074,
+      "loss": 3.1139,
+      "theoretical_loss": 4.497112855999871,
+      "tokens_seen": 155582464
+    },
+    {
+      "epoch": 0.05,
+      "objective/train/docs_used": 96994,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 3.577995777130127,
+      "objective/train/theoretical_loss": 4.496866628826018,
+      "objective/train/tokens_used": 176108000,
+      "theoretical_loss": 4.496866628826018,
+      "tokens_seen": 155648000
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004812337011033099,
+      "loss": 3.366,
+      "theoretical_loss": 4.496866628826018,
+      "tokens_seen": 155648000
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00048122367101303916,
+      "loss": 3.2348,
+      "theoretical_loss": 4.496620534319904,
+      "tokens_seen": 155713536
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00048121364092276834,
+      "loss": 3.1337,
+      "theoretical_loss": 4.496374572354256,
+      "tokens_seen": 155779072
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004812036108324975,
+      "loss": 3.4091,
+      "theoretical_loss": 4.496128742801979,
+      "tokens_seen": 155844608
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004811935807422267,
+      "loss": 3.3832,
+      "theoretical_loss": 4.495883045536148,
+      "tokens_seen": 155910144
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004811835506519559,
+      "loss": 3.1527,
+      "theoretical_loss": 4.495637480430018,
+      "tokens_seen": 155975680
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00048117352056168506,
+      "loss": 3.3667,
+      "theoretical_loss": 4.495392047357015,
+      "tokens_seen": 156041216
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004811634904714143,
+      "loss": 3.1819,
+      "theoretical_loss": 4.4951467461907395,
+      "tokens_seen": 156106752
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004811534603811434,
+      "loss": 3.2472,
+      "theoretical_loss": 4.494901576804969,
+      "tokens_seen": 156172288
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00048114343029087266,
+      "loss": 3.3371,
+      "theoretical_loss": 4.4946565390736515,
+      "tokens_seen": 156237824
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004811334002006018,
+      "loss": 3.3338,
+      "theoretical_loss": 4.494411632870911,
+      "tokens_seen": 156303360
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.000481123370110331,
+      "loss": 3.3003,
+      "theoretical_loss": 4.494166858071041,
+      "tokens_seen": 156368896
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004811133400200602,
+      "loss": 3.0684,
+      "theoretical_loss": 4.493922214548512,
+      "tokens_seen": 156434432
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004811033099297894,
+      "loss": 3.1874,
+      "theoretical_loss": 4.493677702177964,
+      "tokens_seen": 156499968
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00048109327983951857,
+      "loss": 3.2209,
+      "theoretical_loss": 4.493433320834209,
+      "tokens_seen": 156565504
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004810832497492478,
+      "loss": 3.3108,
+      "theoretical_loss": 4.493189070392235,
+      "tokens_seen": 156631040
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00048107321965897693,
+      "loss": 3.0176,
+      "theoretical_loss": 4.492944950727196,
+      "tokens_seen": 156696576
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00048106318956870616,
+      "loss": 3.0247,
+      "theoretical_loss": 4.492700961714419,
+      "tokens_seen": 156762112
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004810531594784353,
+      "loss": 3.1494,
+      "theoretical_loss": 4.492457103229405,
+      "tokens_seen": 156827648
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004810431293881645,
+      "loss": 3.4656,
+      "theoretical_loss": 4.492213375147822,
+      "tokens_seen": 156893184
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004810330992978937,
+      "loss": 3.1795,
+      "theoretical_loss": 4.491969777345509,
+      "tokens_seen": 156958720
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004810230692076229,
+      "loss": 3.2572,
+      "theoretical_loss": 4.491726309698475,
+      "tokens_seen": 157024256
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00048101303911735207,
+      "loss": 3.074,
+      "theoretical_loss": 4.491482972082901,
+      "tokens_seen": 157089792
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00048100300902708125,
+      "loss": 3.3692,
+      "theoretical_loss": 4.491239764375134,
+      "tokens_seen": 157155328
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00048099297893681043,
+      "loss": 3.166,
+      "theoretical_loss": 4.490996686451692,
+      "tokens_seen": 157220864
+    },
+    {
+      "epoch": 0.05,
+      "objective/train/docs_used": 97532,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 3.0817697048187256,
+      "objective/train/theoretical_loss": 4.4907537381892615,
+      "objective/train/tokens_used": 177746400,
+      "theoretical_loss": 4.4907537381892615,
+      "tokens_seen": 157286400
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00048098294884653967,
+      "loss": 3.0902,
+      "theoretical_loss": 4.4907537381892615,
+      "tokens_seen": 157286400
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004809729187562688,
+      "loss": 3.3092,
+      "theoretical_loss": 4.490510919464697,
+      "tokens_seen": 157351936
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00048096288866599803,
+      "loss": 3.1099,
+      "theoretical_loss": 4.4902682301550225,
+      "tokens_seen": 157417472
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00048095285857572716,
+      "loss": 2.9646,
+      "theoretical_loss": 4.4900256701374275,
+      "tokens_seen": 157483008
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004809428284854564,
+      "loss": 3.1576,
+      "theoretical_loss": 4.489783239289271,
+      "tokens_seen": 157548544
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00048093279839518557,
+      "loss": 3.1089,
+      "theoretical_loss": 4.48954093748808,
+      "tokens_seen": 157614080
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00048092276830491475,
+      "loss": 3.4034,
+      "theoretical_loss": 4.489298764611546,
+      "tokens_seen": 157679616
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00048091273821464393,
+      "loss": 2.9096,
+      "theoretical_loss": 4.48905672053753,
+      "tokens_seen": 157745152
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00048090270812437317,
+      "loss": 3.1601,
+      "theoretical_loss": 4.488814805144055,
+      "tokens_seen": 157810688
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004808926780341023,
+      "loss": 3.1648,
+      "theoretical_loss": 4.4885730183093155,
+      "tokens_seen": 157876224
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00048088264794383153,
+      "loss": 3.3537,
+      "theoretical_loss": 4.488331359911669,
+      "tokens_seen": 157941760
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00048087261785356066,
+      "loss": 3.1861,
+      "theoretical_loss": 4.488089829829638,
+      "tokens_seen": 158007296
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004808625877632899,
+      "loss": 3.1146,
+      "theoretical_loss": 4.487848427941911,
+      "tokens_seen": 158072832
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004808525576730191,
+      "loss": 3.4209,
+      "theoretical_loss": 4.487607154127343,
+      "tokens_seen": 158138368
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00048084252758274826,
+      "loss": 3.4348,
+      "theoretical_loss": 4.487366008264949,
+      "tokens_seen": 158203904
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00048083249749247744,
+      "loss": 3.3548,
+      "theoretical_loss": 4.487124990233915,
+      "tokens_seen": 158269440
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004808224674022066,
+      "loss": 3.2445,
+      "theoretical_loss": 4.486884099913585,
+      "tokens_seen": 158334976
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004808124373119358,
+      "loss": 3.5389,
+      "theoretical_loss": 4.486643337183471,
+      "tokens_seen": 158400512
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00048080240722166503,
+      "loss": 2.9913,
+      "theoretical_loss": 4.486402701923245,
+      "tokens_seen": 158466048
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00048079237713139416,
+      "loss": 3.1903,
+      "theoretical_loss": 4.486162194012745,
+      "tokens_seen": 158531584
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004807823470411234,
+      "loss": 3.3898,
+      "theoretical_loss": 4.48592181333197,
+      "tokens_seen": 158597120
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004807723169508525,
+      "loss": 3.134,
+      "theoretical_loss": 4.485681559761082,
+      "tokens_seen": 158662656
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00048076228686058176,
+      "loss": 3.1577,
+      "theoretical_loss": 4.485441433180408,
+      "tokens_seen": 158728192
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00048075225677031094,
+      "loss": 3.0772,
+      "theoretical_loss": 4.485201433470432,
+      "tokens_seen": 158793728
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004807422266800401,
+      "loss": 3.3051,
+      "theoretical_loss": 4.484961560511804,
+      "tokens_seen": 158859264
+    },
+    {
+      "epoch": 0.05,
+      "objective/train/docs_used": 98333,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 3.562363386154175,
+      "objective/train/theoretical_loss": 4.484721814185333,
+      "objective/train/tokens_used": 179384800,
+      "theoretical_loss": 4.484721814185333,
+      "tokens_seen": 158924800
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004807321965897693,
+      "loss": 3.3407,
+      "theoretical_loss": 4.484721814185333,
+      "tokens_seen": 158924800
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00048072216649949854,
+      "loss": 3.439,
+      "theoretical_loss": 4.48448219437199,
+      "tokens_seen": 158990336
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00048071213640922766,
+      "loss": 3.5068,
+      "theoretical_loss": 4.484242700952908,
+      "tokens_seen": 159055872
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004807021063189569,
+      "loss": 3.4014,
+      "theoretical_loss": 4.48400333380938,
+      "tokens_seen": 159121408
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.000480692076228686,
+      "loss": 3.3618,
+      "theoretical_loss": 4.483764092822857,
+      "tokens_seen": 159186944
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00048068204613841526,
+      "loss": 3.2863,
+      "theoretical_loss": 4.483524977874953,
+      "tokens_seen": 159252480
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00048067201604814444,
+      "loss": 3.2132,
+      "theoretical_loss": 4.48328598884744,
+      "tokens_seen": 159318016
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004806619859578736,
+      "loss": 3.2951,
+      "theoretical_loss": 4.483047125622251,
+      "tokens_seen": 159383552
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004806519558676028,
+      "loss": 3.3459,
+      "theoretical_loss": 4.482808388081477,
+      "tokens_seen": 159449088
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.000480641925777332,
+      "loss": 3.1496,
+      "theoretical_loss": 4.482569776107368,
+      "tokens_seen": 159514624
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00048063189568706117,
+      "loss": 3.2453,
+      "theoretical_loss": 4.482331289582334,
+      "tokens_seen": 159580160
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004806218655967904,
+      "loss": 3.1819,
+      "theoretical_loss": 4.482092928388941,
+      "tokens_seen": 159645696
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00048061183550651953,
+      "loss": 3.2609,
+      "theoretical_loss": 4.4818546924099145,
+      "tokens_seen": 159711232
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00048060180541624877,
+      "loss": 3.2747,
+      "theoretical_loss": 4.481616581528138,
+      "tokens_seen": 159776768
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00048059177532597795,
+      "loss": 3.1228,
+      "theoretical_loss": 4.481378595626652,
+      "tokens_seen": 159842304
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00048058174523570713,
+      "loss": 3.0437,
+      "theoretical_loss": 4.481140734588655,
+      "tokens_seen": 159907840
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004805717151454363,
+      "loss": 3.3693,
+      "theoretical_loss": 4.4809029982975,
+      "tokens_seen": 159973376
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004805616850551655,
+      "loss": 3.4456,
+      "theoretical_loss": 4.480665386636701,
+      "tokens_seen": 160038912
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00048055165496489467,
+      "loss": 3.1342,
+      "theoretical_loss": 4.480427899489925,
+      "tokens_seen": 160104448
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004805416248746239,
+      "loss": 2.9679,
+      "theoretical_loss": 4.480190536740995,
+      "tokens_seen": 160169984
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00048053159478435303,
+      "loss": 3.4332,
+      "theoretical_loss": 4.479953298273893,
+      "tokens_seen": 160235520
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00048052156469408227,
+      "loss": 3.2521,
+      "theoretical_loss": 4.479716183972753,
+      "tokens_seen": 160301056
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004805115346038114,
+      "loss": 3.5047,
+      "theoretical_loss": 4.4794791937218665,
+      "tokens_seen": 160366592
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00048050150451354063,
+      "loss": 3.3783,
+      "theoretical_loss": 4.47924232740568,
+      "tokens_seen": 160432128
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004804914744232698,
+      "loss": 3.3839,
+      "theoretical_loss": 4.479005584908793,
+      "tokens_seen": 160497664
+    },
+    {
+      "epoch": 0.05,
+      "objective/train/docs_used": 99019,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 3.4908287525177,
+      "objective/train/theoretical_loss": 4.478768966115963,
+      "objective/train/tokens_used": 181023200,
+      "theoretical_loss": 4.478768966115963,
+      "tokens_seen": 160563200
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.000480481444332999,
+      "loss": 3.1208,
+      "theoretical_loss": 4.478768966115963,
+      "tokens_seen": 160563200
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00048047141424272823,
+      "loss": 3.2379,
+      "theoretical_loss": 4.478532470912096,
+      "tokens_seen": 160628736
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00048046138415245736,
+      "loss": 3.0995,
+      "theoretical_loss": 4.478296099182261,
+      "tokens_seen": 160694272
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004804513540621866,
+      "loss": 3.2414,
+      "theoretical_loss": 4.47805985081167,
+      "tokens_seen": 160759808
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00048044132397191577,
+      "loss": 3.1472,
+      "theoretical_loss": 4.477823725685695,
+      "tokens_seen": 160825344
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00048043129388164495,
+      "loss": 3.2584,
+      "theoretical_loss": 4.477587723689862,
+      "tokens_seen": 160890880
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00048042126379137413,
+      "loss": 3.2151,
+      "theoretical_loss": 4.477351844709846,
+      "tokens_seen": 160956416
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00048041123370110337,
+      "loss": 3.464,
+      "theoretical_loss": 4.477116088631476,
+      "tokens_seen": 161021952
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004804012036108325,
+      "loss": 3.2482,
+      "theoretical_loss": 4.476880455340735,
+      "tokens_seen": 161087488
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00048039117352056173,
+      "loss": 3.347,
+      "theoretical_loss": 4.4766449447237555,
+      "tokens_seen": 161153024
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00048038114343029086,
+      "loss": 3.2773,
+      "theoretical_loss": 4.476409556666825,
+      "tokens_seen": 161218560
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004803711133400201,
+      "loss": 3.0332,
+      "theoretical_loss": 4.476174291056379,
+      "tokens_seen": 161284096
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004803610832497493,
+      "loss": 3.2546,
+      "theoretical_loss": 4.475939147779008,
+      "tokens_seen": 161349632
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00048035105315947846,
+      "loss": 3.0797,
+      "theoretical_loss": 4.47570412672145,
+      "tokens_seen": 161415168
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00048034102306920764,
+      "loss": 3.1193,
+      "theoretical_loss": 4.475469227770596,
+      "tokens_seen": 161480704
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004803309929789368,
+      "loss": 3.222,
+      "theoretical_loss": 4.475234450813487,
+      "tokens_seen": 161546240
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.000480320962888666,
+      "loss": 3.3821,
+      "theoretical_loss": 4.474999795737315,
+      "tokens_seen": 161611776
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00048031093279839523,
+      "loss": 3.17,
+      "theoretical_loss": 4.47476526242942,
+      "tokens_seen": 161677312
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00048030090270812436,
+      "loss": 3.0239,
+      "theoretical_loss": 4.474530850777294,
+      "tokens_seen": 161742848
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004802908726178536,
+      "loss": 3.2871,
+      "theoretical_loss": 4.474296560668577,
+      "tokens_seen": 161808384
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004802808425275827,
+      "loss": 3.0498,
+      "theoretical_loss": 4.474062391991058,
+      "tokens_seen": 161873920
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00048027081243731196,
+      "loss": 3.0457,
+      "theoretical_loss": 4.473828344632677,
+      "tokens_seen": 161939456
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00048026078234704114,
+      "loss": 3.114,
+      "theoretical_loss": 4.473594418481522,
+      "tokens_seen": 162004992
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004802507522567703,
+      "loss": 3.3295,
+      "theoretical_loss": 4.473360613425827,
+      "tokens_seen": 162070528
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004802407221664995,
+      "loss": 3.046,
+      "theoretical_loss": 4.473126929353977,
+      "tokens_seen": 162136064
+    },
+    {
+      "epoch": 0.05,
+      "objective/train/docs_used": 100000,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 2.690746545791626,
+      "objective/train/theoretical_loss": 4.472893366154503,
+      "objective/train/tokens_used": 182661600,
+      "theoretical_loss": 4.472893366154503,
+      "tokens_seen": 162201600
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00048023069207622874,
+      "loss": 2.9009,
+      "theoretical_loss": 4.472893366154503,
+      "tokens_seen": 162201600
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00048022066198595786,
+      "loss": 3.0755,
+      "theoretical_loss": 4.472659923716087,
+      "tokens_seen": 162267136
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004802106318956871,
+      "loss": 3.0381,
+      "theoretical_loss": 4.472426601927554,
+      "tokens_seen": 162332672
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00048020060180541623,
+      "loss": 3.0383,
+      "theoretical_loss": 4.472193400677878,
+      "tokens_seen": 162398208
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00048019057171514546,
+      "loss": 3.207,
+      "theoretical_loss": 4.471960319856182,
+      "tokens_seen": 162463744
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00048018054162487464,
+      "loss": 3.0581,
+      "theoretical_loss": 4.471727359351734,
+      "tokens_seen": 162529280
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004801705115346038,
+      "loss": 3.323,
+      "theoretical_loss": 4.4714945190539455,
+      "tokens_seen": 162594816
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.000480160481444333,
+      "loss": 3.3514,
+      "theoretical_loss": 4.47126179885238,
+      "tokens_seen": 162660352
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004801504513540622,
+      "loss": 3.0515,
+      "theoretical_loss": 4.471029198636741,
+      "tokens_seen": 162725888
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00048014042126379137,
+      "loss": 2.9545,
+      "theoretical_loss": 4.470796718296884,
+      "tokens_seen": 162791424
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004801303911735206,
+      "loss": 3.0233,
+      "theoretical_loss": 4.470564357722804,
+      "tokens_seen": 162856960
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00048012036108324973,
+      "loss": 3.258,
+      "theoretical_loss": 4.470332116804644,
+      "tokens_seen": 162922496
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00048011033099297897,
+      "loss": 3.2388,
+      "theoretical_loss": 4.470099995432692,
+      "tokens_seen": 162988032
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00048010030090270815,
+      "loss": 3.0379,
+      "theoretical_loss": 4.469867993497379,
+      "tokens_seen": 163053568
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00048009027081243733,
+      "loss": 3.0891,
+      "theoretical_loss": 4.469636110889283,
+      "tokens_seen": 163119104
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004800802407221665,
+      "loss": 3.2988,
+      "theoretical_loss": 4.4694043474991245,
+      "tokens_seen": 163184640
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004800702106318957,
+      "loss": 3.0297,
+      "theoretical_loss": 4.469172703217767,
+      "tokens_seen": 163250176
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00048006018054162487,
+      "loss": 3.0857,
+      "theoretical_loss": 4.46894117793622,
+      "tokens_seen": 163315712
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004800501504513541,
+      "loss": 3.1744,
+      "theoretical_loss": 4.468709771545634,
+      "tokens_seen": 163381248
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00048004012036108323,
+      "loss": 3.0103,
+      "theoretical_loss": 4.4684784839373055,
+      "tokens_seen": 163446784
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00048003009027081247,
+      "loss": 3.2331,
+      "theoretical_loss": 4.468247315002671,
+      "tokens_seen": 163512320
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004800200601805416,
+      "loss": 3.1392,
+      "theoretical_loss": 4.468016264633311,
+      "tokens_seen": 163577856
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00048001003009027083,
+      "loss": 3.1491,
+      "theoretical_loss": 4.4677853327209505,
+      "tokens_seen": 163643392
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00048,
+      "loss": 3.3932,
+      "theoretical_loss": 4.467554519157452,
+      "tokens_seen": 163708928
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004799899699097292,
+      "loss": 3.0553,
+      "theoretical_loss": 4.467323823834824,
+      "tokens_seen": 163774464
+    },
+    {
+      "debugging/Self-BLEU-5": 0.4946583130096161,
+      "debugging/distinct-1-grams": 0.815790683224271,
+      "debugging/distinct-2-grams": 0.9751385384195338,
+      "debugging/entropy-1-grams": 5.884314010514224,
+      "debugging/entropy-2-grams": 6.564196933680893,
+      "debugging/length": 580.8,
+      "debugging/num_segments": 10,
+      "epoch": 0.05,
+      "objective/train/docs_used": 101353,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 3.959970474243164,
+      "objective/train/theoretical_loss": 4.467093246645215,
+      "objective/train/tokens_used": 184300000,
+      "theoretical_loss": 4.467093246645215,
+      "tokens_seen": 163840000
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004799799398194584,
+      "loss": 3.3758,
+      "theoretical_loss": 4.467093246645215,
+      "tokens_seen": 163840000
+    }
+  ],
+  "max_steps": 50354,
+  "num_train_epochs": 9223372036854775807,
+  "total_flos": 8.361345024e+16,
+  "trial_name": null,
+  "trial_params": null
+}