diff --git "a/checkpoint-1500/trainer_state.json" "b/checkpoint-1500/trainer_state.json"
new file mode 100644--- /dev/null
+++ "b/checkpoint-1500/trainer_state.json"
@@ -0,0 +1,11176 @@
+{
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 0.029789093220002383,
+  "global_step": 1500,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.0,
+      "learning_rate": 9.92063492063492e-07,
+      "loss": 10.8568,
+      "theoretical_loss": 20.812814784551147,
+      "tokens_seen": 65536
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 1.984126984126984e-06,
+      "loss": 10.8727,
+      "theoretical_loss": 17.56619808733264,
+      "tokens_seen": 131072
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 2.9761904761904763e-06,
+      "loss": 10.7356,
+      "theoretical_loss": 15.939474075840563,
+      "tokens_seen": 196608
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 3.968253968253968e-06,
+      "loss": 10.7113,
+      "theoretical_loss": 14.892313738992565,
+      "tokens_seen": 262144
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 4.96031746031746e-06,
+      "loss": 10.3498,
+      "theoretical_loss": 14.13621392076697,
+      "tokens_seen": 327680
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 5.9523809523809525e-06,
+      "loss": 10.1826,
+      "theoretical_loss": 13.552558455554218,
+      "tokens_seen": 393216
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 6.944444444444444e-06,
+      "loss": 10.0176,
+      "theoretical_loss": 13.081805984405184,
+      "tokens_seen": 458752
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 7.936507936507936e-06,
+      "loss": 9.6515,
+      "theoretical_loss": 12.690126608487319,
+      "tokens_seen": 524288
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 8.928571428571428e-06,
+      "loss": 9.2598,
+      "theoretical_loss": 12.356589446877619,
+      "tokens_seen": 589824
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 9.92063492063492e-06,
+      "loss": 9.1219,
+      "theoretical_loss": 12.067409590039073,
+      "tokens_seen": 655360
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 1.0912698412698412e-05,
+      "loss": 9.2934,
+      "theoretical_loss": 11.813063214105672,
+      "tokens_seen": 720896
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 1.1904761904761905e-05,
+      "loss": 9.0973,
+      "theoretical_loss": 11.586716191710725,
+      "tokens_seen": 786432
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 1.2896825396825396e-05,
+      "loss": 9.4804,
+      "theoretical_loss": 11.383311123190783,
+      "tokens_seen": 851968
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 1.3888888888888888e-05,
+      "loss": 9.3208,
+      "theoretical_loss": 11.199008685115867,
+      "tokens_seen": 917504
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 1.4880952380952381e-05,
+      "loss": 9.1864,
+      "theoretical_loss": 11.030830900981908,
+      "tokens_seen": 983040
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 1.5873015873015872e-05,
+      "loss": 9.398,
+      "theoretical_loss": 10.876425069460945,
+      "tokens_seen": 1048576
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 1.6865079365079364e-05,
+      "loss": 8.9394,
+      "theoretical_loss": 10.73390272306672,
+      "tokens_seen": 1114112
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 1.7857142857142855e-05,
+      "loss": 9.0636,
+      "theoretical_loss": 10.601726859234274,
+      "tokens_seen": 1179648
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 1.884920634920635e-05,
+      "loss": 9.1583,
+      "theoretical_loss": 10.478631155360636,
+      "tokens_seen": 1245184
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 1.984126984126984e-05,
+      "loss": 9.1124,
+      "theoretical_loss": 10.363560926767326,
+      "tokens_seen": 1310720
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 2.0833333333333333e-05,
+      "loss": 9.0087,
+      "theoretical_loss": 10.25562920390074,
+      "tokens_seen": 1376256
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 2.1825396825396824e-05,
+      "loss": 9.0788,
+      "theoretical_loss": 10.154083536274014,
+      "tokens_seen": 1441792
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 2.2817460317460315e-05,
+      "loss": 8.4406,
+      "theoretical_loss": 10.058280544736593,
+      "tokens_seen": 1507328
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 2.380952380952381e-05,
+      "loss": 8.9754,
+      "theoretical_loss": 9.967666161844274,
+      "tokens_seen": 1572864
+    },
+    {
+      "epoch": 0.0,
+      "objective/train/docs_used": 12267,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 8.854658126831055,
+      "objective/train/theoretical_loss": 9.881760109397105,
+      "objective/train/tokens_used": 22098400,
+      "theoretical_loss": 9.881760109397105,
+      "tokens_seen": 1638400
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 2.48015873015873e-05,
+      "loss": 8.9164,
+      "theoretical_loss": 9.881760109397105,
+      "tokens_seen": 1638400
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 2.5793650793650793e-05,
+      "loss": 7.9452,
+      "theoretical_loss": 9.800143574544556,
+      "tokens_seen": 1703936
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 2.6785714285714284e-05,
+      "loss": 8.81,
+      "theoretical_loss": 9.722449329911441,
+      "tokens_seen": 1769472
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 2.7777777777777776e-05,
+      "loss": 9.0741,
+      "theoretical_loss": 9.648353742085542,
+      "tokens_seen": 1835008
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 2.876984126984127e-05,
+      "loss": 8.8256,
+      "theoretical_loss": 9.577570254149634,
+      "tokens_seen": 1900544
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 2.9761904761904762e-05,
+      "loss": 8.3274,
+      "theoretical_loss": 9.509844029768848,
+      "tokens_seen": 1966080
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 3.075396825396825e-05,
+      "loss": 8.55,
+      "theoretical_loss": 9.44494752063593,
+      "tokens_seen": 2031616
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 3.1746031746031745e-05,
+      "loss": 8.0979,
+      "theoretical_loss": 9.382676773914453,
+      "tokens_seen": 2097152
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 3.273809523809524e-05,
+      "loss": 8.3374,
+      "theoretical_loss": 9.322848337237977,
+      "tokens_seen": 2162688
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 3.373015873015873e-05,
+      "loss": 8.5177,
+      "theoretical_loss": 9.26529664966427,
+      "tokens_seen": 2228224
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 3.472222222222222e-05,
+      "loss": 8.6686,
+      "theoretical_loss": 9.209871830448751,
+      "tokens_seen": 2293760
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 3.571428571428571e-05,
+      "loss": 8.675,
+      "theoretical_loss": 9.156437795512288,
+      "tokens_seen": 2359296
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 3.670634920634921e-05,
+      "loss": 8.4694,
+      "theoretical_loss": 9.104870645417346,
+      "tokens_seen": 2424832
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 3.76984126984127e-05,
+      "loss": 8.2407,
+      "theoretical_loss": 9.05505727953773,
+      "tokens_seen": 2490368
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 3.8690476190476195e-05,
+      "loss": 8.5934,
+      "theoretical_loss": 9.006894199647824,
+      "tokens_seen": 2555904
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 3.968253968253968e-05,
+      "loss": 7.8491,
+      "theoretical_loss": 8.960286472913351,
+      "tokens_seen": 2621440
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 4.067460317460318e-05,
+      "loss": 7.8615,
+      "theoretical_loss": 8.915146829644607,
+      "tokens_seen": 2686976
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 4.1666666666666665e-05,
+      "loss": 7.7589,
+      "theoretical_loss": 8.871394875482219,
+      "tokens_seen": 2752512
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 4.265873015873016e-05,
+      "loss": 8.0014,
+      "theoretical_loss": 8.828956401157495,
+      "tokens_seen": 2818048
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 4.365079365079365e-05,
+      "loss": 7.8309,
+      "theoretical_loss": 8.787762775782408,
+      "tokens_seen": 2883584
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 4.464285714285714e-05,
+      "loss": 8.259,
+      "theoretical_loss": 8.747750411915451,
+      "tokens_seen": 2949120
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 4.563492063492063e-05,
+      "loss": 8.1107,
+      "theoretical_loss": 8.708860292524829,
+      "tokens_seen": 3014656
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 4.6626984126984126e-05,
+      "loss": 8.2947,
+      "theoretical_loss": 8.671037551512843,
+      "tokens_seen": 3080192
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 4.761904761904762e-05,
+      "loss": 7.764,
+      "theoretical_loss": 8.63423110073947,
+      "tokens_seen": 3145728
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 4.8611111111111115e-05,
+      "loss": 8.1678,
+      "theoretical_loss": 8.59839329754032,
+      "tokens_seen": 3211264
+    },
+    {
+      "epoch": 0.0,
+      "objective/train/docs_used": 13483,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 8.108365058898926,
+      "objective/train/theoretical_loss": 8.563479647615063,
+      "objective/train/tokens_used": 23736800,
+      "theoretical_loss": 8.563479647615063,
+      "tokens_seen": 3276800
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 4.96031746031746e-05,
+      "loss": 7.5801,
+      "theoretical_loss": 8.563479647615063,
+      "tokens_seen": 3276800
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 5.05952380952381e-05,
+      "loss": 7.8434,
+      "theoretical_loss": 8.52944853889911,
+      "tokens_seen": 3342336
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 5.1587301587301586e-05,
+      "loss": 7.8492,
+      "theoretical_loss": 8.496261002649998,
+      "tokens_seen": 3407872
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 5.257936507936508e-05,
+      "loss": 7.8312,
+      "theoretical_loss": 8.463880498501183,
+      "tokens_seen": 3473408
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 5.357142857142857e-05,
+      "loss": 7.3015,
+      "theoretical_loss": 8.432272720676774,
+      "tokens_seen": 3538944
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 5.4563492063492063e-05,
+      "loss": 7.9474,
+      "theoretical_loss": 8.401405422934712,
+      "tokens_seen": 3604480
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 5.555555555555555e-05,
+      "loss": 7.6148,
+      "theoretical_loss": 8.371248260124204,
+      "tokens_seen": 3670016
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 5.6547619047619046e-05,
+      "loss": 7.5108,
+      "theoretical_loss": 8.341772644515071,
+      "tokens_seen": 3735552
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 5.753968253968254e-05,
+      "loss": 7.6168,
+      "theoretical_loss": 8.312951615289325,
+      "tokens_seen": 3801088
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 5.8531746031746036e-05,
+      "loss": 7.8436,
+      "theoretical_loss": 8.284759719785177,
+      "tokens_seen": 3866624
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 5.9523809523809524e-05,
+      "loss": 7.0651,
+      "theoretical_loss": 8.257172905255858,
+      "tokens_seen": 3932160
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 6.051587301587302e-05,
+      "loss": 7.4859,
+      "theoretical_loss": 8.23016842005411,
+      "tokens_seen": 3997696
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 6.15079365079365e-05,
+      "loss": 7.531,
+      "theoretical_loss": 8.203724723281965,
+      "tokens_seen": 4063232
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 6.25e-05,
+      "loss": 7.5815,
+      "theoretical_loss": 8.177821402057042,
+      "tokens_seen": 4128768
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 6.349206349206349e-05,
+      "loss": 7.2471,
+      "theoretical_loss": 8.152439095643611,
+      "tokens_seen": 4194304
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 6.448412698412699e-05,
+      "loss": 7.4488,
+      "theoretical_loss": 8.127559425781495,
+      "tokens_seen": 4259840
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 6.547619047619048e-05,
+      "loss": 7.4517,
+      "theoretical_loss": 8.103164932619705,
+      "tokens_seen": 4325376
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 6.646825396825397e-05,
+      "loss": 7.1039,
+      "theoretical_loss": 8.079239015726635,
+      "tokens_seen": 4390912
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 6.746031746031745e-05,
+      "loss": 7.1279,
+      "theoretical_loss": 8.055765879705412,
+      "tokens_seen": 4456448
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 6.845238095238096e-05,
+      "loss": 7.3354,
+      "theoretical_loss": 8.032730483993003,
+      "tokens_seen": 4521984
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 6.944444444444444e-05,
+      "loss": 7.1485,
+      "theoretical_loss": 8.01011849646583,
+      "tokens_seen": 4587520
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 7.043650793650793e-05,
+      "loss": 6.9851,
+      "theoretical_loss": 7.987916250513374,
+      "tokens_seen": 4653056
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 7.142857142857142e-05,
+      "loss": 6.9432,
+      "theoretical_loss": 7.966110705275796,
+      "tokens_seen": 4718592
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 7.242063492063492e-05,
+      "loss": 6.9933,
+      "theoretical_loss": 7.944689408771984,
+      "tokens_seen": 4784128
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 7.341269841269842e-05,
+      "loss": 7.0325,
+      "theoretical_loss": 7.923640463671514,
+      "tokens_seen": 4849664
+    },
+    {
+      "epoch": 0.0,
+      "objective/train/docs_used": 14033,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 6.057826519012451,
+      "objective/train/theoretical_loss": 7.902952495488062,
+      "objective/train/tokens_used": 25375200,
+      "theoretical_loss": 7.902952495488062,
+      "tokens_seen": 4915200
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 7.440476190476191e-05,
+      "loss": 6.7791,
+      "theoretical_loss": 7.902952495488062,
+      "tokens_seen": 4915200
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 7.53968253968254e-05,
+      "loss": 6.7978,
+      "theoretical_loss": 7.882614622993199,
+      "tokens_seen": 4980736
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 7.63888888888889e-05,
+      "loss": 6.8558,
+      "theoretical_loss": 7.8626164306686235,
+      "tokens_seen": 5046272
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 7.738095238095239e-05,
+      "loss": 6.9141,
+      "theoretical_loss": 7.842947943031932,
+      "tokens_seen": 5111808
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 7.837301587301588e-05,
+      "loss": 6.9255,
+      "theoretical_loss": 7.8235996006863076,
+      "tokens_seen": 5177344
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 7.936507936507937e-05,
+      "loss": 6.7895,
+      "theoretical_loss": 7.8045622379581605,
+      "tokens_seen": 5242880
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 8.035714285714287e-05,
+      "loss": 6.7059,
+      "theoretical_loss": 7.785827061999068,
+      "tokens_seen": 5308416
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 8.134920634920635e-05,
+      "loss": 6.815,
+      "theoretical_loss": 7.767385633239359,
+      "tokens_seen": 5373952
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 8.234126984126984e-05,
+      "loss": 6.8445,
+      "theoretical_loss": 7.749229847090614,
+      "tokens_seen": 5439488
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 8.333333333333333e-05,
+      "loss": 6.7849,
+      "theoretical_loss": 7.731351916803314,
+      "tokens_seen": 5505024
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 8.432539682539683e-05,
+      "loss": 6.5349,
+      "theoretical_loss": 7.7137443573939155,
+      "tokens_seen": 5570560
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 8.531746031746032e-05,
+      "loss": 6.5103,
+      "theoretical_loss": 7.696399970562929,
+      "tokens_seen": 5636096
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 8.630952380952381e-05,
+      "loss": 6.9307,
+      "theoretical_loss": 7.679311830532177,
+      "tokens_seen": 5701632
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 8.73015873015873e-05,
+      "loss": 6.8115,
+      "theoretical_loss": 7.662473270735324,
+      "tokens_seen": 5767168
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 8.82936507936508e-05,
+      "loss": 6.6601,
+      "theoretical_loss": 7.645877871301275,
+      "tokens_seen": 5832704
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 8.928571428571429e-05,
+      "loss": 6.2238,
+      "theoretical_loss": 7.6295194472748555,
+      "tokens_seen": 5898240
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 9.027777777777777e-05,
+      "loss": 6.6176,
+      "theoretical_loss": 7.613392037523692,
+      "tokens_seen": 5963776
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 9.126984126984126e-05,
+      "loss": 6.5932,
+      "theoretical_loss": 7.597489894284275,
+      "tokens_seen": 6029312
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 9.226190476190476e-05,
+      "loss": 6.51,
+      "theoretical_loss": 7.581807473303883,
+      "tokens_seen": 6094848
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 9.325396825396825e-05,
+      "loss": 6.3768,
+      "theoretical_loss": 7.566339424538366,
+      "tokens_seen": 6160384
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 9.424603174603175e-05,
+      "loss": 6.5465,
+      "theoretical_loss": 7.551080583368945,
+      "tokens_seen": 6225920
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 9.523809523809524e-05,
+      "loss": 6.8931,
+      "theoretical_loss": 7.536025962303915,
+      "tokens_seen": 6291456
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 9.623015873015874e-05,
+      "loss": 6.5266,
+      "theoretical_loss": 7.521170743133757,
+      "tokens_seen": 6356992
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 9.722222222222223e-05,
+      "loss": 6.4856,
+      "theoretical_loss": 7.506510269510493,
+      "tokens_seen": 6422528
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 9.821428571428572e-05,
+      "loss": 6.4763,
+      "theoretical_loss": 7.492040039924245,
+      "tokens_seen": 6488064
+    },
+    {
+      "epoch": 0.0,
+      "objective/train/docs_used": 15357,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 6.699707984924316,
+      "objective/train/theoretical_loss": 7.4777557010520255,
+      "objective/train/tokens_used": 27013600,
+      "theoretical_loss": 7.4777557010520255,
+      "tokens_seen": 6553600
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 9.92063492063492e-05,
+      "loss": 6.6532,
+      "theoretical_loss": 7.4777557010520255,
+      "tokens_seen": 6553600
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00010019841269841271,
+      "loss": 6.4,
+      "theoretical_loss": 7.463653041455457,
+      "tokens_seen": 6619136
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.0001011904761904762,
+      "loss": 6.2462,
+      "theoretical_loss": 7.449727985605911,
+      "tokens_seen": 6684672
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00010218253968253968,
+      "loss": 6.6399,
+      "theoretical_loss": 7.435976588217015,
+      "tokens_seen": 6750208
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00010317460317460317,
+      "loss": 6.1887,
+      "theoretical_loss": 7.4223950288659,
+      "tokens_seen": 6815744
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00010416666666666667,
+      "loss": 6.4129,
+      "theoretical_loss": 7.40897960688587,
+      "tokens_seen": 6881280
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00010515873015873016,
+      "loss": 6.2296,
+      "theoretical_loss": 7.395726736514341,
+      "tokens_seen": 6946816
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00010615079365079365,
+      "loss": 6.3505,
+      "theoretical_loss": 7.3826329422810275,
+      "tokens_seen": 7012352
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00010714285714285714,
+      "loss": 6.5043,
+      "theoretical_loss": 7.369694854622368,
+      "tokens_seen": 7077888
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00010813492063492064,
+      "loss": 6.3689,
+      "theoretical_loss": 7.3569092057090835,
+      "tokens_seen": 7143424
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00010912698412698413,
+      "loss": 6.5583,
+      "theoretical_loss": 7.3442728254746825,
+      "tokens_seen": 7208960
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00011011904761904761,
+      "loss": 6.3757,
+      "theoretical_loss": 7.3317826378335145,
+      "tokens_seen": 7274496
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.0001111111111111111,
+      "loss": 6.3164,
+      "theoretical_loss": 7.319435657077673,
+      "tokens_seen": 7340032
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.0001121031746031746,
+      "loss": 6.205,
+      "theoretical_loss": 7.307228984442819,
+      "tokens_seen": 7405568
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00011309523809523809,
+      "loss": 6.4975,
+      "theoretical_loss": 7.29515980483356,
+      "tokens_seen": 7471104
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00011408730158730158,
+      "loss": 6.087,
+      "theoretical_loss": 7.283225383699648,
+      "tokens_seen": 7536640
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00011507936507936508,
+      "loss": 6.0785,
+      "theoretical_loss": 7.271423064054828,
+      "tokens_seen": 7602176
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00011607142857142858,
+      "loss": 6.4949,
+      "theoretical_loss": 7.259750263630618,
+      "tokens_seen": 7667712
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00011706349206349207,
+      "loss": 6.2155,
+      "theoretical_loss": 7.248204472157866,
+      "tokens_seen": 7733248
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00011805555555555556,
+      "loss": 6.0229,
+      "theoretical_loss": 7.236783248769257,
+      "tokens_seen": 7798784
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00011904761904761905,
+      "loss": 6.6244,
+      "theoretical_loss": 7.225484219516492,
+      "tokens_seen": 7864320
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00012003968253968255,
+      "loss": 5.9387,
+      "theoretical_loss": 7.2143050749961155,
+      "tokens_seen": 7929856
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00012103174603174604,
+      "loss": 6.2842,
+      "theoretical_loss": 7.2032435680783955,
+      "tokens_seen": 7995392
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00012202380952380953,
+      "loss": 6.2047,
+      "theoretical_loss": 7.19229751173401,
+      "tokens_seen": 8060928
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.000123015873015873,
+      "loss": 6.2571,
+      "theoretical_loss": 7.181464776953511,
+      "tokens_seen": 8126464
+    },
+    {
+      "epoch": 0.0,
+      "objective/train/docs_used": 16711,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 6.2001729011535645,
+      "objective/train/theoretical_loss": 7.1707432907549595,
+      "objective/train/tokens_used": 28652000,
+      "theoretical_loss": 7.1707432907549595,
+      "tokens_seen": 8192000
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.0001240079365079365,
+      "loss": 6.0533,
+      "theoretical_loss": 7.1707432907549595,
+      "tokens_seen": 8192000
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.000125,
+      "loss": 6.0526,
+      "theoretical_loss": 7.160131034275267,
+      "tokens_seen": 8257536
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.0001259920634920635,
+      "loss": 6.0532,
+      "theoretical_loss": 7.149626040941133,
+      "tokens_seen": 8323072
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00012698412698412698,
+      "loss": 5.9719,
+      "theoretical_loss": 7.139226394715633,
+      "tokens_seen": 8388608
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00012797619047619048,
+      "loss": 6.0403,
+      "theoretical_loss": 7.12893022841679,
+      "tokens_seen": 8454144
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00012896825396825398,
+      "loss": 5.7847,
+      "theoretical_loss": 7.118735722104612,
+      "tokens_seen": 8519680
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00012996031746031748,
+      "loss": 5.9094,
+      "theoretical_loss": 7.10864110153334,
+      "tokens_seen": 8585216
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00013095238095238096,
+      "loss": 6.0872,
+      "theoretical_loss": 7.098644636665766,
+      "tokens_seen": 8650752
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00013194444444444446,
+      "loss": 6.0185,
+      "theoretical_loss": 7.088744640246688,
+      "tokens_seen": 8716288
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00013293650793650793,
+      "loss": 6.0768,
+      "theoretical_loss": 7.078939466432745,
+      "tokens_seen": 8781824
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00013392857142857144,
+      "loss": 6.3022,
+      "theoretical_loss": 7.069227509475962,
+      "tokens_seen": 8847360
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.0001349206349206349,
+      "loss": 6.0881,
+      "theoretical_loss": 7.059607202458563,
+      "tokens_seen": 8912896
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.0001359126984126984,
+      "loss": 5.7381,
+      "theoretical_loss": 7.0500770160766635,
+      "tokens_seen": 8978432
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.0001369047619047619,
+      "loss": 5.9916,
+      "theoretical_loss": 7.04063545747062,
+      "tokens_seen": 9043968
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00013789682539682541,
+      "loss": 6.0983,
+      "theoretical_loss": 7.031281069099929,
+      "tokens_seen": 9109504
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.0001388888888888889,
+      "loss": 6.0662,
+      "theoretical_loss": 7.022012427660673,
+      "tokens_seen": 9175040
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.0001398809523809524,
+      "loss": 5.7078,
+      "theoretical_loss": 7.012828143043604,
+      "tokens_seen": 9240576
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00014087301587301586,
+      "loss": 6.2455,
+      "theoretical_loss": 7.003726857331067,
+      "tokens_seen": 9306112
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00014186507936507937,
+      "loss": 5.9609,
+      "theoretical_loss": 6.994707243831053,
+      "tokens_seen": 9371648
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00014285714285714284,
+      "loss": 5.9423,
+      "theoretical_loss": 6.985768006146776,
+      "tokens_seen": 9437184
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00014384920634920634,
+      "loss": 6.0329,
+      "theoretical_loss": 6.976907877280185,
+      "tokens_seen": 9502720
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00014484126984126984,
+      "loss": 5.7799,
+      "theoretical_loss": 6.96812561876801,
+      "tokens_seen": 9568256
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00014583333333333335,
+      "loss": 5.9063,
+      "theoretical_loss": 6.95942001984889,
+      "tokens_seen": 9633792
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00014682539682539685,
+      "loss": 5.9317,
+      "theoretical_loss": 6.950789896660304,
+      "tokens_seen": 9699328
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00014781746031746032,
+      "loss": 5.8925,
+      "theoretical_loss": 6.942234091464025,
+      "tokens_seen": 9764864
+    },
+    {
+      "epoch": 0.0,
+      "objective/train/docs_used": 17166,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 6.479184627532959,
+      "objective/train/theoretical_loss": 6.933751471898896,
+      "objective/train/tokens_used": 30290400,
+      "theoretical_loss": 6.933751471898896,
+      "tokens_seen": 9830400
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00014880952380952382,
+      "loss": 6.2539,
+      "theoretical_loss": 6.933751471898896,
+      "tokens_seen": 9830400
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.0001498015873015873,
+      "loss": 5.996,
+      "theoretical_loss": 6.925340930259813,
+      "tokens_seen": 9895936
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.0001507936507936508,
+      "loss": 5.8197,
+      "theoretical_loss": 6.917001382801793,
+      "tokens_seen": 9961472
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00015178571428571427,
+      "loss": 6.0534,
+      "theoretical_loss": 6.908731769068142,
+      "tokens_seen": 10027008
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.0001527777777777778,
+      "loss": 5.7838,
+      "theoretical_loss": 6.900531051241684,
+      "tokens_seen": 10092544
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00015376984126984128,
+      "loss": 5.7799,
+      "theoretical_loss": 6.892398213518156,
+      "tokens_seen": 10158080
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00015476190476190478,
+      "loss": 6.1056,
+      "theoretical_loss": 6.884332261500866,
+      "tokens_seen": 10223616
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00015575396825396825,
+      "loss": 5.5432,
+      "theoretical_loss": 6.876332221615746,
+      "tokens_seen": 10289152
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00015674603174603175,
+      "loss": 5.8572,
+      "theoretical_loss": 6.8683971405459925,
+      "tokens_seen": 10354688
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00015773809523809523,
+      "loss": 6.226,
+      "theoretical_loss": 6.860526084685546,
+      "tokens_seen": 10420224
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00015873015873015873,
+      "loss": 5.7117,
+      "theoretical_loss": 6.852718139610625,
+      "tokens_seen": 10485760
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.0001597222222222222,
+      "loss": 5.6684,
+      "theoretical_loss": 6.844972409568637,
+      "tokens_seen": 10551296
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00016071428571428573,
+      "loss": 5.8166,
+      "theoretical_loss": 6.8372880169837895,
+      "tokens_seen": 10616832
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.0001617063492063492,
+      "loss": 5.971,
+      "theoretical_loss": 6.829664101978745,
+      "tokens_seen": 10682368
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.0001626984126984127,
+      "loss": 5.7343,
+      "theoretical_loss": 6.8220998219117135,
+      "tokens_seen": 10747904
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00016369047619047618,
+      "loss": 5.8061,
+      "theoretical_loss": 6.81459435092839,
+      "tokens_seen": 10813440
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00016468253968253969,
+      "loss": 6.0029,
+      "theoretical_loss": 6.807146879528176,
+      "tokens_seen": 10878976
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00016567460317460316,
+      "loss": 5.5367,
+      "theoretical_loss": 6.79975661414414,
+      "tokens_seen": 10944512
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00016666666666666666,
+      "loss": 5.6256,
+      "theoretical_loss": 6.79242277673622,
+      "tokens_seen": 11010048
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00016765873015873016,
+      "loss": 5.7868,
+      "theoretical_loss": 6.785144604397143,
+      "tokens_seen": 11075584
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00016865079365079366,
+      "loss": 5.6046,
+      "theoretical_loss": 6.777921348970633,
+      "tokens_seen": 11141120
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00016964285714285717,
+      "loss": 5.9753,
+      "theoretical_loss": 6.770752276681419,
+      "tokens_seen": 11206656
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00017063492063492064,
+      "loss": 5.9022,
+      "theoretical_loss": 6.76363666777662,
+      "tokens_seen": 11272192
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00017162698412698414,
+      "loss": 5.8921,
+      "theoretical_loss": 6.756573816178118,
+      "tokens_seen": 11337728
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00017261904761904762,
+      "loss": 5.9722,
+      "theoretical_loss": 6.749563029145481,
+      "tokens_seen": 11403264
+    },
+    {
+      "epoch": 0.0,
+      "objective/train/docs_used": 18369,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 5.672413349151611,
+      "objective/train/theoretical_loss": 6.742603626949085,
+      "objective/train/tokens_used": 31928800,
+      "theoretical_loss": 6.742603626949085,
+      "tokens_seen": 11468800
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00017361111111111112,
+      "loss": 5.4506,
+      "theoretical_loss": 6.742603626949085,
+      "tokens_seen": 11468800
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.0001746031746031746,
+      "loss": 6.0078,
+      "theoretical_loss": 6.735694942553071,
+      "tokens_seen": 11534336
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.0001755952380952381,
+      "loss": 5.8091,
+      "theoretical_loss": 6.728836321307757,
+      "tokens_seen": 11599872
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.0001765873015873016,
+      "loss": 5.6113,
+      "theoretical_loss": 6.722027120651221,
+      "tokens_seen": 11665408
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.0001775793650793651,
+      "loss": 5.6758,
+      "theoretical_loss": 6.715266709819684,
+      "tokens_seen": 11730944
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00017857142857142857,
+      "loss": 5.6378,
+      "theoretical_loss": 6.708554469566408,
+      "tokens_seen": 11796480
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00017956349206349207,
+      "loss": 5.7411,
+      "theoretical_loss": 6.701889791888819,
+      "tokens_seen": 11862016
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00018055555555555555,
+      "loss": 5.8311,
+      "theoretical_loss": 6.695272079763555,
+      "tokens_seen": 11927552
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00018154761904761905,
+      "loss": 5.2846,
+      "theoretical_loss": 6.688700746889175,
+      "tokens_seen": 11993088
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00018253968253968252,
+      "loss": 5.7687,
+      "theoretical_loss": 6.682175217436269,
+      "tokens_seen": 12058624
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00018353174603174602,
+      "loss": 5.7687,
+      "theoretical_loss": 6.675694925804711,
+      "tokens_seen": 12124160
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00018452380952380953,
+      "loss": 5.6209,
+      "theoretical_loss": 6.66925931638781,
+      "tokens_seen": 12189696
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00018551587301587303,
+      "loss": 5.9193,
+      "theoretical_loss": 6.662867843343154,
+      "tokens_seen": 12255232
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.0001865079365079365,
+      "loss": 5.6754,
+      "theoretical_loss": 6.6565199703698745,
+      "tokens_seen": 12320768
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.0001875,
+      "loss": 5.8597,
+      "theoretical_loss": 6.6502151704921655,
+      "tokens_seen": 12386304
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.0001884920634920635,
+      "loss": 5.709,
+      "theoretical_loss": 6.643952925848826,
+      "tokens_seen": 12451840
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00018948412698412698,
+      "loss": 5.7054,
+      "theoretical_loss": 6.637732727488622,
+      "tokens_seen": 12517376
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00019047619047619048,
+      "loss": 5.5972,
+      "theoretical_loss": 6.6315540751713,
+      "tokens_seen": 12582912
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00019146825396825398,
+      "loss": 5.5887,
+      "theoretical_loss": 6.625416477174044,
+      "tokens_seen": 12648448
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00019246031746031748,
+      "loss": 5.7082,
+      "theoretical_loss": 6.619319450103218,
+      "tokens_seen": 12713984
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00019345238095238096,
+      "loss": 5.5405,
+      "theoretical_loss": 6.613262518711206,
+      "tokens_seen": 12779520
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00019444444444444446,
+      "loss": 5.5806,
+      "theoretical_loss": 6.607245215718208,
+      "tokens_seen": 12845056
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00019543650793650793,
+      "loss": 5.4612,
+      "theoretical_loss": 6.601267081638806,
+      "tokens_seen": 12910592
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00019642857142857144,
+      "loss": 5.6405,
+      "theoretical_loss": 6.595327664613176,
+      "tokens_seen": 12976128
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.0001974206349206349,
+      "loss": 5.3968,
+      "theoretical_loss": 6.58942652024278,
+      "tokens_seen": 13041664
+    },
+    {
+      "epoch": 0.0,
+      "objective/train/docs_used": 19017,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 5.398904323577881,
+      "objective/train/theoretical_loss": 6.583563211430409,
+      "objective/train/tokens_used": 33567200,
+      "theoretical_loss": 6.583563211430409,
+      "tokens_seen": 13107200
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.0001984126984126984,
+      "loss": 5.5392,
+      "theoretical_loss": 6.583563211430409,
+      "tokens_seen": 13107200
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00019940476190476191,
+      "loss": 5.3995,
+      "theoretical_loss": 6.577737308224426,
+      "tokens_seen": 13172736
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00020039682539682542,
+      "loss": 5.5665,
+      "theoretical_loss": 6.571948387667094,
+      "tokens_seen": 13238272
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.0002013888888888889,
+      "loss": 5.5745,
+      "theoretical_loss": 6.566196033646859,
+      "tokens_seen": 13303808
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.0002023809523809524,
+      "loss": 5.4756,
+      "theoretical_loss": 6.560479836754459,
+      "tokens_seen": 13369344
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00020337301587301587,
+      "loss": 5.7895,
+      "theoretical_loss": 6.554799394142741,
+      "tokens_seen": 13434880
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00020436507936507937,
+      "loss": 5.3965,
+      "theoretical_loss": 6.549154309390087,
+      "tokens_seen": 13500416
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00020535714285714284,
+      "loss": 5.494,
+      "theoretical_loss": 6.5435441923673245,
+      "tokens_seen": 13565952
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00020634920634920634,
+      "loss": 5.5559,
+      "theoretical_loss": 6.537968659108021,
+      "tokens_seen": 13631488
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00020734126984126985,
+      "loss": 5.9125,
+      "theoretical_loss": 6.532427331682063,
+      "tokens_seen": 13697024
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00020833333333333335,
+      "loss": 5.5235,
+      "theoretical_loss": 6.5269198380724145,
+      "tokens_seen": 13762560
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00020932539682539685,
+      "loss": 5.6307,
+      "theoretical_loss": 6.5214458120549725,
+      "tokens_seen": 13828096
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00021031746031746032,
+      "loss": 5.3661,
+      "theoretical_loss": 6.516004893081412,
+      "tokens_seen": 13893632
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00021130952380952382,
+      "loss": 5.2823,
+      "theoretical_loss": 6.510596726164945,
+      "tokens_seen": 13959168
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.0002123015873015873,
+      "loss": 5.4241,
+      "theoretical_loss": 6.505220961768906,
+      "tokens_seen": 14024704
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.0002132936507936508,
+      "loss": 5.3679,
+      "theoretical_loss": 6.499877255698063,
+      "tokens_seen": 14090240
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00021428571428571427,
+      "loss": 5.2005,
+      "theoretical_loss": 6.494565268992613,
+      "tokens_seen": 14155776
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.0002152777777777778,
+      "loss": 5.6155,
+      "theoretical_loss": 6.489284667824741,
+      "tokens_seen": 14221312
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00021626984126984128,
+      "loss": 5.6027,
+      "theoretical_loss": 6.484035123397694,
+      "tokens_seen": 14286848
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00021726190476190478,
+      "loss": 5.5959,
+      "theoretical_loss": 6.478816311847305,
+      "tokens_seen": 14352384
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00021825396825396825,
+      "loss": 5.7842,
+      "theoretical_loss": 6.473627914145865,
+      "tokens_seen": 14417920
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00021924603174603176,
+      "loss": 5.4174,
+      "theoretical_loss": 6.468469616008303,
+      "tokens_seen": 14483456
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00022023809523809523,
+      "loss": 5.4423,
+      "theoretical_loss": 6.463341107800611,
+      "tokens_seen": 14548992
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00022123015873015873,
+      "loss": 5.2413,
+      "theoretical_loss": 6.458242084450426,
+      "tokens_seen": 14614528
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.0002222222222222222,
+      "loss": 5.4841,
+      "theoretical_loss": 6.453172245359726,
+      "tokens_seen": 14680064
+    },
+    {
+      "epoch": 0.0,
+      "objective/train/docs_used": 20187,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 5.4210638999938965,
+      "objective/train/theoretical_loss": 6.448131294319589,
+      "objective/train/tokens_used": 35205600,
+      "theoretical_loss": 6.448131294319589,
+      "tokens_seen": 14745600
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00022321428571428573,
+      "loss": 5.318,
+      "theoretical_loss": 6.448131294319589,
+      "tokens_seen": 14745600
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.0002242063492063492,
+      "loss": 5.5298,
+      "theoretical_loss": 6.443118939426935,
+      "tokens_seen": 14811136
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.0002251984126984127,
+      "loss": 5.5104,
+      "theoretical_loss": 6.438134893003209,
+      "tokens_seen": 14876672
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00022619047619047618,
+      "loss": 5.4089,
+      "theoretical_loss": 6.43317887151496,
+      "tokens_seen": 14942208
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00022718253968253969,
+      "loss": 5.5783,
+      "theoretical_loss": 6.428250595496234,
+      "tokens_seen": 15007744
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00022817460317460316,
+      "loss": 5.4029,
+      "theoretical_loss": 6.4233497894727805,
+      "tokens_seen": 15073280
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00022916666666666666,
+      "loss": 5.6043,
+      "theoretical_loss": 6.418476181887964,
+      "tokens_seen": 15138816
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00023015873015873016,
+      "loss": 5.5123,
+      "theoretical_loss": 6.413629505030386,
+      "tokens_seen": 15204352
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00023115079365079367,
+      "loss": 5.5084,
+      "theoretical_loss": 6.408809494963135,
+      "tokens_seen": 15269888
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00023214285714285717,
+      "loss": 5.269,
+      "theoretical_loss": 6.404015891454651,
+      "tokens_seen": 15335424
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00023313492063492064,
+      "loss": 5.4379,
+      "theoretical_loss": 6.399248437911128,
+      "tokens_seen": 15400960
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00023412698412698414,
+      "loss": 5.4346,
+      "theoretical_loss": 6.394506881310447,
+      "tokens_seen": 15466496
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00023511904761904762,
+      "loss": 5.7191,
+      "theoretical_loss": 6.38979097213757,
+      "tokens_seen": 15532032
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00023611111111111112,
+      "loss": 5.5484,
+      "theoretical_loss": 6.385100464321383,
+      "tokens_seen": 15597568
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.0002371031746031746,
+      "loss": 5.4379,
+      "theoretical_loss": 6.380435115172919,
+      "tokens_seen": 15663104
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.0002380952380952381,
+      "loss": 5.4785,
+      "theoretical_loss": 6.375794685324962,
+      "tokens_seen": 15728640
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.0002390873015873016,
+      "loss": 5.2088,
+      "theoretical_loss": 6.371178938672962,
+      "tokens_seen": 15794176
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.0002400793650793651,
+      "loss": 5.5076,
+      "theoretical_loss": 6.366587642317243,
+      "tokens_seen": 15859712
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00024107142857142857,
+      "loss": 5.6351,
+      "theoretical_loss": 6.362020566506475,
+      "tokens_seen": 15925248
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00024206349206349207,
+      "loss": 5.6029,
+      "theoretical_loss": 6.3574774845823665,
+      "tokens_seen": 15990784
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00024305555555555555,
+      "loss": 5.1969,
+      "theoretical_loss": 6.352958172925549,
+      "tokens_seen": 16056320
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00024404761904761905,
+      "loss": 5.285,
+      "theoretical_loss": 6.348462410902625,
+      "tokens_seen": 16121856
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00024503968253968255,
+      "loss": 5.6805,
+      "theoretical_loss": 6.343989980814362,
+      "tokens_seen": 16187392
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.000246031746031746,
+      "loss": 5.5855,
+      "theoretical_loss": 6.339540667844965,
+      "tokens_seen": 16252928
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00024702380952380955,
+      "loss": 5.3559,
+      "theoretical_loss": 6.335114260012455,
+      "tokens_seen": 16318464
+    },
+    {
+      "epoch": 0.0,
+      "objective/train/docs_used": 20737,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 5.463630199432373,
+      "objective/train/theoretical_loss": 6.330710548120079,
+      "objective/train/tokens_used": 36844000,
+      "theoretical_loss": 6.330710548120079,
+      "tokens_seen": 16384000
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.000248015873015873,
+      "loss": 5.515,
+      "theoretical_loss": 6.330710548120079,
+      "tokens_seen": 16384000
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.0002490079365079365,
+      "loss": 5.3596,
+      "theoretical_loss": 6.326329325708746,
+      "tokens_seen": 16449536
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00025,
+      "loss": 5.249,
+      "theoretical_loss": 6.321970389010465,
+      "tokens_seen": 16515072
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0002509920634920635,
+      "loss": 5.3675,
+      "theoretical_loss": 6.317633536902759,
+      "tokens_seen": 16580608
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.000251984126984127,
+      "loss": 5.0919,
+      "theoretical_loss": 6.313318570864016,
+      "tokens_seen": 16646144
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00025297619047619046,
+      "loss": 5.1228,
+      "theoretical_loss": 6.30902529492978,
+      "tokens_seen": 16711680
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00025396825396825396,
+      "loss": 5.4573,
+      "theoretical_loss": 6.304753515649935,
+      "tokens_seen": 16777216
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00025496031746031746,
+      "loss": 5.1499,
+      "theoretical_loss": 6.30050304204677,
+      "tokens_seen": 16842752
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00025595238095238096,
+      "loss": 5.6432,
+      "theoretical_loss": 6.296273685573913,
+      "tokens_seen": 16908288
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0002569444444444444,
+      "loss": 5.439,
+      "theoretical_loss": 6.292065260076094,
+      "tokens_seen": 16973824
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00025793650793650796,
+      "loss": 5.2959,
+      "theoretical_loss": 6.287877581749726,
+      "tokens_seen": 17039360
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00025892857142857146,
+      "loss": 5.3502,
+      "theoretical_loss": 6.2837104691042915,
+      "tokens_seen": 17104896
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00025992063492063497,
+      "loss": 5.5134,
+      "theoretical_loss": 6.279563742924502,
+      "tokens_seen": 17170432
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0002609126984126984,
+      "loss": 5.4871,
+      "theoretical_loss": 6.275437226233224,
+      "tokens_seen": 17235968
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0002619047619047619,
+      "loss": 5.3132,
+      "theoretical_loss": 6.271330744255137,
+      "tokens_seen": 17301504
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0002628968253968254,
+      "loss": 5.1796,
+      "theoretical_loss": 6.267244124381133,
+      "tokens_seen": 17367040
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0002638888888888889,
+      "loss": 5.2656,
+      "theoretical_loss": 6.2631771961334035,
+      "tokens_seen": 17432576
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00026488095238095237,
+      "loss": 5.5896,
+      "theoretical_loss": 6.259129791131242,
+      "tokens_seen": 17498112
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00026587301587301587,
+      "loss": 5.14,
+      "theoretical_loss": 6.255101743057493,
+      "tokens_seen": 17563648
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00026686507936507937,
+      "loss": 5.3606,
+      "theoretical_loss": 6.251092887625685,
+      "tokens_seen": 17629184
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00026785714285714287,
+      "loss": 5.487,
+      "theoretical_loss": 6.247103062547796,
+      "tokens_seen": 17694720
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0002688492063492063,
+      "loss": 5.2887,
+      "theoretical_loss": 6.243132107502647,
+      "tokens_seen": 17760256
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0002698412698412698,
+      "loss": 5.4254,
+      "theoretical_loss": 6.239179864104911,
+      "tokens_seen": 17825792
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0002708333333333333,
+      "loss": 5.4668,
+      "theoretical_loss": 6.235246175874727,
+      "tokens_seen": 17891328
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0002718253968253968,
+      "loss": 5.3584,
+      "theoretical_loss": 6.231330888207894,
+      "tokens_seen": 17956864
+    },
+    {
+      "epoch": 0.01,
+      "objective/train/docs_used": 21782,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 5.119491100311279,
+      "objective/train/theoretical_loss": 6.227433848346639,
+      "objective/train/tokens_used": 38482400,
+      "theoretical_loss": 6.227433848346639,
+      "tokens_seen": 18022400
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0002728174603174603,
+      "loss": 5.1069,
+      "theoretical_loss": 6.227433848346639,
+      "tokens_seen": 18022400
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0002738095238095238,
+      "loss": 5.1568,
+      "theoretical_loss": 6.22355490535095,
+      "tokens_seen": 18087936
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0002748015873015873,
+      "loss": 5.4409,
+      "theoretical_loss": 6.219693910070452,
+      "tokens_seen": 18153472
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00027579365079365083,
+      "loss": 5.1939,
+      "theoretical_loss": 6.215850715116817,
+      "tokens_seen": 18219008
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00027678571428571433,
+      "loss": 5.2274,
+      "theoretical_loss": 6.212025174836697,
+      "tokens_seen": 18284544
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0002777777777777778,
+      "loss": 5.5299,
+      "theoretical_loss": 6.208217145285173,
+      "tokens_seen": 18350080
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0002787698412698413,
+      "loss": 5.4441,
+      "theoretical_loss": 6.204426484199696,
+      "tokens_seen": 18415616
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0002797619047619048,
+      "loss": 5.3135,
+      "theoretical_loss": 6.200653050974525,
+      "tokens_seen": 18481152
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0002807539682539683,
+      "loss": 5.189,
+      "theoretical_loss": 6.196896706635635,
+      "tokens_seen": 18546688
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00028174603174603173,
+      "loss": 5.0786,
+      "theoretical_loss": 6.1931573138160925,
+      "tokens_seen": 18612224
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00028273809523809523,
+      "loss": 5.0434,
+      "theoretical_loss": 6.189434736731897,
+      "tokens_seen": 18677760
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00028373015873015873,
+      "loss": 5.0132,
+      "theoretical_loss": 6.185728841158257,
+      "tokens_seen": 18743296
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00028472222222222223,
+      "loss": 5.1159,
+      "theoretical_loss": 6.182039494406309,
+      "tokens_seen": 18808832
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0002857142857142857,
+      "loss": 5.1786,
+      "theoretical_loss": 6.178366565300266,
+      "tokens_seen": 18874368
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0002867063492063492,
+      "loss": 5.1947,
+      "theoretical_loss": 6.174709924154978,
+      "tokens_seen": 18939904
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0002876984126984127,
+      "loss": 5.2396,
+      "theoretical_loss": 6.171069442753909,
+      "tokens_seen": 19005440
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0002886904761904762,
+      "loss": 5.3947,
+      "theoretical_loss": 6.1674449943275045,
+      "tokens_seen": 19070976
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0002896825396825397,
+      "loss": 4.8732,
+      "theoretical_loss": 6.16383645353196,
+      "tokens_seen": 19136512
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0002906746031746032,
+      "loss": 5.3178,
+      "theoretical_loss": 6.160243696428367,
+      "tokens_seen": 19202048
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0002916666666666667,
+      "loss": 5.1441,
+      "theoretical_loss": 6.156666600462238,
+      "tokens_seen": 19267584
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0002926587301587302,
+      "loss": 5.2695,
+      "theoretical_loss": 6.153105044443393,
+      "tokens_seen": 19333120
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0002936507936507937,
+      "loss": 5.3181,
+      "theoretical_loss": 6.149558908526206,
+      "tokens_seen": 19398656
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00029464285714285714,
+      "loss": 5.2188,
+      "theoretical_loss": 6.146028074190217,
+      "tokens_seen": 19464192
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00029563492063492064,
+      "loss": 5.1652,
+      "theoretical_loss": 6.14251242422106,
+      "tokens_seen": 19529728
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00029662698412698414,
+      "loss": 5.3475,
+      "theoretical_loss": 6.139011842691756,
+      "tokens_seen": 19595264
+    },
+    {
+      "epoch": 0.01,
+      "objective/train/docs_used": 22379,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 5.661308765411377,
+      "objective/train/theoretical_loss": 6.135526214944321,
+      "objective/train/tokens_used": 40120800,
+      "theoretical_loss": 6.135526214944321,
+      "tokens_seen": 19660800
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00029761904761904765,
+      "loss": 5.6218,
+      "theoretical_loss": 6.135526214944321,
+      "tokens_seen": 19660800
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0002986111111111111,
+      "loss": 5.4528,
+      "theoretical_loss": 6.1320554275717,
+      "tokens_seen": 19726336
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0002996031746031746,
+      "loss": 5.1971,
+      "theoretical_loss": 6.128599368400017,
+      "tokens_seen": 19791872
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0003005952380952381,
+      "loss": 5.2823,
+      "theoretical_loss": 6.125157926471134,
+      "tokens_seen": 19857408
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0003015873015873016,
+      "loss": 5.1154,
+      "theoretical_loss": 6.121730992025516,
+      "tokens_seen": 19922944
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00030257936507936505,
+      "loss": 4.8293,
+      "theoretical_loss": 6.118318456485394,
+      "tokens_seen": 19988480
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00030357142857142855,
+      "loss": 5.0591,
+      "theoretical_loss": 6.114920212438209,
+      "tokens_seen": 20054016
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00030456349206349205,
+      "loss": 5.2452,
+      "theoretical_loss": 6.111536153620355,
+      "tokens_seen": 20119552
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0003055555555555556,
+      "loss": 5.1158,
+      "theoretical_loss": 6.108166174901191,
+      "tokens_seen": 20185088
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00030654761904761905,
+      "loss": 5.0927,
+      "theoretical_loss": 6.104810172267331,
+      "tokens_seen": 20250624
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00030753968253968255,
+      "loss": 5.1097,
+      "theoretical_loss": 6.101468042807199,
+      "tokens_seen": 20316160
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00030853174603174605,
+      "loss": 5.0109,
+      "theoretical_loss": 6.098139684695851,
+      "tokens_seen": 20381696
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00030952380952380956,
+      "loss": 5.2,
+      "theoretical_loss": 6.094824997180048,
+      "tokens_seen": 20447232
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.000310515873015873,
+      "loss": 4.919,
+      "theoretical_loss": 6.091523880563589,
+      "tokens_seen": 20512768
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0003115079365079365,
+      "loss": 4.7726,
+      "theoretical_loss": 6.088236236192881,
+      "tokens_seen": 20578304
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0003125,
+      "loss": 5.1579,
+      "theoretical_loss": 6.0849619664427586,
+      "tokens_seen": 20643840
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0003134920634920635,
+      "loss": 5.2649,
+      "theoretical_loss": 6.0817009747025415,
+      "tokens_seen": 20709376
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.000314484126984127,
+      "loss": 4.9338,
+      "theoretical_loss": 6.07845316536232,
+      "tokens_seen": 20774912
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00031547619047619046,
+      "loss": 5.2449,
+      "theoretical_loss": 6.075218443799468,
+      "tokens_seen": 20840448
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00031646825396825396,
+      "loss": 5.2222,
+      "theoretical_loss": 6.071996716365382,
+      "tokens_seen": 20905984
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00031746031746031746,
+      "loss": 4.8122,
+      "theoretical_loss": 6.068787890372443,
+      "tokens_seen": 20971520
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00031845238095238096,
+      "loss": 4.983,
+      "theoretical_loss": 6.0655918740811865,
+      "tokens_seen": 21037056
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0003194444444444444,
+      "loss": 4.9428,
+      "theoretical_loss": 6.062408576687682,
+      "tokens_seen": 21102592
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00032043650793650796,
+      "loss": 5.0493,
+      "theoretical_loss": 6.059237908311129,
+      "tokens_seen": 21168128
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00032142857142857147,
+      "loss": 4.8675,
+      "theoretical_loss": 6.056079779981644,
+      "tokens_seen": 21233664
+    },
+    {
+      "epoch": 0.01,
+      "objective/train/docs_used": 23601,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 5.199455738067627,
+      "objective/train/theoretical_loss": 6.052934103628253,
+      "objective/train/tokens_used": 41759200,
+      "theoretical_loss": 6.052934103628253,
+      "tokens_seen": 21299200
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00032242063492063497,
+      "loss": 5.2637,
+      "theoretical_loss": 6.052934103628253,
+      "tokens_seen": 21299200
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0003234126984126984,
+      "loss": 5.4325,
+      "theoretical_loss": 6.049800792067078,
+      "tokens_seen": 21364736
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0003244047619047619,
+      "loss": 4.8274,
+      "theoretical_loss": 6.046679758989709,
+      "tokens_seen": 21430272
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0003253968253968254,
+      "loss": 5.2104,
+      "theoretical_loss": 6.043570918951775,
+      "tokens_seen": 21495808
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0003263888888888889,
+      "loss": 5.1036,
+      "theoretical_loss": 6.040474187361681,
+      "tokens_seen": 21561344
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00032738095238095237,
+      "loss": 4.8811,
+      "theoretical_loss": 6.037389480469548,
+      "tokens_seen": 21626880
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00032837301587301587,
+      "loss": 5.0504,
+      "theoretical_loss": 6.034316715356304,
+      "tokens_seen": 21692416
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00032936507936507937,
+      "loss": 5.0898,
+      "theoretical_loss": 6.031255809922974,
+      "tokens_seen": 21757952
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00033035714285714287,
+      "loss": 5.0121,
+      "theoretical_loss": 6.0282066828801195,
+      "tokens_seen": 21823488
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0003313492063492063,
+      "loss": 5.1363,
+      "theoretical_loss": 6.0251692537374595,
+      "tokens_seen": 21889024
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0003323412698412698,
+      "loss": 5.1503,
+      "theoretical_loss": 6.022143442793643,
+      "tokens_seen": 21954560
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0003333333333333333,
+      "loss": 5.1066,
+      "theoretical_loss": 6.0191291711261945,
+      "tokens_seen": 22020096
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0003343253968253968,
+      "loss": 5.0889,
+      "theoretical_loss": 6.016126360581609,
+      "tokens_seen": 22085632
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0003353174603174603,
+      "loss": 4.9077,
+      "theoretical_loss": 6.013134933765605,
+      "tokens_seen": 22151168
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0003363095238095238,
+      "loss": 5.2813,
+      "theoretical_loss": 6.010154814033528,
+      "tokens_seen": 22216704
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00033730158730158733,
+      "loss": 4.6999,
+      "theoretical_loss": 6.007185925480902,
+      "tokens_seen": 22282240
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00033829365079365083,
+      "loss": 5.0728,
+      "theoretical_loss": 6.0042281929341375,
+      "tokens_seen": 22347776
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00033928571428571433,
+      "loss": 5.1319,
+      "theoretical_loss": 6.001281541941363,
+      "tokens_seen": 22413312
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0003402777777777778,
+      "loss": 5.0915,
+      "theoretical_loss": 5.998345898763421,
+      "tokens_seen": 22478848
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0003412698412698413,
+      "loss": 4.8912,
+      "theoretical_loss": 5.995421190364983,
+      "tokens_seen": 22544384
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0003422619047619048,
+      "loss": 5.0317,
+      "theoretical_loss": 5.992507344405814,
+      "tokens_seen": 22609920
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0003432539682539683,
+      "loss": 5.1603,
+      "theoretical_loss": 5.9896042892321635,
+      "tokens_seen": 22675456
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00034424603174603173,
+      "loss": 4.9822,
+      "theoretical_loss": 5.986711953868287,
+      "tokens_seen": 22740992
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00034523809523809523,
+      "loss": 5.3948,
+      "theoretical_loss": 5.983830268008107,
+      "tokens_seen": 22806528
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00034623015873015873,
+      "loss": 5.222,
+      "theoretical_loss": 5.980959162006979,
+      "tokens_seen": 22872064
+    },
+    {
+      "epoch": 0.01,
+      "objective/train/docs_used": 24859,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 4.439995288848877,
+      "objective/train/theoretical_loss": 5.978098566873603,
+      "objective/train/tokens_used": 43397600,
+      "theoretical_loss": 5.978098566873603,
+      "tokens_seen": 22937600
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00034722222222222224,
+      "loss": 4.9666,
+      "theoretical_loss": 5.978098566873603,
+      "tokens_seen": 22937600
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0003482142857142857,
+      "loss": 4.7873,
+      "theoretical_loss": 5.975248414262053,
+      "tokens_seen": 23003136
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0003492063492063492,
+      "loss": 5.1429,
+      "theoretical_loss": 5.972408636463909,
+      "tokens_seen": 23068672
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0003501984126984127,
+      "loss": 4.8387,
+      "theoretical_loss": 5.969579166400528,
+      "tokens_seen": 23134208
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0003511904761904762,
+      "loss": 5.2741,
+      "theoretical_loss": 5.966759937615427,
+      "tokens_seen": 23199744
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0003521825396825397,
+      "loss": 4.9993,
+      "theoretical_loss": 5.96395088426676,
+      "tokens_seen": 23265280
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0003531746031746032,
+      "loss": 4.838,
+      "theoretical_loss": 5.961151941119932,
+      "tokens_seen": 23330816
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0003541666666666667,
+      "loss": 4.8937,
+      "theoretical_loss": 5.95836304354031,
+      "tokens_seen": 23396352
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0003551587301587302,
+      "loss": 5.1755,
+      "theoretical_loss": 5.9555841274860395,
+      "tokens_seen": 23461888
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0003561507936507937,
+      "loss": 5.0192,
+      "theoretical_loss": 5.952815129500973,
+      "tokens_seen": 23527424
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00035714285714285714,
+      "loss": 5.1528,
+      "theoretical_loss": 5.950055986707699,
+      "tokens_seen": 23592960
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00035813492063492064,
+      "loss": 5.2473,
+      "theoretical_loss": 5.9473066368006755,
+      "tokens_seen": 23658496
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00035912698412698415,
+      "loss": 5.1431,
+      "theoretical_loss": 5.944567018039454,
+      "tokens_seen": 23724032
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00036011904761904765,
+      "loss": 5.2078,
+      "theoretical_loss": 5.941837069242023,
+      "tokens_seen": 23789568
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0003611111111111111,
+      "loss": 5.2074,
+      "theoretical_loss": 5.939116729778224,
+      "tokens_seen": 23855104
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0003621031746031746,
+      "loss": 5.045,
+      "theoretical_loss": 5.936405939563279,
+      "tokens_seen": 23920640
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0003630952380952381,
+      "loss": 5.0272,
+      "theoretical_loss": 5.93370463905141,
+      "tokens_seen": 23986176
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0003640873015873016,
+      "loss": 5.0628,
+      "theoretical_loss": 5.931012769229536,
+      "tokens_seen": 24051712
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00036507936507936505,
+      "loss": 4.6127,
+      "theoretical_loss": 5.928330271611081,
+      "tokens_seen": 24117248
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00036607142857142855,
+      "loss": 4.8408,
+      "theoretical_loss": 5.925657088229862,
+      "tokens_seen": 24182784
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00036706349206349205,
+      "loss": 4.9603,
+      "theoretical_loss": 5.9229931616340545,
+      "tokens_seen": 24248320
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0003680555555555556,
+      "loss": 5.0127,
+      "theoretical_loss": 5.920338434880263,
+      "tokens_seen": 24313856
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00036904761904761905,
+      "loss": 5.3699,
+      "theoretical_loss": 5.9176928515276535,
+      "tokens_seen": 24379392
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00037003968253968255,
+      "loss": 4.9974,
+      "theoretical_loss": 5.915056355632197,
+      "tokens_seen": 24444928
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00037103174603174606,
+      "loss": 4.9348,
+      "theoretical_loss": 5.912428891740967,
+      "tokens_seen": 24510464
+    },
+    {
+      "epoch": 0.01,
+      "objective/train/docs_used": 25411,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 4.414747714996338,
+      "objective/train/theoretical_loss": 5.90981040488653,
+      "objective/train/tokens_used": 45036000,
+      "theoretical_loss": 5.90981040488653,
+      "tokens_seen": 24576000
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00037202380952380956,
+      "loss": 5.0376,
+      "theoretical_loss": 5.90981040488653,
+      "tokens_seen": 24576000
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.000373015873015873,
+      "loss": 5.0724,
+      "theoretical_loss": 5.907200840581417,
+      "tokens_seen": 24641536
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0003740079365079365,
+      "loss": 5.2247,
+      "theoretical_loss": 5.904600144812672,
+      "tokens_seen": 24707072
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.000375,
+      "loss": 5.4811,
+      "theoretical_loss": 5.902008264036468,
+      "tokens_seen": 24772608
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0003759920634920635,
+      "loss": 4.9107,
+      "theoretical_loss": 5.899425145172803,
+      "tokens_seen": 24838144
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.000376984126984127,
+      "loss": 5.0933,
+      "theoretical_loss": 5.896850735600281,
+      "tokens_seen": 24903680
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00037797619047619046,
+      "loss": 5.1802,
+      "theoretical_loss": 5.8942849831509445,
+      "tokens_seen": 24969216
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00037896825396825396,
+      "loss": 5.2521,
+      "theoretical_loss": 5.891727836105194,
+      "tokens_seen": 25034752
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00037996031746031746,
+      "loss": 4.8607,
+      "theoretical_loss": 5.889179243186776,
+      "tokens_seen": 25100288
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00038095238095238096,
+      "loss": 5.0973,
+      "theoretical_loss": 5.886639153557828,
+      "tokens_seen": 25165824
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0003819444444444444,
+      "loss": 5.3397,
+      "theoretical_loss": 5.8841075168140105,
+      "tokens_seen": 25231360
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00038293650793650797,
+      "loss": 4.7777,
+      "theoretical_loss": 5.88158428297969,
+      "tokens_seen": 25296896
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00038392857142857147,
+      "loss": 4.7219,
+      "theoretical_loss": 5.879069402503189,
+      "tokens_seen": 25362432
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00038492063492063497,
+      "loss": 5.0299,
+      "theoretical_loss": 5.876562826252119,
+      "tokens_seen": 25427968
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0003859126984126984,
+      "loss": 4.8369,
+      "theoretical_loss": 5.874064505508748,
+      "tokens_seen": 25493504
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0003869047619047619,
+      "loss": 5.0017,
+      "theoretical_loss": 5.871574391965453,
+      "tokens_seen": 25559040
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0003878968253968254,
+      "loss": 4.9705,
+      "theoretical_loss": 5.8690924377202265,
+      "tokens_seen": 25624576
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0003888888888888889,
+      "loss": 5.034,
+      "theoretical_loss": 5.866618595272241,
+      "tokens_seen": 25690112
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00038988095238095237,
+      "loss": 4.7181,
+      "theoretical_loss": 5.864152817517482,
+      "tokens_seen": 25755648
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00039087301587301587,
+      "loss": 4.9562,
+      "theoretical_loss": 5.8616950577444245,
+      "tokens_seen": 25821184
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00039186507936507937,
+      "loss": 4.9405,
+      "theoretical_loss": 5.859245269629783,
+      "tokens_seen": 25886720
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0003928571428571429,
+      "loss": 5.099,
+      "theoretical_loss": 5.85680340723431,
+      "tokens_seen": 25952256
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0003938492063492063,
+      "loss": 4.6503,
+      "theoretical_loss": 5.85436942499865,
+      "tokens_seen": 26017792
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0003948412698412698,
+      "loss": 4.8672,
+      "theoretical_loss": 5.851943277739253,
+      "tokens_seen": 26083328
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0003958333333333333,
+      "loss": 5.0201,
+      "theoretical_loss": 5.84952492064434,
+      "tokens_seen": 26148864
+    },
+    {
+      "epoch": 0.01,
+      "objective/train/docs_used": 26668,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 5.337137222290039,
+      "objective/train/theoretical_loss": 5.847114309269919,
+      "objective/train/tokens_used": 46674400,
+      "theoretical_loss": 5.847114309269919,
+      "tokens_seen": 26214400
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0003968253968253968,
+      "loss": 5.1138,
+      "theoretical_loss": 5.847114309269919,
+      "tokens_seen": 26214400
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0003978174603174603,
+      "loss": 4.8239,
+      "theoretical_loss": 5.844711399535855,
+      "tokens_seen": 26279936
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00039880952380952383,
+      "loss": 4.7304,
+      "theoretical_loss": 5.842316147722,
+      "tokens_seen": 26345472
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00039980158730158733,
+      "loss": 5.0092,
+      "theoretical_loss": 5.839928510464356,
+      "tokens_seen": 26411008
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00040079365079365083,
+      "loss": 4.4343,
+      "theoretical_loss": 5.837548444751306,
+      "tokens_seen": 26476544
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00040178571428571433,
+      "loss": 4.9309,
+      "theoretical_loss": 5.835175907919885,
+      "tokens_seen": 26542080
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004027777777777778,
+      "loss": 5.0148,
+      "theoretical_loss": 5.832810857652097,
+      "tokens_seen": 26607616
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004037698412698413,
+      "loss": 4.9709,
+      "theoretical_loss": 5.830453251971296,
+      "tokens_seen": 26673152
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004047619047619048,
+      "loss": 5.1225,
+      "theoretical_loss": 5.8281030492385835,
+      "tokens_seen": 26738688
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004057539682539683,
+      "loss": 4.9439,
+      "theoretical_loss": 5.82576020814929,
+      "tokens_seen": 26804224
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00040674603174603173,
+      "loss": 5.154,
+      "theoretical_loss": 5.823424687729469,
+      "tokens_seen": 26869760
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00040773809523809523,
+      "loss": 4.6484,
+      "theoretical_loss": 5.821096447332456,
+      "tokens_seen": 26935296
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00040873015873015874,
+      "loss": 4.6262,
+      "theoretical_loss": 5.818775446635469,
+      "tokens_seen": 27000832
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00040972222222222224,
+      "loss": 5.0236,
+      "theoretical_loss": 5.816461645636238,
+      "tokens_seen": 27066368
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004107142857142857,
+      "loss": 5.1234,
+      "theoretical_loss": 5.814155004649702,
+      "tokens_seen": 27131904
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004117063492063492,
+      "loss": 4.6509,
+      "theoretical_loss": 5.811855484304724,
+      "tokens_seen": 27197440
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004126984126984127,
+      "loss": 4.9814,
+      "theoretical_loss": 5.809563045540864,
+      "tokens_seen": 27262976
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004136904761904762,
+      "loss": 5.0187,
+      "theoretical_loss": 5.8072776496051866,
+      "tokens_seen": 27328512
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004146825396825397,
+      "loss": 4.9255,
+      "theoretical_loss": 5.804999258049106,
+      "tokens_seen": 27394048
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004156746031746032,
+      "loss": 5.0265,
+      "theoretical_loss": 5.802727832725283,
+      "tokens_seen": 27459584
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004166666666666667,
+      "loss": 4.8827,
+      "theoretical_loss": 5.800463335784541,
+      "tokens_seen": 27525120
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004176587301587302,
+      "loss": 5.0962,
+      "theoretical_loss": 5.798205729672842,
+      "tokens_seen": 27590656
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004186507936507937,
+      "loss": 5.0409,
+      "theoretical_loss": 5.795954977128286,
+      "tokens_seen": 27656192
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00041964285714285714,
+      "loss": 4.8332,
+      "theoretical_loss": 5.7937110411781525,
+      "tokens_seen": 27721728
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00042063492063492065,
+      "loss": 4.813,
+      "theoretical_loss": 5.79147388513598,
+      "tokens_seen": 27787264
+    },
+    {
+      "epoch": 0.01,
+      "objective/train/docs_used": 27295,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 4.60736083984375,
+      "objective/train/theoretical_loss": 5.789243472598683,
+      "objective/train/tokens_used": 48312800,
+      "theoretical_loss": 5.789243472598683,
+      "tokens_seen": 27852800
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00042162698412698415,
+      "loss": 4.735,
+      "theoretical_loss": 5.789243472598683,
+      "tokens_seen": 27852800
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00042261904761904765,
+      "loss": 5.0103,
+      "theoretical_loss": 5.787019767443696,
+      "tokens_seen": 27918336
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004236111111111111,
+      "loss": 4.9487,
+      "theoretical_loss": 5.784802733826166,
+      "tokens_seen": 27983872
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004246031746031746,
+      "loss": 5.0393,
+      "theoretical_loss": 5.782592336176171,
+      "tokens_seen": 28049408
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004255952380952381,
+      "loss": 4.8239,
+      "theoretical_loss": 5.780388539195972,
+      "tokens_seen": 28114944
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004265873015873016,
+      "loss": 4.8301,
+      "theoretical_loss": 5.778191307857307,
+      "tokens_seen": 28180480
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00042757936507936505,
+      "loss": 4.8818,
+      "theoretical_loss": 5.776000607398707,
+      "tokens_seen": 28246016
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00042857142857142855,
+      "loss": 5.0365,
+      "theoretical_loss": 5.773816403322854,
+      "tokens_seen": 28311552
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00042956349206349205,
+      "loss": 4.8806,
+      "theoretical_loss": 5.7716386613939665,
+      "tokens_seen": 28377088
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004305555555555556,
+      "loss": 4.8212,
+      "theoretical_loss": 5.769467347635221,
+      "tokens_seen": 28442624
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00043154761904761905,
+      "loss": 4.6856,
+      "theoretical_loss": 5.767302428326196,
+      "tokens_seen": 28508160
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00043253968253968256,
+      "loss": 5.1099,
+      "theoretical_loss": 5.765143870000358,
+      "tokens_seen": 28573696
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00043353174603174606,
+      "loss": 4.9942,
+      "theoretical_loss": 5.762991639442574,
+      "tokens_seen": 28639232
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00043452380952380956,
+      "loss": 4.7269,
+      "theoretical_loss": 5.7608457036866465,
+      "tokens_seen": 28704768
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.000435515873015873,
+      "loss": 4.7079,
+      "theoretical_loss": 5.758706030012889,
+      "tokens_seen": 28770304
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004365079365079365,
+      "loss": 5.0535,
+      "theoretical_loss": 5.756572585945728,
+      "tokens_seen": 28835840
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004375,
+      "loss": 4.9953,
+      "theoretical_loss": 5.754445339251326,
+      "tokens_seen": 28901376
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004384920634920635,
+      "loss": 4.8239,
+      "theoretical_loss": 5.752324257935244,
+      "tokens_seen": 28966912
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.000439484126984127,
+      "loss": 4.8738,
+      "theoretical_loss": 5.750209310240125,
+      "tokens_seen": 29032448
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00044047619047619046,
+      "loss": 4.5774,
+      "theoretical_loss": 5.7481004646434055,
+      "tokens_seen": 29097984
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00044146825396825396,
+      "loss": 4.7821,
+      "theoretical_loss": 5.745997689855058,
+      "tokens_seen": 29163520
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00044246031746031746,
+      "loss": 4.9197,
+      "theoretical_loss": 5.743900954815356,
+      "tokens_seen": 29229056
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00044345238095238096,
+      "loss": 4.9178,
+      "theoretical_loss": 5.741810228692663,
+      "tokens_seen": 29294592
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004444444444444444,
+      "loss": 4.9811,
+      "theoretical_loss": 5.739725480881262,
+      "tokens_seen": 29360128
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00044543650793650797,
+      "loss": 4.595,
+      "theoretical_loss": 5.737646680999193,
+      "tokens_seen": 29425664
+    },
+    {
+      "epoch": 0.01,
+      "objective/train/docs_used": 28553,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 4.193212032318115,
+      "objective/train/theoretical_loss": 5.73557379888612,
+      "objective/train/tokens_used": 49951200,
+      "theoretical_loss": 5.73557379888612,
+      "tokens_seen": 29491200
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00044642857142857147,
+      "loss": 4.6289,
+      "theoretical_loss": 5.73557379888612,
+      "tokens_seen": 29491200
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00044742063492063497,
+      "loss": 4.5175,
+      "theoretical_loss": 5.733506804601236,
+      "tokens_seen": 29556736
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004484126984126984,
+      "loss": 4.97,
+      "theoretical_loss": 5.7314456684211725,
+      "tokens_seen": 29622272
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004494047619047619,
+      "loss": 4.9007,
+      "theoretical_loss": 5.729390360837952,
+      "tokens_seen": 29687808
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004503968253968254,
+      "loss": 5.0718,
+      "theoretical_loss": 5.7273408525569485,
+      "tokens_seen": 29753344
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004513888888888889,
+      "loss": 4.8076,
+      "theoretical_loss": 5.725297114494884,
+      "tokens_seen": 29818880
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00045238095238095237,
+      "loss": 4.9151,
+      "theoretical_loss": 5.7232591177778405,
+      "tokens_seen": 29884416
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00045337301587301587,
+      "loss": 4.7611,
+      "theoretical_loss": 5.7212268337393,
+      "tokens_seen": 29949952
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00045436507936507937,
+      "loss": 4.6533,
+      "theoretical_loss": 5.719200233918203,
+      "tokens_seen": 30015488
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004553571428571429,
+      "loss": 4.7689,
+      "theoretical_loss": 5.717179290057032,
+      "tokens_seen": 30081024
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004563492063492063,
+      "loss": 4.753,
+      "theoretical_loss": 5.715163974099917,
+      "tokens_seen": 30146560
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004573412698412698,
+      "loss": 4.9783,
+      "theoretical_loss": 5.713154258190757,
+      "tokens_seen": 30212096
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004583333333333333,
+      "loss": 4.8919,
+      "theoretical_loss": 5.711150114671375,
+      "tokens_seen": 30277632
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004593253968253968,
+      "loss": 4.6246,
+      "theoretical_loss": 5.709151516079683,
+      "tokens_seen": 30343168
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00046031746031746033,
+      "loss": 4.8047,
+      "theoretical_loss": 5.707158435147875,
+      "tokens_seen": 30408704
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00046130952380952383,
+      "loss": 4.7619,
+      "theoretical_loss": 5.705170844800628,
+      "tokens_seen": 30474240
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00046230158730158733,
+      "loss": 5.0389,
+      "theoretical_loss": 5.703188718153347,
+      "tokens_seen": 30539776
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00046329365079365083,
+      "loss": 4.7978,
+      "theoretical_loss": 5.7012120285104055,
+      "tokens_seen": 30605312
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00046428571428571433,
+      "loss": 4.8182,
+      "theoretical_loss": 5.699240749363417,
+      "tokens_seen": 30670848
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004652777777777778,
+      "loss": 5.1046,
+      "theoretical_loss": 5.69727485438953,
+      "tokens_seen": 30736384
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004662698412698413,
+      "loss": 4.9301,
+      "theoretical_loss": 5.695314317449732,
+      "tokens_seen": 30801920
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004672619047619048,
+      "loss": 4.9513,
+      "theoretical_loss": 5.69335911258718,
+      "tokens_seen": 30867456
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004682539682539683,
+      "loss": 4.7506,
+      "theoretical_loss": 5.691409214025544,
+      "tokens_seen": 30932992
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00046924603174603173,
+      "loss": 4.9203,
+      "theoretical_loss": 5.689464596167383,
+      "tokens_seen": 30998528
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00047023809523809523,
+      "loss": 4.6588,
+      "theoretical_loss": 5.687525233592513,
+      "tokens_seen": 31064064
+    },
+    {
+      "epoch": 0.01,
+      "objective/train/docs_used": 29210,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 4.980744361877441,
+      "objective/train/theoretical_loss": 5.685591101056422,
+      "objective/train/tokens_used": 51589600,
+      "theoretical_loss": 5.685591101056422,
+      "tokens_seen": 31129600
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00047123015873015874,
+      "loss": 4.8131,
+      "theoretical_loss": 5.685591101056422,
+      "tokens_seen": 31129600
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00047222222222222224,
+      "loss": 4.9729,
+      "theoretical_loss": 5.683662173488678,
+      "tokens_seen": 31195136
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004732142857142857,
+      "loss": 4.9988,
+      "theoretical_loss": 5.681738425991377,
+      "tokens_seen": 31260672
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004742063492063492,
+      "loss": 4.9296,
+      "theoretical_loss": 5.679819833837586,
+      "tokens_seen": 31326208
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004751984126984127,
+      "loss": 4.8866,
+      "theoretical_loss": 5.677906372469826,
+      "tokens_seen": 31391744
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004761904761904762,
+      "loss": 4.8619,
+      "theoretical_loss": 5.675998017498549,
+      "tokens_seen": 31457280
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004771825396825397,
+      "loss": 4.4881,
+      "theoretical_loss": 5.674094744700648,
+      "tokens_seen": 31522816
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004781746031746032,
+      "loss": 4.7385,
+      "theoretical_loss": 5.672196530017979,
+      "tokens_seen": 31588352
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004791666666666667,
+      "loss": 4.9446,
+      "theoretical_loss": 5.670303349555893,
+      "tokens_seen": 31653888
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004801587301587302,
+      "loss": 4.5805,
+      "theoretical_loss": 5.668415179581795,
+      "tokens_seen": 31719424
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004811507936507937,
+      "loss": 4.943,
+      "theoretical_loss": 5.666531996523711,
+      "tokens_seen": 31784960
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00048214285714285715,
+      "loss": 4.9994,
+      "theoretical_loss": 5.664653776968867,
+      "tokens_seen": 31850496
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00048313492063492065,
+      "loss": 4.9432,
+      "theoretical_loss": 5.6627804976622995,
+      "tokens_seen": 31916032
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00048412698412698415,
+      "loss": 4.4612,
+      "theoretical_loss": 5.660912135505461,
+      "tokens_seen": 31981568
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00048511904761904765,
+      "loss": 5.0296,
+      "theoretical_loss": 5.659048667554853,
+      "tokens_seen": 32047104
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004861111111111111,
+      "loss": 4.7342,
+      "theoretical_loss": 5.657190071020672,
+      "tokens_seen": 32112640
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004871031746031746,
+      "loss": 5.0271,
+      "theoretical_loss": 5.65533632326547,
+      "tokens_seen": 32178176
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004880952380952381,
+      "loss": 4.5208,
+      "theoretical_loss": 5.6534874018028205,
+      "tokens_seen": 32243712
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004890873015873016,
+      "loss": 4.8883,
+      "theoretical_loss": 5.651643284296018,
+      "tokens_seen": 32309248
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004900793650793651,
+      "loss": 4.5918,
+      "theoretical_loss": 5.6498039485567695,
+      "tokens_seen": 32374784
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004910714285714286,
+      "loss": 4.8362,
+      "theoretical_loss": 5.6479693725439155,
+      "tokens_seen": 32440320
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.000492063492063492,
+      "loss": 4.4816,
+      "theoretical_loss": 5.646139534362161,
+      "tokens_seen": 32505856
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004930555555555556,
+      "loss": 4.7304,
+      "theoretical_loss": 5.644314412260815,
+      "tokens_seen": 32571392
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004940476190476191,
+      "loss": 4.7639,
+      "theoretical_loss": 5.642493984632544,
+      "tokens_seen": 32636928
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004950396825396826,
+      "loss": 4.486,
+      "theoretical_loss": 5.640678230012151,
+      "tokens_seen": 32702464
+    },
+    {
+      "epoch": 0.01,
+      "objective/train/docs_used": 30192,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 4.205135822296143,
+      "objective/train/theoretical_loss": 5.638867127075349,
+      "objective/train/tokens_used": 53228000,
+      "theoretical_loss": 5.638867127075349,
+      "tokens_seen": 32768000
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.000496031746031746,
+      "loss": 4.6383,
+      "theoretical_loss": 5.638867127075349,
+      "tokens_seen": 32768000
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004970238095238095,
+      "loss": 4.7665,
+      "theoretical_loss": 5.63706065463756,
+      "tokens_seen": 32833536
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.000498015873015873,
+      "loss": 4.6651,
+      "theoretical_loss": 5.635258791652723,
+      "tokens_seen": 32899072
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004990079365079365,
+      "loss": 4.6032,
+      "theoretical_loss": 5.633461517212115,
+      "tokens_seen": 32964608
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0005,
+      "loss": 4.8611,
+      "theoretical_loss": 5.631668810543181,
+      "tokens_seen": 33030144
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004999899699097292,
+      "loss": 4.4403,
+      "theoretical_loss": 5.629880651008384,
+      "tokens_seen": 33095680
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004999799398194584,
+      "loss": 4.6342,
+      "theoretical_loss": 5.628097018104057,
+      "tokens_seen": 33161216
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004999699097291876,
+      "loss": 4.789,
+      "theoretical_loss": 5.626317891459275,
+      "tokens_seen": 33226752
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004999598796389167,
+      "loss": 4.4669,
+      "theoretical_loss": 5.6245432508347335,
+      "tokens_seen": 33292288
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.000499949849548646,
+      "loss": 4.5265,
+      "theoretical_loss": 5.622773076121648,
+      "tokens_seen": 33357824
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004999398194583751,
+      "loss": 4.8668,
+      "theoretical_loss": 5.621007347340646,
+      "tokens_seen": 33423360
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004999297893681044,
+      "loss": 4.663,
+      "theoretical_loss": 5.619246044640693,
+      "tokens_seen": 33488896
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004999197592778335,
+      "loss": 4.51,
+      "theoretical_loss": 5.617489148298015,
+      "tokens_seen": 33554432
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004999097291875627,
+      "loss": 4.4827,
+      "theoretical_loss": 5.615736638715033,
+      "tokens_seen": 33619968
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004998996990972919,
+      "loss": 4.6552,
+      "theoretical_loss": 5.613988496419315,
+      "tokens_seen": 33685504
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004998896690070211,
+      "loss": 4.5444,
+      "theoretical_loss": 5.6122447020625374,
+      "tokens_seen": 33751040
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004998796389167503,
+      "loss": 4.7462,
+      "theoretical_loss": 5.610505236419448,
+      "tokens_seen": 33816576
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004998696088264795,
+      "loss": 4.6343,
+      "theoretical_loss": 5.608770080386854,
+      "tokens_seen": 33882112
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004998595787362087,
+      "loss": 4.3194,
+      "theoretical_loss": 5.6070392149826045,
+      "tokens_seen": 33947648
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004998495486459378,
+      "loss": 4.7376,
+      "theoretical_loss": 5.605312621344602,
+      "tokens_seen": 34013184
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.000499839518555667,
+      "loss": 4.6319,
+      "theoretical_loss": 5.603590280729803,
+      "tokens_seen": 34078720
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004998294884653962,
+      "loss": 4.3936,
+      "theoretical_loss": 5.601872174513245,
+      "tokens_seen": 34144256
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004998194583751254,
+      "loss": 4.8045,
+      "theoretical_loss": 5.60015828418708,
+      "tokens_seen": 34209792
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004998094282848546,
+      "loss": 4.8346,
+      "theoretical_loss": 5.59844859135961,
+      "tokens_seen": 34275328
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004997993981945837,
+      "loss": 4.4444,
+      "theoretical_loss": 5.596743077754337,
+      "tokens_seen": 34340864
+    },
+    {
+      "epoch": 0.01,
+      "objective/train/docs_used": 30758,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 4.619051933288574,
+      "objective/train/theoretical_loss": 5.5950417252090325,
+      "objective/train/tokens_used": 54866400,
+      "theoretical_loss": 5.5950417252090325,
+      "tokens_seen": 34406400
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004997893681043129,
+      "loss": 4.5684,
+      "theoretical_loss": 5.5950417252090325,
+      "tokens_seen": 34406400
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004997793380140421,
+      "loss": 4.7403,
+      "theoretical_loss": 5.593344515674795,
+      "tokens_seen": 34471936
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004997693079237714,
+      "loss": 4.7682,
+      "theoretical_loss": 5.591651431215139,
+      "tokens_seen": 34537472
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004997592778335005,
+      "loss": 4.9194,
+      "theoretical_loss": 5.589962454005072,
+      "tokens_seen": 34603008
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004997492477432298,
+      "loss": 4.7926,
+      "theoretical_loss": 5.588277566330207,
+      "tokens_seen": 34668544
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004997392176529588,
+      "loss": 4.5919,
+      "theoretical_loss": 5.586596750585855,
+      "tokens_seen": 34734080
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004997291875626881,
+      "loss": 4.8471,
+      "theoretical_loss": 5.584919989276146,
+      "tokens_seen": 34799616
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004997191574724173,
+      "loss": 4.8823,
+      "theoretical_loss": 5.583247265013155,
+      "tokens_seen": 34865152
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004997091273821465,
+      "loss": 4.7952,
+      "theoretical_loss": 5.581578560516027,
+      "tokens_seen": 34930688
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004996990972918757,
+      "loss": 4.8327,
+      "theoretical_loss": 5.57991385861013,
+      "tokens_seen": 34996224
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004996890672016048,
+      "loss": 4.3745,
+      "theoretical_loss": 5.578253142226191,
+      "tokens_seen": 35061760
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.000499679037111334,
+      "loss": 4.6921,
+      "theoretical_loss": 5.576596394399467,
+      "tokens_seen": 35127296
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004996690070210632,
+      "loss": 4.6698,
+      "theoretical_loss": 5.574943598268902,
+      "tokens_seen": 35192832
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004996589769307924,
+      "loss": 4.9374,
+      "theoretical_loss": 5.573294737076306,
+      "tokens_seen": 35258368
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004996489468405216,
+      "loss": 4.5814,
+      "theoretical_loss": 5.571649794165537,
+      "tokens_seen": 35323904
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004996389167502507,
+      "loss": 4.6064,
+      "theoretical_loss": 5.570008752981688,
+      "tokens_seen": 35389440
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004996288866599799,
+      "loss": 4.7137,
+      "theoretical_loss": 5.5683715970702945,
+      "tokens_seen": 35454976
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004996188565697091,
+      "loss": 4.7919,
+      "theoretical_loss": 5.566738310076531,
+      "tokens_seen": 35520512
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004996088264794383,
+      "loss": 4.5989,
+      "theoretical_loss": 5.5651088757444285,
+      "tokens_seen": 35586048
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004995987963891675,
+      "loss": 4.4273,
+      "theoretical_loss": 5.5634832779161,
+      "tokens_seen": 35651584
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004995887662988968,
+      "loss": 4.6147,
+      "theoretical_loss": 5.561861500530965,
+      "tokens_seen": 35717120
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004995787362086258,
+      "loss": 4.5803,
+      "theoretical_loss": 5.560243527624989,
+      "tokens_seen": 35782656
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004995687061183551,
+      "loss": 4.762,
+      "theoretical_loss": 5.558629343329924,
+      "tokens_seen": 35848192
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004995586760280842,
+      "loss": 4.7117,
+      "theoretical_loss": 5.557018931872566,
+      "tokens_seen": 35913728
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004995486459378135,
+      "loss": 4.7915,
+      "theoretical_loss": 5.555412277574007,
+      "tokens_seen": 35979264
+    },
+    {
+      "epoch": 0.01,
+      "objective/train/docs_used": 31707,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 4.547565460205078,
+      "objective/train/theoretical_loss": 5.553809364848902,
+      "objective/train/tokens_used": 56504800,
+      "theoretical_loss": 5.553809364848902,
+      "tokens_seen": 36044800
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004995386158475427,
+      "loss": 4.6425,
+      "theoretical_loss": 5.553809364848902,
+      "tokens_seen": 36044800
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004995285857572718,
+      "loss": 4.5963,
+      "theoretical_loss": 5.552210178204751,
+      "tokens_seen": 36110336
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.000499518555667001,
+      "loss": 4.6793,
+      "theoretical_loss": 5.550614702241162,
+      "tokens_seen": 36175872
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004995085255767302,
+      "loss": 4.7561,
+      "theoretical_loss": 5.549022921649151,
+      "tokens_seen": 36241408
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004994984954864594,
+      "loss": 4.6607,
+      "theoretical_loss": 5.54743482121043,
+      "tokens_seen": 36306944
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004994884653961886,
+      "loss": 4.6252,
+      "theoretical_loss": 5.545850385796712,
+      "tokens_seen": 36372480
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004994784353059178,
+      "loss": 4.4331,
+      "theoretical_loss": 5.544269600369009,
+      "tokens_seen": 36438016
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004994684052156469,
+      "loss": 4.643,
+      "theoretical_loss": 5.5426924499769505,
+      "tokens_seen": 36503552
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004994583751253761,
+      "loss": 4.7548,
+      "theoretical_loss": 5.541118919758107,
+      "tokens_seen": 36569088
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004994483450351053,
+      "loss": 4.7337,
+      "theoretical_loss": 5.539548994937308,
+      "tokens_seen": 36634624
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004994383149448345,
+      "loss": 4.3641,
+      "theoretical_loss": 5.537982660825981,
+      "tokens_seen": 36700160
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004994282848545637,
+      "loss": 4.5599,
+      "theoretical_loss": 5.536419902821491,
+      "tokens_seen": 36765696
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004994182547642928,
+      "loss": 4.6875,
+      "theoretical_loss": 5.534860706406477,
+      "tokens_seen": 36831232
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004994082246740221,
+      "loss": 4.7352,
+      "theoretical_loss": 5.533305057148215,
+      "tokens_seen": 36896768
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004993981945837512,
+      "loss": 4.6149,
+      "theoretical_loss": 5.531752940697968,
+      "tokens_seen": 36962304
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004993881644934805,
+      "loss": 4.5067,
+      "theoretical_loss": 5.5302043427903484,
+      "tokens_seen": 37027840
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004993781344032096,
+      "loss": 4.9474,
+      "theoretical_loss": 5.528659249242693,
+      "tokens_seen": 37093376
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004993681043129389,
+      "loss": 4.4017,
+      "theoretical_loss": 5.527117645954435,
+      "tokens_seen": 37158912
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.000499358074222668,
+      "loss": 4.5323,
+      "theoretical_loss": 5.525579518906484,
+      "tokens_seen": 37224448
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004993480441323972,
+      "loss": 4.4802,
+      "theoretical_loss": 5.524044854160613,
+      "tokens_seen": 37289984
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004993380140421264,
+      "loss": 4.4531,
+      "theoretical_loss": 5.522513637858855,
+      "tokens_seen": 37355520
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004993279839518556,
+      "loss": 4.6216,
+      "theoretical_loss": 5.520985856222893,
+      "tokens_seen": 37421056
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004993179538615848,
+      "loss": 4.5407,
+      "theoretical_loss": 5.519461495553473,
+      "tokens_seen": 37486592
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004993079237713139,
+      "loss": 4.4097,
+      "theoretical_loss": 5.5179405422298045,
+      "tokens_seen": 37552128
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004992978936810431,
+      "loss": 4.406,
+      "theoretical_loss": 5.516422982708983,
+      "tokens_seen": 37617664
+    },
+    {
+      "epoch": 0.01,
+      "objective/train/docs_used": 32203,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 4.049692630767822,
+      "objective/train/theoretical_loss": 5.514908803525402,
+      "objective/train/tokens_used": 58143200,
+      "theoretical_loss": 5.514908803525402,
+      "tokens_seen": 37683200
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004992878635907723,
+      "loss": 4.3638,
+      "theoretical_loss": 5.514908803525402,
+      "tokens_seen": 37683200
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004992778335005015,
+      "loss": 4.6329,
+      "theoretical_loss": 5.513397991290185,
+      "tokens_seen": 37748736
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004992678034102307,
+      "loss": 4.6538,
+      "theoretical_loss": 5.511890532690611,
+      "tokens_seen": 37814272
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004992577733199598,
+      "loss": 4.7513,
+      "theoretical_loss": 5.510386414489556,
+      "tokens_seen": 37879808
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.000499247743229689,
+      "loss": 4.6591,
+      "theoretical_loss": 5.5088856235249235,
+      "tokens_seen": 37945344
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004992377131394183,
+      "loss": 4.4618,
+      "theoretical_loss": 5.507388146709105,
+      "tokens_seen": 38010880
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004992276830491475,
+      "loss": 4.4231,
+      "theoretical_loss": 5.505893971028418,
+      "tokens_seen": 38076416
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004992176529588767,
+      "loss": 4.2713,
+      "theoretical_loss": 5.504403083542568,
+      "tokens_seen": 38141952
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004992076228686059,
+      "loss": 4.4092,
+      "theoretical_loss": 5.502915471384111,
+      "tokens_seen": 38207488
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.000499197592778335,
+      "loss": 4.7235,
+      "theoretical_loss": 5.501431121757914,
+      "tokens_seen": 38273024
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004991875626880642,
+      "loss": 4.3762,
+      "theoretical_loss": 5.49995002194063,
+      "tokens_seen": 38338560
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004991775325977934,
+      "loss": 4.6923,
+      "theoretical_loss": 5.498472159280171,
+      "tokens_seen": 38404096
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004991675025075226,
+      "loss": 4.8392,
+      "theoretical_loss": 5.4969975211951905,
+      "tokens_seen": 38469632
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004991574724172518,
+      "loss": 4.5345,
+      "theoretical_loss": 5.495526095174563,
+      "tokens_seen": 38535168
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.000499147442326981,
+      "loss": 4.6407,
+      "theoretical_loss": 5.494057868776883,
+      "tokens_seen": 38600704
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004991374122367101,
+      "loss": 4.5119,
+      "theoretical_loss": 5.492592829629946,
+      "tokens_seen": 38666240
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004991273821464393,
+      "loss": 4.6456,
+      "theoretical_loss": 5.491130965430262,
+      "tokens_seen": 38731776
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004991173520561685,
+      "loss": 4.2981,
+      "theoretical_loss": 5.489672263942543,
+      "tokens_seen": 38797312
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004991073219658977,
+      "loss": 4.7488,
+      "theoretical_loss": 5.488216712999222,
+      "tokens_seen": 38862848
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.000499097291875627,
+      "loss": 4.5654,
+      "theoretical_loss": 5.486764300499962,
+      "tokens_seen": 38928384
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.000499087261785356,
+      "loss": 4.3462,
+      "theoretical_loss": 5.4853150144111655,
+      "tokens_seen": 38993920
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004990772316950853,
+      "loss": 4.6389,
+      "theoretical_loss": 5.4838688427655065,
+      "tokens_seen": 39059456
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004990672016048144,
+      "loss": 4.5166,
+      "theoretical_loss": 5.482425773661445,
+      "tokens_seen": 39124992
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004990571715145437,
+      "loss": 4.5858,
+      "theoretical_loss": 5.480985795262759,
+      "tokens_seen": 39190528
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004990471414242729,
+      "loss": 4.6098,
+      "theoretical_loss": 5.479548895798082,
+      "tokens_seen": 39256064
+    },
+    {
+      "epoch": 0.01,
+      "objective/train/docs_used": 33134,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 4.761120319366455,
+      "objective/train/theoretical_loss": 5.478115063560433,
+      "objective/train/tokens_used": 59781600,
+      "theoretical_loss": 5.478115063560433,
+      "tokens_seen": 39321600
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.000499037111334002,
+      "loss": 4.4957,
+      "theoretical_loss": 5.478115063560433,
+      "tokens_seen": 39321600
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004990270812437312,
+      "loss": 4.7046,
+      "theoretical_loss": 5.476684286906764,
+      "tokens_seen": 39387136
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004990170511534604,
+      "loss": 4.2948,
+      "theoretical_loss": 5.475256554257497,
+      "tokens_seen": 39452672
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004990070210631896,
+      "loss": 4.5969,
+      "theoretical_loss": 5.473831854096085,
+      "tokens_seen": 39518208
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004989969909729188,
+      "loss": 4.5511,
+      "theoretical_loss": 5.472410174968553,
+      "tokens_seen": 39583744
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.000498986960882648,
+      "loss": 4.4106,
+      "theoretical_loss": 5.470991505483065,
+      "tokens_seen": 39649280
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004989769307923771,
+      "loss": 4.623,
+      "theoretical_loss": 5.469575834309477,
+      "tokens_seen": 39714816
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004989669007021063,
+      "loss": 4.5077,
+      "theoretical_loss": 5.468163150178907,
+      "tokens_seen": 39780352
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004989568706118355,
+      "loss": 4.475,
+      "theoretical_loss": 5.466753441883302,
+      "tokens_seen": 39845888
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004989468405215647,
+      "loss": 4.6451,
+      "theoretical_loss": 5.465346698275008,
+      "tokens_seen": 39911424
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004989368104312939,
+      "loss": 4.3366,
+      "theoretical_loss": 5.46394290826635,
+      "tokens_seen": 39976960
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.000498926780341023,
+      "loss": 4.4816,
+      "theoretical_loss": 5.462542060829209,
+      "tokens_seen": 40042496
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004989167502507523,
+      "loss": 4.5005,
+      "theoretical_loss": 5.461144144994606,
+      "tokens_seen": 40108032
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004989067201604814,
+      "loss": 4.5075,
+      "theoretical_loss": 5.459749149852287,
+      "tokens_seen": 40173568
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004988966900702107,
+      "loss": 4.555,
+      "theoretical_loss": 5.4583570645503166,
+      "tokens_seen": 40239104
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004988866599799398,
+      "loss": 4.411,
+      "theoretical_loss": 5.45696787829467,
+      "tokens_seen": 40304640
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004988766298896691,
+      "loss": 4.479,
+      "theoretical_loss": 5.455581580348831,
+      "tokens_seen": 40370176
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004988665997993982,
+      "loss": 4.424,
+      "theoretical_loss": 5.454198160033391,
+      "tokens_seen": 40435712
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004988565697091274,
+      "loss": 4.6216,
+      "theoretical_loss": 5.452817606725657,
+      "tokens_seen": 40501248
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004988465396188566,
+      "loss": 4.5915,
+      "theoretical_loss": 5.451439909859255,
+      "tokens_seen": 40566784
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004988365095285858,
+      "loss": 4.4225,
+      "theoretical_loss": 5.450065058923747,
+      "tokens_seen": 40632320
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.000498826479438315,
+      "loss": 4.4659,
+      "theoretical_loss": 5.448693043464235,
+      "tokens_seen": 40697856
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004988164493480441,
+      "loss": 4.7231,
+      "theoretical_loss": 5.447323853080991,
+      "tokens_seen": 40763392
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004988064192577733,
+      "loss": 4.561,
+      "theoretical_loss": 5.445957477429067,
+      "tokens_seen": 40828928
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004987963891675025,
+      "loss": 4.4559,
+      "theoretical_loss": 5.444593906217927,
+      "tokens_seen": 40894464
+    },
+    {
+      "epoch": 0.01,
+      "objective/train/docs_used": 33820,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 4.709114074707031,
+      "objective/train/theoretical_loss": 5.443233129211069,
+      "objective/train/tokens_used": 61420000,
+      "theoretical_loss": 5.443233129211069,
+      "tokens_seen": 40960000
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004987863590772317,
+      "loss": 4.5353,
+      "theoretical_loss": 5.443233129211069,
+      "tokens_seen": 40960000
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004987763289869609,
+      "loss": 4.5888,
+      "theoretical_loss": 5.441875136225658,
+      "tokens_seen": 41025536
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00049876629889669,
+      "loss": 4.6609,
+      "theoretical_loss": 5.440519917132159,
+      "tokens_seen": 41091072
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004987562688064192,
+      "loss": 4.6782,
+      "theoretical_loss": 5.439167461853971,
+      "tokens_seen": 41156608
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004987462387161484,
+      "loss": 4.3218,
+      "theoretical_loss": 5.437817760367073,
+      "tokens_seen": 41222144
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004987362086258777,
+      "loss": 4.6102,
+      "theoretical_loss": 5.436470802699661,
+      "tokens_seen": 41287680
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004987261785356068,
+      "loss": 4.4044,
+      "theoretical_loss": 5.43512657893179,
+      "tokens_seen": 41353216
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004987161484453361,
+      "loss": 4.5902,
+      "theoretical_loss": 5.433785079195035,
+      "tokens_seen": 41418752
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004987061183550651,
+      "loss": 4.4395,
+      "theoretical_loss": 5.432446293672129,
+      "tokens_seen": 41484288
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004986960882647944,
+      "loss": 4.6268,
+      "theoretical_loss": 5.43111021259663,
+      "tokens_seen": 41549824
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004986860581745236,
+      "loss": 4.7247,
+      "theoretical_loss": 5.429776826252565,
+      "tokens_seen": 41615360
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004986760280842528,
+      "loss": 4.7844,
+      "theoretical_loss": 5.428446124974103,
+      "tokens_seen": 41680896
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.000498665997993982,
+      "loss": 4.748,
+      "theoretical_loss": 5.427118099145207,
+      "tokens_seen": 41746432
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004986559679037111,
+      "loss": 4.4521,
+      "theoretical_loss": 5.425792739199306,
+      "tokens_seen": 41811968
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004986459378134403,
+      "loss": 4.5162,
+      "theoretical_loss": 5.424470035618963,
+      "tokens_seen": 41877504
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004986359077231695,
+      "loss": 4.5787,
+      "theoretical_loss": 5.423149978935547,
+      "tokens_seen": 41943040
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004986258776328987,
+      "loss": 4.4447,
+      "theoretical_loss": 5.421832559728902,
+      "tokens_seen": 42008576
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004986158475426279,
+      "loss": 4.5436,
+      "theoretical_loss": 5.420517768627027,
+      "tokens_seen": 42074112
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.000498605817452357,
+      "loss": 4.5508,
+      "theoretical_loss": 5.419205596305758,
+      "tokens_seen": 42139648
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004985957873620862,
+      "loss": 4.3851,
+      "theoretical_loss": 5.417896033488446,
+      "tokens_seen": 42205184
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004985857572718154,
+      "loss": 4.4232,
+      "theoretical_loss": 5.416589070945641,
+      "tokens_seen": 42270720
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004985757271815446,
+      "loss": 4.4672,
+      "theoretical_loss": 5.415284699494782,
+      "tokens_seen": 42336256
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004985656970912738,
+      "loss": 4.3494,
+      "theoretical_loss": 5.413982909999887,
+      "tokens_seen": 42401792
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004985556670010031,
+      "loss": 4.246,
+      "theoretical_loss": 5.412683693371241,
+      "tokens_seen": 42467328
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004985456369107321,
+      "loss": 4.5265,
+      "theoretical_loss": 5.411387040565092,
+      "tokens_seen": 42532864
+    },
+    {
+      "epoch": 0.01,
+      "objective/train/docs_used": 34938,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 3.999885082244873,
+      "objective/train/theoretical_loss": 5.410092942583357,
+      "objective/train/tokens_used": 63058400,
+      "theoretical_loss": 5.410092942583357,
+      "tokens_seen": 42598400
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004985356068204614,
+      "loss": 4.5919,
+      "theoretical_loss": 5.410092942583357,
+      "tokens_seen": 42598400
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004985255767301905,
+      "loss": 4.603,
+      "theoretical_loss": 5.408801390473304,
+      "tokens_seen": 42663936
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004985155466399198,
+      "loss": 4.546,
+      "theoretical_loss": 5.407512375327272,
+      "tokens_seen": 42729472
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.000498505516549649,
+      "loss": 4.5326,
+      "theoretical_loss": 5.406225888282363,
+      "tokens_seen": 42795008
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004984954864593782,
+      "loss": 4.3889,
+      "theoretical_loss": 5.404941920520156,
+      "tokens_seen": 42860544
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004984854563691073,
+      "loss": 4.1785,
+      "theoretical_loss": 5.403660463266413,
+      "tokens_seen": 42926080
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004984754262788365,
+      "loss": 4.5382,
+      "theoretical_loss": 5.402381507790792,
+      "tokens_seen": 42991616
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004984653961885657,
+      "loss": 4.5693,
+      "theoretical_loss": 5.401105045406557,
+      "tokens_seen": 43057152
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004984553660982949,
+      "loss": 4.4403,
+      "theoretical_loss": 5.399831067470302,
+      "tokens_seen": 43122688
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004984453360080241,
+      "loss": 4.7214,
+      "theoretical_loss": 5.398559565381662,
+      "tokens_seen": 43188224
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004984353059177532,
+      "loss": 4.4442,
+      "theoretical_loss": 5.397290530583037,
+      "tokens_seen": 43253760
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004984252758274825,
+      "loss": 4.309,
+      "theoretical_loss": 5.396023954559315,
+      "tokens_seen": 43319296
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004984152457372116,
+      "loss": 4.3474,
+      "theoretical_loss": 5.394759828837596,
+      "tokens_seen": 43384832
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004984052156469409,
+      "loss": 4.5364,
+      "theoretical_loss": 5.393498144986921,
+      "tokens_seen": 43450368
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00049839518555667,
+      "loss": 4.647,
+      "theoretical_loss": 5.392238894618,
+      "tokens_seen": 43515904
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004983851554663993,
+      "loss": 4.4688,
+      "theoretical_loss": 5.390982069382944,
+      "tokens_seen": 43581440
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004983751253761284,
+      "loss": 4.555,
+      "theoretical_loss": 5.389727660974998,
+      "tokens_seen": 43646976
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004983650952858576,
+      "loss": 4.2673,
+      "theoretical_loss": 5.38847566112828,
+      "tokens_seen": 43712512
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004983550651955868,
+      "loss": 4.223,
+      "theoretical_loss": 5.387226061617516,
+      "tokens_seen": 43778048
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.000498345035105316,
+      "loss": 4.5944,
+      "theoretical_loss": 5.38597885425778,
+      "tokens_seen": 43843584
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004983350050150452,
+      "loss": 4.2395,
+      "theoretical_loss": 5.384734030904238,
+      "tokens_seen": 43909120
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004983249749247743,
+      "loss": 4.2047,
+      "theoretical_loss": 5.383491583451892,
+      "tokens_seen": 43974656
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004983149448345035,
+      "loss": 4.6784,
+      "theoretical_loss": 5.382251503835324,
+      "tokens_seen": 44040192
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004983049147442327,
+      "loss": 4.5094,
+      "theoretical_loss": 5.381013784028445,
+      "tokens_seen": 44105728
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004982948846539619,
+      "loss": 4.2719,
+      "theoretical_loss": 5.379778416044247,
+      "tokens_seen": 44171264
+    },
+    {
+      "epoch": 0.01,
+      "objective/train/docs_used": 35534,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 4.2067718505859375,
+      "objective/train/theoretical_loss": 5.378545391934553,
+      "objective/train/tokens_used": 64696800,
+      "theoretical_loss": 5.378545391934553,
+      "tokens_seen": 44236800
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004982848545636911,
+      "loss": 4.3011,
+      "theoretical_loss": 5.378545391934553,
+      "tokens_seen": 44236800
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004982748244734202,
+      "loss": 4.3593,
+      "theoretical_loss": 5.377314703789773,
+      "tokens_seen": 44302336
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004982647943831494,
+      "loss": 4.5546,
+      "theoretical_loss": 5.376086343738656,
+      "tokens_seen": 44367872
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004982547642928786,
+      "loss": 4.2113,
+      "theoretical_loss": 5.3748603039480525,
+      "tokens_seen": 44433408
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004982447342026079,
+      "loss": 4.6548,
+      "theoretical_loss": 5.373636576622671,
+      "tokens_seen": 44498944
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.000498234704112337,
+      "loss": 4.6067,
+      "theoretical_loss": 5.372415154004842,
+      "tokens_seen": 44564480
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004982246740220663,
+      "loss": 4.5434,
+      "theoretical_loss": 5.371196028374278,
+      "tokens_seen": 44630016
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004982146439317953,
+      "loss": 4.219,
+      "theoretical_loss": 5.369979192047846,
+      "tokens_seen": 44695552
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004982046138415246,
+      "loss": 4.4616,
+      "theoretical_loss": 5.368764637379322,
+      "tokens_seen": 44761088
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004981945837512538,
+      "loss": 4.3927,
+      "theoretical_loss": 5.367552356759175,
+      "tokens_seen": 44826624
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.000498184553660983,
+      "loss": 4.3436,
+      "theoretical_loss": 5.366342342614322,
+      "tokens_seen": 44892160
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004981745235707122,
+      "loss": 4.3641,
+      "theoretical_loss": 5.365134587407918,
+      "tokens_seen": 44957696
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004981644934804413,
+      "loss": 4.3878,
+      "theoretical_loss": 5.363929083639112,
+      "tokens_seen": 45023232
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004981544633901705,
+      "loss": 4.252,
+      "theoretical_loss": 5.362725823842838,
+      "tokens_seen": 45088768
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004981444332998997,
+      "loss": 4.1657,
+      "theoretical_loss": 5.361524800589581,
+      "tokens_seen": 45154304
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004981344032096289,
+      "loss": 4.0417,
+      "theoretical_loss": 5.360326006485165,
+      "tokens_seen": 45219840
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004981243731193581,
+      "loss": 4.6023,
+      "theoretical_loss": 5.359129434170529,
+      "tokens_seen": 45285376
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004981143430290873,
+      "loss": 4.3602,
+      "theoretical_loss": 5.357935076321514,
+      "tokens_seen": 45350912
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004981043129388164,
+      "loss": 4.4446,
+      "theoretical_loss": 5.35674292564864,
+      "tokens_seen": 45416448
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004980942828485456,
+      "loss": 4.4122,
+      "theoretical_loss": 5.3555529748969,
+      "tokens_seen": 45481984
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004980842527582748,
+      "loss": 4.465,
+      "theoretical_loss": 5.354365216845546,
+      "tokens_seen": 45547520
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.000498074222668004,
+      "loss": 4.1501,
+      "theoretical_loss": 5.353179644307868,
+      "tokens_seen": 45613056
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004980641925777333,
+      "loss": 4.6183,
+      "theoretical_loss": 5.351996250131005,
+      "tokens_seen": 45678592
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004980541624874623,
+      "loss": 4.4826,
+      "theoretical_loss": 5.350815027195717,
+      "tokens_seen": 45744128
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004980441323971916,
+      "loss": 4.0969,
+      "theoretical_loss": 5.349635968416189,
+      "tokens_seen": 45809664
+    },
+    {
+      "epoch": 0.01,
+      "objective/train/docs_used": 36791,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 3.983624219894409,
+      "objective/train/theoretical_loss": 5.34845906673983,
+      "objective/train/tokens_used": 66335200,
+      "theoretical_loss": 5.34845906673983,
+      "tokens_seen": 45875200
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004980341023069207,
+      "loss": 4.1114,
+      "theoretical_loss": 5.34845906673983,
+      "tokens_seen": 45875200
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00049802407221665,
+      "loss": 4.3748,
+      "theoretical_loss": 5.347284315147059,
+      "tokens_seen": 45940736
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004980140421263792,
+      "loss": 4.3086,
+      "theoretical_loss": 5.346111706651114,
+      "tokens_seen": 46006272
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004980040120361084,
+      "loss": 4.5521,
+      "theoretical_loss": 5.344941234297847,
+      "tokens_seen": 46071808
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004979939819458375,
+      "loss": 4.4492,
+      "theoretical_loss": 5.343772891165527,
+      "tokens_seen": 46137344
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004979839518555667,
+      "loss": 4.4528,
+      "theoretical_loss": 5.34260667036464,
+      "tokens_seen": 46202880
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004979739217652959,
+      "loss": 4.6275,
+      "theoretical_loss": 5.3414425650377,
+      "tokens_seen": 46268416
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004979638916750251,
+      "loss": 4.5955,
+      "theoretical_loss": 5.3402805683590495,
+      "tokens_seen": 46333952
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004979538615847543,
+      "loss": 4.5274,
+      "theoretical_loss": 5.339120673534668,
+      "tokens_seen": 46399488
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004979438314944834,
+      "loss": 4.1984,
+      "theoretical_loss": 5.337962873801984,
+      "tokens_seen": 46465024
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004979338014042126,
+      "loss": 4.4472,
+      "theoretical_loss": 5.336807162429681,
+      "tokens_seen": 46530560
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004979237713139418,
+      "loss": 4.2688,
+      "theoretical_loss": 5.335653532717512,
+      "tokens_seen": 46596096
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.000497913741223671,
+      "loss": 4.5101,
+      "theoretical_loss": 5.33450197799611,
+      "tokens_seen": 46661632
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004979037111334002,
+      "loss": 4.4218,
+      "theoretical_loss": 5.33335249162681,
+      "tokens_seen": 46727168
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004978936810431293,
+      "loss": 4.365,
+      "theoretical_loss": 5.332205067001454,
+      "tokens_seen": 46792704
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004978836509528586,
+      "loss": 4.4012,
+      "theoretical_loss": 5.331059697542216,
+      "tokens_seen": 46858240
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004978736208625877,
+      "loss": 4.178,
+      "theoretical_loss": 5.329916376701417,
+      "tokens_seen": 46923776
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.000497863590772317,
+      "loss": 4.404,
+      "theoretical_loss": 5.328775097961346,
+      "tokens_seen": 46989312
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004978535606820461,
+      "loss": 4.3063,
+      "theoretical_loss": 5.327635854834084,
+      "tokens_seen": 47054848
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004978435305917754,
+      "loss": 4.3593,
+      "theoretical_loss": 5.326498640861321,
+      "tokens_seen": 47120384
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004978335005015045,
+      "loss": 4.458,
+      "theoretical_loss": 5.325363449614185,
+      "tokens_seen": 47185920
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004978234704112337,
+      "loss": 4.4505,
+      "theoretical_loss": 5.324230274693065,
+      "tokens_seen": 47251456
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004978134403209629,
+      "loss": 4.2784,
+      "theoretical_loss": 5.323099109727434,
+      "tokens_seen": 47316992
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004978034102306921,
+      "loss": 4.3645,
+      "theoretical_loss": 5.321969948375687,
+      "tokens_seen": 47382528
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004977933801404213,
+      "loss": 4.3349,
+      "theoretical_loss": 5.320842784324954,
+      "tokens_seen": 47448064
+    },
+    {
+      "epoch": 0.01,
+      "objective/train/docs_used": 38014,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 4.285874366760254,
+      "objective/train/theoretical_loss": 5.319717611290951,
+      "objective/train/tokens_used": 67973600,
+      "theoretical_loss": 5.319717611290951,
+      "tokens_seen": 47513600
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004977833500501504,
+      "loss": 4.2487,
+      "theoretical_loss": 5.319717611290951,
+      "tokens_seen": 47513600
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004977733199598796,
+      "loss": 4.227,
+      "theoretical_loss": 5.318594423017791,
+      "tokens_seen": 47579136
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004977632898696088,
+      "loss": 4.5068,
+      "theoretical_loss": 5.317473213277827,
+      "tokens_seen": 47644672
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.000497753259779338,
+      "loss": 4.4264,
+      "theoretical_loss": 5.316353975871486,
+      "tokens_seen": 47710208
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004977432296890672,
+      "loss": 4.4848,
+      "theoretical_loss": 5.315236704627106,
+      "tokens_seen": 47775744
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004977331995987965,
+      "loss": 4.4288,
+      "theoretical_loss": 5.314121393400763,
+      "tokens_seen": 47841280
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004977231695085255,
+      "loss": 4.0077,
+      "theoretical_loss": 5.313008036076115,
+      "tokens_seen": 47906816
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004977131394182548,
+      "loss": 4.3322,
+      "theoretical_loss": 5.311896626564246,
+      "tokens_seen": 47972352
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.000497703109327984,
+      "loss": 4.1105,
+      "theoretical_loss": 5.3107871588034925,
+      "tokens_seen": 48037888
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004976930792377132,
+      "loss": 4.3437,
+      "theoretical_loss": 5.309679626759296,
+      "tokens_seen": 48103424
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004976830491474424,
+      "loss": 4.292,
+      "theoretical_loss": 5.308574024424041,
+      "tokens_seen": 48168960
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004976730190571715,
+      "loss": 4.3868,
+      "theoretical_loss": 5.3074703458168955,
+      "tokens_seen": 48234496
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004976629889669007,
+      "loss": 4.436,
+      "theoretical_loss": 5.3063685849836615,
+      "tokens_seen": 48300032
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004976529588766299,
+      "loss": 4.368,
+      "theoretical_loss": 5.305268735996615,
+      "tokens_seen": 48365568
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004976429287863591,
+      "loss": 4.2524,
+      "theoretical_loss": 5.304170792954354,
+      "tokens_seen": 48431104
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004976328986960883,
+      "loss": 4.5041,
+      "theoretical_loss": 5.303074749981648,
+      "tokens_seen": 48496640
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004976228686058175,
+      "loss": 4.412,
+      "theoretical_loss": 5.301980601229285,
+      "tokens_seen": 48562176
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004976128385155466,
+      "loss": 4.3484,
+      "theoretical_loss": 5.300888340873924,
+      "tokens_seen": 48627712
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004976028084252758,
+      "loss": 4.4722,
+      "theoretical_loss": 5.299797963117941,
+      "tokens_seen": 48693248
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.000497592778335005,
+      "loss": 4.1945,
+      "theoretical_loss": 5.298709462189283,
+      "tokens_seen": 48758784
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004975827482447342,
+      "loss": 4.2003,
+      "theoretical_loss": 5.297622832341327,
+      "tokens_seen": 48824320
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004975727181544635,
+      "loss": 4.0957,
+      "theoretical_loss": 5.2965380678527225,
+      "tokens_seen": 48889856
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004975626880641925,
+      "loss": 4.2418,
+      "theoretical_loss": 5.295455163027258,
+      "tokens_seen": 48955392
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004975526579739218,
+      "loss": 4.3033,
+      "theoretical_loss": 5.29437411219371,
+      "tokens_seen": 49020928
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004975426278836509,
+      "loss": 4.2827,
+      "theoretical_loss": 5.293294909705702,
+      "tokens_seen": 49086464
+    },
+    {
+      "epoch": 0.01,
+      "objective/train/docs_used": 38689,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 4.5455546379089355,
+      "objective/train/theoretical_loss": 5.292217549941562,
+      "objective/train/tokens_used": 69612000,
+      "theoretical_loss": 5.292217549941562,
+      "tokens_seen": 49152000
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004975325977933802,
+      "loss": 4.2752,
+      "theoretical_loss": 5.292217549941562,
+      "tokens_seen": 49152000
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004975225677031094,
+      "loss": 4.1889,
+      "theoretical_loss": 5.291142027304185,
+      "tokens_seen": 49217536
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004975125376128386,
+      "loss": 3.9788,
+      "theoretical_loss": 5.29006833622089,
+      "tokens_seen": 49283072
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004975025075225677,
+      "loss": 4.4666,
+      "theoretical_loss": 5.28899647114328,
+      "tokens_seen": 49348608
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004974924774322969,
+      "loss": 4.3282,
+      "theoretical_loss": 5.287926426547106,
+      "tokens_seen": 49414144
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004974824473420261,
+      "loss": 4.3775,
+      "theoretical_loss": 5.2868581969321315,
+      "tokens_seen": 49479680
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004974724172517553,
+      "loss": 4.3022,
+      "theoretical_loss": 5.285791776821995,
+      "tokens_seen": 49545216
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004974623871614845,
+      "loss": 4.559,
+      "theoretical_loss": 5.2847271607640725,
+      "tokens_seen": 49610752
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004974523570712136,
+      "loss": 4.5333,
+      "theoretical_loss": 5.283664343329346,
+      "tokens_seen": 49676288
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004974423269809428,
+      "loss": 4.2267,
+      "theoretical_loss": 5.282603319112273,
+      "tokens_seen": 49741824
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.000497432296890672,
+      "loss": 4.3567,
+      "theoretical_loss": 5.281544082730649,
+      "tokens_seen": 49807360
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004974222668004012,
+      "loss": 4.2938,
+      "theoretical_loss": 5.280486628825479,
+      "tokens_seen": 49872896
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004974122367101304,
+      "loss": 4.3754,
+      "theoretical_loss": 5.279430952060843,
+      "tokens_seen": 49938432
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004974022066198595,
+      "loss": 4.3361,
+      "theoretical_loss": 5.278377047123777,
+      "tokens_seen": 50003968
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004973921765295888,
+      "loss": 4.2186,
+      "theoretical_loss": 5.277324908724132,
+      "tokens_seen": 50069504
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004973821464393179,
+      "loss": 4.2978,
+      "theoretical_loss": 5.276274531594453,
+      "tokens_seen": 50135040
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004973721163490472,
+      "loss": 4.3052,
+      "theoretical_loss": 5.27522591048985,
+      "tokens_seen": 50200576
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004973620862587763,
+      "loss": 4.352,
+      "theoretical_loss": 5.274179040187876,
+      "tokens_seen": 50266112
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004973520561685056,
+      "loss": 4.0427,
+      "theoretical_loss": 5.273133915488394,
+      "tokens_seen": 50331648
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004973420260782347,
+      "loss": 4.3078,
+      "theoretical_loss": 5.272090531213462,
+      "tokens_seen": 50397184
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004973319959879639,
+      "loss": 4.0645,
+      "theoretical_loss": 5.271048882207202,
+      "tokens_seen": 50462720
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004973219658976931,
+      "loss": 4.6409,
+      "theoretical_loss": 5.27000896333568,
+      "tokens_seen": 50528256
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004973119358074223,
+      "loss": 4.0496,
+      "theoretical_loss": 5.268970769486789,
+      "tokens_seen": 50593792
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004973019057171515,
+      "loss": 4.4574,
+      "theoretical_loss": 5.267934295570118,
+      "tokens_seen": 50659328
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004972918756268806,
+      "loss": 4.2343,
+      "theoretical_loss": 5.266899536516842,
+      "tokens_seen": 50724864
+    },
+    {
+      "epoch": 0.02,
+      "objective/train/docs_used": 39885,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 4.198443412780762,
+      "objective/train/theoretical_loss": 5.265866487279597,
+      "objective/train/tokens_used": 71250400,
+      "theoretical_loss": 5.265866487279597,
+      "tokens_seen": 50790400
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004972818455366098,
+      "loss": 4.2608,
+      "theoretical_loss": 5.265866487279597,
+      "tokens_seen": 50790400
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.000497271815446339,
+      "loss": 4.2763,
+      "theoretical_loss": 5.264835142832364,
+      "tokens_seen": 50855936
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004972617853560682,
+      "loss": 4.1906,
+      "theoretical_loss": 5.263805498170351,
+      "tokens_seen": 50921472
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004972517552657974,
+      "loss": 4.1761,
+      "theoretical_loss": 5.26277754830987,
+      "tokens_seen": 50987008
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004972417251755266,
+      "loss": 4.2526,
+      "theoretical_loss": 5.261751288288236,
+      "tokens_seen": 51052544
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004972316950852557,
+      "loss": 4.275,
+      "theoretical_loss": 5.260726713163637,
+      "tokens_seen": 51118080
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004972216649949849,
+      "loss": 4.3822,
+      "theoretical_loss": 5.259703818015023,
+      "tokens_seen": 51183616
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004972116349047142,
+      "loss": 4.235,
+      "theoretical_loss": 5.258682597941998,
+      "tokens_seen": 51249152
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004972016048144433,
+      "loss": 4.1933,
+      "theoretical_loss": 5.257663048064704,
+      "tokens_seen": 51314688
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004971915747241726,
+      "loss": 4.0819,
+      "theoretical_loss": 5.256645163523704,
+      "tokens_seen": 51380224
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004971815446339017,
+      "loss": 4.2302,
+      "theoretical_loss": 5.255628939479877,
+      "tokens_seen": 51445760
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004971715145436309,
+      "loss": 4.1741,
+      "theoretical_loss": 5.254614371114306,
+      "tokens_seen": 51511296
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004971614844533601,
+      "loss": 4.0105,
+      "theoretical_loss": 5.253601453628165,
+      "tokens_seen": 51576832
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004971514543630893,
+      "loss": 4.1921,
+      "theoretical_loss": 5.2525901822426135,
+      "tokens_seen": 51642368
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004971414242728185,
+      "loss": 4.1033,
+      "theoretical_loss": 5.251580552198687,
+      "tokens_seen": 51707904
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004971313941825477,
+      "loss": 4.5223,
+      "theoretical_loss": 5.250572558757186,
+      "tokens_seen": 51773440
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004971213640922768,
+      "loss": 4.2685,
+      "theoretical_loss": 5.2495661971985745,
+      "tokens_seen": 51838976
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.000497111334002006,
+      "loss": 4.4627,
+      "theoretical_loss": 5.248561462822871,
+      "tokens_seen": 51904512
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004971013039117352,
+      "loss": 4.2511,
+      "theoretical_loss": 5.247558350949539,
+      "tokens_seen": 51970048
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004970912738214644,
+      "loss": 4.0554,
+      "theoretical_loss": 5.246556856917392,
+      "tokens_seen": 52035584
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004970812437311936,
+      "loss": 4.1646,
+      "theoretical_loss": 5.24555697608448,
+      "tokens_seen": 52101120
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004970712136409227,
+      "loss": 4.2295,
+      "theoretical_loss": 5.244558703827986,
+      "tokens_seen": 52166656
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004970611835506519,
+      "loss": 4.1183,
+      "theoretical_loss": 5.243562035544132,
+      "tokens_seen": 52232192
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004970511534603811,
+      "loss": 4.4799,
+      "theoretical_loss": 5.24256696664807,
+      "tokens_seen": 52297728
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004970411233701103,
+      "loss": 4.1907,
+      "theoretical_loss": 5.24157349257378,
+      "tokens_seen": 52363264
+    },
+    {
+      "epoch": 0.02,
+      "objective/train/docs_used": 40418,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 3.979517936706543,
+      "objective/train/theoretical_loss": 5.240581608773973,
+      "objective/train/tokens_used": 72888800,
+      "theoretical_loss": 5.240581608773973,
+      "tokens_seen": 52428800
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004970310932798396,
+      "loss": 4.0593,
+      "theoretical_loss": 5.240581608773973,
+      "tokens_seen": 52428800
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004970210631895686,
+      "loss": 4.2029,
+      "theoretical_loss": 5.239591310719987,
+      "tokens_seen": 52494336
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004970110330992979,
+      "loss": 4.0965,
+      "theoretical_loss": 5.238602593901694,
+      "tokens_seen": 52559872
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004970010030090271,
+      "loss": 4.0066,
+      "theoretical_loss": 5.23761545382739,
+      "tokens_seen": 52625408
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004969909729187563,
+      "loss": 4.1287,
+      "theoretical_loss": 5.236629886023708,
+      "tokens_seen": 52690944
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004969809428284855,
+      "loss": 4.1441,
+      "theoretical_loss": 5.235645886035517,
+      "tokens_seen": 52756480
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004969709127382147,
+      "loss": 4.2684,
+      "theoretical_loss": 5.2346634494258195,
+      "tokens_seen": 52822016
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004969608826479438,
+      "loss": 4.1129,
+      "theoretical_loss": 5.233682571775664,
+      "tokens_seen": 52887552
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.000496950852557673,
+      "loss": 4.44,
+      "theoretical_loss": 5.232703248684045,
+      "tokens_seen": 52953088
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004969408224674022,
+      "loss": 4.3805,
+      "theoretical_loss": 5.231725475767806,
+      "tokens_seen": 53018624
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004969307923771314,
+      "loss": 4.012,
+      "theoretical_loss": 5.2307492486615494,
+      "tokens_seen": 53084160
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004969207622868606,
+      "loss": 4.3914,
+      "theoretical_loss": 5.229774563017541,
+      "tokens_seen": 53149696
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004969107321965897,
+      "loss": 4.2825,
+      "theoretical_loss": 5.2288014145056145,
+      "tokens_seen": 53215232
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.000496900702106319,
+      "loss": 3.9345,
+      "theoretical_loss": 5.227829798813083,
+      "tokens_seen": 53280768
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004968906720160481,
+      "loss": 4.1139,
+      "theoretical_loss": 5.2268597116446465,
+      "tokens_seen": 53346304
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004968806419257774,
+      "loss": 4.2213,
+      "theoretical_loss": 5.225891148722296,
+      "tokens_seen": 53411840
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004968706118355065,
+      "loss": 4.1851,
+      "theoretical_loss": 5.224924105785227,
+      "tokens_seen": 53477376
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004968605817452358,
+      "loss": 4.1925,
+      "theoretical_loss": 5.22395857858975,
+      "tokens_seen": 53542912
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004968505516549649,
+      "loss": 3.8839,
+      "theoretical_loss": 5.222994562909199,
+      "tokens_seen": 53608448
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004968405215646941,
+      "loss": 4.1607,
+      "theoretical_loss": 5.222032054533841,
+      "tokens_seen": 53673984
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004968304914744233,
+      "loss": 4.2475,
+      "theoretical_loss": 5.2210710492707895,
+      "tokens_seen": 53739520
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004968204613841525,
+      "loss": 3.881,
+      "theoretical_loss": 5.220111542943918,
+      "tokens_seen": 53805056
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004968104312938817,
+      "loss": 4.0171,
+      "theoretical_loss": 5.21915353139377,
+      "tokens_seen": 53870592
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004968004012036108,
+      "loss": 4.0926,
+      "theoretical_loss": 5.218197010477477,
+      "tokens_seen": 53936128
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.00049679037111334,
+      "loss": 4.3292,
+      "theoretical_loss": 5.21724197606866,
+      "tokens_seen": 54001664
+    },
+    {
+      "epoch": 0.02,
+      "objective/train/docs_used": 41741,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 4.073785305023193,
+      "objective/train/theoretical_loss": 5.216288424057362,
+      "objective/train/tokens_used": 74527200,
+      "theoretical_loss": 5.216288424057362,
+      "tokens_seen": 54067200
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004967803410230692,
+      "loss": 4.2491,
+      "theoretical_loss": 5.216288424057362,
+      "tokens_seen": 54067200
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004967703109327984,
+      "loss": 4.2446,
+      "theoretical_loss": 5.21533635034995,
+      "tokens_seen": 54132736
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004967602808425276,
+      "loss": 4.2944,
+      "theoretical_loss": 5.214385750869031,
+      "tokens_seen": 54198272
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004967502507522568,
+      "loss": 4.1296,
+      "theoretical_loss": 5.213436621553377,
+      "tokens_seen": 54263808
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004967402206619859,
+      "loss": 4.17,
+      "theoretical_loss": 5.212488958357831,
+      "tokens_seen": 54329344
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004967301905717151,
+      "loss": 4.0717,
+      "theoretical_loss": 5.211542757253229,
+      "tokens_seen": 54394880
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004967201604814444,
+      "loss": 4.4436,
+      "theoretical_loss": 5.210598014226319,
+      "tokens_seen": 54460416
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004967101303911735,
+      "loss": 4.1394,
+      "theoretical_loss": 5.209654725279678,
+      "tokens_seen": 54525952
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004967001003009028,
+      "loss": 4.4479,
+      "theoretical_loss": 5.208712886431626,
+      "tokens_seen": 54591488
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.000496690070210632,
+      "loss": 3.9774,
+      "theoretical_loss": 5.207772493716154,
+      "tokens_seen": 54657024
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004966800401203611,
+      "loss": 4.0348,
+      "theoretical_loss": 5.206833543182833,
+      "tokens_seen": 54722560
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004966700100300903,
+      "loss": 4.1275,
+      "theoretical_loss": 5.205896030896748,
+      "tokens_seen": 54788096
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004966599799398195,
+      "loss": 4.034,
+      "theoretical_loss": 5.2049599529384025,
+      "tokens_seen": 54853632
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004966499498495487,
+      "loss": 4.2901,
+      "theoretical_loss": 5.204025305403653,
+      "tokens_seen": 54919168
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004966399197592779,
+      "loss": 4.0859,
+      "theoretical_loss": 5.2030920844036235,
+      "tokens_seen": 54984704
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.000496629889669007,
+      "loss": 4.2186,
+      "theoretical_loss": 5.202160286064628,
+      "tokens_seen": 55050240
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004966198595787362,
+      "loss": 4.1753,
+      "theoretical_loss": 5.2012299065281,
+      "tokens_seen": 55115776
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004966098294884654,
+      "loss": 4.2037,
+      "theoretical_loss": 5.2003009419505055,
+      "tokens_seen": 55181312
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004965997993981946,
+      "loss": 4.3058,
+      "theoretical_loss": 5.199373388503272,
+      "tokens_seen": 55246848
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004965897693079238,
+      "loss": 4.3641,
+      "theoretical_loss": 5.198447242372716,
+      "tokens_seen": 55312384
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004965797392176529,
+      "loss": 4.1662,
+      "theoretical_loss": 5.197522499759961,
+      "tokens_seen": 55377920
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004965697091273821,
+      "loss": 4.1908,
+      "theoretical_loss": 5.196599156880863,
+      "tokens_seen": 55443456
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004965596790371113,
+      "loss": 3.9863,
+      "theoretical_loss": 5.195677209965942,
+      "tokens_seen": 55508992
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004965496489468405,
+      "loss": 4.5222,
+      "theoretical_loss": 5.194756655260305,
+      "tokens_seen": 55574528
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004965396188565698,
+      "loss": 3.9698,
+      "theoretical_loss": 5.193837489023565,
+      "tokens_seen": 55640064
+    },
+    {
+      "epoch": 0.02,
+      "objective/train/docs_used": 42409,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 4.305491924285889,
+      "objective/train/theoretical_loss": 5.192919707529784,
+      "objective/train/tokens_used": 76165600,
+      "theoretical_loss": 5.192919707529784,
+      "tokens_seen": 55705600
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004965295887662988,
+      "loss": 4.121,
+      "theoretical_loss": 5.192919707529784,
+      "tokens_seen": 55705600
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004965195586760281,
+      "loss": 4.108,
+      "theoretical_loss": 5.1920033070673846,
+      "tokens_seen": 55771136
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004965095285857573,
+      "loss": 3.9647,
+      "theoretical_loss": 5.191088283939086,
+      "tokens_seen": 55836672
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004964994984954865,
+      "loss": 3.9453,
+      "theoretical_loss": 5.190174634461828,
+      "tokens_seen": 55902208
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004964894684052157,
+      "loss": 3.9193,
+      "theoretical_loss": 5.189262354966708,
+      "tokens_seen": 55967744
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004964794383149449,
+      "loss": 4.0925,
+      "theoretical_loss": 5.188351441798897,
+      "tokens_seen": 56033280
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.000496469408224674,
+      "loss": 4.1099,
+      "theoretical_loss": 5.187441891317581,
+      "tokens_seen": 56098816
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004964593781344032,
+      "loss": 4.0055,
+      "theoretical_loss": 5.186533699895888,
+      "tokens_seen": 56164352
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004964493480441324,
+      "loss": 4.1547,
+      "theoretical_loss": 5.18562686392081,
+      "tokens_seen": 56229888
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004964393179538616,
+      "loss": 4.1385,
+      "theoretical_loss": 5.184721379793145,
+      "tokens_seen": 56295424
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004964292878635908,
+      "loss": 4.1137,
+      "theoretical_loss": 5.183817243927425,
+      "tokens_seen": 56360960
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.00049641925777332,
+      "loss": 3.988,
+      "theoretical_loss": 5.182914452751846,
+      "tokens_seen": 56426496
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004964092276830491,
+      "loss": 3.9087,
+      "theoretical_loss": 5.1820130027081985,
+      "tokens_seen": 56492032
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004963991975927783,
+      "loss": 4.1078,
+      "theoretical_loss": 5.181112890251804,
+      "tokens_seen": 56557568
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004963891675025075,
+      "loss": 3.9657,
+      "theoretical_loss": 5.180214111851447,
+      "tokens_seen": 56623104
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004963791374122367,
+      "loss": 3.8969,
+      "theoretical_loss": 5.179316663989305,
+      "tokens_seen": 56688640
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004963691073219659,
+      "loss": 4.0938,
+      "theoretical_loss": 5.178420543160889,
+      "tokens_seen": 56754176
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004963590772316951,
+      "loss": 4.0717,
+      "theoretical_loss": 5.177525745874968,
+      "tokens_seen": 56819712
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004963490471414242,
+      "loss": 4.0293,
+      "theoretical_loss": 5.176632268653516,
+      "tokens_seen": 56885248
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004963390170511535,
+      "loss": 4.0914,
+      "theoretical_loss": 5.175740108031633,
+      "tokens_seen": 56950784
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004963289869608827,
+      "loss": 4.3386,
+      "theoretical_loss": 5.174849260557494,
+      "tokens_seen": 57016320
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004963189568706119,
+      "loss": 4.1438,
+      "theoretical_loss": 5.173959722792272,
+      "tokens_seen": 57081856
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.000496308926780341,
+      "loss": 4.1502,
+      "theoretical_loss": 5.173071491310086,
+      "tokens_seen": 57147392
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004962988966900702,
+      "loss": 3.9246,
+      "theoretical_loss": 5.172184562697928,
+      "tokens_seen": 57212928
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004962888665997994,
+      "loss": 4.0249,
+      "theoretical_loss": 5.171298933555606,
+      "tokens_seen": 57278464
+    },
+    {
+      "epoch": 0.02,
+      "objective/train/docs_used": 43695,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 3.649152994155884,
+      "objective/train/theoretical_loss": 5.170414600495677,
+      "objective/train/tokens_used": 77804000,
+      "theoretical_loss": 5.170414600495677,
+      "tokens_seen": 57344000
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004962788365095286,
+      "loss": 4.0093,
+      "theoretical_loss": 5.170414600495677,
+      "tokens_seen": 57344000
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004962688064192578,
+      "loss": 4.1786,
+      "theoretical_loss": 5.16953156014339,
+      "tokens_seen": 57409536
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.000496258776328987,
+      "loss": 4.0607,
+      "theoretical_loss": 5.1686498091366175,
+      "tokens_seen": 57475072
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004962487462387161,
+      "loss": 3.6266,
+      "theoretical_loss": 5.1677693441258,
+      "tokens_seen": 57540608
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004962387161484453,
+      "loss": 3.8942,
+      "theoretical_loss": 5.16689016177388,
+      "tokens_seen": 57606144
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004962286860581746,
+      "loss": 3.9167,
+      "theoretical_loss": 5.166012258756243,
+      "tokens_seen": 57671680
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004962186559679037,
+      "loss": 4.2399,
+      "theoretical_loss": 5.16513563176066,
+      "tokens_seen": 57737216
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.000496208625877633,
+      "loss": 3.7377,
+      "theoretical_loss": 5.164260277487221,
+      "tokens_seen": 57802752
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004961985957873621,
+      "loss": 4.0589,
+      "theoretical_loss": 5.163386192648282,
+      "tokens_seen": 57868288
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004961885656970913,
+      "loss": 4.2436,
+      "theoretical_loss": 5.162513373968403,
+      "tokens_seen": 57933824
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004961785356068205,
+      "loss": 4.2688,
+      "theoretical_loss": 5.1616418181842825,
+      "tokens_seen": 57999360
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004961685055165497,
+      "loss": 3.9842,
+      "theoretical_loss": 5.160771522044712,
+      "tokens_seen": 58064896
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004961584754262789,
+      "loss": 4.202,
+      "theoretical_loss": 5.159902482310506,
+      "tokens_seen": 58130432
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004961484453360081,
+      "loss": 4.0252,
+      "theoretical_loss": 5.15903469575445,
+      "tokens_seen": 58195968
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004961384152457372,
+      "loss": 4.0653,
+      "theoretical_loss": 5.1581681591612405,
+      "tokens_seen": 58261504
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004961283851554664,
+      "loss": 4.1853,
+      "theoretical_loss": 5.15730286932743,
+      "tokens_seen": 58327040
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004961183550651956,
+      "loss": 4.0287,
+      "theoretical_loss": 5.156438823061366,
+      "tokens_seen": 58392576
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004961083249749248,
+      "loss": 3.8706,
+      "theoretical_loss": 5.15557601718314,
+      "tokens_seen": 58458112
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.000496098294884654,
+      "loss": 3.4818,
+      "theoretical_loss": 5.154714448524525,
+      "tokens_seen": 58523648
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004960882647943831,
+      "loss": 4.1847,
+      "theoretical_loss": 5.153854113928924,
+      "tokens_seen": 58589184
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004960782347041123,
+      "loss": 4.0123,
+      "theoretical_loss": 5.152995010251313,
+      "tokens_seen": 58654720
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004960682046138415,
+      "loss": 3.8546,
+      "theoretical_loss": 5.1521371343581865,
+      "tokens_seen": 58720256
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004960581745235707,
+      "loss": 3.9651,
+      "theoretical_loss": 5.1512804831275,
+      "tokens_seen": 58785792
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004960481444333,
+      "loss": 3.9367,
+      "theoretical_loss": 5.1504250534486165,
+      "tokens_seen": 58851328
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.000496038114343029,
+      "loss": 4.3063,
+      "theoretical_loss": 5.149570842222257,
+      "tokens_seen": 58916864
+    },
+    {
+      "epoch": 0.02,
+      "objective/train/docs_used": 44395,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 4.063347339630127,
+      "objective/train/theoretical_loss": 5.1487178463604355,
+      "objective/train/tokens_used": 79442400,
+      "theoretical_loss": 5.1487178463604355,
+      "tokens_seen": 58982400
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004960280842527583,
+      "loss": 3.7506,
+      "theoretical_loss": 5.1487178463604355,
+      "tokens_seen": 58982400
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004960180541624875,
+      "loss": 3.9872,
+      "theoretical_loss": 5.147866062786418,
+      "tokens_seen": 59047936
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004960080240722167,
+      "loss": 3.9911,
+      "theoretical_loss": 5.147015488434661,
+      "tokens_seen": 59113472
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004959979939819459,
+      "loss": 3.8278,
+      "theoretical_loss": 5.14616612025076,
+      "tokens_seen": 59179008
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004959879638916751,
+      "loss": 3.9528,
+      "theoretical_loss": 5.145317955191397,
+      "tokens_seen": 59244544
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004959779338014042,
+      "loss": 3.7919,
+      "theoretical_loss": 5.144470990224288,
+      "tokens_seen": 59310080
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004959679037111334,
+      "loss": 3.9634,
+      "theoretical_loss": 5.143625222328134,
+      "tokens_seen": 59375616
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004959578736208626,
+      "loss": 4.096,
+      "theoretical_loss": 5.142780648492563,
+      "tokens_seen": 59441152
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004959478435305918,
+      "loss": 3.9749,
+      "theoretical_loss": 5.141937265718084,
+      "tokens_seen": 59506688
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.000495937813440321,
+      "loss": 4.0214,
+      "theoretical_loss": 5.141095071016032,
+      "tokens_seen": 59572224
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004959277833500501,
+      "loss": 3.9143,
+      "theoretical_loss": 5.14025406140852,
+      "tokens_seen": 59637760
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004959177532597793,
+      "loss": 4.0642,
+      "theoretical_loss": 5.139414233928385,
+      "tokens_seen": 59703296
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004959077231695085,
+      "loss": 3.9695,
+      "theoretical_loss": 5.138575585619142,
+      "tokens_seen": 59768832
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004958976930792377,
+      "loss": 4.1946,
+      "theoretical_loss": 5.13773811353493,
+      "tokens_seen": 59834368
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004958876629889669,
+      "loss": 4.0784,
+      "theoretical_loss": 5.1369018147404635,
+      "tokens_seen": 59899904
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.000495877632898696,
+      "loss": 4.0826,
+      "theoretical_loss": 5.136066686310986,
+      "tokens_seen": 59965440
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004958676028084253,
+      "loss": 4.0869,
+      "theoretical_loss": 5.135232725332212,
+      "tokens_seen": 60030976
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004958575727181544,
+      "loss": 3.9758,
+      "theoretical_loss": 5.134399928900292,
+      "tokens_seen": 60096512
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004958475426278837,
+      "loss": 4.0519,
+      "theoretical_loss": 5.13356829412175,
+      "tokens_seen": 60162048
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004958375125376129,
+      "loss": 4.1545,
+      "theoretical_loss": 5.132737818113444,
+      "tokens_seen": 60227584
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004958274824473421,
+      "loss": 4.2355,
+      "theoretical_loss": 5.131908498002513,
+      "tokens_seen": 60293120
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004958174523570712,
+      "loss": 3.8189,
+      "theoretical_loss": 5.131080330926334,
+      "tokens_seen": 60358656
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004958074222668004,
+      "loss": 4.1869,
+      "theoretical_loss": 5.130253314032469,
+      "tokens_seen": 60424192
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004957973921765296,
+      "loss": 3.728,
+      "theoretical_loss": 5.1294274444786225,
+      "tokens_seen": 60489728
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004957873620862588,
+      "loss": 4.1881,
+      "theoretical_loss": 5.128602719432592,
+      "tokens_seen": 60555264
+    },
+    {
+      "epoch": 0.02,
+      "objective/train/docs_used": 45571,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 4.731624603271484,
+      "objective/train/theoretical_loss": 5.12777913607222,
+      "objective/train/tokens_used": 81080800,
+      "theoretical_loss": 5.12777913607222,
+      "tokens_seen": 60620800
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.000495777331995988,
+      "loss": 4.3579,
+      "theoretical_loss": 5.12777913607222,
+      "tokens_seen": 60620800
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004957673019057172,
+      "loss": 3.8709,
+      "theoretical_loss": 5.126956691585351,
+      "tokens_seen": 60686336
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004957572718154463,
+      "loss": 3.9854,
+      "theoretical_loss": 5.1261353831697845,
+      "tokens_seen": 60751872
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004957472417251755,
+      "loss": 4.2274,
+      "theoretical_loss": 5.125315208033226,
+      "tokens_seen": 60817408
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004957372116349047,
+      "loss": 3.8993,
+      "theoretical_loss": 5.124496163393244,
+      "tokens_seen": 60882944
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004957271815446339,
+      "loss": 3.7806,
+      "theoretical_loss": 5.123678246477226,
+      "tokens_seen": 60948480
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004957171514543631,
+      "loss": 3.977,
+      "theoretical_loss": 5.122861454522329,
+      "tokens_seen": 61014016
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004957071213640923,
+      "loss": 3.9739,
+      "theoretical_loss": 5.122045784775439,
+      "tokens_seen": 61079552
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004956970912738214,
+      "loss": 3.8256,
+      "theoretical_loss": 5.1212312344931235,
+      "tokens_seen": 61145088
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004956870611835507,
+      "loss": 4.0096,
+      "theoretical_loss": 5.120417800941587,
+      "tokens_seen": 61210624
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004956770310932798,
+      "loss": 4.0399,
+      "theoretical_loss": 5.119605481396629,
+      "tokens_seen": 61276160
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004956670010030091,
+      "loss": 3.9718,
+      "theoretical_loss": 5.1187942731436,
+      "tokens_seen": 61341696
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004956569709127383,
+      "loss": 3.9982,
+      "theoretical_loss": 5.117984173477356,
+      "tokens_seen": 61407232
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004956469408224674,
+      "loss": 4.0777,
+      "theoretical_loss": 5.117175179702215,
+      "tokens_seen": 61472768
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004956369107321966,
+      "loss": 3.9865,
+      "theoretical_loss": 5.116367289131917,
+      "tokens_seen": 61538304
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004956268806419258,
+      "loss": 4.1915,
+      "theoretical_loss": 5.115560499089575,
+      "tokens_seen": 61603840
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.000495616850551655,
+      "loss": 3.8813,
+      "theoretical_loss": 5.1147548069076425,
+      "tokens_seen": 61669376
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004956068204613842,
+      "loss": 3.97,
+      "theoretical_loss": 5.113950209927859,
+      "tokens_seen": 61734912
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004955967903711133,
+      "loss": 3.5137,
+      "theoretical_loss": 5.113146705501217,
+      "tokens_seen": 61800448
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004955867602808425,
+      "loss": 3.7782,
+      "theoretical_loss": 5.112344290987915,
+      "tokens_seen": 61865984
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004955767301905717,
+      "loss": 4.1055,
+      "theoretical_loss": 5.1115429637573175,
+      "tokens_seen": 61931520
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004955667001003009,
+      "loss": 3.9588,
+      "theoretical_loss": 5.110742721187915,
+      "tokens_seen": 61997056
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004955566700100301,
+      "loss": 3.8558,
+      "theoretical_loss": 5.109943560667279,
+      "tokens_seen": 62062592
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004955466399197592,
+      "loss": 3.8969,
+      "theoretical_loss": 5.109145479592028,
+      "tokens_seen": 62128128
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004955366098294884,
+      "loss": 4.1078,
+      "theoretical_loss": 5.108348475367774,
+      "tokens_seen": 62193664
+    },
+    {
+      "epoch": 0.02,
+      "objective/train/docs_used": 46598,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 4.215669631958008,
+      "objective/train/theoretical_loss": 5.107552545409097,
+      "objective/train/tokens_used": 82719200,
+      "theoretical_loss": 5.107552545409097,
+      "tokens_seen": 62259200
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004955265797392177,
+      "loss": 4.056,
+      "theoretical_loss": 5.107552545409097,
+      "tokens_seen": 62259200
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004955165496489468,
+      "loss": 3.9227,
+      "theoretical_loss": 5.106757687139494,
+      "tokens_seen": 62324736
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004955065195586761,
+      "loss": 4.2246,
+      "theoretical_loss": 5.1059638979913435,
+      "tokens_seen": 62390272
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004954964894684052,
+      "loss": 4.0438,
+      "theoretical_loss": 5.105171175405864,
+      "tokens_seen": 62455808
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004954864593781344,
+      "loss": 4.0275,
+      "theoretical_loss": 5.104379516833075,
+      "tokens_seen": 62521344
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004954764292878636,
+      "loss": 4.005,
+      "theoretical_loss": 5.103588919731757,
+      "tokens_seen": 62586880
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004954663991975928,
+      "loss": 3.937,
+      "theoretical_loss": 5.1027993815694135,
+      "tokens_seen": 62652416
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.000495456369107322,
+      "loss": 4.1536,
+      "theoretical_loss": 5.10201089982223,
+      "tokens_seen": 62717952
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004954463390170512,
+      "loss": 3.971,
+      "theoretical_loss": 5.101223471975038,
+      "tokens_seen": 62783488
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004954363089267803,
+      "loss": 3.9062,
+      "theoretical_loss": 5.100437095521272,
+      "tokens_seen": 62849024
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004954262788365095,
+      "loss": 3.8866,
+      "theoretical_loss": 5.099651767962936,
+      "tokens_seen": 62914560
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004954162487462387,
+      "loss": 3.8535,
+      "theoretical_loss": 5.098867486810562,
+      "tokens_seen": 62980096
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004954062186559679,
+      "loss": 3.7859,
+      "theoretical_loss": 5.098084249583175,
+      "tokens_seen": 63045632
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004953961885656971,
+      "loss": 3.9093,
+      "theoretical_loss": 5.097302053808251,
+      "tokens_seen": 63111168
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004953861584754263,
+      "loss": 3.5672,
+      "theoretical_loss": 5.0965208970216835,
+      "tokens_seen": 63176704
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004953761283851555,
+      "loss": 3.8846,
+      "theoretical_loss": 5.095740776767743,
+      "tokens_seen": 63242240
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004953660982948846,
+      "loss": 3.9469,
+      "theoretical_loss": 5.0949616905990425,
+      "tokens_seen": 63307776
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004953560682046139,
+      "loss": 4.2765,
+      "theoretical_loss": 5.094183636076501,
+      "tokens_seen": 63373312
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004953460381143431,
+      "loss": 4.015,
+      "theoretical_loss": 5.093406610769302,
+      "tokens_seen": 63438848
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004953360080240723,
+      "loss": 4.2246,
+      "theoretical_loss": 5.092630612254862,
+      "tokens_seen": 63504384
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004953259779338014,
+      "loss": 3.9173,
+      "theoretical_loss": 5.091855638118792,
+      "tokens_seen": 63569920
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004953159478435306,
+      "loss": 4.0249,
+      "theoretical_loss": 5.091081685954863,
+      "tokens_seen": 63635456
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004953059177532598,
+      "loss": 3.7978,
+      "theoretical_loss": 5.090308753364967,
+      "tokens_seen": 63700992
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.000495295887662989,
+      "loss": 3.8605,
+      "theoretical_loss": 5.089536837959084,
+      "tokens_seen": 63766528
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004952858575727182,
+      "loss": 3.9657,
+      "theoretical_loss": 5.0887659373552445,
+      "tokens_seen": 63832064
+    },
+    {
+      "epoch": 0.02,
+      "objective/train/docs_used": 47189,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 3.3733479976654053,
+      "objective/train/theoretical_loss": 5.087996049179497,
+      "objective/train/tokens_used": 84357600,
+      "theoretical_loss": 5.087996049179497,
+      "tokens_seen": 63897600
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004952758274824474,
+      "loss": 3.6824,
+      "theoretical_loss": 5.087996049179497,
+      "tokens_seen": 63897600
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004952657973921765,
+      "loss": 3.7677,
+      "theoretical_loss": 5.087227171065869,
+      "tokens_seen": 63963136
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004952557673019057,
+      "loss": 3.9197,
+      "theoretical_loss": 5.086459300656337,
+      "tokens_seen": 64028672
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004952457372116349,
+      "loss": 3.927,
+      "theoretical_loss": 5.085692435600783,
+      "tokens_seen": 64094208
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004952357071213641,
+      "loss": 3.8821,
+      "theoretical_loss": 5.084926573556972,
+      "tokens_seen": 64159744
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004952256770310933,
+      "loss": 3.7931,
+      "theoretical_loss": 5.084161712190507,
+      "tokens_seen": 64225280
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004952156469408225,
+      "loss": 3.7647,
+      "theoretical_loss": 5.083397849174801,
+      "tokens_seen": 64290816
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004952056168505516,
+      "loss": 3.9431,
+      "theoretical_loss": 5.082634982191042,
+      "tokens_seen": 64356352
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004951955867602809,
+      "loss": 3.9173,
+      "theoretical_loss": 5.081873108928154,
+      "tokens_seen": 64421888
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.00049518555667001,
+      "loss": 4.0204,
+      "theoretical_loss": 5.081112227082771,
+      "tokens_seen": 64487424
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004951755265797393,
+      "loss": 3.816,
+      "theoretical_loss": 5.0803523343591985,
+      "tokens_seen": 64552960
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004951654964894685,
+      "loss": 3.8761,
+      "theoretical_loss": 5.079593428469382,
+      "tokens_seen": 64618496
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004951554663991976,
+      "loss": 4.0756,
+      "theoretical_loss": 5.078835507132873,
+      "tokens_seen": 64684032
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004951454363089268,
+      "loss": 3.8136,
+      "theoretical_loss": 5.078078568076798,
+      "tokens_seen": 64749568
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.000495135406218656,
+      "loss": 4.0581,
+      "theoretical_loss": 5.077322609035821,
+      "tokens_seen": 64815104
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004951253761283852,
+      "loss": 3.8215,
+      "theoretical_loss": 5.076567627752118,
+      "tokens_seen": 64880640
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004951153460381144,
+      "loss": 3.8355,
+      "theoretical_loss": 5.075813621975337,
+      "tokens_seen": 64946176
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004951053159478435,
+      "loss": 3.9719,
+      "theoretical_loss": 5.075060589462572,
+      "tokens_seen": 65011712
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004950952858575727,
+      "loss": 3.7383,
+      "theoretical_loss": 5.074308527978326,
+      "tokens_seen": 65077248
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004950852557673019,
+      "loss": 3.8755,
+      "theoretical_loss": 5.073557435294486,
+      "tokens_seen": 65142784
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004950752256770311,
+      "loss": 3.6252,
+      "theoretical_loss": 5.07280730919028,
+      "tokens_seen": 65208320
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004950651955867603,
+      "loss": 3.7529,
+      "theoretical_loss": 5.072058147452257,
+      "tokens_seen": 65273856
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004950551654964894,
+      "loss": 3.679,
+      "theoretical_loss": 5.071309947874248,
+      "tokens_seen": 65339392
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004950451354062186,
+      "loss": 3.5894,
+      "theoretical_loss": 5.070562708257339,
+      "tokens_seen": 65404928
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004950351053159479,
+      "loss": 3.9951,
+      "theoretical_loss": 5.069816426409837,
+      "tokens_seen": 65470464
+    },
+    {
+      "epoch": 0.02,
+      "objective/train/docs_used": 48433,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 3.8971009254455566,
+      "objective/train/theoretical_loss": 5.069071100147241,
+      "objective/train/tokens_used": 85996000,
+      "theoretical_loss": 5.069071100147241,
+      "tokens_seen": 65536000
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.000495025075225677,
+      "loss": 3.9133,
+      "theoretical_loss": 5.069071100147241,
+      "tokens_seen": 65536000
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004950150451354063,
+      "loss": 4.2034,
+      "theoretical_loss": 5.068326727292211,
+      "tokens_seen": 65601536
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004950050150451354,
+      "loss": 3.7703,
+      "theoretical_loss": 5.067583305674537,
+      "tokens_seen": 65667072
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004949949849548646,
+      "loss": 3.8678,
+      "theoretical_loss": 5.066840833131106,
+      "tokens_seen": 65732608
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004949849548645938,
+      "loss": 3.8256,
+      "theoretical_loss": 5.066099307505878,
+      "tokens_seen": 65798144
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.000494974924774323,
+      "loss": 3.6242,
+      "theoretical_loss": 5.06535872664985,
+      "tokens_seen": 65863680
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004949648946840522,
+      "loss": 3.9033,
+      "theoretical_loss": 5.064619088421029,
+      "tokens_seen": 65929216
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004949548645937814,
+      "loss": 3.5217,
+      "theoretical_loss": 5.0638803906844,
+      "tokens_seen": 65994752
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004949448345035105,
+      "loss": 3.8469,
+      "theoretical_loss": 5.0631426313119,
+      "tokens_seen": 66060288
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004949348044132397,
+      "loss": 3.7472,
+      "theoretical_loss": 5.062405808182383,
+      "tokens_seen": 66125824
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004949247743229689,
+      "loss": 3.4883,
+      "theoretical_loss": 5.061669919181599,
+      "tokens_seen": 66191360
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004949147442326981,
+      "loss": 3.8441,
+      "theoretical_loss": 5.060934962202155,
+      "tokens_seen": 66256896
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004949047141424273,
+      "loss": 3.9786,
+      "theoretical_loss": 5.060200935143492,
+      "tokens_seen": 66322432
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004948946840521565,
+      "loss": 4.0608,
+      "theoretical_loss": 5.059467835911857,
+      "tokens_seen": 66387968
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004948846539618856,
+      "loss": 3.7521,
+      "theoretical_loss": 5.058735662420268,
+      "tokens_seen": 66453504
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004948746238716148,
+      "loss": 3.6167,
+      "theoretical_loss": 5.0580044125884935,
+      "tokens_seen": 66519040
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.000494864593781344,
+      "loss": 3.76,
+      "theoretical_loss": 5.057274084343018,
+      "tokens_seen": 66584576
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004948545636910733,
+      "loss": 3.8083,
+      "theoretical_loss": 5.056544675617017,
+      "tokens_seen": 66650112
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004948445336008024,
+      "loss": 3.7601,
+      "theoretical_loss": 5.055816184350326,
+      "tokens_seen": 66715648
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004948345035105316,
+      "loss": 3.8069,
+      "theoretical_loss": 5.055088608489417,
+      "tokens_seen": 66781184
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004948244734202607,
+      "loss": 3.3711,
+      "theoretical_loss": 5.054361945987363,
+      "tokens_seen": 66846720
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.00049481444332999,
+      "loss": 3.3429,
+      "theoretical_loss": 5.05363619480382,
+      "tokens_seen": 66912256
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004948044132397192,
+      "loss": 4.0226,
+      "theoretical_loss": 5.052911352904992,
+      "tokens_seen": 66977792
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004947943831494484,
+      "loss": 3.7436,
+      "theoretical_loss": 5.052187418263609,
+      "tokens_seen": 67043328
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004947843530591776,
+      "loss": 3.8333,
+      "theoretical_loss": 5.051464388858893,
+      "tokens_seen": 67108864
+    },
+    {
+      "epoch": 0.02,
+      "objective/train/docs_used": 48877,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 3.7897393703460693,
+      "objective/train/theoretical_loss": 5.050742262676536,
+      "objective/train/tokens_used": 87634400,
+      "theoretical_loss": 5.050742262676536,
+      "tokens_seen": 67174400
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004947743229689067,
+      "loss": 3.7583,
+      "theoretical_loss": 5.050742262676536,
+      "tokens_seen": 67174400
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004947642928786359,
+      "loss": 4.0895,
+      "theoretical_loss": 5.050021037708673,
+      "tokens_seen": 67239936
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004947542627883651,
+      "loss": 3.642,
+      "theoretical_loss": 5.049300711953854,
+      "tokens_seen": 67305472
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004947442326980943,
+      "loss": 3.7434,
+      "theoretical_loss": 5.048581283417015,
+      "tokens_seen": 67371008
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004947342026078235,
+      "loss": 3.7182,
+      "theoretical_loss": 5.047862750109456,
+      "tokens_seen": 67436544
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004947241725175527,
+      "loss": 3.9326,
+      "theoretical_loss": 5.047145110048811,
+      "tokens_seen": 67502080
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004947141424272818,
+      "loss": 3.638,
+      "theoretical_loss": 5.046428361259022,
+      "tokens_seen": 67567616
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004947041123370111,
+      "loss": 3.7471,
+      "theoretical_loss": 5.045712501770317,
+      "tokens_seen": 67633152
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004946940822467402,
+      "loss": 3.699,
+      "theoretical_loss": 5.0449975296191765,
+      "tokens_seen": 67698688
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004946840521564695,
+      "loss": 3.8026,
+      "theoretical_loss": 5.0442834428483145,
+      "tokens_seen": 67764224
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004946740220661987,
+      "loss": 3.7185,
+      "theoretical_loss": 5.043570239506648,
+      "tokens_seen": 67829760
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004946639919759278,
+      "loss": 3.8962,
+      "theoretical_loss": 5.042857917649275,
+      "tokens_seen": 67895296
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.000494653961885657,
+      "loss": 3.7692,
+      "theoretical_loss": 5.042146475337448,
+      "tokens_seen": 67960832
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004946439317953862,
+      "loss": 3.5141,
+      "theoretical_loss": 5.041435910638544,
+      "tokens_seen": 68026368
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004946339017051154,
+      "loss": 3.8396,
+      "theoretical_loss": 5.040726221626048,
+      "tokens_seen": 68091904
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004946238716148446,
+      "loss": 3.8633,
+      "theoretical_loss": 5.040017406379521,
+      "tokens_seen": 68157440
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004946138415245737,
+      "loss": 3.6866,
+      "theoretical_loss": 5.0393094629845745,
+      "tokens_seen": 68222976
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004946038114343029,
+      "loss": 3.5706,
+      "theoretical_loss": 5.038602389532853,
+      "tokens_seen": 68288512
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004945937813440321,
+      "loss": 3.4773,
+      "theoretical_loss": 5.037896184122,
+      "tokens_seen": 68354048
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004945837512537613,
+      "loss": 3.7606,
+      "theoretical_loss": 5.037190844855641,
+      "tokens_seen": 68419584
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004945737211634905,
+      "loss": 3.8062,
+      "theoretical_loss": 5.036486369843355,
+      "tokens_seen": 68485120
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004945636910732196,
+      "loss": 3.6164,
+      "theoretical_loss": 5.03578275720065,
+      "tokens_seen": 68550656
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004945536609829488,
+      "loss": 3.7046,
+      "theoretical_loss": 5.03508000504894,
+      "tokens_seen": 68616192
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004945436308926781,
+      "loss": 3.7478,
+      "theoretical_loss": 5.034378111515521,
+      "tokens_seen": 68681728
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004945336008024072,
+      "loss": 3.663,
+      "theoretical_loss": 5.0336770747335455,
+      "tokens_seen": 68747264
+    },
+    {
+      "epoch": 0.02,
+      "objective/train/docs_used": 49922,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 2.9727299213409424,
+      "objective/train/theoretical_loss": 5.032976892842002,
+      "objective/train/tokens_used": 89272800,
+      "theoretical_loss": 5.032976892842002,
+      "tokens_seen": 68812800
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004945235707121365,
+      "loss": 3.6731,
+      "theoretical_loss": 5.032976892842002,
+      "tokens_seen": 68812800
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004945135406218656,
+      "loss": 3.8125,
+      "theoretical_loss": 5.032277563985687,
+      "tokens_seen": 68878336
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004945035105315948,
+      "loss": 3.8008,
+      "theoretical_loss": 5.031579086315183,
+      "tokens_seen": 68943872
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.000494493480441324,
+      "loss": 3.9551,
+      "theoretical_loss": 5.030881457986837,
+      "tokens_seen": 69009408
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004944834503510532,
+      "loss": 3.843,
+      "theoretical_loss": 5.030184677162734,
+      "tokens_seen": 69074944
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004944734202607824,
+      "loss": 3.5448,
+      "theoretical_loss": 5.029488742010676,
+      "tokens_seen": 69140480
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004944633901705116,
+      "loss": 3.8376,
+      "theoretical_loss": 5.028793650704154,
+      "tokens_seen": 69206016
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004944533600802407,
+      "loss": 3.9125,
+      "theoretical_loss": 5.028099401422336,
+      "tokens_seen": 69271552
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004944433299899699,
+      "loss": 3.6831,
+      "theoretical_loss": 5.0274059923500305,
+      "tokens_seen": 69337088
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004944332998996991,
+      "loss": 3.7007,
+      "theoretical_loss": 5.026713421677673,
+      "tokens_seen": 69402624
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004944232698094283,
+      "loss": 3.6241,
+      "theoretical_loss": 5.0260216876013,
+      "tokens_seen": 69468160
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004944132397191575,
+      "loss": 3.7006,
+      "theoretical_loss": 5.025330788322526,
+      "tokens_seen": 69533696
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004944032096288867,
+      "loss": 3.9117,
+      "theoretical_loss": 5.024640722048522,
+      "tokens_seen": 69599232
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004943931795386158,
+      "loss": 3.4086,
+      "theoretical_loss": 5.023951486991994,
+      "tokens_seen": 69664768
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.000494383149448345,
+      "loss": 3.9368,
+      "theoretical_loss": 5.023263081371157,
+      "tokens_seen": 69730304
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004943731193580742,
+      "loss": 3.8784,
+      "theoretical_loss": 5.022575503409717,
+      "tokens_seen": 69795840
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004943630892678035,
+      "loss": 3.941,
+      "theoretical_loss": 5.021888751336848,
+      "tokens_seen": 69861376
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004943530591775326,
+      "loss": 3.8625,
+      "theoretical_loss": 5.02120282338717,
+      "tokens_seen": 69926912
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004943430290872618,
+      "loss": 3.6762,
+      "theoretical_loss": 5.020517717800723,
+      "tokens_seen": 69992448
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004943329989969909,
+      "loss": 3.4889,
+      "theoretical_loss": 5.019833432822953,
+      "tokens_seen": 70057984
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004943229689067202,
+      "loss": 3.5122,
+      "theoretical_loss": 5.019149966704683,
+      "tokens_seen": 70123520
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004943129388164494,
+      "loss": 3.6293,
+      "theoretical_loss": 5.018467317702097,
+      "tokens_seen": 70189056
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004943029087261786,
+      "loss": 3.5085,
+      "theoretical_loss": 5.017785484076715,
+      "tokens_seen": 70254592
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004942928786359078,
+      "loss": 3.6348,
+      "theoretical_loss": 5.017104464095375,
+      "tokens_seen": 70320128
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004942828485456369,
+      "loss": 3.6907,
+      "theoretical_loss": 5.0164242560302075,
+      "tokens_seen": 70385664
+    },
+    {
+      "epoch": 0.02,
+      "objective/train/docs_used": 51260,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 3.9014105796813965,
+      "objective/train/theoretical_loss": 5.015744858158618,
+      "objective/train/tokens_used": 90911200,
+      "theoretical_loss": 5.015744858158618,
+      "tokens_seen": 70451200
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004942728184553661,
+      "loss": 3.8765,
+      "theoretical_loss": 5.015744858158618,
+      "tokens_seen": 70451200
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004942627883650953,
+      "loss": 3.8516,
+      "theoretical_loss": 5.015066268763264,
+      "tokens_seen": 70516736
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004942527582748245,
+      "loss": 3.6519,
+      "theoretical_loss": 5.014388486132036,
+      "tokens_seen": 70582272
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004942427281845537,
+      "loss": 3.9132,
+      "theoretical_loss": 5.0137115085580355,
+      "tokens_seen": 70647808
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004942326980942828,
+      "loss": 3.8834,
+      "theoretical_loss": 5.013035334339554,
+      "tokens_seen": 70713344
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.000494222668004012,
+      "loss": 3.8794,
+      "theoretical_loss": 5.012359961780053,
+      "tokens_seen": 70778880
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004942126379137412,
+      "loss": 3.8483,
+      "theoretical_loss": 5.011685389188142,
+      "tokens_seen": 70844416
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004942026078234704,
+      "loss": 3.4989,
+      "theoretical_loss": 5.011011614877562,
+      "tokens_seen": 70909952
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004941925777331996,
+      "loss": 3.7672,
+      "theoretical_loss": 5.010338637167162,
+      "tokens_seen": 70975488
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004941825476429289,
+      "loss": 3.6359,
+      "theoretical_loss": 5.009666454380877,
+      "tokens_seen": 71041024
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004941725175526579,
+      "loss": 3.8619,
+      "theoretical_loss": 5.008995064847716,
+      "tokens_seen": 71106560
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004941624874623872,
+      "loss": 3.8008,
+      "theoretical_loss": 5.0083244669017315,
+      "tokens_seen": 71172096
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004941524573721163,
+      "loss": 3.9147,
+      "theoretical_loss": 5.0076546588820054,
+      "tokens_seen": 71237632
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004941424272818456,
+      "loss": 3.3797,
+      "theoretical_loss": 5.00698563913263,
+      "tokens_seen": 71303168
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004941323971915748,
+      "loss": 3.617,
+      "theoretical_loss": 5.006317406002687,
+      "tokens_seen": 71368704
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004941223671013039,
+      "loss": 3.5556,
+      "theoretical_loss": 5.0056499578462255,
+      "tokens_seen": 71434240
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004941123370110331,
+      "loss": 3.4481,
+      "theoretical_loss": 5.0049832930222475,
+      "tokens_seen": 71499776
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004941023069207623,
+      "loss": 3.826,
+      "theoretical_loss": 5.004317409894682,
+      "tokens_seen": 71565312
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004940922768304915,
+      "loss": 3.6718,
+      "theoretical_loss": 5.003652306832372,
+      "tokens_seen": 71630848
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004940822467402207,
+      "loss": 3.8578,
+      "theoretical_loss": 5.002987982209052,
+      "tokens_seen": 71696384
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004940722166499498,
+      "loss": 3.758,
+      "theoretical_loss": 5.00232443440333,
+      "tokens_seen": 71761920
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.000494062186559679,
+      "loss": 3.4768,
+      "theoretical_loss": 5.001661661798666,
+      "tokens_seen": 71827456
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004940521564694082,
+      "loss": 3.6442,
+      "theoretical_loss": 5.000999662783359,
+      "tokens_seen": 71892992
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004940421263791374,
+      "loss": 3.7435,
+      "theoretical_loss": 5.0003384357505185,
+      "tokens_seen": 71958528
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004940320962888666,
+      "loss": 3.8745,
+      "theoretical_loss": 4.999677979098056,
+      "tokens_seen": 72024064
+    },
+    {
+      "epoch": 0.02,
+      "objective/train/docs_used": 51878,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 3.4065515995025635,
+      "objective/train/theoretical_loss": 4.999018291228659,
+      "objective/train/tokens_used": 92549600,
+      "theoretical_loss": 4.999018291228659,
+      "tokens_seen": 72089600
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004940220661985958,
+      "loss": 3.5808,
+      "theoretical_loss": 4.999018291228659,
+      "tokens_seen": 72089600
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004940120361083249,
+      "loss": 3.8043,
+      "theoretical_loss": 4.998359370549778,
+      "tokens_seen": 72155136
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004940020060180542,
+      "loss": 3.649,
+      "theoretical_loss": 4.997701215473601,
+      "tokens_seen": 72220672
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004939919759277834,
+      "loss": 3.6107,
+      "theoretical_loss": 4.997043824417044,
+      "tokens_seen": 72286208
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004939819458375126,
+      "loss": 3.6793,
+      "theoretical_loss": 4.996387195801723,
+      "tokens_seen": 72351744
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004939719157472418,
+      "loss": 3.6529,
+      "theoretical_loss": 4.995731328053944,
+      "tokens_seen": 72417280
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.000493961885656971,
+      "loss": 3.7394,
+      "theoretical_loss": 4.995076219604682,
+      "tokens_seen": 72482816
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004939518555667001,
+      "loss": 3.6256,
+      "theoretical_loss": 4.994421868889559,
+      "tokens_seen": 72548352
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004939418254764293,
+      "loss": 3.6209,
+      "theoretical_loss": 4.9937682743488345,
+      "tokens_seen": 72613888
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004939317953861585,
+      "loss": 3.5004,
+      "theoretical_loss": 4.993115434427376,
+      "tokens_seen": 72679424
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004939217652958877,
+      "loss": 3.5103,
+      "theoretical_loss": 4.992463347574654,
+      "tokens_seen": 72744960
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004939117352056169,
+      "loss": 3.7514,
+      "theoretical_loss": 4.991812012244717,
+      "tokens_seen": 72810496
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.000493901705115346,
+      "loss": 3.5319,
+      "theoretical_loss": 4.99116142689617,
+      "tokens_seen": 72876032
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004938916750250752,
+      "loss": 3.8628,
+      "theoretical_loss": 4.990511589992169,
+      "tokens_seen": 72941568
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004938816449348044,
+      "loss": 4.0163,
+      "theoretical_loss": 4.989862500000392,
+      "tokens_seen": 73007104
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004938716148445337,
+      "loss": 3.7264,
+      "theoretical_loss": 4.989214155393025,
+      "tokens_seen": 73072640
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004938615847542628,
+      "loss": 3.9026,
+      "theoretical_loss": 4.988566554646752,
+      "tokens_seen": 73138176
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.000493851554663992,
+      "loss": 3.4925,
+      "theoretical_loss": 4.9879196962427255,
+      "tokens_seen": 73203712
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004938415245737211,
+      "loss": 3.5623,
+      "theoretical_loss": 4.9872735786665565,
+      "tokens_seen": 73269248
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004938314944834504,
+      "loss": 3.415,
+      "theoretical_loss": 4.986628200408299,
+      "tokens_seen": 73334784
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004938214643931796,
+      "loss": 3.5253,
+      "theoretical_loss": 4.985983559962429,
+      "tokens_seen": 73400320
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004938114343029088,
+      "loss": 3.5016,
+      "theoretical_loss": 4.985339655827827,
+      "tokens_seen": 73465856
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.000493801404212638,
+      "loss": 3.8156,
+      "theoretical_loss": 4.984696486507767,
+      "tokens_seen": 73531392
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004937913741223671,
+      "loss": 3.7367,
+      "theoretical_loss": 4.984054050509893,
+      "tokens_seen": 73596928
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004937813440320963,
+      "loss": 3.6885,
+      "theoretical_loss": 4.983412346346206,
+      "tokens_seen": 73662464
+    },
+    {
+      "epoch": 0.02,
+      "objective/train/docs_used": 53017,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 3.889392137527466,
+      "objective/train/theoretical_loss": 4.982771372533048,
+      "objective/train/tokens_used": 94188000,
+      "theoretical_loss": 4.982771372533048,
+      "tokens_seen": 73728000
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004937713139418255,
+      "loss": 3.6721,
+      "theoretical_loss": 4.982771372533048,
+      "tokens_seen": 73728000
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004937612838515547,
+      "loss": 3.9426,
+      "theoretical_loss": 4.982131127591083,
+      "tokens_seen": 73793536
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004937512537612839,
+      "loss": 3.6135,
+      "theoretical_loss": 4.981491610045282,
+      "tokens_seen": 73859072
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.000493741223671013,
+      "loss": 3.6257,
+      "theoretical_loss": 4.980852818424905,
+      "tokens_seen": 73924608
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004937311935807422,
+      "loss": 3.6675,
+      "theoretical_loss": 4.980214751263491,
+      "tokens_seen": 73990144
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004937211634904714,
+      "loss": 3.6453,
+      "theoretical_loss": 4.979577407098831,
+      "tokens_seen": 74055680
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004937111334002006,
+      "loss": 3.7323,
+      "theoretical_loss": 4.978940784472962,
+      "tokens_seen": 74121216
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004937011033099298,
+      "loss": 3.4667,
+      "theoretical_loss": 4.978304881932145,
+      "tokens_seen": 74186752
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004936910732196591,
+      "loss": 3.9063,
+      "theoretical_loss": 4.97766969802685,
+      "tokens_seen": 74252288
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004936810431293881,
+      "loss": 3.3322,
+      "theoretical_loss": 4.977035231311745,
+      "tokens_seen": 74317824
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004936710130391174,
+      "loss": 3.6439,
+      "theoretical_loss": 4.976401480345672,
+      "tokens_seen": 74383360
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004936609829488465,
+      "loss": 3.5552,
+      "theoretical_loss": 4.975768443691636,
+      "tokens_seen": 74448896
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004936509528585758,
+      "loss": 3.5095,
+      "theoretical_loss": 4.97513611991679,
+      "tokens_seen": 74514432
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.000493640922768305,
+      "loss": 3.7738,
+      "theoretical_loss": 4.974504507592419,
+      "tokens_seen": 74579968
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004936308926780341,
+      "loss": 3.8278,
+      "theoretical_loss": 4.973873605293922,
+      "tokens_seen": 74645504
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004936208625877633,
+      "loss": 3.7248,
+      "theoretical_loss": 4.973243411600797,
+      "tokens_seen": 74711040
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004936108324974925,
+      "loss": 3.8762,
+      "theoretical_loss": 4.9726139250966295,
+      "tokens_seen": 74776576
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004936008024072217,
+      "loss": 3.6798,
+      "theoretical_loss": 4.971985144369073,
+      "tokens_seen": 74842112
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004935907723169509,
+      "loss": 3.6282,
+      "theoretical_loss": 4.9713570680098345,
+      "tokens_seen": 74907648
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.00049358074222668,
+      "loss": 3.7215,
+      "theoretical_loss": 4.970729694614662,
+      "tokens_seen": 74973184
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004935707121364092,
+      "loss": 3.5638,
+      "theoretical_loss": 4.970103022783325,
+      "tokens_seen": 75038720
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004935606820461384,
+      "loss": 3.5533,
+      "theoretical_loss": 4.969477051119606,
+      "tokens_seen": 75104256
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004935506519558676,
+      "loss": 3.496,
+      "theoretical_loss": 4.968851778231276,
+      "tokens_seen": 75169792
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004935406218655968,
+      "loss": 3.6459,
+      "theoretical_loss": 4.9682272027300876,
+      "tokens_seen": 75235328
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.000493530591775326,
+      "loss": 3.5601,
+      "theoretical_loss": 4.96760332323176,
+      "tokens_seen": 75300864
+    },
+    {
+      "epoch": 0.02,
+      "objective/train/docs_used": 53591,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 3.8066093921661377,
+      "objective/train/theoretical_loss": 4.966980138355957,
+      "objective/train/tokens_used": 95826400,
+      "theoretical_loss": 4.966980138355957,
+      "tokens_seen": 75366400
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004935205616850551,
+      "loss": 3.7377,
+      "theoretical_loss": 4.966980138355957,
+      "tokens_seen": 75366400
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004935105315947844,
+      "loss": 3.6051,
+      "theoretical_loss": 4.966357646726282,
+      "tokens_seen": 75431936
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004935005015045135,
+      "loss": 3.5847,
+      "theoretical_loss": 4.965735846970255,
+      "tokens_seen": 75497472
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004934904714142428,
+      "loss": 3.7027,
+      "theoretical_loss": 4.965114737719302,
+      "tokens_seen": 75563008
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004934804413239719,
+      "loss": 3.648,
+      "theoretical_loss": 4.964494317608743,
+      "tokens_seen": 75628544
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004934704112337011,
+      "loss": 3.4989,
+      "theoretical_loss": 4.963874585277772,
+      "tokens_seen": 75694080
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004934603811434303,
+      "loss": 3.8889,
+      "theoretical_loss": 4.963255539369445,
+      "tokens_seen": 75759616
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004934503510531595,
+      "loss": 3.5961,
+      "theoretical_loss": 4.9626371785306675,
+      "tokens_seen": 75825152
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004934403209628887,
+      "loss": 3.4261,
+      "theoretical_loss": 4.962019501412178,
+      "tokens_seen": 75890688
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004934302908726179,
+      "loss": 3.7473,
+      "theoretical_loss": 4.961402506668535,
+      "tokens_seen": 75956224
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.000493420260782347,
+      "loss": 3.4055,
+      "theoretical_loss": 4.960786192958103,
+      "tokens_seen": 76021760
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004934102306920762,
+      "loss": 3.6034,
+      "theoretical_loss": 4.960170558943037,
+      "tokens_seen": 76087296
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004934002006018054,
+      "loss": 3.4187,
+      "theoretical_loss": 4.959555603289269,
+      "tokens_seen": 76152832
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004933901705115346,
+      "loss": 3.6193,
+      "theoretical_loss": 4.9589413246664975,
+      "tokens_seen": 76218368
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004933801404212638,
+      "loss": 3.794,
+      "theoretical_loss": 4.958327721748168,
+      "tokens_seen": 76283904
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.000493370110330993,
+      "loss": 3.3982,
+      "theoretical_loss": 4.957714793211462,
+      "tokens_seen": 76349440
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004933600802407221,
+      "loss": 3.3334,
+      "theoretical_loss": 4.957102537737284,
+      "tokens_seen": 76414976
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004933500501504513,
+      "loss": 3.6421,
+      "theoretical_loss": 4.956490954010249,
+      "tokens_seen": 76480512
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004933400200601805,
+      "loss": 3.5119,
+      "theoretical_loss": 4.9558800407186645,
+      "tokens_seen": 76546048
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004933299899699098,
+      "loss": 3.4933,
+      "theoretical_loss": 4.95526979655452,
+      "tokens_seen": 76611584
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004933199598796389,
+      "loss": 3.7028,
+      "theoretical_loss": 4.954660220213473,
+      "tokens_seen": 76677120
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004933099297893682,
+      "loss": 3.7165,
+      "theoretical_loss": 4.954051310394836,
+      "tokens_seen": 76742656
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004932998996990972,
+      "loss": 3.5507,
+      "theoretical_loss": 4.953443065801563,
+      "tokens_seen": 76808192
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004932898696088265,
+      "loss": 3.4901,
+      "theoretical_loss": 4.952835485140236,
+      "tokens_seen": 76873728
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004932798395185557,
+      "loss": 3.2798,
+      "theoretical_loss": 4.952228567121052,
+      "tokens_seen": 76939264
+    },
+    {
+      "epoch": 0.02,
+      "objective/train/docs_used": 54447,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 3.6758010387420654,
+      "objective/train/theoretical_loss": 4.951622310457807,
+      "objective/train/tokens_used": 97464800,
+      "theoretical_loss": 4.951622310457807,
+      "tokens_seen": 77004800
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004932698094282849,
+      "loss": 3.4817,
+      "theoretical_loss": 4.951622310457807,
+      "tokens_seen": 77004800
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004932597793380141,
+      "loss": 3.5561,
+      "theoretical_loss": 4.95101671386789,
+      "tokens_seen": 77070336
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004932497492477432,
+      "loss": 3.6278,
+      "theoretical_loss": 4.950411776072261,
+      "tokens_seen": 77135872
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004932397191574724,
+      "loss": 3.7609,
+      "theoretical_loss": 4.949807495795445,
+      "tokens_seen": 77201408
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004932296890672016,
+      "loss": 3.6186,
+      "theoretical_loss": 4.949203871765516,
+      "tokens_seen": 77266944
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004932196589769308,
+      "loss": 3.6535,
+      "theoretical_loss": 4.948600902714083,
+      "tokens_seen": 77332480
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.00049320962888666,
+      "loss": 3.4356,
+      "theoretical_loss": 4.947998587376282,
+      "tokens_seen": 77398016
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004931995987963893,
+      "loss": 3.7607,
+      "theoretical_loss": 4.947396924490757,
+      "tokens_seen": 77463552
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004931895687061183,
+      "loss": 3.9334,
+      "theoretical_loss": 4.946795912799653,
+      "tokens_seen": 77529088
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004931795386158476,
+      "loss": 3.4391,
+      "theoretical_loss": 4.946195551048597,
+      "tokens_seen": 77594624
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004931695085255767,
+      "loss": 3.4097,
+      "theoretical_loss": 4.945595837986693,
+      "tokens_seen": 77660160
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.000493159478435306,
+      "loss": 3.4903,
+      "theoretical_loss": 4.944996772366503,
+      "tokens_seen": 77725696
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004931494483450352,
+      "loss": 3.5634,
+      "theoretical_loss": 4.944398352944038,
+      "tokens_seen": 77791232
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004931394182547643,
+      "loss": 3.5448,
+      "theoretical_loss": 4.943800578478745,
+      "tokens_seen": 77856768
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004931293881644935,
+      "loss": 3.4788,
+      "theoretical_loss": 4.9432034477334925,
+      "tokens_seen": 77922304
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004931193580742227,
+      "loss": 3.4963,
+      "theoretical_loss": 4.942606959474562,
+      "tokens_seen": 77987840
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004931093279839519,
+      "loss": 3.4853,
+      "theoretical_loss": 4.942011112471633,
+      "tokens_seen": 78053376
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004930992978936811,
+      "loss": 3.3058,
+      "theoretical_loss": 4.94141590549777,
+      "tokens_seen": 78118912
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004930892678034102,
+      "loss": 3.7623,
+      "theoretical_loss": 4.940821337329414,
+      "tokens_seen": 78184448
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004930792377131394,
+      "loss": 3.324,
+      "theoretical_loss": 4.940227406746368,
+      "tokens_seen": 78249984
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004930692076228686,
+      "loss": 3.595,
+      "theoretical_loss": 4.939634112531784,
+      "tokens_seen": 78315520
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004930591775325978,
+      "loss": 3.6571,
+      "theoretical_loss": 4.939041453472152,
+      "tokens_seen": 78381056
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.000493049147442327,
+      "loss": 3.546,
+      "theoretical_loss": 4.9384494283572895,
+      "tokens_seen": 78446592
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004930391173520562,
+      "loss": 3.7395,
+      "theoretical_loss": 4.937858035980327,
+      "tokens_seen": 78512128
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004930290872617853,
+      "loss": 3.5258,
+      "theoretical_loss": 4.9372672751377,
+      "tokens_seen": 78577664
+    },
+    {
+      "epoch": 0.02,
+      "objective/train/docs_used": 54900,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 2.9137701988220215,
+      "objective/train/theoretical_loss": 4.936677144629131,
+      "objective/train/tokens_used": 99103200,
+      "theoretical_loss": 4.936677144629131,
+      "tokens_seen": 78643200
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004930190571715146,
+      "loss": 3.2102,
+      "theoretical_loss": 4.936677144629131,
+      "tokens_seen": 78643200
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004930090270812437,
+      "loss": 3.7118,
+      "theoretical_loss": 4.936087643257625,
+      "tokens_seen": 78708736
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.000492998996990973,
+      "loss": 3.5876,
+      "theoretical_loss": 4.935498769829453,
+      "tokens_seen": 78774272
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004929889669007021,
+      "loss": 3.7919,
+      "theoretical_loss": 4.934910523154139,
+      "tokens_seen": 78839808
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004929789368104313,
+      "loss": 3.8801,
+      "theoretical_loss": 4.934322902044457,
+      "tokens_seen": 78905344
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004929689067201605,
+      "loss": 3.5096,
+      "theoretical_loss": 4.933735905316408,
+      "tokens_seen": 78970880
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004929588766298897,
+      "loss": 3.4217,
+      "theoretical_loss": 4.933149531789217,
+      "tokens_seen": 79036416
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004929488465396189,
+      "loss": 3.6953,
+      "theoretical_loss": 4.932563780285319,
+      "tokens_seen": 79101952
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004929388164493481,
+      "loss": 3.6487,
+      "theoretical_loss": 4.931978649630346,
+      "tokens_seen": 79167488
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004929287863590773,
+      "loss": 3.6641,
+      "theoretical_loss": 4.931394138653117,
+      "tokens_seen": 79233024
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004929187562688064,
+      "loss": 3.8094,
+      "theoretical_loss": 4.930810246185626,
+      "tokens_seen": 79298560
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004929087261785356,
+      "loss": 3.5257,
+      "theoretical_loss": 4.930226971063035,
+      "tokens_seen": 79364096
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004928986960882648,
+      "loss": 3.4985,
+      "theoretical_loss": 4.929644312123655,
+      "tokens_seen": 79429632
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.000492888665997994,
+      "loss": 3.3258,
+      "theoretical_loss": 4.929062268208941,
+      "tokens_seen": 79495168
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004928786359077232,
+      "loss": 3.2647,
+      "theoretical_loss": 4.92848083816348,
+      "tokens_seen": 79560704
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004928686058174523,
+      "loss": 3.8228,
+      "theoretical_loss": 4.927900020834979,
+      "tokens_seen": 79626240
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004928585757271815,
+      "loss": 3.3943,
+      "theoretical_loss": 4.927319815074252,
+      "tokens_seen": 79691776
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004928485456369107,
+      "loss": 3.4187,
+      "theoretical_loss": 4.926740219735214,
+      "tokens_seen": 79757312
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.00049283851554664,
+      "loss": 3.6324,
+      "theoretical_loss": 4.926161233674863,
+      "tokens_seen": 79822848
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004928284854563691,
+      "loss": 3.627,
+      "theoretical_loss": 4.925582855753279,
+      "tokens_seen": 79888384
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004928184553660984,
+      "loss": 3.4882,
+      "theoretical_loss": 4.925005084833603,
+      "tokens_seen": 79953920
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004928084252758274,
+      "loss": 3.8262,
+      "theoretical_loss": 4.924427919782033,
+      "tokens_seen": 80019456
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004927983951855567,
+      "loss": 3.5192,
+      "theoretical_loss": 4.923851359467811,
+      "tokens_seen": 80084992
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004927883650952859,
+      "loss": 3.714,
+      "theoretical_loss": 4.923275402763213,
+      "tokens_seen": 80150528
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004927783350050151,
+      "loss": 3.4187,
+      "theoretical_loss": 4.922700048543536,
+      "tokens_seen": 80216064
+    },
+    {
+      "epoch": 0.02,
+      "objective/train/docs_used": 56090,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 3.1410176753997803,
+      "objective/train/theoretical_loss": 4.922125295687091,
+      "objective/train/tokens_used": 100741600,
+      "theoretical_loss": 4.922125295687091,
+      "tokens_seen": 80281600
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004927683049147443,
+      "loss": 3.5733,
+      "theoretical_loss": 4.922125295687091,
+      "tokens_seen": 80281600
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004927582748244734,
+      "loss": 3.396,
+      "theoretical_loss": 4.92155114307519,
+      "tokens_seen": 80347136
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004927482447342026,
+      "loss": 3.6166,
+      "theoretical_loss": 4.9209775895921375,
+      "tokens_seen": 80412672
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004927382146439318,
+      "loss": 3.4859,
+      "theoretical_loss": 4.920404634125217,
+      "tokens_seen": 80478208
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.000492728184553661,
+      "loss": 3.2846,
+      "theoretical_loss": 4.919832275564684,
+      "tokens_seen": 80543744
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004927181544633902,
+      "loss": 3.653,
+      "theoretical_loss": 4.9192605128037545,
+      "tokens_seen": 80609280
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004927081243731193,
+      "loss": 3.5658,
+      "theoretical_loss": 4.918689344738594,
+      "tokens_seen": 80674816
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004926980942828485,
+      "loss": 3.5587,
+      "theoretical_loss": 4.918118770268305,
+      "tokens_seen": 80740352
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004926880641925777,
+      "loss": 3.3881,
+      "theoretical_loss": 4.917548788294924,
+      "tokens_seen": 80805888
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004926780341023069,
+      "loss": 3.4548,
+      "theoretical_loss": 4.916979397723403,
+      "tokens_seen": 80871424
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004926680040120361,
+      "loss": 3.6868,
+      "theoretical_loss": 4.916410597461606,
+      "tokens_seen": 80936960
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004926579739217654,
+      "loss": 3.6518,
+      "theoretical_loss": 4.9158423864202945,
+      "tokens_seen": 81002496
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004926479438314944,
+      "loss": 3.4168,
+      "theoretical_loss": 4.915274763513119,
+      "tokens_seen": 81068032
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004926379137412237,
+      "loss": 3.353,
+      "theoretical_loss": 4.914707727656609,
+      "tokens_seen": 81133568
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004926278836509528,
+      "loss": 3.3831,
+      "theoretical_loss": 4.914141277770164,
+      "tokens_seen": 81199104
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004926178535606821,
+      "loss": 3.1851,
+      "theoretical_loss": 4.913575412776042,
+      "tokens_seen": 81264640
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004926078234704113,
+      "loss": 3.7187,
+      "theoretical_loss": 4.91301013159935,
+      "tokens_seen": 81330176
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004925977933801404,
+      "loss": 3.5052,
+      "theoretical_loss": 4.912445433168036,
+      "tokens_seen": 81395712
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004925877632898696,
+      "loss": 3.5131,
+      "theoretical_loss": 4.911881316412876,
+      "tokens_seen": 81461248
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004925777331995988,
+      "loss": 3.7694,
+      "theoretical_loss": 4.911317780267467,
+      "tokens_seen": 81526784
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.000492567703109328,
+      "loss": 3.3832,
+      "theoretical_loss": 4.910754823668215,
+      "tokens_seen": 81592320
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004925576730190572,
+      "loss": 3.8546,
+      "theoretical_loss": 4.910192445554329,
+      "tokens_seen": 81657856
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004925476429287864,
+      "loss": 3.6127,
+      "theoretical_loss": 4.909630644867807,
+      "tokens_seen": 81723392
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004925376128385155,
+      "loss": 3.3545,
+      "theoretical_loss": 4.909069420553429,
+      "tokens_seen": 81788928
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004925275827482447,
+      "loss": 3.5589,
+      "theoretical_loss": 4.908508771558748,
+      "tokens_seen": 81854464
+    },
+    {
+      "epoch": 0.02,
+      "objective/train/docs_used": 57069,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 3.6827940940856934,
+      "objective/train/theoretical_loss": 4.907948696834077,
+      "objective/train/tokens_used": 102380000,
+      "theoretical_loss": 4.907948696834077,
+      "tokens_seen": 81920000
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004925175526579739,
+      "loss": 3.7195,
+      "theoretical_loss": 4.907948696834077,
+      "tokens_seen": 81920000
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004925075225677031,
+      "loss": 3.5548,
+      "theoretical_loss": 4.907389195332484,
+      "tokens_seen": 81985536
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004924974924774323,
+      "loss": 3.807,
+      "theoretical_loss": 4.906830266009781,
+      "tokens_seen": 82051072
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004924874623871615,
+      "loss": 3.4511,
+      "theoretical_loss": 4.90627190782451,
+      "tokens_seen": 82116608
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004924774322968907,
+      "loss": 3.7151,
+      "theoretical_loss": 4.905714119737945,
+      "tokens_seen": 82182144
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004924674022066199,
+      "loss": 3.5237,
+      "theoretical_loss": 4.9051569007140685,
+      "tokens_seen": 82247680
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004924573721163491,
+      "loss": 3.332,
+      "theoretical_loss": 4.904600249719573,
+      "tokens_seen": 82313216
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004924473420260783,
+      "loss": 3.504,
+      "theoretical_loss": 4.90404416572385,
+      "tokens_seen": 82378752
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004924373119358075,
+      "loss": 3.7408,
+      "theoretical_loss": 4.903488647698972,
+      "tokens_seen": 82444288
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004924272818455366,
+      "loss": 3.334,
+      "theoretical_loss": 4.902933694619698,
+      "tokens_seen": 82509824
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004924172517552658,
+      "loss": 3.2703,
+      "theoretical_loss": 4.902379305463451,
+      "tokens_seen": 82575360
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.000492407221664995,
+      "loss": 3.5645,
+      "theoretical_loss": 4.901825479210317,
+      "tokens_seen": 82640896
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004923971915747242,
+      "loss": 3.3278,
+      "theoretical_loss": 4.9012722148430345,
+      "tokens_seen": 82706432
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004923871614844534,
+      "loss": 3.6277,
+      "theoretical_loss": 4.900719511346983,
+      "tokens_seen": 82771968
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004923771313941825,
+      "loss": 3.6241,
+      "theoretical_loss": 4.900167367710176,
+      "tokens_seen": 82837504
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004923671013039117,
+      "loss": 3.5454,
+      "theoretical_loss": 4.899615782923251,
+      "tokens_seen": 82903040
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004923570712136409,
+      "loss": 3.6864,
+      "theoretical_loss": 4.899064755979464,
+      "tokens_seen": 82968576
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004923470411233702,
+      "loss": 3.3855,
+      "theoretical_loss": 4.898514285874674,
+      "tokens_seen": 83034112
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004923370110330993,
+      "loss": 3.4924,
+      "theoretical_loss": 4.897964371607341,
+      "tokens_seen": 83099648
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004923269809428286,
+      "loss": 3.4934,
+      "theoretical_loss": 4.897415012178515,
+      "tokens_seen": 83165184
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004923169508525576,
+      "loss": 3.3992,
+      "theoretical_loss": 4.896866206591823,
+      "tokens_seen": 83230720
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004923069207622869,
+      "loss": 3.5205,
+      "theoretical_loss": 4.896317953853467,
+      "tokens_seen": 83296256
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004922968906720161,
+      "loss": 3.624,
+      "theoretical_loss": 4.8957702529722145,
+      "tokens_seen": 83361792
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004922868605817453,
+      "loss": 3.5903,
+      "theoretical_loss": 4.895223102959381,
+      "tokens_seen": 83427328
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004922768304914745,
+      "loss": 3.7871,
+      "theoretical_loss": 4.894676502828836,
+      "tokens_seen": 83492864
+    },
+    {
+      "epoch": 0.03,
+      "objective/train/docs_used": 57750,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 3.804002046585083,
+      "objective/train/theoretical_loss": 4.894130451596979,
+      "objective/train/tokens_used": 104018400,
+      "theoretical_loss": 4.894130451596979,
+      "tokens_seen": 83558400
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004922668004012036,
+      "loss": 3.4882,
+      "theoretical_loss": 4.894130451596979,
+      "tokens_seen": 83558400
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004922567703109328,
+      "loss": 3.2586,
+      "theoretical_loss": 4.893584948282745,
+      "tokens_seen": 83623936
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.000492246740220662,
+      "loss": 3.5408,
+      "theoretical_loss": 4.893039991907587,
+      "tokens_seen": 83689472
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004922367101303912,
+      "loss": 3.6106,
+      "theoretical_loss": 4.892495581495468,
+      "tokens_seen": 83755008
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004922266800401204,
+      "loss": 3.2746,
+      "theoretical_loss": 4.8919517160728585,
+      "tokens_seen": 83820544
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004922166499498495,
+      "loss": 3.5637,
+      "theoretical_loss": 4.891408394668723,
+      "tokens_seen": 83886080
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004922066198595787,
+      "loss": 3.5945,
+      "theoretical_loss": 4.89086561631451,
+      "tokens_seen": 83951616
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004921965897693079,
+      "loss": 3.6297,
+      "theoretical_loss": 4.890323380044153,
+      "tokens_seen": 84017152
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004921865596790371,
+      "loss": 3.6036,
+      "theoretical_loss": 4.889781684894051,
+      "tokens_seen": 84082688
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004921765295887663,
+      "loss": 3.7009,
+      "theoretical_loss": 4.889240529903066,
+      "tokens_seen": 84148224
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004921664994984956,
+      "loss": 3.4759,
+      "theoretical_loss": 4.888699914112516,
+      "tokens_seen": 84213760
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004921564694082246,
+      "loss": 3.4833,
+      "theoretical_loss": 4.888159836566162,
+      "tokens_seen": 84279296
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004921464393179539,
+      "loss": 3.7335,
+      "theoretical_loss": 4.887620296310205,
+      "tokens_seen": 84344832
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.000492136409227683,
+      "loss": 3.8101,
+      "theoretical_loss": 4.887081292393276,
+      "tokens_seen": 84410368
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004921263791374123,
+      "loss": 3.6913,
+      "theoretical_loss": 4.886542823866424,
+      "tokens_seen": 84475904
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004921163490471415,
+      "loss": 3.5273,
+      "theoretical_loss": 4.886004889783116,
+      "tokens_seen": 84541440
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004921063189568706,
+      "loss": 3.5718,
+      "theoretical_loss": 4.885467489199222,
+      "tokens_seen": 84606976
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004920962888665998,
+      "loss": 3.6503,
+      "theoretical_loss": 4.884930621173009,
+      "tokens_seen": 84672512
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.000492086258776329,
+      "loss": 3.5557,
+      "theoretical_loss": 4.884394284765135,
+      "tokens_seen": 84738048
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004920762286860582,
+      "loss": 3.5114,
+      "theoretical_loss": 4.88385847903864,
+      "tokens_seen": 84803584
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004920661985957874,
+      "loss": 3.8069,
+      "theoretical_loss": 4.883323203058936,
+      "tokens_seen": 84869120
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004920561685055166,
+      "loss": 3.529,
+      "theoretical_loss": 4.882788455893802,
+      "tokens_seen": 84934656
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004920461384152457,
+      "loss": 3.5584,
+      "theoretical_loss": 4.882254236613376,
+      "tokens_seen": 85000192
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004920361083249749,
+      "loss": 3.7302,
+      "theoretical_loss": 4.881720544290144,
+      "tokens_seen": 85065728
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004920260782347041,
+      "loss": 3.4352,
+      "theoretical_loss": 4.881187377998938,
+      "tokens_seen": 85131264
+    },
+    {
+      "epoch": 0.03,
+      "objective/train/docs_used": 58904,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 3.6649792194366455,
+      "objective/train/theoretical_loss": 4.880654736816922,
+      "objective/train/tokens_used": 105656800,
+      "theoretical_loss": 4.880654736816922,
+      "tokens_seen": 85196800
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004920160481444333,
+      "loss": 3.7428,
+      "theoretical_loss": 4.880654736816922,
+      "tokens_seen": 85196800
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004920060180541625,
+      "loss": 3.3637,
+      "theoretical_loss": 4.880122619823589,
+      "tokens_seen": 85262336
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004919959879638916,
+      "loss": 3.4179,
+      "theoretical_loss": 4.87959102610075,
+      "tokens_seen": 85327872
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004919859578736209,
+      "loss": 3.3888,
+      "theoretical_loss": 4.879059954732529,
+      "tokens_seen": 85393408
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.00049197592778335,
+      "loss": 3.7427,
+      "theoretical_loss": 4.878529404805356,
+      "tokens_seen": 85458944
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004919658976930793,
+      "loss": 3.7709,
+      "theoretical_loss": 4.8779993754079545,
+      "tokens_seen": 85524480
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004919558676028084,
+      "loss": 3.5494,
+      "theoretical_loss": 4.877469865631338,
+      "tokens_seen": 85590016
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004919458375125377,
+      "loss": 3.5295,
+      "theoretical_loss": 4.876940874568804,
+      "tokens_seen": 85655552
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004919358074222668,
+      "loss": 3.5878,
+      "theoretical_loss": 4.876412401315923,
+      "tokens_seen": 85721088
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.000491925777331996,
+      "loss": 3.57,
+      "theoretical_loss": 4.8758844449705325,
+      "tokens_seen": 85786624
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004919157472417252,
+      "loss": 3.5246,
+      "theoretical_loss": 4.875357004632728,
+      "tokens_seen": 85852160
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004919057171514544,
+      "loss": 3.3963,
+      "theoretical_loss": 4.87483007940486,
+      "tokens_seen": 85917696
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004918956870611836,
+      "loss": 3.6586,
+      "theoretical_loss": 4.874303668391521,
+      "tokens_seen": 85983232
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004918856569709127,
+      "loss": 3.6073,
+      "theoretical_loss": 4.873777770699542,
+      "tokens_seen": 86048768
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004918756268806419,
+      "loss": 3.5692,
+      "theoretical_loss": 4.873252385437985,
+      "tokens_seen": 86114304
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004918655967903711,
+      "loss": 3.5077,
+      "theoretical_loss": 4.872727511718134,
+      "tokens_seen": 86179840
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004918555667001003,
+      "loss": 3.6584,
+      "theoretical_loss": 4.8722031486534885,
+      "tokens_seen": 86245376
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004918455366098295,
+      "loss": 3.4986,
+      "theoretical_loss": 4.8716792953597565,
+      "tokens_seen": 86310912
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004918355065195586,
+      "loss": 3.7466,
+      "theoretical_loss": 4.871155950954847,
+      "tokens_seen": 86376448
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004918254764292878,
+      "loss": 3.627,
+      "theoretical_loss": 4.870633114558865,
+      "tokens_seen": 86441984
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.000491815446339017,
+      "loss": 3.5282,
+      "theoretical_loss": 4.8701107852941,
+      "tokens_seen": 86507520
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004918054162487463,
+      "loss": 3.1841,
+      "theoretical_loss": 4.869588962285023,
+      "tokens_seen": 86573056
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004917953861584754,
+      "loss": 3.6871,
+      "theoretical_loss": 4.869067644658276,
+      "tokens_seen": 86638592
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004917853560682047,
+      "loss": 3.6656,
+      "theoretical_loss": 4.86854683154267,
+      "tokens_seen": 86704128
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004917753259779337,
+      "loss": 3.5156,
+      "theoretical_loss": 4.868026522069172,
+      "tokens_seen": 86769664
+    },
+    {
+      "epoch": 0.03,
+      "objective/train/docs_used": 59520,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 3.652179718017578,
+      "objective/train/theoretical_loss": 4.867506715370903,
+      "objective/train/tokens_used": 107295200,
+      "theoretical_loss": 4.867506715370903,
+      "tokens_seen": 86835200
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.000491765295887663,
+      "loss": 3.4928,
+      "theoretical_loss": 4.867506715370903,
+      "tokens_seen": 86835200
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004917552657973922,
+      "loss": 3.4802,
+      "theoretical_loss": 4.866987410583125,
+      "tokens_seen": 86900736
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004917452357071214,
+      "loss": 3.5752,
+      "theoretical_loss": 4.8664686068432434,
+      "tokens_seen": 86966272
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004917352056168506,
+      "loss": 3.5258,
+      "theoretical_loss": 4.8659503032907905,
+      "tokens_seen": 87031808
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004917251755265797,
+      "loss": 3.7263,
+      "theoretical_loss": 4.865432499067425,
+      "tokens_seen": 87097344
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004917151454363089,
+      "loss": 3.7067,
+      "theoretical_loss": 4.864915193316922,
+      "tokens_seen": 87162880
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004917051153460381,
+      "loss": 3.4247,
+      "theoretical_loss": 4.864398385185169,
+      "tokens_seen": 87228416
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004916950852557673,
+      "loss": 3.5719,
+      "theoretical_loss": 4.863882073820154,
+      "tokens_seen": 87293952
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004916850551654965,
+      "loss": 3.5349,
+      "theoretical_loss": 4.863366258371963,
+      "tokens_seen": 87359488
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004916750250752258,
+      "loss": 3.5815,
+      "theoretical_loss": 4.862850937992777,
+      "tokens_seen": 87425024
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004916649949849548,
+      "loss": 3.5233,
+      "theoretical_loss": 4.862336111836852,
+      "tokens_seen": 87490560
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004916549648946841,
+      "loss": 3.5181,
+      "theoretical_loss": 4.861821779060529,
+      "tokens_seen": 87556096
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004916449348044132,
+      "loss": 3.369,
+      "theoretical_loss": 4.861307938822215,
+      "tokens_seen": 87621632
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004916349047141425,
+      "loss": 3.3532,
+      "theoretical_loss": 4.86079459028238,
+      "tokens_seen": 87687168
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004916248746238717,
+      "loss": 3.4634,
+      "theoretical_loss": 4.860281732603555,
+      "tokens_seen": 87752704
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004916148445336008,
+      "loss": 3.64,
+      "theoretical_loss": 4.859769364950319,
+      "tokens_seen": 87818240
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.00049160481444333,
+      "loss": 3.5164,
+      "theoretical_loss": 4.859257486489293,
+      "tokens_seen": 87883776
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004915947843530592,
+      "loss": 3.5639,
+      "theoretical_loss": 4.858746096389139,
+      "tokens_seen": 87949312
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004915847542627884,
+      "loss": 3.4632,
+      "theoretical_loss": 4.8582351938205495,
+      "tokens_seen": 88014848
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004915747241725176,
+      "loss": 3.2569,
+      "theoretical_loss": 4.857724777956237,
+      "tokens_seen": 88080384
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004915646940822468,
+      "loss": 3.4019,
+      "theoretical_loss": 4.857214847970939,
+      "tokens_seen": 88145920
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004915546639919759,
+      "loss": 3.266,
+      "theoretical_loss": 4.856705403041397,
+      "tokens_seen": 88211456
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004915446339017051,
+      "loss": 3.7935,
+      "theoretical_loss": 4.8561964423463655,
+      "tokens_seen": 88276992
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004915346038114343,
+      "loss": 3.7385,
+      "theoretical_loss": 4.8556879650665925,
+      "tokens_seen": 88342528
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004915245737211635,
+      "loss": 3.3908,
+      "theoretical_loss": 4.855179970384819,
+      "tokens_seen": 88408064
+    },
+    {
+      "epoch": 0.03,
+      "objective/train/docs_used": 60543,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 3.110377788543701,
+      "objective/train/theoretical_loss": 4.8546724574857745,
+      "objective/train/tokens_used": 108933600,
+      "theoretical_loss": 4.8546724574857745,
+      "tokens_seen": 88473600
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004915145436308927,
+      "loss": 3.4761,
+      "theoretical_loss": 4.8546724574857745,
+      "tokens_seen": 88473600
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004915045135406218,
+      "loss": 3.442,
+      "theoretical_loss": 4.854165425556166,
+      "tokens_seen": 88539136
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004914944834503511,
+      "loss": 3.613,
+      "theoretical_loss": 4.853658873784675,
+      "tokens_seen": 88604672
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004914844533600802,
+      "loss": 3.7501,
+      "theoretical_loss": 4.853152801361952,
+      "tokens_seen": 88670208
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004914744232698095,
+      "loss": 3.4899,
+      "theoretical_loss": 4.852647207480604,
+      "tokens_seen": 88735744
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004914643931795386,
+      "loss": 3.6797,
+      "theoretical_loss": 4.8521420913352,
+      "tokens_seen": 88801280
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004914543630892679,
+      "loss": 3.2736,
+      "theoretical_loss": 4.851637452122251,
+      "tokens_seen": 88866816
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.000491444332998997,
+      "loss": 3.5697,
+      "theoretical_loss": 4.851133289040214,
+      "tokens_seen": 88932352
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004914343029087262,
+      "loss": 3.6628,
+      "theoretical_loss": 4.850629601289482,
+      "tokens_seen": 88997888
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004914242728184554,
+      "loss": 3.4603,
+      "theoretical_loss": 4.850126388072379,
+      "tokens_seen": 89063424
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004914142427281846,
+      "loss": 3.5286,
+      "theoretical_loss": 4.849623648593151,
+      "tokens_seen": 89128960
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004914042126379138,
+      "loss": 3.4864,
+      "theoretical_loss": 4.849121382057964,
+      "tokens_seen": 89194496
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004913941825476429,
+      "loss": 3.5269,
+      "theoretical_loss": 4.848619587674898,
+      "tokens_seen": 89260032
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004913841524573721,
+      "loss": 3.4857,
+      "theoretical_loss": 4.8481182646539365,
+      "tokens_seen": 89325568
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004913741223671013,
+      "loss": 3.6437,
+      "theoretical_loss": 4.847617412206963,
+      "tokens_seen": 89391104
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004913640922768305,
+      "loss": 3.4963,
+      "theoretical_loss": 4.847117029547758,
+      "tokens_seen": 89456640
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004913540621865597,
+      "loss": 3.5452,
+      "theoretical_loss": 4.846617115891988,
+      "tokens_seen": 89522176
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004913440320962888,
+      "loss": 3.2749,
+      "theoretical_loss": 4.8461176704572,
+      "tokens_seen": 89587712
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.000491334002006018,
+      "loss": 3.319,
+      "theoretical_loss": 4.845618692462826,
+      "tokens_seen": 89653248
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004913239719157472,
+      "loss": 3.4855,
+      "theoretical_loss": 4.845120181130158,
+      "tokens_seen": 89718784
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004913139418254765,
+      "loss": 3.7812,
+      "theoretical_loss": 4.844622135682359,
+      "tokens_seen": 89784320
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004913039117352056,
+      "loss": 3.1217,
+      "theoretical_loss": 4.84412455534445,
+      "tokens_seen": 89849856
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004912938816449349,
+      "loss": 3.3833,
+      "theoretical_loss": 4.843627439343306,
+      "tokens_seen": 89915392
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004912838515546639,
+      "loss": 3.5363,
+      "theoretical_loss": 4.843130786907647,
+      "tokens_seen": 89980928
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004912738214643932,
+      "loss": 3.4395,
+      "theoretical_loss": 4.842634597268036,
+      "tokens_seen": 90046464
+    },
+    {
+      "epoch": 0.03,
+      "objective/train/docs_used": 60961,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 3.3933932781219482,
+      "objective/train/theoretical_loss": 4.842138869656871,
+      "objective/train/tokens_used": 110572000,
+      "theoretical_loss": 4.842138869656871,
+      "tokens_seen": 90112000
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004912637913741224,
+      "loss": 3.5361,
+      "theoretical_loss": 4.842138869656871,
+      "tokens_seen": 90112000
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004912537612838516,
+      "loss": 3.1717,
+      "theoretical_loss": 4.841643603308384,
+      "tokens_seen": 90177536
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004912437311935808,
+      "loss": 3.6274,
+      "theoretical_loss": 4.841148797458627,
+      "tokens_seen": 90243072
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.00049123370110331,
+      "loss": 3.5958,
+      "theoretical_loss": 4.840654451345472,
+      "tokens_seen": 90308608
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004912236710130391,
+      "loss": 3.6702,
+      "theoretical_loss": 4.840160564208606,
+      "tokens_seen": 90374144
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004912136409227683,
+      "loss": 3.4133,
+      "theoretical_loss": 4.839667135289522,
+      "tokens_seen": 90439680
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004912036108324975,
+      "loss": 3.3868,
+      "theoretical_loss": 4.839174163831515,
+      "tokens_seen": 90505216
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004911935807422267,
+      "loss": 3.4971,
+      "theoretical_loss": 4.838681649079677,
+      "tokens_seen": 90570752
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004911835506519559,
+      "loss": 3.581,
+      "theoretical_loss": 4.838189590280891,
+      "tokens_seen": 90636288
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.000491173520561685,
+      "loss": 3.5547,
+      "theoretical_loss": 4.837697986683825,
+      "tokens_seen": 90701824
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004911634904714142,
+      "loss": 3.6512,
+      "theoretical_loss": 4.837206837538925,
+      "tokens_seen": 90767360
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004911534603811434,
+      "loss": 3.189,
+      "theoretical_loss": 4.836716142098417,
+      "tokens_seen": 90832896
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004911434302908726,
+      "loss": 3.4003,
+      "theoretical_loss": 4.836225899616288,
+      "tokens_seen": 90898432
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004911334002006019,
+      "loss": 3.5343,
+      "theoretical_loss": 4.835736109348293,
+      "tokens_seen": 90963968
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004911233701103309,
+      "loss": 3.491,
+      "theoretical_loss": 4.8352467705519455,
+      "tokens_seen": 91029504
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004911133400200602,
+      "loss": 3.543,
+      "theoretical_loss": 4.83475788248651,
+      "tokens_seen": 91095040
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004911033099297893,
+      "loss": 3.5829,
+      "theoretical_loss": 4.834269444412996,
+      "tokens_seen": 91160576
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004910932798395186,
+      "loss": 3.5038,
+      "theoretical_loss": 4.83378145559416,
+      "tokens_seen": 91226112
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004910832497492478,
+      "loss": 3.6003,
+      "theoretical_loss": 4.83329391529449,
+      "tokens_seen": 91291648
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.000491073219658977,
+      "loss": 3.5657,
+      "theoretical_loss": 4.832806822780208,
+      "tokens_seen": 91357184
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004910631895687061,
+      "loss": 3.3872,
+      "theoretical_loss": 4.832320177319261,
+      "tokens_seen": 91422720
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004910531594784353,
+      "loss": 3.4167,
+      "theoretical_loss": 4.831833978181314,
+      "tokens_seen": 91488256
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004910431293881645,
+      "loss": 3.2479,
+      "theoretical_loss": 4.831348224637752,
+      "tokens_seen": 91553792
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004910330992978937,
+      "loss": 3.5555,
+      "theoretical_loss": 4.830862915961665,
+      "tokens_seen": 91619328
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004910230692076229,
+      "loss": 3.4961,
+      "theoretical_loss": 4.830378051427852,
+      "tokens_seen": 91684864
+    },
+    {
+      "epoch": 0.03,
+      "objective/train/docs_used": 62148,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 3.0938072204589844,
+      "objective/train/theoretical_loss": 4.829893630312808,
+      "objective/train/tokens_used": 112210400,
+      "theoretical_loss": 4.829893630312808,
+      "tokens_seen": 91750400
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.000491013039117352,
+      "loss": 3.1482,
+      "theoretical_loss": 4.829893630312808,
+      "tokens_seen": 91750400
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004910030090270812,
+      "loss": 3.4885,
+      "theoretical_loss": 4.829409651894725,
+      "tokens_seen": 91815936
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004909929789368104,
+      "loss": 3.5385,
+      "theoretical_loss": 4.8289261154534815,
+      "tokens_seen": 91881472
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004909829488465397,
+      "loss": 3.2172,
+      "theoretical_loss": 4.828443020270642,
+      "tokens_seen": 91947008
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004909729187562688,
+      "loss": 3.351,
+      "theoretical_loss": 4.827960365629449,
+      "tokens_seen": 92012544
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004909628886659981,
+      "loss": 3.6262,
+      "theoretical_loss": 4.827478150814821,
+      "tokens_seen": 92078080
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004909528585757272,
+      "loss": 3.4536,
+      "theoretical_loss": 4.826996375113339,
+      "tokens_seen": 92143616
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004909428284854564,
+      "loss": 3.4569,
+      "theoretical_loss": 4.826515037813255,
+      "tokens_seen": 92209152
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004909327983951856,
+      "loss": 3.7922,
+      "theoretical_loss": 4.826034138204474,
+      "tokens_seen": 92274688
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004909227683049148,
+      "loss": 3.4368,
+      "theoretical_loss": 4.825553675578557,
+      "tokens_seen": 92340224
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.000490912738214644,
+      "loss": 3.3922,
+      "theoretical_loss": 4.825073649228713,
+      "tokens_seen": 92405760
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004909027081243731,
+      "loss": 3.5078,
+      "theoretical_loss": 4.824594058449794,
+      "tokens_seen": 92471296
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004908926780341023,
+      "loss": 3.3106,
+      "theoretical_loss": 4.824114902538292,
+      "tokens_seen": 92536832
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004908826479438315,
+      "loss": 3.136,
+      "theoretical_loss": 4.823636180792329,
+      "tokens_seen": 92602368
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004908726178535607,
+      "loss": 3.3286,
+      "theoretical_loss": 4.823157892511661,
+      "tokens_seen": 92667904
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004908625877632899,
+      "loss": 3.6266,
+      "theoretical_loss": 4.82268003699766,
+      "tokens_seen": 92733440
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.000490852557673019,
+      "loss": 3.5695,
+      "theoretical_loss": 4.8222026135533245,
+      "tokens_seen": 92798976
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004908425275827482,
+      "loss": 3.487,
+      "theoretical_loss": 4.821725621483262,
+      "tokens_seen": 92864512
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004908324974924774,
+      "loss": 3.503,
+      "theoretical_loss": 4.821249060093692,
+      "tokens_seen": 92930048
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004908224674022067,
+      "loss": 3.3758,
+      "theoretical_loss": 4.820772928692433,
+      "tokens_seen": 92995584
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004908124373119358,
+      "loss": 3.4497,
+      "theoretical_loss": 4.820297226588908,
+      "tokens_seen": 93061120
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004908024072216651,
+      "loss": 3.6942,
+      "theoretical_loss": 4.819821953094133,
+      "tokens_seen": 93126656
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004907923771313941,
+      "loss": 3.6533,
+      "theoretical_loss": 4.819347107520713,
+      "tokens_seen": 93192192
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004907823470411234,
+      "loss": 3.3973,
+      "theoretical_loss": 4.818872689182836,
+      "tokens_seen": 93257728
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004907723169508526,
+      "loss": 3.2205,
+      "theoretical_loss": 4.818398697396274,
+      "tokens_seen": 93323264
+    },
+    {
+      "epoch": 0.03,
+      "objective/train/docs_used": 62815,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 3.5069005489349365,
+      "objective/train/theoretical_loss": 4.817925131478374,
+      "objective/train/tokens_used": 113848800,
+      "theoretical_loss": 4.817925131478374,
+      "tokens_seen": 93388800
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004907622868605818,
+      "loss": 3.5578,
+      "theoretical_loss": 4.817925131478374,
+      "tokens_seen": 93388800
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.000490752256770311,
+      "loss": 3.5471,
+      "theoretical_loss": 4.817451990748047,
+      "tokens_seen": 93454336
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004907422266800401,
+      "loss": 3.6923,
+      "theoretical_loss": 4.81697927452578,
+      "tokens_seen": 93519872
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004907321965897693,
+      "loss": 3.5227,
+      "theoretical_loss": 4.816506982133613,
+      "tokens_seen": 93585408
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004907221664994985,
+      "loss": 3.2295,
+      "theoretical_loss": 4.816035112895147,
+      "tokens_seen": 93650944
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004907121364092277,
+      "loss": 3.1312,
+      "theoretical_loss": 4.815563666135532,
+      "tokens_seen": 93716480
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004907021063189569,
+      "loss": 3.6798,
+      "theoretical_loss": 4.815092641181467,
+      "tokens_seen": 93782016
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.000490692076228686,
+      "loss": 3.7669,
+      "theoretical_loss": 4.814622037361195,
+      "tokens_seen": 93847552
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004906820461384152,
+      "loss": 3.6417,
+      "theoretical_loss": 4.814151854004493,
+      "tokens_seen": 93913088
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004906720160481444,
+      "loss": 3.2937,
+      "theoretical_loss": 4.813682090442674,
+      "tokens_seen": 93978624
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004906619859578736,
+      "loss": 3.2833,
+      "theoretical_loss": 4.81321274600858,
+      "tokens_seen": 94044160
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004906519558676028,
+      "loss": 3.2788,
+      "theoretical_loss": 4.812743820036577,
+      "tokens_seen": 94109696
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004906419257773321,
+      "loss": 3.5618,
+      "theoretical_loss": 4.81227531186255,
+      "tokens_seen": 94175232
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004906318956870611,
+      "loss": 3.5329,
+      "theoretical_loss": 4.811807220823899,
+      "tokens_seen": 94240768
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004906218655967904,
+      "loss": 3.6156,
+      "theoretical_loss": 4.811339546259536,
+      "tokens_seen": 94306304
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004906118355065195,
+      "loss": 3.3777,
+      "theoretical_loss": 4.810872287509877,
+      "tokens_seen": 94371840
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004906018054162488,
+      "loss": 3.5883,
+      "theoretical_loss": 4.810405443916842,
+      "tokens_seen": 94437376
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.000490591775325978,
+      "loss": 3.6907,
+      "theoretical_loss": 4.809939014823849,
+      "tokens_seen": 94502912
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004905817452357072,
+      "loss": 3.6602,
+      "theoretical_loss": 4.809472999575805,
+      "tokens_seen": 94568448
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004905717151454363,
+      "loss": 3.6322,
+      "theoretical_loss": 4.809007397519108,
+      "tokens_seen": 94633984
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004905616850551655,
+      "loss": 3.4204,
+      "theoretical_loss": 4.80854220800164,
+      "tokens_seen": 94699520
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004905516549648947,
+      "loss": 3.5405,
+      "theoretical_loss": 4.808077430372762,
+      "tokens_seen": 94765056
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004905416248746239,
+      "loss": 3.4647,
+      "theoretical_loss": 4.807613063983311,
+      "tokens_seen": 94830592
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004905315947843531,
+      "loss": 3.7424,
+      "theoretical_loss": 4.807149108185593,
+      "tokens_seen": 94896128
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004905215646940822,
+      "loss": 3.3372,
+      "theoretical_loss": 4.806685562333383,
+      "tokens_seen": 94961664
+    },
+    {
+      "epoch": 0.03,
+      "objective/train/docs_used": 64162,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 3.0013229846954346,
+      "objective/train/theoretical_loss": 4.806222425781916,
+      "objective/train/tokens_used": 115487200,
+      "theoretical_loss": 4.806222425781916,
+      "tokens_seen": 95027200
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004905115346038114,
+      "loss": 3.4672,
+      "theoretical_loss": 4.806222425781916,
+      "tokens_seen": 95027200
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004905015045135406,
+      "loss": 3.4124,
+      "theoretical_loss": 4.805759697887886,
+      "tokens_seen": 95092736
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004904914744232698,
+      "loss": 3.5337,
+      "theoretical_loss": 4.80529737800944,
+      "tokens_seen": 95158272
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.000490481444332999,
+      "loss": 3.1987,
+      "theoretical_loss": 4.804835465506176,
+      "tokens_seen": 95223808
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004904714142427281,
+      "loss": 3.3264,
+      "theoretical_loss": 4.8043739597391335,
+      "tokens_seen": 95289344
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004904613841524574,
+      "loss": 3.421,
+      "theoretical_loss": 4.803912860070795,
+      "tokens_seen": 95354880
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004904513540621865,
+      "loss": 3.5238,
+      "theoretical_loss": 4.803452165865082,
+      "tokens_seen": 95420416
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004904413239719158,
+      "loss": 3.4263,
+      "theoretical_loss": 4.802991876487344,
+      "tokens_seen": 95485952
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004904312938816449,
+      "loss": 3.3975,
+      "theoretical_loss": 4.802531991304359,
+      "tokens_seen": 95551488
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004904212637913742,
+      "loss": 3.5391,
+      "theoretical_loss": 4.802072509684331,
+      "tokens_seen": 95617024
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004904112337011033,
+      "loss": 3.4524,
+      "theoretical_loss": 4.801613430996884,
+      "tokens_seen": 95682560
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004904012036108325,
+      "loss": 3.5944,
+      "theoretical_loss": 4.801154754613056,
+      "tokens_seen": 95748096
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004903911735205617,
+      "loss": 3.2372,
+      "theoretical_loss": 4.800696479905296,
+      "tokens_seen": 95813632
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004903811434302909,
+      "loss": 3.1815,
+      "theoretical_loss": 4.800238606247462,
+      "tokens_seen": 95879168
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004903711133400201,
+      "loss": 3.4315,
+      "theoretical_loss": 4.799781133014815,
+      "tokens_seen": 95944704
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004903610832497492,
+      "loss": 3.2851,
+      "theoretical_loss": 4.799324059584013,
+      "tokens_seen": 96010240
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004903510531594784,
+      "loss": 3.6757,
+      "theoretical_loss": 4.798867385333111,
+      "tokens_seen": 96075776
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004903410230692076,
+      "loss": 3.3841,
+      "theoretical_loss": 4.798411109641553,
+      "tokens_seen": 96141312
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004903309929789368,
+      "loss": 3.6343,
+      "theoretical_loss": 4.797955231890175,
+      "tokens_seen": 96206848
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.000490320962888666,
+      "loss": 3.3026,
+      "theoretical_loss": 4.797499751461188,
+      "tokens_seen": 96272384
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004903109327983952,
+      "loss": 3.3258,
+      "theoretical_loss": 4.797044667738188,
+      "tokens_seen": 96337920
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004903009027081243,
+      "loss": 3.4414,
+      "theoretical_loss": 4.796589980106143,
+      "tokens_seen": 96403456
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004902908726178535,
+      "loss": 3.2512,
+      "theoretical_loss": 4.796135687951393,
+      "tokens_seen": 96468992
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004902808425275828,
+      "loss": 3.4072,
+      "theoretical_loss": 4.795681790661644,
+      "tokens_seen": 96534528
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004902708124373119,
+      "loss": 3.4574,
+      "theoretical_loss": 4.795228287625964,
+      "tokens_seen": 96600064
+    },
+    {
+      "epoch": 0.03,
+      "objective/train/docs_used": 64775,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 3.715846538543701,
+      "objective/train/theoretical_loss": 4.794775178234782,
+      "objective/train/tokens_used": 117125600,
+      "theoretical_loss": 4.794775178234782,
+      "tokens_seen": 96665600
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004902607823470412,
+      "loss": 3.622,
+      "theoretical_loss": 4.794775178234782,
+      "tokens_seen": 96665600
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004902507522567703,
+      "loss": 3.3135,
+      "theoretical_loss": 4.794322461879881,
+      "tokens_seen": 96731136
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004902407221664995,
+      "loss": 3.3367,
+      "theoretical_loss": 4.793870137954395,
+      "tokens_seen": 96796672
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004902306920762287,
+      "loss": 3.3923,
+      "theoretical_loss": 4.793418205852804,
+      "tokens_seen": 96862208
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004902206619859579,
+      "loss": 3.2944,
+      "theoretical_loss": 4.7929666649709315,
+      "tokens_seen": 96927744
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004902106318956871,
+      "loss": 3.4677,
+      "theoretical_loss": 4.7925155147059435,
+      "tokens_seen": 96993280
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004902006018054163,
+      "loss": 3.5354,
+      "theoretical_loss": 4.792064754456337,
+      "tokens_seen": 97058816
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004901905717151454,
+      "loss": 3.4739,
+      "theoretical_loss": 4.791614383621943,
+      "tokens_seen": 97124352
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004901805416248746,
+      "loss": 3.4712,
+      "theoretical_loss": 4.79116440160392,
+      "tokens_seen": 97189888
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004901705115346038,
+      "loss": 3.6806,
+      "theoretical_loss": 4.790714807804751,
+      "tokens_seen": 97255424
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.000490160481444333,
+      "loss": 3.2497,
+      "theoretical_loss": 4.790265601628235,
+      "tokens_seen": 97320960
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004901504513540623,
+      "loss": 3.3692,
+      "theoretical_loss": 4.789816782479494,
+      "tokens_seen": 97386496
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004901404212637913,
+      "loss": 3.576,
+      "theoretical_loss": 4.789368349764956,
+      "tokens_seen": 97452032
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004901303911735206,
+      "loss": 3.4743,
+      "theoretical_loss": 4.788920302892365,
+      "tokens_seen": 97517568
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004901203610832497,
+      "loss": 3.1291,
+      "theoretical_loss": 4.7884726412707606,
+      "tokens_seen": 97583104
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.000490110330992979,
+      "loss": 3.5709,
+      "theoretical_loss": 4.788025364310492,
+      "tokens_seen": 97648640
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004901003009027082,
+      "loss": 3.3566,
+      "theoretical_loss": 4.787578471423203,
+      "tokens_seen": 97714176
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004900902708124374,
+      "loss": 3.6399,
+      "theoretical_loss": 4.787131962021827,
+      "tokens_seen": 97779712
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004900802407221665,
+      "loss": 3.577,
+      "theoretical_loss": 4.786685835520595,
+      "tokens_seen": 97845248
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004900702106318957,
+      "loss": 3.4237,
+      "theoretical_loss": 4.786240091335019,
+      "tokens_seen": 97910784
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004900601805416249,
+      "loss": 3.3724,
+      "theoretical_loss": 4.785794728881895,
+      "tokens_seen": 97976320
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004900501504513541,
+      "loss": 3.3342,
+      "theoretical_loss": 4.785349747579299,
+      "tokens_seen": 98041856
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004900401203610833,
+      "loss": 3.5862,
+      "theoretical_loss": 4.784905146846581,
+      "tokens_seen": 98107392
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004900300902708124,
+      "loss": 3.3465,
+      "theoretical_loss": 4.784460926104362,
+      "tokens_seen": 98172928
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004900200601805416,
+      "loss": 3.3709,
+      "theoretical_loss": 4.7840170847745345,
+      "tokens_seen": 98238464
+    },
+    {
+      "epoch": 0.03,
+      "objective/train/docs_used": 65599,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 3.469925880432129,
+      "objective/train/theoretical_loss": 4.783573622280253,
+      "objective/train/tokens_used": 118764000,
+      "theoretical_loss": 4.783573622280253,
+      "tokens_seen": 98304000
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004900100300902708,
+      "loss": 3.5123,
+      "theoretical_loss": 4.783573622280253,
+      "tokens_seen": 98304000
+    }
+  ],
+  "max_steps": 50354,
+  "num_train_epochs": 9223372036854775807,
+  "total_flos": 5.0168070144e+16,
+  "trial_name": null,
+  "trial_params": null
+}