diff --git "a/checkpoint-5000/trainer_state.json" "b/checkpoint-5000/trainer_state.json"
new file mode 100644--- /dev/null
+++ "b/checkpoint-5000/trainer_state.json"
@@ -0,0 +1,35566 @@
+{
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 0.09929697740000794,
+  "global_step": 5000,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.0,
+      "learning_rate": 1.5873015873015873e-06,
+      "loss": 10.5355,
+      "theoretical_loss": 20.841284642178216,
+      "tokens_seen": 65536
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 3.1746031746031746e-06,
+      "loss": 10.5405,
+      "theoretical_loss": 17.59466794495971,
+      "tokens_seen": 131072
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 4.7619047619047615e-06,
+      "loss": 10.4096,
+      "theoretical_loss": 15.967943933467634,
+      "tokens_seen": 196608
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 6.349206349206349e-06,
+      "loss": 10.1794,
+      "theoretical_loss": 14.920783596619636,
+      "tokens_seen": 262144
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 7.936507936507936e-06,
+      "loss": 9.9009,
+      "theoretical_loss": 14.164683778394041,
+      "tokens_seen": 327680
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 9.523809523809523e-06,
+      "loss": 9.6355,
+      "theoretical_loss": 13.581028313181289,
+      "tokens_seen": 393216
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 1.1111111111111112e-05,
+      "loss": 9.3831,
+      "theoretical_loss": 13.110275842032255,
+      "tokens_seen": 458752
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 1.2698412698412699e-05,
+      "loss": 9.1719,
+      "theoretical_loss": 12.71859646611439,
+      "tokens_seen": 524288
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 1.4285714285714285e-05,
+      "loss": 9.0143,
+      "theoretical_loss": 12.38505930450469,
+      "tokens_seen": 589824
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 1.5873015873015872e-05,
+      "loss": 8.8536,
+      "theoretical_loss": 12.095879447666144,
+      "tokens_seen": 655360
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 1.746031746031746e-05,
+      "loss": 8.8086,
+      "theoretical_loss": 11.841533071732742,
+      "tokens_seen": 720896
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 1.9047619047619046e-05,
+      "loss": 8.74,
+      "theoretical_loss": 11.615186049337796,
+      "tokens_seen": 786432
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 2.0634920634920636e-05,
+      "loss": 8.5264,
+      "theoretical_loss": 11.411780980817854,
+      "tokens_seen": 851968
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 2.2222222222222223e-05,
+      "loss": 8.5191,
+      "theoretical_loss": 11.227478542742938,
+      "tokens_seen": 917504
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 2.380952380952381e-05,
+      "loss": 8.4946,
+      "theoretical_loss": 11.059300758608979,
+      "tokens_seen": 983040
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 2.5396825396825397e-05,
+      "loss": 8.3993,
+      "theoretical_loss": 10.904894927088016,
+      "tokens_seen": 1048576
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 2.6984126984126984e-05,
+      "loss": 8.339,
+      "theoretical_loss": 10.76237258069379,
+      "tokens_seen": 1114112
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 2.857142857142857e-05,
+      "loss": 8.3136,
+      "theoretical_loss": 10.630196716861345,
+      "tokens_seen": 1179648
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 3.0158730158730158e-05,
+      "loss": 8.2016,
+      "theoretical_loss": 10.507101012987707,
+      "tokens_seen": 1245184
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 3.1746031746031745e-05,
+      "loss": 8.2822,
+      "theoretical_loss": 10.392030784394397,
+      "tokens_seen": 1310720
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 3.3333333333333335e-05,
+      "loss": 8.1899,
+      "theoretical_loss": 10.284099061527812,
+      "tokens_seen": 1376256
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 3.492063492063492e-05,
+      "loss": 8.0618,
+      "theoretical_loss": 10.182553393901085,
+      "tokens_seen": 1441792
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 3.650793650793651e-05,
+      "loss": 8.1167,
+      "theoretical_loss": 10.086750402363664,
+      "tokens_seen": 1507328
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 3.809523809523809e-05,
+      "loss": 7.9888,
+      "theoretical_loss": 9.996136019471344,
+      "tokens_seen": 1572864
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 3.968253968253968e-05,
+      "loss": 7.9389,
+      "theoretical_loss": 9.910229967024176,
+      "tokens_seen": 1638400
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 4.126984126984127e-05,
+      "loss": 7.9611,
+      "theoretical_loss": 9.828613432171625,
+      "tokens_seen": 1703936
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 4.2857142857142856e-05,
+      "loss": 7.8176,
+      "theoretical_loss": 9.75091918753851,
+      "tokens_seen": 1769472
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 4.4444444444444447e-05,
+      "loss": 7.782,
+      "theoretical_loss": 9.676823599712613,
+      "tokens_seen": 1835008
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 4.603174603174603e-05,
+      "loss": 7.8388,
+      "theoretical_loss": 9.606040111776705,
+      "tokens_seen": 1900544
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 4.761904761904762e-05,
+      "loss": 7.7238,
+      "theoretical_loss": 9.538313887395919,
+      "tokens_seen": 1966080
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 4.9206349206349204e-05,
+      "loss": 7.6459,
+      "theoretical_loss": 9.473417378263001,
+      "tokens_seen": 2031616
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 5.0793650793650794e-05,
+      "loss": 7.4806,
+      "theoretical_loss": 9.411146631541524,
+      "tokens_seen": 2097152
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 5.2380952380952384e-05,
+      "loss": 7.462,
+      "theoretical_loss": 9.351318194865048,
+      "tokens_seen": 2162688
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 5.396825396825397e-05,
+      "loss": 7.4201,
+      "theoretical_loss": 9.293766507291341,
+      "tokens_seen": 2228224
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 5.555555555555556e-05,
+      "loss": 7.38,
+      "theoretical_loss": 9.238341688075822,
+      "tokens_seen": 2293760
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 5.714285714285714e-05,
+      "loss": 7.2879,
+      "theoretical_loss": 9.184907653139359,
+      "tokens_seen": 2359296
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 5.873015873015873e-05,
+      "loss": 7.3045,
+      "theoretical_loss": 9.133340503044415,
+      "tokens_seen": 2424832
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 6.0317460317460316e-05,
+      "loss": 7.1794,
+      "theoretical_loss": 9.0835271371648,
+      "tokens_seen": 2490368
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 6.19047619047619e-05,
+      "loss": 7.1184,
+      "theoretical_loss": 9.035364057274894,
+      "tokens_seen": 2555904
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 6.349206349206349e-05,
+      "loss": 7.0666,
+      "theoretical_loss": 8.988756330540422,
+      "tokens_seen": 2621440
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 6.507936507936509e-05,
+      "loss": 6.982,
+      "theoretical_loss": 8.943616687271678,
+      "tokens_seen": 2686976
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 6.666666666666667e-05,
+      "loss": 7.1173,
+      "theoretical_loss": 8.89986473310929,
+      "tokens_seen": 2752512
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 6.825396825396825e-05,
+      "loss": 7.0194,
+      "theoretical_loss": 8.857426258784564,
+      "tokens_seen": 2818048
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 6.984126984126984e-05,
+      "loss": 6.9244,
+      "theoretical_loss": 8.816232633409479,
+      "tokens_seen": 2883584
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 7.142857142857143e-05,
+      "loss": 6.9294,
+      "theoretical_loss": 8.776220269542522,
+      "tokens_seen": 2949120
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 7.301587301587302e-05,
+      "loss": 6.7326,
+      "theoretical_loss": 8.737330150151898,
+      "tokens_seen": 3014656
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 7.460317460317461e-05,
+      "loss": 6.8233,
+      "theoretical_loss": 8.699507409139912,
+      "tokens_seen": 3080192
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 7.619047619047618e-05,
+      "loss": 6.7542,
+      "theoretical_loss": 8.662700958366539,
+      "tokens_seen": 3145728
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 7.777777777777778e-05,
+      "loss": 6.6128,
+      "theoretical_loss": 8.62686315516739,
+      "tokens_seen": 3211264
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 7.936507936507937e-05,
+      "loss": 6.6336,
+      "theoretical_loss": 8.591949505242134,
+      "tokens_seen": 3276800
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 8.095238095238096e-05,
+      "loss": 6.5844,
+      "theoretical_loss": 8.557918396526182,
+      "tokens_seen": 3342336
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 8.253968253968255e-05,
+      "loss": 6.3935,
+      "theoretical_loss": 8.524730860277067,
+      "tokens_seen": 3407872
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 8.412698412698413e-05,
+      "loss": 6.4484,
+      "theoretical_loss": 8.492350356128252,
+      "tokens_seen": 3473408
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 8.571428571428571e-05,
+      "loss": 6.3623,
+      "theoretical_loss": 8.460742578303845,
+      "tokens_seen": 3538944
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 8.730158730158731e-05,
+      "loss": 6.4215,
+      "theoretical_loss": 8.429875280561783,
+      "tokens_seen": 3604480
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 8.888888888888889e-05,
+      "loss": 6.222,
+      "theoretical_loss": 8.399718117751275,
+      "tokens_seen": 3670016
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 9.047619047619048e-05,
+      "loss": 6.3531,
+      "theoretical_loss": 8.370242502142142,
+      "tokens_seen": 3735552
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 9.206349206349206e-05,
+      "loss": 6.2966,
+      "theoretical_loss": 8.341421472916394,
+      "tokens_seen": 3801088
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 9.365079365079366e-05,
+      "loss": 6.2777,
+      "theoretical_loss": 8.313229577412248,
+      "tokens_seen": 3866624
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 9.523809523809524e-05,
+      "loss": 6.1319,
+      "theoretical_loss": 8.28564276288293,
+      "tokens_seen": 3932160
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 9.682539682539682e-05,
+      "loss": 6.1204,
+      "theoretical_loss": 8.258638277681179,
+      "tokens_seen": 3997696
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 9.841269841269841e-05,
+      "loss": 6.2434,
+      "theoretical_loss": 8.232194580909036,
+      "tokens_seen": 4063232
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.0001,
+      "loss": 6.1409,
+      "theoretical_loss": 8.20629125968411,
+      "tokens_seen": 4128768
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00010158730158730159,
+      "loss": 5.9151,
+      "theoretical_loss": 8.180908953270682,
+      "tokens_seen": 4194304
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00010317460317460319,
+      "loss": 5.9821,
+      "theoretical_loss": 8.156029283408564,
+      "tokens_seen": 4259840
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00010476190476190477,
+      "loss": 5.8359,
+      "theoretical_loss": 8.131634790246775,
+      "tokens_seen": 4325376
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00010634920634920635,
+      "loss": 5.9882,
+      "theoretical_loss": 8.107708873353706,
+      "tokens_seen": 4390912
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00010793650793650794,
+      "loss": 5.9366,
+      "theoretical_loss": 8.084235737332481,
+      "tokens_seen": 4456448
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00010952380952380953,
+      "loss": 5.7864,
+      "theoretical_loss": 8.061200341620074,
+      "tokens_seen": 4521984
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00011111111111111112,
+      "loss": 5.7847,
+      "theoretical_loss": 8.038588354092902,
+      "tokens_seen": 4587520
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.0001126984126984127,
+      "loss": 5.7529,
+      "theoretical_loss": 8.016386108140445,
+      "tokens_seen": 4653056
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00011428571428571428,
+      "loss": 5.8326,
+      "theoretical_loss": 7.994580562902867,
+      "tokens_seen": 4718592
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.0001158730158730159,
+      "loss": 5.7328,
+      "theoretical_loss": 7.973159266399054,
+      "tokens_seen": 4784128
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00011746031746031746,
+      "loss": 5.6849,
+      "theoretical_loss": 7.952110321298584,
+      "tokens_seen": 4849664
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00011904761904761905,
+      "loss": 5.641,
+      "theoretical_loss": 7.931422353115133,
+      "tokens_seen": 4915200
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00012063492063492063,
+      "loss": 5.7055,
+      "theoretical_loss": 7.911084480620269,
+      "tokens_seen": 4980736
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00012222222222222224,
+      "loss": 5.7157,
+      "theoretical_loss": 7.891086288295694,
+      "tokens_seen": 5046272
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.0001238095238095238,
+      "loss": 5.7148,
+      "theoretical_loss": 7.871417800659003,
+      "tokens_seen": 5111808
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.0001253968253968254,
+      "loss": 5.5534,
+      "theoretical_loss": 7.8520694583133785,
+      "tokens_seen": 5177344
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00012698412698412698,
+      "loss": 5.7642,
+      "theoretical_loss": 7.833032095585231,
+      "tokens_seen": 5242880
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00012857142857142858,
+      "loss": 5.6881,
+      "theoretical_loss": 7.8142969196261385,
+      "tokens_seen": 5308416
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00013015873015873017,
+      "loss": 5.5251,
+      "theoretical_loss": 7.79585549086643,
+      "tokens_seen": 5373952
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00013174603174603174,
+      "loss": 5.5142,
+      "theoretical_loss": 7.777699704717685,
+      "tokens_seen": 5439488
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00013333333333333334,
+      "loss": 5.5252,
+      "theoretical_loss": 7.759821774430384,
+      "tokens_seen": 5505024
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00013492063492063494,
+      "loss": 5.6046,
+      "theoretical_loss": 7.742214215020986,
+      "tokens_seen": 5570560
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.0001365079365079365,
+      "loss": 5.4653,
+      "theoretical_loss": 7.72486982819,
+      "tokens_seen": 5636096
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.0001380952380952381,
+      "loss": 5.6301,
+      "theoretical_loss": 7.707781688159247,
+      "tokens_seen": 5701632
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00013968253968253967,
+      "loss": 5.5605,
+      "theoretical_loss": 7.690943128362394,
+      "tokens_seen": 5767168
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.0001412698412698413,
+      "loss": 5.54,
+      "theoretical_loss": 7.674347728928345,
+      "tokens_seen": 5832704
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00014285714285714287,
+      "loss": 5.574,
+      "theoretical_loss": 7.6579893049019265,
+      "tokens_seen": 5898240
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00014444444444444444,
+      "loss": 5.5315,
+      "theoretical_loss": 7.641861895150762,
+      "tokens_seen": 5963776
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00014603174603174603,
+      "loss": 5.4421,
+      "theoretical_loss": 7.625959751911346,
+      "tokens_seen": 6029312
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00014761904761904763,
+      "loss": 5.4408,
+      "theoretical_loss": 7.610277330930954,
+      "tokens_seen": 6094848
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00014920634920634923,
+      "loss": 5.4034,
+      "theoretical_loss": 7.5948092821654365,
+      "tokens_seen": 6160384
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.0001507936507936508,
+      "loss": 5.2766,
+      "theoretical_loss": 7.579550440996015,
+      "tokens_seen": 6225920
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00015238095238095237,
+      "loss": 5.3709,
+      "theoretical_loss": 7.564495819930985,
+      "tokens_seen": 6291456
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.000153968253968254,
+      "loss": 5.3441,
+      "theoretical_loss": 7.549640600760828,
+      "tokens_seen": 6356992
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00015555555555555556,
+      "loss": 5.3515,
+      "theoretical_loss": 7.534980127137563,
+      "tokens_seen": 6422528
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00015714285714285716,
+      "loss": 5.394,
+      "theoretical_loss": 7.520509897551315,
+      "tokens_seen": 6488064
+    },
+    {
+      "epoch": 0.0,
+      "objective/train/docs_used": 10124,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 5.372183799743652,
+      "objective/train/theoretical_loss": 7.5062255586790965,
+      "objective/train/tokens_used": 27013600,
+      "theoretical_loss": 7.5062255586790965,
+      "tokens_seen": 6553600
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00015873015873015873,
+      "loss": 5.3722,
+      "theoretical_loss": 7.5062255586790965,
+      "tokens_seen": 6553600
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00016031746031746033,
+      "loss": 5.3875,
+      "theoretical_loss": 7.492122899082528,
+      "tokens_seen": 6619136
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00016190476190476192,
+      "loss": 5.4057,
+      "theoretical_loss": 7.478197843232982,
+      "tokens_seen": 6684672
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.0001634920634920635,
+      "loss": 5.3362,
+      "theoretical_loss": 7.464446445844085,
+      "tokens_seen": 6750208
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.0001650793650793651,
+      "loss": 5.3139,
+      "theoretical_loss": 7.450864886492971,
+      "tokens_seen": 6815744
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.0001666666666666667,
+      "loss": 5.2791,
+      "theoretical_loss": 7.437449464512941,
+      "tokens_seen": 6881280
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00016825396825396826,
+      "loss": 5.2369,
+      "theoretical_loss": 7.424196594141411,
+      "tokens_seen": 6946816
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00016984126984126986,
+      "loss": 5.2822,
+      "theoretical_loss": 7.4111027999080985,
+      "tokens_seen": 7012352
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00017142857142857143,
+      "loss": 5.3693,
+      "theoretical_loss": 7.398164712249439,
+      "tokens_seen": 7077888
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00017301587301587302,
+      "loss": 5.4055,
+      "theoretical_loss": 7.3853790633361545,
+      "tokens_seen": 7143424
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00017460317460317462,
+      "loss": 5.1771,
+      "theoretical_loss": 7.3727426831017535,
+      "tokens_seen": 7208960
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.0001761904761904762,
+      "loss": 5.3257,
+      "theoretical_loss": 7.360252495460585,
+      "tokens_seen": 7274496
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00017777777777777779,
+      "loss": 5.1899,
+      "theoretical_loss": 7.347905514704743,
+      "tokens_seen": 7340032
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00017936507936507938,
+      "loss": 5.2891,
+      "theoretical_loss": 7.33569884206989,
+      "tokens_seen": 7405568
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00018095238095238095,
+      "loss": 5.2128,
+      "theoretical_loss": 7.32362966246063,
+      "tokens_seen": 7471104
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00018253968253968255,
+      "loss": 5.1534,
+      "theoretical_loss": 7.311695241326718,
+      "tokens_seen": 7536640
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00018412698412698412,
+      "loss": 5.302,
+      "theoretical_loss": 7.299892921681898,
+      "tokens_seen": 7602176
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00018571428571428572,
+      "loss": 5.2996,
+      "theoretical_loss": 7.288220121257689,
+      "tokens_seen": 7667712
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00018730158730158731,
+      "loss": 5.2726,
+      "theoretical_loss": 7.276674329784936,
+      "tokens_seen": 7733248
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00018888888888888888,
+      "loss": 5.1882,
+      "theoretical_loss": 7.265253106396328,
+      "tokens_seen": 7798784
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00019047619047619048,
+      "loss": 5.0924,
+      "theoretical_loss": 7.253954077143563,
+      "tokens_seen": 7864320
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00019206349206349208,
+      "loss": 5.282,
+      "theoretical_loss": 7.242774932623186,
+      "tokens_seen": 7929856
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00019365079365079365,
+      "loss": 5.0143,
+      "theoretical_loss": 7.2317134257054665,
+      "tokens_seen": 7995392
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00019523809523809525,
+      "loss": 5.1009,
+      "theoretical_loss": 7.220767369361081,
+      "tokens_seen": 8060928
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00019682539682539682,
+      "loss": 4.9736,
+      "theoretical_loss": 7.2099346345805815,
+      "tokens_seen": 8126464
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00019841269841269844,
+      "loss": 5.2631,
+      "theoretical_loss": 7.1992131483820305,
+      "tokens_seen": 8192000
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.0002,
+      "loss": 5.1442,
+      "theoretical_loss": 7.188600891902338,
+      "tokens_seen": 8257536
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00020158730158730158,
+      "loss": 5.146,
+      "theoretical_loss": 7.178095898568204,
+      "tokens_seen": 8323072
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00020317460317460318,
+      "loss": 5.2826,
+      "theoretical_loss": 7.167696252342704,
+      "tokens_seen": 8388608
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00020476190476190475,
+      "loss": 5.1134,
+      "theoretical_loss": 7.15740008604386,
+      "tokens_seen": 8454144
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00020634920634920637,
+      "loss": 5.1957,
+      "theoretical_loss": 7.147205579731682,
+      "tokens_seen": 8519680
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00020793650793650797,
+      "loss": 5.0839,
+      "theoretical_loss": 7.1371109591604105,
+      "tokens_seen": 8585216
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00020952380952380954,
+      "loss": 5.2144,
+      "theoretical_loss": 7.127114494292836,
+      "tokens_seen": 8650752
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00021111111111111113,
+      "loss": 5.0342,
+      "theoretical_loss": 7.117214497873759,
+      "tokens_seen": 8716288
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.0002126984126984127,
+      "loss": 4.9848,
+      "theoretical_loss": 7.107409324059815,
+      "tokens_seen": 8781824
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.0002142857142857143,
+      "loss": 5.1246,
+      "theoretical_loss": 7.097697367103032,
+      "tokens_seen": 8847360
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00021587301587301587,
+      "loss": 5.1589,
+      "theoretical_loss": 7.0880770600856335,
+      "tokens_seen": 8912896
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00021746031746031744,
+      "loss": 5.1326,
+      "theoretical_loss": 7.0785468737037345,
+      "tokens_seen": 8978432
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00021904761904761907,
+      "loss": 4.9242,
+      "theoretical_loss": 7.069105315097691,
+      "tokens_seen": 9043968
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00022063492063492066,
+      "loss": 4.9897,
+      "theoretical_loss": 7.059750926726999,
+      "tokens_seen": 9109504
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00022222222222222223,
+      "loss": 4.936,
+      "theoretical_loss": 7.050482285287744,
+      "tokens_seen": 9175040
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00022380952380952383,
+      "loss": 4.9534,
+      "theoretical_loss": 7.041298000670675,
+      "tokens_seen": 9240576
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.0002253968253968254,
+      "loss": 4.972,
+      "theoretical_loss": 7.032196714958137,
+      "tokens_seen": 9306112
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.000226984126984127,
+      "loss": 5.0425,
+      "theoretical_loss": 7.023177101458123,
+      "tokens_seen": 9371648
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00022857142857142857,
+      "loss": 5.1426,
+      "theoretical_loss": 7.014237863773846,
+      "tokens_seen": 9437184
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00023015873015873016,
+      "loss": 4.9935,
+      "theoretical_loss": 7.005377734907255,
+      "tokens_seen": 9502720
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.0002317460317460318,
+      "loss": 4.9806,
+      "theoretical_loss": 6.996595476395081,
+      "tokens_seen": 9568256
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00023333333333333336,
+      "loss": 4.9519,
+      "theoretical_loss": 6.98788987747596,
+      "tokens_seen": 9633792
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00023492063492063493,
+      "loss": 4.9623,
+      "theoretical_loss": 6.979259754287375,
+      "tokens_seen": 9699328
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00023650793650793653,
+      "loss": 4.9584,
+      "theoretical_loss": 6.970703949091095,
+      "tokens_seen": 9764864
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.0002380952380952381,
+      "loss": 5.0187,
+      "theoretical_loss": 6.962221329525966,
+      "tokens_seen": 9830400
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.0002396825396825397,
+      "loss": 4.9565,
+      "theoretical_loss": 6.953810787886883,
+      "tokens_seen": 9895936
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00024126984126984126,
+      "loss": 4.8708,
+      "theoretical_loss": 6.945471240428864,
+      "tokens_seen": 9961472
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00024285714285714286,
+      "loss": 4.8598,
+      "theoretical_loss": 6.937201626695213,
+      "tokens_seen": 10027008
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.0002444444444444445,
+      "loss": 4.978,
+      "theoretical_loss": 6.929000908868754,
+      "tokens_seen": 10092544
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00024603174603174605,
+      "loss": 5.0203,
+      "theoretical_loss": 6.920868071145226,
+      "tokens_seen": 10158080
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.0002476190476190476,
+      "loss": 5.0491,
+      "theoretical_loss": 6.912802119127937,
+      "tokens_seen": 10223616
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00024920634920634925,
+      "loss": 4.8428,
+      "theoretical_loss": 6.904802079242816,
+      "tokens_seen": 10289152
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.0002507936507936508,
+      "loss": 4.8803,
+      "theoretical_loss": 6.896866998173063,
+      "tokens_seen": 10354688
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.0002523809523809524,
+      "loss": 4.9369,
+      "theoretical_loss": 6.888995942312617,
+      "tokens_seen": 10420224
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00025396825396825396,
+      "loss": 4.8271,
+      "theoretical_loss": 6.881187997237696,
+      "tokens_seen": 10485760
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00025555555555555553,
+      "loss": 4.872,
+      "theoretical_loss": 6.873442267195708,
+      "tokens_seen": 10551296
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00025714285714285715,
+      "loss": 5.0061,
+      "theoretical_loss": 6.8657578746108605,
+      "tokens_seen": 10616832
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.0002587301587301588,
+      "loss": 4.9092,
+      "theoretical_loss": 6.858133959605815,
+      "tokens_seen": 10682368
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00026031746031746035,
+      "loss": 5.0769,
+      "theoretical_loss": 6.8505696795387845,
+      "tokens_seen": 10747904
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.0002619047619047619,
+      "loss": 4.8411,
+      "theoretical_loss": 6.843064208555461,
+      "tokens_seen": 10813440
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.0002634920634920635,
+      "loss": 4.9565,
+      "theoretical_loss": 6.835616737155247,
+      "tokens_seen": 10878976
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.0002650793650793651,
+      "loss": 4.7779,
+      "theoretical_loss": 6.828226471771211,
+      "tokens_seen": 10944512
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.0002666666666666667,
+      "loss": 4.9304,
+      "theoretical_loss": 6.8208926343632905,
+      "tokens_seen": 11010048
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00026825396825396825,
+      "loss": 4.9079,
+      "theoretical_loss": 6.813614462024214,
+      "tokens_seen": 11075584
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.0002698412698412699,
+      "loss": 4.8657,
+      "theoretical_loss": 6.806391206597704,
+      "tokens_seen": 11141120
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00027142857142857144,
+      "loss": 4.8902,
+      "theoretical_loss": 6.799222134308489,
+      "tokens_seen": 11206656
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.000273015873015873,
+      "loss": 4.8231,
+      "theoretical_loss": 6.792106525403691,
+      "tokens_seen": 11272192
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00027460317460317464,
+      "loss": 4.9382,
+      "theoretical_loss": 6.785043673805189,
+      "tokens_seen": 11337728
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.0002761904761904762,
+      "loss": 4.8235,
+      "theoretical_loss": 6.778032886772552,
+      "tokens_seen": 11403264
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.0002777777777777778,
+      "loss": 4.7207,
+      "theoretical_loss": 6.771073484576156,
+      "tokens_seen": 11468800
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00027936507936507935,
+      "loss": 4.9425,
+      "theoretical_loss": 6.764164800180141,
+      "tokens_seen": 11534336
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00028095238095238097,
+      "loss": 4.7528,
+      "theoretical_loss": 6.757306178934827,
+      "tokens_seen": 11599872
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.0002825396825396826,
+      "loss": 4.935,
+      "theoretical_loss": 6.750496978278292,
+      "tokens_seen": 11665408
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00028412698412698417,
+      "loss": 4.7528,
+      "theoretical_loss": 6.743736567446755,
+      "tokens_seen": 11730944
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00028571428571428574,
+      "loss": 4.8485,
+      "theoretical_loss": 6.737024327193478,
+      "tokens_seen": 11796480
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.0002873015873015873,
+      "loss": 4.8715,
+      "theoretical_loss": 6.73035964951589,
+      "tokens_seen": 11862016
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.0002888888888888889,
+      "loss": 4.7815,
+      "theoretical_loss": 6.723741937390625,
+      "tokens_seen": 11927552
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.0002904761904761905,
+      "loss": 4.7251,
+      "theoretical_loss": 6.717170604516246,
+      "tokens_seen": 11993088
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00029206349206349207,
+      "loss": 4.7504,
+      "theoretical_loss": 6.71064507506334,
+      "tokens_seen": 12058624
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00029365079365079364,
+      "loss": 4.7729,
+      "theoretical_loss": 6.704164783431781,
+      "tokens_seen": 12124160
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00029523809523809526,
+      "loss": 4.6926,
+      "theoretical_loss": 6.6977291740148805,
+      "tokens_seen": 12189696
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00029682539682539683,
+      "loss": 4.7032,
+      "theoretical_loss": 6.691337700970224,
+      "tokens_seen": 12255232
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00029841269841269846,
+      "loss": 4.6978,
+      "theoretical_loss": 6.684989827996945,
+      "tokens_seen": 12320768
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00030000000000000003,
+      "loss": 4.913,
+      "theoretical_loss": 6.6786850281192365,
+      "tokens_seen": 12386304
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.0003015873015873016,
+      "loss": 4.6724,
+      "theoretical_loss": 6.672422783475897,
+      "tokens_seen": 12451840
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00030317460317460317,
+      "loss": 4.7759,
+      "theoretical_loss": 6.666202585115692,
+      "tokens_seen": 12517376
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00030476190476190474,
+      "loss": 4.7548,
+      "theoretical_loss": 6.66002393279837,
+      "tokens_seen": 12582912
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00030634920634920636,
+      "loss": 4.6869,
+      "theoretical_loss": 6.653886334801115,
+      "tokens_seen": 12648448
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.000307936507936508,
+      "loss": 4.7677,
+      "theoretical_loss": 6.647789307730289,
+      "tokens_seen": 12713984
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00030952380952380956,
+      "loss": 4.7758,
+      "theoretical_loss": 6.641732376338277,
+      "tokens_seen": 12779520
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.0003111111111111111,
+      "loss": 4.8743,
+      "theoretical_loss": 6.6357150733452785,
+      "tokens_seen": 12845056
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.0003126984126984127,
+      "loss": 4.7179,
+      "theoretical_loss": 6.629736939265876,
+      "tokens_seen": 12910592
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.0003142857142857143,
+      "loss": 4.9144,
+      "theoretical_loss": 6.623797522240246,
+      "tokens_seen": 12976128
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.0003158730158730159,
+      "loss": 4.8827,
+      "theoretical_loss": 6.617896377869851,
+      "tokens_seen": 13041664
+    },
+    {
+      "epoch": 0.0,
+      "objective/train/docs_used": 12399,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 4.735827445983887,
+      "objective/train/theoretical_loss": 6.61203306905748,
+      "objective/train/tokens_used": 33567200,
+      "theoretical_loss": 6.61203306905748,
+      "tokens_seen": 13107200
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00031746031746031746,
+      "loss": 4.7358,
+      "theoretical_loss": 6.61203306905748,
+      "tokens_seen": 13107200
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.0003190476190476191,
+      "loss": 4.7847,
+      "theoretical_loss": 6.6062071658514965,
+      "tokens_seen": 13172736
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00032063492063492065,
+      "loss": 4.7583,
+      "theoretical_loss": 6.600418245294164,
+      "tokens_seen": 13238272
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.0003222222222222222,
+      "loss": 4.6434,
+      "theoretical_loss": 6.594665891273929,
+      "tokens_seen": 13303808
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00032380952380952385,
+      "loss": 4.8477,
+      "theoretical_loss": 6.588949694381529,
+      "tokens_seen": 13369344
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.0003253968253968254,
+      "loss": 4.6934,
+      "theoretical_loss": 6.583269251769811,
+      "tokens_seen": 13434880
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.000326984126984127,
+      "loss": 4.6676,
+      "theoretical_loss": 6.577624167017157,
+      "tokens_seen": 13500416
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00032857142857142856,
+      "loss": 4.6634,
+      "theoretical_loss": 6.5720140499943955,
+      "tokens_seen": 13565952
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.0003301587301587302,
+      "loss": 4.6706,
+      "theoretical_loss": 6.566438516735092,
+      "tokens_seen": 13631488
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.0003317460317460318,
+      "loss": 4.6321,
+      "theoretical_loss": 6.560897189309134,
+      "tokens_seen": 13697024
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.0003333333333333334,
+      "loss": 4.659,
+      "theoretical_loss": 6.5553896956994855,
+      "tokens_seen": 13762560
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00033492063492063495,
+      "loss": 4.7745,
+      "theoretical_loss": 6.5499156696820435,
+      "tokens_seen": 13828096
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.0003365079365079365,
+      "loss": 4.6713,
+      "theoretical_loss": 6.544474750708482,
+      "tokens_seen": 13893632
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.0003380952380952381,
+      "loss": 4.6694,
+      "theoretical_loss": 6.539066583792016,
+      "tokens_seen": 13959168
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.0003396825396825397,
+      "loss": 4.6684,
+      "theoretical_loss": 6.533690819395976,
+      "tokens_seen": 14024704
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.0003412698412698413,
+      "loss": 4.6629,
+      "theoretical_loss": 6.528347113325134,
+      "tokens_seen": 14090240
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00034285714285714285,
+      "loss": 4.8538,
+      "theoretical_loss": 6.523035126619684,
+      "tokens_seen": 14155776
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.0003444444444444445,
+      "loss": 4.63,
+      "theoretical_loss": 6.517754525451811,
+      "tokens_seen": 14221312
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00034603174603174604,
+      "loss": 4.8216,
+      "theoretical_loss": 6.512504981024765,
+      "tokens_seen": 14286848
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00034761904761904767,
+      "loss": 4.7503,
+      "theoretical_loss": 6.507286169474376,
+      "tokens_seen": 14352384
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00034920634920634924,
+      "loss": 4.6495,
+      "theoretical_loss": 6.502097771772935,
+      "tokens_seen": 14417920
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.0003507936507936508,
+      "loss": 4.6212,
+      "theoretical_loss": 6.496939473635374,
+      "tokens_seen": 14483456
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.0003523809523809524,
+      "loss": 4.6616,
+      "theoretical_loss": 6.491810965427682,
+      "tokens_seen": 14548992
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00035396825396825395,
+      "loss": 4.5403,
+      "theoretical_loss": 6.486711942077496,
+      "tokens_seen": 14614528
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00035555555555555557,
+      "loss": 4.5692,
+      "theoretical_loss": 6.481642102986796,
+      "tokens_seen": 14680064
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.0003571428571428572,
+      "loss": 4.6113,
+      "theoretical_loss": 6.476601151946659,
+      "tokens_seen": 14745600
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00035873015873015877,
+      "loss": 4.6895,
+      "theoretical_loss": 6.471588797054005,
+      "tokens_seen": 14811136
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00036031746031746034,
+      "loss": 4.6877,
+      "theoretical_loss": 6.46660475063028,
+      "tokens_seen": 14876672
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.0003619047619047619,
+      "loss": 4.5206,
+      "theoretical_loss": 6.46164872914203,
+      "tokens_seen": 14942208
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00036349206349206353,
+      "loss": 4.725,
+      "theoretical_loss": 6.456720453123305,
+      "tokens_seen": 15007744
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.0003650793650793651,
+      "loss": 4.5871,
+      "theoretical_loss": 6.4518196470998515,
+      "tokens_seen": 15073280
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00036666666666666667,
+      "loss": 4.6798,
+      "theoretical_loss": 6.446946039515035,
+      "tokens_seen": 15138816
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00036825396825396824,
+      "loss": 4.5425,
+      "theoretical_loss": 6.442099362657457,
+      "tokens_seen": 15204352
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00036984126984126986,
+      "loss": 4.5059,
+      "theoretical_loss": 6.437279352590206,
+      "tokens_seen": 15269888
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00037142857142857143,
+      "loss": 4.6036,
+      "theoretical_loss": 6.432485749081721,
+      "tokens_seen": 15335424
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00037301587301587306,
+      "loss": 4.6586,
+      "theoretical_loss": 6.427718295538199,
+      "tokens_seen": 15400960
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00037460317460317463,
+      "loss": 4.6508,
+      "theoretical_loss": 6.422976738937518,
+      "tokens_seen": 15466496
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.0003761904761904762,
+      "loss": 4.592,
+      "theoretical_loss": 6.418260829764641,
+      "tokens_seen": 15532032
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00037777777777777777,
+      "loss": 4.5609,
+      "theoretical_loss": 6.413570321948453,
+      "tokens_seen": 15597568
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.0003793650793650794,
+      "loss": 4.5308,
+      "theoretical_loss": 6.40890497279999,
+      "tokens_seen": 15663104
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00038095238095238096,
+      "loss": 4.4842,
+      "theoretical_loss": 6.404264542952033,
+      "tokens_seen": 15728640
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.0003825396825396826,
+      "loss": 4.5147,
+      "theoretical_loss": 6.399648796300032,
+      "tokens_seen": 15794176
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00038412698412698416,
+      "loss": 4.579,
+      "theoretical_loss": 6.395057499944313,
+      "tokens_seen": 15859712
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.0003857142857142857,
+      "loss": 4.5359,
+      "theoretical_loss": 6.390490424133546,
+      "tokens_seen": 15925248
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.0003873015873015873,
+      "loss": 4.6826,
+      "theoretical_loss": 6.3859473422094375,
+      "tokens_seen": 15990784
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.0003888888888888889,
+      "loss": 4.6003,
+      "theoretical_loss": 6.381428030552619,
+      "tokens_seen": 16056320
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.0003904761904761905,
+      "loss": 4.4455,
+      "theoretical_loss": 6.376932268529695,
+      "tokens_seen": 16121856
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00039206349206349206,
+      "loss": 4.6933,
+      "theoretical_loss": 6.372459838441432,
+      "tokens_seen": 16187392
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00039365079365079363,
+      "loss": 4.6425,
+      "theoretical_loss": 6.368010525472036,
+      "tokens_seen": 16252928
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00039523809523809526,
+      "loss": 4.4457,
+      "theoretical_loss": 6.363584117639526,
+      "tokens_seen": 16318464
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.0003968253968253969,
+      "loss": 4.4832,
+      "theoretical_loss": 6.35918040574715,
+      "tokens_seen": 16384000
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00039841269841269845,
+      "loss": 4.6614,
+      "theoretical_loss": 6.354799183335816,
+      "tokens_seen": 16449536
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004,
+      "loss": 4.5142,
+      "theoretical_loss": 6.350440246637536,
+      "tokens_seen": 16515072
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004015873015873016,
+      "loss": 4.4264,
+      "theoretical_loss": 6.3461033945298295,
+      "tokens_seen": 16580608
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00040317460317460316,
+      "loss": 4.3929,
+      "theoretical_loss": 6.341788428491087,
+      "tokens_seen": 16646144
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004047619047619048,
+      "loss": 4.5313,
+      "theoretical_loss": 6.337495152556851,
+      "tokens_seen": 16711680
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00040634920634920635,
+      "loss": 4.5278,
+      "theoretical_loss": 6.333223373277006,
+      "tokens_seen": 16777216
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004079365079365079,
+      "loss": 4.5302,
+      "theoretical_loss": 6.32897289967384,
+      "tokens_seen": 16842752
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004095238095238095,
+      "loss": 4.5178,
+      "theoretical_loss": 6.324743543200984,
+      "tokens_seen": 16908288
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004111111111111111,
+      "loss": 4.4393,
+      "theoretical_loss": 6.320535117703164,
+      "tokens_seen": 16973824
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00041269841269841274,
+      "loss": 4.4453,
+      "theoretical_loss": 6.316347439376797,
+      "tokens_seen": 17039360
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00041428571428571437,
+      "loss": 4.4908,
+      "theoretical_loss": 6.312180326731362,
+      "tokens_seen": 17104896
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00041587301587301594,
+      "loss": 4.4296,
+      "theoretical_loss": 6.308033600551573,
+      "tokens_seen": 17170432
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004174603174603175,
+      "loss": 4.598,
+      "theoretical_loss": 6.303907083860294,
+      "tokens_seen": 17235968
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004190476190476191,
+      "loss": 4.616,
+      "theoretical_loss": 6.2998006018822075,
+      "tokens_seen": 17301504
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00042063492063492065,
+      "loss": 4.5066,
+      "theoretical_loss": 6.295713982008203,
+      "tokens_seen": 17367040
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00042222222222222227,
+      "loss": 4.3981,
+      "theoretical_loss": 6.2916470537604745,
+      "tokens_seen": 17432576
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00042380952380952384,
+      "loss": 4.4611,
+      "theoretical_loss": 6.287599648758312,
+      "tokens_seen": 17498112
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004253968253968254,
+      "loss": 4.3655,
+      "theoretical_loss": 6.283571600684564,
+      "tokens_seen": 17563648
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.000426984126984127,
+      "loss": 4.4687,
+      "theoretical_loss": 6.279562745252756,
+      "tokens_seen": 17629184
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004285714285714286,
+      "loss": 4.5314,
+      "theoretical_loss": 6.2755729201748665,
+      "tokens_seen": 17694720
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004301587301587302,
+      "loss": 4.3542,
+      "theoretical_loss": 6.271601965129717,
+      "tokens_seen": 17760256
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00043174603174603174,
+      "loss": 4.4021,
+      "theoretical_loss": 6.267649721731981,
+      "tokens_seen": 17825792
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004333333333333333,
+      "loss": 4.5365,
+      "theoretical_loss": 6.263716033501797,
+      "tokens_seen": 17891328
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004349206349206349,
+      "loss": 4.3897,
+      "theoretical_loss": 6.259800745834964,
+      "tokens_seen": 17956864
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004365079365079365,
+      "loss": 4.3846,
+      "theoretical_loss": 6.255903705973709,
+      "tokens_seen": 18022400
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00043809523809523813,
+      "loss": 4.4128,
+      "theoretical_loss": 6.252024762978021,
+      "tokens_seen": 18087936
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00043968253968253976,
+      "loss": 4.2714,
+      "theoretical_loss": 6.248163767697523,
+      "tokens_seen": 18153472
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004412698412698413,
+      "loss": 4.4651,
+      "theoretical_loss": 6.244320572743888,
+      "tokens_seen": 18219008
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004428571428571429,
+      "loss": 4.2786,
+      "theoretical_loss": 6.240495032463768,
+      "tokens_seen": 18284544
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00044444444444444447,
+      "loss": 4.3311,
+      "theoretical_loss": 6.236687002912243,
+      "tokens_seen": 18350080
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004460317460317461,
+      "loss": 4.4895,
+      "theoretical_loss": 6.232896341826766,
+      "tokens_seen": 18415616
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00044761904761904766,
+      "loss": 4.3794,
+      "theoretical_loss": 6.229122908601596,
+      "tokens_seen": 18481152
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00044920634920634923,
+      "loss": 4.3418,
+      "theoretical_loss": 6.225366564262705,
+      "tokens_seen": 18546688
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004507936507936508,
+      "loss": 4.4627,
+      "theoretical_loss": 6.2216271714431635,
+      "tokens_seen": 18612224
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00045238095238095237,
+      "loss": 4.3152,
+      "theoretical_loss": 6.217904594358968,
+      "tokens_seen": 18677760
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.000453968253968254,
+      "loss": 4.4017,
+      "theoretical_loss": 6.214198698785328,
+      "tokens_seen": 18743296
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00045555555555555556,
+      "loss": 4.2326,
+      "theoretical_loss": 6.210509352033379,
+      "tokens_seen": 18808832
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00045714285714285713,
+      "loss": 4.3124,
+      "theoretical_loss": 6.206836422927337,
+      "tokens_seen": 18874368
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004587301587301587,
+      "loss": 4.4355,
+      "theoretical_loss": 6.203179781782049,
+      "tokens_seen": 18939904
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00046031746031746033,
+      "loss": 4.2566,
+      "theoretical_loss": 6.199539300380979,
+      "tokens_seen": 19005440
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004619047619047619,
+      "loss": 4.3537,
+      "theoretical_loss": 6.195914851954575,
+      "tokens_seen": 19070976
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004634920634920636,
+      "loss": 4.3399,
+      "theoretical_loss": 6.19230631115903,
+      "tokens_seen": 19136512
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00046507936507936515,
+      "loss": 4.3265,
+      "theoretical_loss": 6.188713554055438,
+      "tokens_seen": 19202048
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004666666666666667,
+      "loss": 4.3022,
+      "theoretical_loss": 6.185136458089309,
+      "tokens_seen": 19267584
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004682539682539683,
+      "loss": 4.3129,
+      "theoretical_loss": 6.181574902070463,
+      "tokens_seen": 19333120
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00046984126984126986,
+      "loss": 4.2721,
+      "theoretical_loss": 6.178028766153277,
+      "tokens_seen": 19398656
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004714285714285715,
+      "loss": 4.2608,
+      "theoretical_loss": 6.174497931817287,
+      "tokens_seen": 19464192
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00047301587301587305,
+      "loss": 4.2924,
+      "theoretical_loss": 6.17098228184813,
+      "tokens_seen": 19529728
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004746031746031746,
+      "loss": 4.3237,
+      "theoretical_loss": 6.167481700318827,
+      "tokens_seen": 19595264
+    },
+    {
+      "epoch": 0.01,
+      "objective/train/docs_used": 14832,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 4.324337482452393,
+      "objective/train/theoretical_loss": 6.163996072571392,
+      "objective/train/tokens_used": 40120800,
+      "theoretical_loss": 6.163996072571392,
+      "tokens_seen": 19660800
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004761904761904762,
+      "loss": 4.3243,
+      "theoretical_loss": 6.163996072571392,
+      "tokens_seen": 19660800
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004777777777777778,
+      "loss": 4.262,
+      "theoretical_loss": 6.160525285198771,
+      "tokens_seen": 19726336
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004793650793650794,
+      "loss": 4.413,
+      "theoretical_loss": 6.157069226027087,
+      "tokens_seen": 19791872
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00048095238095238095,
+      "loss": 4.4588,
+      "theoretical_loss": 6.153627784098204,
+      "tokens_seen": 19857408
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004825396825396825,
+      "loss": 4.1598,
+      "theoretical_loss": 6.150200849652586,
+      "tokens_seen": 19922944
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004841269841269841,
+      "loss": 4.2043,
+      "theoretical_loss": 6.146788314112464,
+      "tokens_seen": 19988480
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004857142857142857,
+      "loss": 4.2098,
+      "theoretical_loss": 6.143390070065279,
+      "tokens_seen": 20054016
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004873015873015873,
+      "loss": 4.2947,
+      "theoretical_loss": 6.140006011247426,
+      "tokens_seen": 20119552
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.000488888888888889,
+      "loss": 4.3071,
+      "theoretical_loss": 6.136636032528262,
+      "tokens_seen": 20185088
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004904761904761905,
+      "loss": 4.2235,
+      "theoretical_loss": 6.133280029894401,
+      "tokens_seen": 20250624
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004920634920634921,
+      "loss": 4.2626,
+      "theoretical_loss": 6.129937900434269,
+      "tokens_seen": 20316160
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004936507936507937,
+      "loss": 4.1477,
+      "theoretical_loss": 6.126609542322921,
+      "tokens_seen": 20381696
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004952380952380952,
+      "loss": 4.1866,
+      "theoretical_loss": 6.123294854807119,
+      "tokens_seen": 20447232
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004968253968253969,
+      "loss": 4.2563,
+      "theoretical_loss": 6.11999373819066,
+      "tokens_seen": 20512768
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004984126984126985,
+      "loss": 4.3367,
+      "theoretical_loss": 6.116706093819951,
+      "tokens_seen": 20578304
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0005,
+      "loss": 4.0307,
+      "theoretical_loss": 6.113431824069829,
+      "tokens_seen": 20643840
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0005015873015873016,
+      "loss": 4.2187,
+      "theoretical_loss": 6.110170832329612,
+      "tokens_seen": 20709376
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0005031746031746032,
+      "loss": 4.2966,
+      "theoretical_loss": 6.10692302298939,
+      "tokens_seen": 20774912
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0005047619047619048,
+      "loss": 4.4374,
+      "theoretical_loss": 6.103688301426538,
+      "tokens_seen": 20840448
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0005063492063492064,
+      "loss": 4.2097,
+      "theoretical_loss": 6.100466573992453,
+      "tokens_seen": 20905984
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0005079365079365079,
+      "loss": 4.2785,
+      "theoretical_loss": 6.097257747999514,
+      "tokens_seen": 20971520
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0005095238095238095,
+      "loss": 4.2787,
+      "theoretical_loss": 6.094061731708257,
+      "tokens_seen": 21037056
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0005111111111111111,
+      "loss": 4.2814,
+      "theoretical_loss": 6.090878434314753,
+      "tokens_seen": 21102592
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0005126984126984128,
+      "loss": 4.0601,
+      "theoretical_loss": 6.0877077659382,
+      "tokens_seen": 21168128
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0005142857142857143,
+      "loss": 4.3214,
+      "theoretical_loss": 6.084549637608715,
+      "tokens_seen": 21233664
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0005158730158730159,
+      "loss": 4.0764,
+      "theoretical_loss": 6.081403961255324,
+      "tokens_seen": 21299200
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0005174603174603176,
+      "loss": 4.0942,
+      "theoretical_loss": 6.078270649694149,
+      "tokens_seen": 21364736
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0005190476190476191,
+      "loss": 4.1596,
+      "theoretical_loss": 6.07514961661678,
+      "tokens_seen": 21430272
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0005206349206349207,
+      "loss": 4.1015,
+      "theoretical_loss": 6.072040776578845,
+      "tokens_seen": 21495808
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0005222222222222222,
+      "loss": 4.2186,
+      "theoretical_loss": 6.068944044988752,
+      "tokens_seen": 21561344
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0005238095238095238,
+      "loss": 4.122,
+      "theoretical_loss": 6.065859338096619,
+      "tokens_seen": 21626880
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0005253968253968255,
+      "loss": 4.0439,
+      "theoretical_loss": 6.0627865729833745,
+      "tokens_seen": 21692416
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.000526984126984127,
+      "loss": 4.2066,
+      "theoretical_loss": 6.059725667550044,
+      "tokens_seen": 21757952
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0005285714285714286,
+      "loss": 4.1576,
+      "theoretical_loss": 6.0566765405071905,
+      "tokens_seen": 21823488
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0005301587301587302,
+      "loss": 4.0447,
+      "theoretical_loss": 6.0536391113645305,
+      "tokens_seen": 21889024
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0005317460317460317,
+      "loss": 4.2702,
+      "theoretical_loss": 6.050613300420713,
+      "tokens_seen": 21954560
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0005333333333333334,
+      "loss": 4.1029,
+      "theoretical_loss": 6.047599028753265,
+      "tokens_seen": 22020096
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0005349206349206349,
+      "loss": 4.1342,
+      "theoretical_loss": 6.04459621820868,
+      "tokens_seen": 22085632
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0005365079365079365,
+      "loss": 4.2107,
+      "theoretical_loss": 6.041604791392675,
+      "tokens_seen": 22151168
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0005380952380952381,
+      "loss": 4.1167,
+      "theoretical_loss": 6.038624671660598,
+      "tokens_seen": 22216704
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0005396825396825397,
+      "loss": 4.2148,
+      "theoretical_loss": 6.035655783107973,
+      "tokens_seen": 22282240
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0005412698412698414,
+      "loss": 4.1704,
+      "theoretical_loss": 6.0326980505612084,
+      "tokens_seen": 22347776
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0005428571428571429,
+      "loss": 4.1784,
+      "theoretical_loss": 6.029751399568434,
+      "tokens_seen": 22413312
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0005444444444444445,
+      "loss": 4.065,
+      "theoretical_loss": 6.026815756390491,
+      "tokens_seen": 22478848
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.000546031746031746,
+      "loss": 3.9732,
+      "theoretical_loss": 6.023891047992053,
+      "tokens_seen": 22544384
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0005476190476190477,
+      "loss": 4.2287,
+      "theoretical_loss": 6.020977202032884,
+      "tokens_seen": 22609920
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0005492063492063493,
+      "loss": 4.1212,
+      "theoretical_loss": 6.018074146859234,
+      "tokens_seen": 22675456
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0005507936507936508,
+      "loss": 4.0999,
+      "theoretical_loss": 6.015181811495358,
+      "tokens_seen": 22740992
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0005523809523809524,
+      "loss": 4.1779,
+      "theoretical_loss": 6.012300125635177,
+      "tokens_seen": 22806528
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0005539682539682539,
+      "loss": 4.0847,
+      "theoretical_loss": 6.009429019634049,
+      "tokens_seen": 22872064
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0005555555555555556,
+      "loss": 4.1275,
+      "theoretical_loss": 6.006568424500674,
+      "tokens_seen": 22937600
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0005571428571428572,
+      "loss": 4.042,
+      "theoretical_loss": 6.003718271889123,
+      "tokens_seen": 23003136
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0005587301587301587,
+      "loss": 4.0857,
+      "theoretical_loss": 6.000878494090979,
+      "tokens_seen": 23068672
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0005603174603174603,
+      "loss": 4.1695,
+      "theoretical_loss": 5.998049024027599,
+      "tokens_seen": 23134208
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0005619047619047619,
+      "loss": 4.0201,
+      "theoretical_loss": 5.995229795242498,
+      "tokens_seen": 23199744
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0005634920634920636,
+      "loss": 4.0275,
+      "theoretical_loss": 5.99242074189383,
+      "tokens_seen": 23265280
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0005650793650793652,
+      "loss": 4.0145,
+      "theoretical_loss": 5.989621798747002,
+      "tokens_seen": 23330816
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0005666666666666667,
+      "loss": 4.0395,
+      "theoretical_loss": 5.98683290116738,
+      "tokens_seen": 23396352
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0005682539682539683,
+      "loss": 4.0452,
+      "theoretical_loss": 5.9840539851131105,
+      "tokens_seen": 23461888
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0005698412698412698,
+      "loss": 4.003,
+      "theoretical_loss": 5.981284987128044,
+      "tokens_seen": 23527424
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0005714285714285715,
+      "loss": 3.9503,
+      "theoretical_loss": 5.97852584433477,
+      "tokens_seen": 23592960
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0005730158730158731,
+      "loss": 4.1692,
+      "theoretical_loss": 5.975776494427746,
+      "tokens_seen": 23658496
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0005746031746031746,
+      "loss": 4.0227,
+      "theoretical_loss": 5.973036875666525,
+      "tokens_seen": 23724032
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0005761904761904762,
+      "loss": 3.9862,
+      "theoretical_loss": 5.970306926869093,
+      "tokens_seen": 23789568
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0005777777777777778,
+      "loss": 4.0578,
+      "theoretical_loss": 5.967586587405295,
+      "tokens_seen": 23855104
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0005793650793650794,
+      "loss": 4.0431,
+      "theoretical_loss": 5.96487579719035,
+      "tokens_seen": 23920640
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.000580952380952381,
+      "loss": 3.9127,
+      "theoretical_loss": 5.962174496678481,
+      "tokens_seen": 23986176
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0005825396825396825,
+      "loss": 3.8789,
+      "theoretical_loss": 5.959482626856606,
+      "tokens_seen": 24051712
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0005841269841269841,
+      "loss": 3.9908,
+      "theoretical_loss": 5.956800129238152,
+      "tokens_seen": 24117248
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0005857142857142857,
+      "loss": 3.9941,
+      "theoretical_loss": 5.954126945856933,
+      "tokens_seen": 24182784
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0005873015873015873,
+      "loss": 3.9969,
+      "theoretical_loss": 5.951463019261125,
+      "tokens_seen": 24248320
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0005888888888888889,
+      "loss": 4.1916,
+      "theoretical_loss": 5.948808292507333,
+      "tokens_seen": 24313856
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0005904761904761905,
+      "loss": 4.1251,
+      "theoretical_loss": 5.946162709154724,
+      "tokens_seen": 24379392
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0005920634920634922,
+      "loss": 3.9343,
+      "theoretical_loss": 5.943526213259268,
+      "tokens_seen": 24444928
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0005936507936507937,
+      "loss": 3.9681,
+      "theoretical_loss": 5.940898749368037,
+      "tokens_seen": 24510464
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0005952380952380953,
+      "loss": 3.9707,
+      "theoretical_loss": 5.9382802625136,
+      "tokens_seen": 24576000
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0005968253968253969,
+      "loss": 4.0216,
+      "theoretical_loss": 5.935670698208488,
+      "tokens_seen": 24641536
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0005984126984126984,
+      "loss": 3.9762,
+      "theoretical_loss": 5.933070002439743,
+      "tokens_seen": 24707072
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0006000000000000001,
+      "loss": 3.9851,
+      "theoretical_loss": 5.930478121663538,
+      "tokens_seen": 24772608
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0006015873015873016,
+      "loss": 4.0456,
+      "theoretical_loss": 5.927895002799874,
+      "tokens_seen": 24838144
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0006031746031746032,
+      "loss": 4.0994,
+      "theoretical_loss": 5.925320593227351,
+      "tokens_seen": 24903680
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0006047619047619048,
+      "loss": 3.9298,
+      "theoretical_loss": 5.922754840778015,
+      "tokens_seen": 24969216
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0006063492063492063,
+      "loss": 3.9744,
+      "theoretical_loss": 5.9201976937322645,
+      "tokens_seen": 25034752
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.000607936507936508,
+      "loss": 3.9431,
+      "theoretical_loss": 5.917649100813846,
+      "tokens_seen": 25100288
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0006095238095238095,
+      "loss": 4.1063,
+      "theoretical_loss": 5.915109011184899,
+      "tokens_seen": 25165824
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0006111111111111111,
+      "loss": 3.9948,
+      "theoretical_loss": 5.9125773744410814,
+      "tokens_seen": 25231360
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0006126984126984127,
+      "loss": 3.989,
+      "theoretical_loss": 5.91005414060676,
+      "tokens_seen": 25296896
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0006142857142857143,
+      "loss": 3.8734,
+      "theoretical_loss": 5.90753926013026,
+      "tokens_seen": 25362432
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.000615873015873016,
+      "loss": 4.0352,
+      "theoretical_loss": 5.90503268387919,
+      "tokens_seen": 25427968
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0006174603174603175,
+      "loss": 3.871,
+      "theoretical_loss": 5.9025343631358185,
+      "tokens_seen": 25493504
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0006190476190476191,
+      "loss": 3.9888,
+      "theoretical_loss": 5.900044249592523,
+      "tokens_seen": 25559040
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0006206349206349206,
+      "loss": 3.9204,
+      "theoretical_loss": 5.897562295347297,
+      "tokens_seen": 25624576
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0006222222222222223,
+      "loss": 3.9253,
+      "theoretical_loss": 5.895088452899312,
+      "tokens_seen": 25690112
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0006238095238095239,
+      "loss": 3.8434,
+      "theoretical_loss": 5.892622675144553,
+      "tokens_seen": 25755648
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0006253968253968254,
+      "loss": 3.7913,
+      "theoretical_loss": 5.890164915371495,
+      "tokens_seen": 25821184
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.000626984126984127,
+      "loss": 4.0049,
+      "theoretical_loss": 5.887715127256854,
+      "tokens_seen": 25886720
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0006285714285714286,
+      "loss": 3.8893,
+      "theoretical_loss": 5.885273264861381,
+      "tokens_seen": 25952256
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0006301587301587302,
+      "loss": 3.9636,
+      "theoretical_loss": 5.882839282625721,
+      "tokens_seen": 26017792
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0006317460317460318,
+      "loss": 3.9577,
+      "theoretical_loss": 5.880413135366323,
+      "tokens_seen": 26083328
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0006333333333333333,
+      "loss": 4.0493,
+      "theoretical_loss": 5.877994778271411,
+      "tokens_seen": 26148864
+    },
+    {
+      "epoch": 0.01,
+      "objective/train/docs_used": 17681,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 3.8979604244232178,
+      "objective/train/theoretical_loss": 5.8755841668969895,
+      "objective/train/tokens_used": 46674400,
+      "theoretical_loss": 5.8755841668969895,
+      "tokens_seen": 26214400
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0006349206349206349,
+      "loss": 3.898,
+      "theoretical_loss": 5.8755841668969895,
+      "tokens_seen": 26214400
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0006365079365079365,
+      "loss": 3.9651,
+      "theoretical_loss": 5.873181257162926,
+      "tokens_seen": 26279936
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0006380952380952382,
+      "loss": 3.8692,
+      "theoretical_loss": 5.870786005349071,
+      "tokens_seen": 26345472
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0006396825396825398,
+      "loss": 3.8808,
+      "theoretical_loss": 5.868398368091427,
+      "tokens_seen": 26411008
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0006412698412698413,
+      "loss": 3.9366,
+      "theoretical_loss": 5.866018302378377,
+      "tokens_seen": 26476544
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0006428571428571429,
+      "loss": 3.846,
+      "theoretical_loss": 5.863645765546956,
+      "tokens_seen": 26542080
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0006444444444444444,
+      "loss": 3.7786,
+      "theoretical_loss": 5.861280715279168,
+      "tokens_seen": 26607616
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0006460317460317461,
+      "loss": 3.9345,
+      "theoretical_loss": 5.858923109598366,
+      "tokens_seen": 26673152
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0006476190476190477,
+      "loss": 3.7997,
+      "theoretical_loss": 5.8565729068656545,
+      "tokens_seen": 26738688
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0006492063492063492,
+      "loss": 3.7713,
+      "theoretical_loss": 5.85423006577636,
+      "tokens_seen": 26804224
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0006507936507936508,
+      "loss": 3.8107,
+      "theoretical_loss": 5.851894545356539,
+      "tokens_seen": 26869760
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0006523809523809524,
+      "loss": 3.8922,
+      "theoretical_loss": 5.849566304959527,
+      "tokens_seen": 26935296
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.000653968253968254,
+      "loss": 3.7606,
+      "theoretical_loss": 5.84724530426254,
+      "tokens_seen": 27000832
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0006555555555555556,
+      "loss": 3.7933,
+      "theoretical_loss": 5.844931503263309,
+      "tokens_seen": 27066368
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0006571428571428571,
+      "loss": 3.9081,
+      "theoretical_loss": 5.842624862276773,
+      "tokens_seen": 27131904
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0006587301587301587,
+      "loss": 3.832,
+      "theoretical_loss": 5.840325341931795,
+      "tokens_seen": 27197440
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0006603174603174604,
+      "loss": 3.8188,
+      "theoretical_loss": 5.838032903167934,
+      "tokens_seen": 27262976
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0006619047619047619,
+      "loss": 3.8649,
+      "theoretical_loss": 5.835747507232257,
+      "tokens_seen": 27328512
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0006634920634920636,
+      "loss": 3.8314,
+      "theoretical_loss": 5.833469115676177,
+      "tokens_seen": 27394048
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0006650793650793651,
+      "loss": 3.9019,
+      "theoretical_loss": 5.8311976903523535,
+      "tokens_seen": 27459584
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0006666666666666668,
+      "loss": 3.8903,
+      "theoretical_loss": 5.828933193411611,
+      "tokens_seen": 27525120
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0006682539682539683,
+      "loss": 3.9442,
+      "theoretical_loss": 5.826675587299913,
+      "tokens_seen": 27590656
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0006698412698412699,
+      "loss": 3.7999,
+      "theoretical_loss": 5.824424834755357,
+      "tokens_seen": 27656192
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0006714285714285715,
+      "loss": 3.9527,
+      "theoretical_loss": 5.8221808988052235,
+      "tokens_seen": 27721728
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.000673015873015873,
+      "loss": 3.7933,
+      "theoretical_loss": 5.819943742763051,
+      "tokens_seen": 27787264
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0006746031746031747,
+      "loss": 3.741,
+      "theoretical_loss": 5.817713330225754,
+      "tokens_seen": 27852800
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0006761904761904762,
+      "loss": 3.8241,
+      "theoretical_loss": 5.815489625070766,
+      "tokens_seen": 27918336
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0006777777777777778,
+      "loss": 3.7877,
+      "theoretical_loss": 5.813272591453236,
+      "tokens_seen": 27983872
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0006793650793650794,
+      "loss": 3.6882,
+      "theoretical_loss": 5.811062193803242,
+      "tokens_seen": 28049408
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0006809523809523809,
+      "loss": 3.7131,
+      "theoretical_loss": 5.808858396823043,
+      "tokens_seen": 28114944
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0006825396825396826,
+      "loss": 3.755,
+      "theoretical_loss": 5.806661165484377,
+      "tokens_seen": 28180480
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0006841269841269841,
+      "loss": 3.8132,
+      "theoretical_loss": 5.804470465025777,
+      "tokens_seen": 28246016
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0006857142857142857,
+      "loss": 3.7366,
+      "theoretical_loss": 5.802286260949924,
+      "tokens_seen": 28311552
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0006873015873015873,
+      "loss": 3.7987,
+      "theoretical_loss": 5.8001085190210375,
+      "tokens_seen": 28377088
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.000688888888888889,
+      "loss": 3.6966,
+      "theoretical_loss": 5.797937205262292,
+      "tokens_seen": 28442624
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0006904761904761906,
+      "loss": 3.8535,
+      "theoretical_loss": 5.795772285953266,
+      "tokens_seen": 28508160
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0006920634920634921,
+      "loss": 3.7654,
+      "theoretical_loss": 5.793613727627429,
+      "tokens_seen": 28573696
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0006936507936507937,
+      "loss": 3.8714,
+      "theoretical_loss": 5.791461497069644,
+      "tokens_seen": 28639232
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0006952380952380953,
+      "loss": 3.7254,
+      "theoretical_loss": 5.789315561313717,
+      "tokens_seen": 28704768
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0006968253968253969,
+      "loss": 3.7796,
+      "theoretical_loss": 5.787175887639959,
+      "tokens_seen": 28770304
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0006984126984126985,
+      "loss": 3.7973,
+      "theoretical_loss": 5.785042443572798,
+      "tokens_seen": 28835840
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007,
+      "loss": 3.809,
+      "theoretical_loss": 5.782915196878396,
+      "tokens_seen": 28901376
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007015873015873016,
+      "loss": 3.7745,
+      "theoretical_loss": 5.780794115562314,
+      "tokens_seen": 28966912
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007031746031746032,
+      "loss": 3.8756,
+      "theoretical_loss": 5.778679167867196,
+      "tokens_seen": 29032448
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007047619047619048,
+      "loss": 3.6832,
+      "theoretical_loss": 5.7765703222704765,
+      "tokens_seen": 29097984
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007063492063492064,
+      "loss": 3.6919,
+      "theoretical_loss": 5.7744675474821285,
+      "tokens_seen": 29163520
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007079365079365079,
+      "loss": 3.6528,
+      "theoretical_loss": 5.772370812442426,
+      "tokens_seen": 29229056
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007095238095238095,
+      "loss": 3.7711,
+      "theoretical_loss": 5.770280086319733,
+      "tokens_seen": 29294592
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007111111111111111,
+      "loss": 3.8099,
+      "theoretical_loss": 5.768195338508333,
+      "tokens_seen": 29360128
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007126984126984128,
+      "loss": 3.8639,
+      "theoretical_loss": 5.766116538626263,
+      "tokens_seen": 29425664
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007142857142857144,
+      "loss": 3.8192,
+      "theoretical_loss": 5.7640436565131905,
+      "tokens_seen": 29491200
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007158730158730159,
+      "loss": 3.6903,
+      "theoretical_loss": 5.761976662228307,
+      "tokens_seen": 29556736
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007174603174603175,
+      "loss": 3.8698,
+      "theoretical_loss": 5.759915526048243,
+      "tokens_seen": 29622272
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.000719047619047619,
+      "loss": 3.7103,
+      "theoretical_loss": 5.757860218465023,
+      "tokens_seen": 29687808
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007206349206349207,
+      "loss": 3.7779,
+      "theoretical_loss": 5.755810710184019,
+      "tokens_seen": 29753344
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007222222222222223,
+      "loss": 3.7096,
+      "theoretical_loss": 5.753766972121954,
+      "tokens_seen": 29818880
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007238095238095238,
+      "loss": 3.8155,
+      "theoretical_loss": 5.751728975404911,
+      "tokens_seen": 29884416
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007253968253968254,
+      "loss": 3.6583,
+      "theoretical_loss": 5.7496966913663705,
+      "tokens_seen": 29949952
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007269841269841271,
+      "loss": 3.7968,
+      "theoretical_loss": 5.747670091545274,
+      "tokens_seen": 30015488
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007285714285714286,
+      "loss": 3.6869,
+      "theoretical_loss": 5.745649147684103,
+      "tokens_seen": 30081024
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007301587301587302,
+      "loss": 3.6626,
+      "theoretical_loss": 5.743633831726987,
+      "tokens_seen": 30146560
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007317460317460317,
+      "loss": 3.6203,
+      "theoretical_loss": 5.741624115817827,
+      "tokens_seen": 30212096
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007333333333333333,
+      "loss": 3.6655,
+      "theoretical_loss": 5.739619972298446,
+      "tokens_seen": 30277632
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.000734920634920635,
+      "loss": 3.5931,
+      "theoretical_loss": 5.737621373706753,
+      "tokens_seen": 30343168
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007365079365079365,
+      "loss": 3.8355,
+      "theoretical_loss": 5.735628292774945,
+      "tokens_seen": 30408704
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007380952380952382,
+      "loss": 3.6235,
+      "theoretical_loss": 5.7336407024276985,
+      "tokens_seen": 30474240
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007396825396825397,
+      "loss": 3.7167,
+      "theoretical_loss": 5.731658575780418,
+      "tokens_seen": 30539776
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007412698412698414,
+      "loss": 3.688,
+      "theoretical_loss": 5.729681886137476,
+      "tokens_seen": 30605312
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007428571428571429,
+      "loss": 3.7001,
+      "theoretical_loss": 5.727710606990488,
+      "tokens_seen": 30670848
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007444444444444445,
+      "loss": 3.5595,
+      "theoretical_loss": 5.725744712016601,
+      "tokens_seen": 30736384
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007460317460317461,
+      "loss": 3.689,
+      "theoretical_loss": 5.723784175076802,
+      "tokens_seen": 30801920
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007476190476190476,
+      "loss": 3.6387,
+      "theoretical_loss": 5.72182897021425,
+      "tokens_seen": 30867456
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007492063492063493,
+      "loss": 3.7047,
+      "theoretical_loss": 5.719879071652615,
+      "tokens_seen": 30932992
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007507936507936508,
+      "loss": 3.6256,
+      "theoretical_loss": 5.717934453794454,
+      "tokens_seen": 30998528
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007523809523809524,
+      "loss": 3.6762,
+      "theoretical_loss": 5.715995091219584,
+      "tokens_seen": 31064064
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.000753968253968254,
+      "loss": 3.7207,
+      "theoretical_loss": 5.7140609586834925,
+      "tokens_seen": 31129600
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007555555555555555,
+      "loss": 3.5748,
+      "theoretical_loss": 5.712132031115749,
+      "tokens_seen": 31195136
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007571428571428572,
+      "loss": 3.6296,
+      "theoretical_loss": 5.710208283618447,
+      "tokens_seen": 31260672
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007587301587301588,
+      "loss": 3.6736,
+      "theoretical_loss": 5.708289691464657,
+      "tokens_seen": 31326208
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007603174603174603,
+      "loss": 3.6819,
+      "theoretical_loss": 5.706376230096897,
+      "tokens_seen": 31391744
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007619047619047619,
+      "loss": 3.6571,
+      "theoretical_loss": 5.704467875125619,
+      "tokens_seen": 31457280
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007634920634920635,
+      "loss": 3.6361,
+      "theoretical_loss": 5.702564602327719,
+      "tokens_seen": 31522816
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007650793650793652,
+      "loss": 3.5594,
+      "theoretical_loss": 5.700666387645049,
+      "tokens_seen": 31588352
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007666666666666667,
+      "loss": 3.6851,
+      "theoretical_loss": 5.698773207182963,
+      "tokens_seen": 31653888
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007682539682539683,
+      "loss": 3.6468,
+      "theoretical_loss": 5.696885037208865,
+      "tokens_seen": 31719424
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007698412698412699,
+      "loss": 3.7059,
+      "theoretical_loss": 5.695001854150782,
+      "tokens_seen": 31784960
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007714285714285715,
+      "loss": 3.6026,
+      "theoretical_loss": 5.693123634595938,
+      "tokens_seen": 31850496
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007730158730158731,
+      "loss": 3.6567,
+      "theoretical_loss": 5.69125035528937,
+      "tokens_seen": 31916032
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007746031746031746,
+      "loss": 3.6296,
+      "theoretical_loss": 5.689381993132532,
+      "tokens_seen": 31981568
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007761904761904762,
+      "loss": 3.5629,
+      "theoretical_loss": 5.687518525181924,
+      "tokens_seen": 32047104
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007777777777777778,
+      "loss": 3.6558,
+      "theoretical_loss": 5.685659928647743,
+      "tokens_seen": 32112640
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007793650793650794,
+      "loss": 3.5843,
+      "theoretical_loss": 5.68380618089254,
+      "tokens_seen": 32178176
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.000780952380952381,
+      "loss": 3.7462,
+      "theoretical_loss": 5.6819572594298915,
+      "tokens_seen": 32243712
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007825396825396825,
+      "loss": 3.6714,
+      "theoretical_loss": 5.680113141923088,
+      "tokens_seen": 32309248
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007841269841269841,
+      "loss": 3.6394,
+      "theoretical_loss": 5.67827380618384,
+      "tokens_seen": 32374784
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007857142857142857,
+      "loss": 3.687,
+      "theoretical_loss": 5.6764392301709865,
+      "tokens_seen": 32440320
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007873015873015873,
+      "loss": 3.6331,
+      "theoretical_loss": 5.674609391989232,
+      "tokens_seen": 32505856
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.000788888888888889,
+      "loss": 3.6678,
+      "theoretical_loss": 5.672784269887885,
+      "tokens_seen": 32571392
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007904761904761905,
+      "loss": 3.6251,
+      "theoretical_loss": 5.670963842259615,
+      "tokens_seen": 32636928
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007920634920634921,
+      "loss": 3.5818,
+      "theoretical_loss": 5.669148087639222,
+      "tokens_seen": 32702464
+    },
+    {
+      "epoch": 0.01,
+      "objective/train/docs_used": 20023,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 3.6404943466186523,
+      "objective/train/theoretical_loss": 5.66733698470242,
+      "objective/train/tokens_used": 53228000,
+      "theoretical_loss": 5.66733698470242,
+      "tokens_seen": 32768000
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007936507936507938,
+      "loss": 3.6405,
+      "theoretical_loss": 5.66733698470242,
+      "tokens_seen": 32768000
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007952380952380953,
+      "loss": 3.5114,
+      "theoretical_loss": 5.665530512264631,
+      "tokens_seen": 32833536
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007968253968253969,
+      "loss": 3.7098,
+      "theoretical_loss": 5.663728649279793,
+      "tokens_seen": 32899072
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007984126984126984,
+      "loss": 3.4674,
+      "theoretical_loss": 5.6619313748391855,
+      "tokens_seen": 32964608
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0008,
+      "loss": 3.5626,
+      "theoretical_loss": 5.6601386681702515,
+      "tokens_seen": 33030144
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007999839518555668,
+      "loss": 3.5969,
+      "theoretical_loss": 5.658350508635454,
+      "tokens_seen": 33095680
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007999679037111334,
+      "loss": 3.605,
+      "theoretical_loss": 5.656566875731127,
+      "tokens_seen": 33161216
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007999518555667002,
+      "loss": 3.5132,
+      "theoretical_loss": 5.654787749086345,
+      "tokens_seen": 33226752
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007999358074222668,
+      "loss": 3.5917,
+      "theoretical_loss": 5.6530131084618045,
+      "tokens_seen": 33292288
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007999197592778336,
+      "loss": 3.6407,
+      "theoretical_loss": 5.651242933748718,
+      "tokens_seen": 33357824
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007999037111334002,
+      "loss": 3.6881,
+      "theoretical_loss": 5.649477204967717,
+      "tokens_seen": 33423360
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.000799887662988967,
+      "loss": 3.6215,
+      "theoretical_loss": 5.647715902267764,
+      "tokens_seen": 33488896
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007998716148445336,
+      "loss": 3.4909,
+      "theoretical_loss": 5.645959005925086,
+      "tokens_seen": 33554432
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007998555667001004,
+      "loss": 3.5438,
+      "theoretical_loss": 5.644206496342103,
+      "tokens_seen": 33619968
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.000799839518555667,
+      "loss": 3.4454,
+      "theoretical_loss": 5.642458354046386,
+      "tokens_seen": 33685504
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007998234704112338,
+      "loss": 3.4821,
+      "theoretical_loss": 5.6407145596896076,
+      "tokens_seen": 33751040
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007998074222668004,
+      "loss": 3.4895,
+      "theoretical_loss": 5.638975094046518,
+      "tokens_seen": 33816576
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007997913741223672,
+      "loss": 3.5686,
+      "theoretical_loss": 5.637239938013924,
+      "tokens_seen": 33882112
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007997753259779338,
+      "loss": 3.4646,
+      "theoretical_loss": 5.6355090726096755,
+      "tokens_seen": 33947648
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007997592778335006,
+      "loss": 3.7202,
+      "theoretical_loss": 5.633782478971672,
+      "tokens_seen": 34013184
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007997432296890672,
+      "loss": 3.5494,
+      "theoretical_loss": 5.632060138356873,
+      "tokens_seen": 34078720
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.000799727181544634,
+      "loss": 3.5303,
+      "theoretical_loss": 5.630342032140315,
+      "tokens_seen": 34144256
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007997111334002006,
+      "loss": 3.6338,
+      "theoretical_loss": 5.628628141814151,
+      "tokens_seen": 34209792
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007996950852557674,
+      "loss": 3.6308,
+      "theoretical_loss": 5.62691844898668,
+      "tokens_seen": 34275328
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.000799679037111334,
+      "loss": 3.3829,
+      "theoretical_loss": 5.625212935381407,
+      "tokens_seen": 34340864
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007996629889669008,
+      "loss": 3.4401,
+      "theoretical_loss": 5.6235115828361035,
+      "tokens_seen": 34406400
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007996469408224675,
+      "loss": 3.6042,
+      "theoretical_loss": 5.621814373301866,
+      "tokens_seen": 34471936
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007996308926780342,
+      "loss": 3.567,
+      "theoretical_loss": 5.620121288842209,
+      "tokens_seen": 34537472
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007996148445336009,
+      "loss": 3.5176,
+      "theoretical_loss": 5.618432311632143,
+      "tokens_seen": 34603008
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007995987963891676,
+      "loss": 3.4325,
+      "theoretical_loss": 5.616747423957278,
+      "tokens_seen": 34668544
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007995827482447343,
+      "loss": 3.5763,
+      "theoretical_loss": 5.615066608212926,
+      "tokens_seen": 34734080
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.000799566700100301,
+      "loss": 3.4953,
+      "theoretical_loss": 5.613389846903216,
+      "tokens_seen": 34799616
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007995506519558677,
+      "loss": 3.5708,
+      "theoretical_loss": 5.611717122640225,
+      "tokens_seen": 34865152
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007995346038114344,
+      "loss": 3.7478,
+      "theoretical_loss": 5.610048418143098,
+      "tokens_seen": 34930688
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007995185556670011,
+      "loss": 3.5119,
+      "theoretical_loss": 5.6083837162372,
+      "tokens_seen": 34996224
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007995025075225678,
+      "loss": 3.5977,
+      "theoretical_loss": 5.606722999853261,
+      "tokens_seen": 35061760
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007994864593781345,
+      "loss": 3.4637,
+      "theoretical_loss": 5.605066252026537,
+      "tokens_seen": 35127296
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007994704112337012,
+      "loss": 3.5509,
+      "theoretical_loss": 5.603413455895973,
+      "tokens_seen": 35192832
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007994543630892679,
+      "loss": 3.5226,
+      "theoretical_loss": 5.601764594703377,
+      "tokens_seen": 35258368
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007994383149448346,
+      "loss": 3.5264,
+      "theoretical_loss": 5.600119651792607,
+      "tokens_seen": 35323904
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007994222668004013,
+      "loss": 3.5676,
+      "theoretical_loss": 5.598478610608758,
+      "tokens_seen": 35389440
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.000799406218655968,
+      "loss": 3.3829,
+      "theoretical_loss": 5.596841454697365,
+      "tokens_seen": 35454976
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007993901705115347,
+      "loss": 3.5647,
+      "theoretical_loss": 5.595208167703601,
+      "tokens_seen": 35520512
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007993741223671014,
+      "loss": 3.553,
+      "theoretical_loss": 5.5935787333714995,
+      "tokens_seen": 35586048
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007993580742226681,
+      "loss": 3.4864,
+      "theoretical_loss": 5.59195313554317,
+      "tokens_seen": 35651584
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007993420260782347,
+      "loss": 3.6239,
+      "theoretical_loss": 5.590331358158036,
+      "tokens_seen": 35717120
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007993259779338015,
+      "loss": 3.4815,
+      "theoretical_loss": 5.58871338525206,
+      "tokens_seen": 35782656
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007993099297893681,
+      "loss": 3.5306,
+      "theoretical_loss": 5.587099200956995,
+      "tokens_seen": 35848192
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007992938816449349,
+      "loss": 3.467,
+      "theoretical_loss": 5.585488789499637,
+      "tokens_seen": 35913728
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007992778335005015,
+      "loss": 3.5261,
+      "theoretical_loss": 5.583882135201078,
+      "tokens_seen": 35979264
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007992617853560683,
+      "loss": 3.4974,
+      "theoretical_loss": 5.582279222475973,
+      "tokens_seen": 36044800
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007992457372116349,
+      "loss": 3.2454,
+      "theoretical_loss": 5.580680035831821,
+      "tokens_seen": 36110336
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007992296890672017,
+      "loss": 3.4453,
+      "theoretical_loss": 5.579084559868232,
+      "tokens_seen": 36175872
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007992136409227683,
+      "loss": 3.5094,
+      "theoretical_loss": 5.577492779276222,
+      "tokens_seen": 36241408
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.000799197592778335,
+      "loss": 3.4054,
+      "theoretical_loss": 5.575904678837501,
+      "tokens_seen": 36306944
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007991815446339017,
+      "loss": 3.5655,
+      "theoretical_loss": 5.574320243423783,
+      "tokens_seen": 36372480
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007991654964894684,
+      "loss": 3.3753,
+      "theoretical_loss": 5.572739457996079,
+      "tokens_seen": 36438016
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007991494483450351,
+      "loss": 3.3295,
+      "theoretical_loss": 5.5711623076040215,
+      "tokens_seen": 36503552
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007991334002006018,
+      "loss": 3.3126,
+      "theoretical_loss": 5.569588777385177,
+      "tokens_seen": 36569088
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007991173520561685,
+      "loss": 3.4123,
+      "theoretical_loss": 5.568018852564379,
+      "tokens_seen": 36634624
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007991013039117352,
+      "loss": 3.5944,
+      "theoretical_loss": 5.566452518453051,
+      "tokens_seen": 36700160
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007990852557673019,
+      "loss": 3.3454,
+      "theoretical_loss": 5.564889760448561,
+      "tokens_seen": 36765696
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007990692076228686,
+      "loss": 3.3713,
+      "theoretical_loss": 5.5633305640335475,
+      "tokens_seen": 36831232
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007990531594784353,
+      "loss": 3.4241,
+      "theoretical_loss": 5.561774914775286,
+      "tokens_seen": 36896768
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.000799037111334002,
+      "loss": 3.4546,
+      "theoretical_loss": 5.5602227983250385,
+      "tokens_seen": 36962304
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007990210631895687,
+      "loss": 3.5998,
+      "theoretical_loss": 5.558674200417419,
+      "tokens_seen": 37027840
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007990050150451354,
+      "loss": 3.5237,
+      "theoretical_loss": 5.557129106869764,
+      "tokens_seen": 37093376
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007989889669007021,
+      "loss": 3.3499,
+      "theoretical_loss": 5.555587503581506,
+      "tokens_seen": 37158912
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007989729187562688,
+      "loss": 3.3352,
+      "theoretical_loss": 5.554049376533554,
+      "tokens_seen": 37224448
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007989568706118355,
+      "loss": 3.4595,
+      "theoretical_loss": 5.552514711787683,
+      "tokens_seen": 37289984
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007989408224674022,
+      "loss": 3.3808,
+      "theoretical_loss": 5.550983495485925,
+      "tokens_seen": 37355520
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007989247743229689,
+      "loss": 3.4302,
+      "theoretical_loss": 5.549455713849963,
+      "tokens_seen": 37421056
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007989087261785356,
+      "loss": 3.3324,
+      "theoretical_loss": 5.547931353180543,
+      "tokens_seen": 37486592
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007988926780341023,
+      "loss": 3.3144,
+      "theoretical_loss": 5.546410399856875,
+      "tokens_seen": 37552128
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.000798876629889669,
+      "loss": 3.39,
+      "theoretical_loss": 5.544892840336053,
+      "tokens_seen": 37617664
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007988605817452357,
+      "loss": 3.4015,
+      "theoretical_loss": 5.543378661152472,
+      "tokens_seen": 37683200
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007988445336008024,
+      "loss": 3.4097,
+      "theoretical_loss": 5.5418678489172555,
+      "tokens_seen": 37748736
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007988284854563691,
+      "loss": 3.3702,
+      "theoretical_loss": 5.540360390317682,
+      "tokens_seen": 37814272
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007988124373119358,
+      "loss": 3.4479,
+      "theoretical_loss": 5.538856272116626,
+      "tokens_seen": 37879808
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007987963891675025,
+      "loss": 3.3019,
+      "theoretical_loss": 5.5373554811519945,
+      "tokens_seen": 37945344
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007987803410230693,
+      "loss": 3.4943,
+      "theoretical_loss": 5.535858004336175,
+      "tokens_seen": 38010880
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007987642928786359,
+      "loss": 3.2602,
+      "theoretical_loss": 5.534363828655488,
+      "tokens_seen": 38076416
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007987482447342027,
+      "loss": 3.2579,
+      "theoretical_loss": 5.532872941169639,
+      "tokens_seen": 38141952
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007987321965897693,
+      "loss": 3.4578,
+      "theoretical_loss": 5.531385329011181,
+      "tokens_seen": 38207488
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007987161484453361,
+      "loss": 3.3188,
+      "theoretical_loss": 5.529900979384985,
+      "tokens_seen": 38273024
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007987001003009027,
+      "loss": 3.3775,
+      "theoretical_loss": 5.5284198795677,
+      "tokens_seen": 38338560
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007986840521564695,
+      "loss": 3.3057,
+      "theoretical_loss": 5.526942016907242,
+      "tokens_seen": 38404096
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007986680040120361,
+      "loss": 3.5022,
+      "theoretical_loss": 5.5254673788222615,
+      "tokens_seen": 38469632
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007986519558676029,
+      "loss": 3.3356,
+      "theoretical_loss": 5.523995952801634,
+      "tokens_seen": 38535168
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007986359077231695,
+      "loss": 3.293,
+      "theoretical_loss": 5.522527726403953,
+      "tokens_seen": 38600704
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007986198595787363,
+      "loss": 3.3967,
+      "theoretical_loss": 5.521062687257016,
+      "tokens_seen": 38666240
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007986038114343029,
+      "loss": 3.3977,
+      "theoretical_loss": 5.519600823057332,
+      "tokens_seen": 38731776
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007985877632898697,
+      "loss": 3.4202,
+      "theoretical_loss": 5.518142121569613,
+      "tokens_seen": 38797312
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007985717151454363,
+      "loss": 3.3039,
+      "theoretical_loss": 5.516686570626293,
+      "tokens_seen": 38862848
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007985556670010031,
+      "loss": 3.3701,
+      "theoretical_loss": 5.515234158127032,
+      "tokens_seen": 38928384
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007985396188565697,
+      "loss": 3.3696,
+      "theoretical_loss": 5.5137848720382365,
+      "tokens_seen": 38993920
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007985235707121365,
+      "loss": 3.4991,
+      "theoretical_loss": 5.5123387003925775,
+      "tokens_seen": 39059456
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007985075225677031,
+      "loss": 3.3928,
+      "theoretical_loss": 5.510895631288515,
+      "tokens_seen": 39124992
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007984914744232699,
+      "loss": 3.3664,
+      "theoretical_loss": 5.50945565288983,
+      "tokens_seen": 39190528
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007984754262788365,
+      "loss": 3.3201,
+      "theoretical_loss": 5.508018753425152,
+      "tokens_seen": 39256064
+    },
+    {
+      "epoch": 0.01,
+      "objective/train/docs_used": 22534,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 3.435593366622925,
+      "objective/train/theoretical_loss": 5.506584921187503,
+      "objective/train/tokens_used": 59781600,
+      "theoretical_loss": 5.506584921187503,
+      "tokens_seen": 39321600
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007984593781344033,
+      "loss": 3.4356,
+      "theoretical_loss": 5.506584921187503,
+      "tokens_seen": 39321600
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007984433299899699,
+      "loss": 3.3481,
+      "theoretical_loss": 5.505154144533835,
+      "tokens_seen": 39387136
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007984272818455367,
+      "loss": 3.2633,
+      "theoretical_loss": 5.503726411884568,
+      "tokens_seen": 39452672
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007984112337011033,
+      "loss": 3.3036,
+      "theoretical_loss": 5.502301711723156,
+      "tokens_seen": 39518208
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007983951855566701,
+      "loss": 3.2297,
+      "theoretical_loss": 5.5008800325956235,
+      "tokens_seen": 39583744
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007983791374122367,
+      "loss": 3.3255,
+      "theoretical_loss": 5.499461363110136,
+      "tokens_seen": 39649280
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007983630892678035,
+      "loss": 3.3713,
+      "theoretical_loss": 5.498045691936548,
+      "tokens_seen": 39714816
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007983470411233701,
+      "loss": 3.2234,
+      "theoretical_loss": 5.496633007805977,
+      "tokens_seen": 39780352
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007983309929789369,
+      "loss": 3.1825,
+      "theoretical_loss": 5.495223299510372,
+      "tokens_seen": 39845888
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007983149448345036,
+      "loss": 3.355,
+      "theoretical_loss": 5.493816555902079,
+      "tokens_seen": 39911424
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007982988966900703,
+      "loss": 3.3841,
+      "theoretical_loss": 5.492412765893421,
+      "tokens_seen": 39976960
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.000798282848545637,
+      "loss": 3.3008,
+      "theoretical_loss": 5.49101191845628,
+      "tokens_seen": 40042496
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007982668004012037,
+      "loss": 3.2659,
+      "theoretical_loss": 5.489614002621677,
+      "tokens_seen": 40108032
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007982507522567704,
+      "loss": 3.3211,
+      "theoretical_loss": 5.4882190074793575,
+      "tokens_seen": 40173568
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007982347041123371,
+      "loss": 3.2794,
+      "theoretical_loss": 5.486826922177387,
+      "tokens_seen": 40239104
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007982186559679038,
+      "loss": 3.4319,
+      "theoretical_loss": 5.4854377359217406,
+      "tokens_seen": 40304640
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007982026078234705,
+      "loss": 3.2653,
+      "theoretical_loss": 5.484051437975901,
+      "tokens_seen": 40370176
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007981865596790372,
+      "loss": 3.3589,
+      "theoretical_loss": 5.482668017660462,
+      "tokens_seen": 40435712
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007981705115346039,
+      "loss": 3.2239,
+      "theoretical_loss": 5.481287464352727,
+      "tokens_seen": 40501248
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007981544633901706,
+      "loss": 3.2691,
+      "theoretical_loss": 5.479909767486326,
+      "tokens_seen": 40566784
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007981384152457373,
+      "loss": 3.2705,
+      "theoretical_loss": 5.478534916550817,
+      "tokens_seen": 40632320
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.000798122367101304,
+      "loss": 3.2859,
+      "theoretical_loss": 5.477162901091306,
+      "tokens_seen": 40697856
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007981063189568707,
+      "loss": 3.4215,
+      "theoretical_loss": 5.475793710708061,
+      "tokens_seen": 40763392
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007980902708124374,
+      "loss": 3.1642,
+      "theoretical_loss": 5.474427335056138,
+      "tokens_seen": 40828928
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007980742226680041,
+      "loss": 3.1913,
+      "theoretical_loss": 5.473063763844998,
+      "tokens_seen": 40894464
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007980581745235708,
+      "loss": 3.2397,
+      "theoretical_loss": 5.471702986838139,
+      "tokens_seen": 40960000
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007980421263791375,
+      "loss": 3.1662,
+      "theoretical_loss": 5.470344993852729,
+      "tokens_seen": 41025536
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007980260782347042,
+      "loss": 3.0606,
+      "theoretical_loss": 5.46898977475923,
+      "tokens_seen": 41091072
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007980100300902709,
+      "loss": 3.3902,
+      "theoretical_loss": 5.467637319481042,
+      "tokens_seen": 41156608
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007979939819458376,
+      "loss": 3.0364,
+      "theoretical_loss": 5.466287617994144,
+      "tokens_seen": 41222144
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007979779338014043,
+      "loss": 3.224,
+      "theoretical_loss": 5.464940660326731,
+      "tokens_seen": 41287680
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.000797961885656971,
+      "loss": 3.2479,
+      "theoretical_loss": 5.463596436558861,
+      "tokens_seen": 41353216
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007979458375125376,
+      "loss": 3.139,
+      "theoretical_loss": 5.462254936822105,
+      "tokens_seen": 41418752
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007979297893681044,
+      "loss": 3.2887,
+      "theoretical_loss": 5.4609161512992,
+      "tokens_seen": 41484288
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.000797913741223671,
+      "loss": 3.1795,
+      "theoretical_loss": 5.4595800702237005,
+      "tokens_seen": 41549824
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007978976930792378,
+      "loss": 3.2645,
+      "theoretical_loss": 5.458246683879636,
+      "tokens_seen": 41615360
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007978816449348044,
+      "loss": 3.2391,
+      "theoretical_loss": 5.456915982601173,
+      "tokens_seen": 41680896
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007978655967903712,
+      "loss": 3.1832,
+      "theoretical_loss": 5.455587956772277,
+      "tokens_seen": 41746432
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007978495486459378,
+      "loss": 3.1946,
+      "theoretical_loss": 5.454262596826377,
+      "tokens_seen": 41811968
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007978335005015046,
+      "loss": 3.1349,
+      "theoretical_loss": 5.452939893246034,
+      "tokens_seen": 41877504
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007978174523570712,
+      "loss": 3.2418,
+      "theoretical_loss": 5.451619836562617,
+      "tokens_seen": 41943040
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007978014042126379,
+      "loss": 3.162,
+      "theoretical_loss": 5.450302417355973,
+      "tokens_seen": 42008576
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007977853560682046,
+      "loss": 3.08,
+      "theoretical_loss": 5.448987626254097,
+      "tokens_seen": 42074112
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007977693079237713,
+      "loss": 3.1325,
+      "theoretical_loss": 5.447675453932829,
+      "tokens_seen": 42139648
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.000797753259779338,
+      "loss": 3.2414,
+      "theoretical_loss": 5.446365891115516,
+      "tokens_seen": 42205184
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007977372116349047,
+      "loss": 3.3154,
+      "theoretical_loss": 5.445058928572712,
+      "tokens_seen": 42270720
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007977211634904714,
+      "loss": 3.2803,
+      "theoretical_loss": 5.443754557121853,
+      "tokens_seen": 42336256
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007977051153460381,
+      "loss": 3.2213,
+      "theoretical_loss": 5.442452767626957,
+      "tokens_seen": 42401792
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007976890672016048,
+      "loss": 3.1288,
+      "theoretical_loss": 5.441153550998311,
+      "tokens_seen": 42467328
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007976730190571715,
+      "loss": 3.1906,
+      "theoretical_loss": 5.439856898192163,
+      "tokens_seen": 42532864
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007976569709127382,
+      "loss": 3.305,
+      "theoretical_loss": 5.438562800210427,
+      "tokens_seen": 42598400
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007976409227683049,
+      "loss": 3.19,
+      "theoretical_loss": 5.437271248100375,
+      "tokens_seen": 42663936
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007976248746238716,
+      "loss": 3.2208,
+      "theoretical_loss": 5.435982232954342,
+      "tokens_seen": 42729472
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007976088264794383,
+      "loss": 3.1801,
+      "theoretical_loss": 5.434695745909433,
+      "tokens_seen": 42795008
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.000797592778335005,
+      "loss": 3.1413,
+      "theoretical_loss": 5.433411778147226,
+      "tokens_seen": 42860544
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007975767301905718,
+      "loss": 3.2043,
+      "theoretical_loss": 5.432130320893483,
+      "tokens_seen": 42926080
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007975606820461384,
+      "loss": 3.1159,
+      "theoretical_loss": 5.430851365417862,
+      "tokens_seen": 42991616
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007975446339017052,
+      "loss": 3.252,
+      "theoretical_loss": 5.429574903033627,
+      "tokens_seen": 43057152
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007975285857572718,
+      "loss": 3.0309,
+      "theoretical_loss": 5.428300925097373,
+      "tokens_seen": 43122688
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007975125376128386,
+      "loss": 3.0413,
+      "theoretical_loss": 5.427029423008732,
+      "tokens_seen": 43188224
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007974964894684052,
+      "loss": 3.1815,
+      "theoretical_loss": 5.425760388210108,
+      "tokens_seen": 43253760
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.000797480441323972,
+      "loss": 3.0532,
+      "theoretical_loss": 5.424493812186386,
+      "tokens_seen": 43319296
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007974643931795386,
+      "loss": 3.0778,
+      "theoretical_loss": 5.423229686464667,
+      "tokens_seen": 43384832
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007974483450351054,
+      "loss": 3.1643,
+      "theoretical_loss": 5.421968002613992,
+      "tokens_seen": 43450368
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.000797432296890672,
+      "loss": 3.0246,
+      "theoretical_loss": 5.420708752245071,
+      "tokens_seen": 43515904
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007974162487462388,
+      "loss": 3.084,
+      "theoretical_loss": 5.419451927010014,
+      "tokens_seen": 43581440
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007974002006018054,
+      "loss": 2.9828,
+      "theoretical_loss": 5.418197518602068,
+      "tokens_seen": 43646976
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007973841524573722,
+      "loss": 3.2811,
+      "theoretical_loss": 5.41694551875535,
+      "tokens_seen": 43712512
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007973681043129388,
+      "loss": 3.0724,
+      "theoretical_loss": 5.415695919244586,
+      "tokens_seen": 43778048
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007973520561685056,
+      "loss": 3.2274,
+      "theoretical_loss": 5.41444871188485,
+      "tokens_seen": 43843584
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007973360080240722,
+      "loss": 3.0714,
+      "theoretical_loss": 5.413203888531308,
+      "tokens_seen": 43909120
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.000797319959879639,
+      "loss": 3.1786,
+      "theoretical_loss": 5.411961441078963,
+      "tokens_seen": 43974656
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007973039117352056,
+      "loss": 3.0765,
+      "theoretical_loss": 5.410721361462395,
+      "tokens_seen": 44040192
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007972878635907724,
+      "loss": 2.9274,
+      "theoretical_loss": 5.409483641655516,
+      "tokens_seen": 44105728
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.000797271815446339,
+      "loss": 2.9688,
+      "theoretical_loss": 5.408248273671317,
+      "tokens_seen": 44171264
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007972557673019058,
+      "loss": 3.1094,
+      "theoretical_loss": 5.407015249561624,
+      "tokens_seen": 44236800
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007972397191574724,
+      "loss": 3.0363,
+      "theoretical_loss": 5.405784561416844,
+      "tokens_seen": 44302336
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007972236710130392,
+      "loss": 3.1509,
+      "theoretical_loss": 5.404556201365727,
+      "tokens_seen": 44367872
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007972076228686058,
+      "loss": 3.1461,
+      "theoretical_loss": 5.403330161575123,
+      "tokens_seen": 44433408
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007971915747241726,
+      "loss": 3.2156,
+      "theoretical_loss": 5.402106434249742,
+      "tokens_seen": 44498944
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007971755265797392,
+      "loss": 2.9571,
+      "theoretical_loss": 5.400885011631912,
+      "tokens_seen": 44564480
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.000797159478435306,
+      "loss": 3.115,
+      "theoretical_loss": 5.399665886001349,
+      "tokens_seen": 44630016
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007971434302908726,
+      "loss": 3.095,
+      "theoretical_loss": 5.398449049674916,
+      "tokens_seen": 44695552
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007971273821464394,
+      "loss": 2.8581,
+      "theoretical_loss": 5.397234495006392,
+      "tokens_seen": 44761088
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.000797111334002006,
+      "loss": 3.0027,
+      "theoretical_loss": 5.396022214386246,
+      "tokens_seen": 44826624
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007970952858575728,
+      "loss": 3.0842,
+      "theoretical_loss": 5.394812200241393,
+      "tokens_seen": 44892160
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007970792377131394,
+      "loss": 3.1089,
+      "theoretical_loss": 5.393604445034988,
+      "tokens_seen": 44957696
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007970631895687062,
+      "loss": 2.9782,
+      "theoretical_loss": 5.392398941266183,
+      "tokens_seen": 45023232
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007970471414242728,
+      "loss": 2.982,
+      "theoretical_loss": 5.391195681469908,
+      "tokens_seen": 45088768
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007970310932798396,
+      "loss": 3.0419,
+      "theoretical_loss": 5.389994658216651,
+      "tokens_seen": 45154304
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007970150451354062,
+      "loss": 3.0054,
+      "theoretical_loss": 5.388795864112235,
+      "tokens_seen": 45219840
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.000796998996990973,
+      "loss": 2.9654,
+      "theoretical_loss": 5.387599291797599,
+      "tokens_seen": 45285376
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007969829488465396,
+      "loss": 2.9248,
+      "theoretical_loss": 5.386404933948584,
+      "tokens_seen": 45350912
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007969669007021064,
+      "loss": 2.9716,
+      "theoretical_loss": 5.38521278327571,
+      "tokens_seen": 45416448
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.000796950852557673,
+      "loss": 2.9865,
+      "theoretical_loss": 5.384022832523971,
+      "tokens_seen": 45481984
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007969348044132398,
+      "loss": 3.0551,
+      "theoretical_loss": 5.382835074472616,
+      "tokens_seen": 45547520
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007969187562688065,
+      "loss": 3.0242,
+      "theoretical_loss": 5.381649501934939,
+      "tokens_seen": 45613056
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007969027081243732,
+      "loss": 3.1321,
+      "theoretical_loss": 5.3804661077580755,
+      "tokens_seen": 45678592
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007968866599799399,
+      "loss": 2.9587,
+      "theoretical_loss": 5.379284884822787,
+      "tokens_seen": 45744128
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007968706118355066,
+      "loss": 3.1394,
+      "theoretical_loss": 5.37810582604326,
+      "tokens_seen": 45809664
+    },
+    {
+      "epoch": 0.01,
+      "objective/train/docs_used": 24846,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 3.0651895999908447,
+      "objective/train/theoretical_loss": 5.376928924366901,
+      "objective/train/tokens_used": 66335200,
+      "theoretical_loss": 5.376928924366901,
+      "tokens_seen": 45875200
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007968545636910733,
+      "loss": 3.0652,
+      "theoretical_loss": 5.376928924366901,
+      "tokens_seen": 45875200
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00079683851554664,
+      "loss": 2.9256,
+      "theoretical_loss": 5.37575417277413,
+      "tokens_seen": 45940736
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007968224674022067,
+      "loss": 2.9612,
+      "theoretical_loss": 5.374581564278184,
+      "tokens_seen": 46006272
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007968064192577734,
+      "loss": 2.9538,
+      "theoretical_loss": 5.373411091924918,
+      "tokens_seen": 46071808
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007967903711133401,
+      "loss": 2.986,
+      "theoretical_loss": 5.372242748792598,
+      "tokens_seen": 46137344
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007967743229689068,
+      "loss": 2.8112,
+      "theoretical_loss": 5.371076527991711,
+      "tokens_seen": 46202880
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007967582748244735,
+      "loss": 2.935,
+      "theoretical_loss": 5.36991242266477,
+      "tokens_seen": 46268416
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007967422266800402,
+      "loss": 2.9079,
+      "theoretical_loss": 5.36875042598612,
+      "tokens_seen": 46333952
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007967261785356069,
+      "loss": 3.0976,
+      "theoretical_loss": 5.3675905311617385,
+      "tokens_seen": 46399488
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007967101303911736,
+      "loss": 2.9532,
+      "theoretical_loss": 5.366432731429054,
+      "tokens_seen": 46465024
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007966940822467403,
+      "loss": 3.0112,
+      "theoretical_loss": 5.365277020056752,
+      "tokens_seen": 46530560
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.000796678034102307,
+      "loss": 3.0015,
+      "theoretical_loss": 5.364123390344582,
+      "tokens_seen": 46596096
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007966619859578737,
+      "loss": 2.9189,
+      "theoretical_loss": 5.362971835623181,
+      "tokens_seen": 46661632
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007966459378134404,
+      "loss": 2.8917,
+      "theoretical_loss": 5.361822349253881,
+      "tokens_seen": 46727168
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007966298896690071,
+      "loss": 2.9613,
+      "theoretical_loss": 5.360674924628524,
+      "tokens_seen": 46792704
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007966138415245738,
+      "loss": 2.9993,
+      "theoretical_loss": 5.359529555169287,
+      "tokens_seen": 46858240
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007965977933801405,
+      "loss": 2.9576,
+      "theoretical_loss": 5.358386234328487,
+      "tokens_seen": 46923776
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007965817452357072,
+      "loss": 2.8915,
+      "theoretical_loss": 5.357244955588417,
+      "tokens_seen": 46989312
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007965656970912739,
+      "loss": 2.8678,
+      "theoretical_loss": 5.356105712461154,
+      "tokens_seen": 47054848
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007965496489468405,
+      "loss": 2.9256,
+      "theoretical_loss": 5.354968498488391,
+      "tokens_seen": 47120384
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007965336008024073,
+      "loss": 2.9371,
+      "theoretical_loss": 5.353833307241256,
+      "tokens_seen": 47185920
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007965175526579739,
+      "loss": 3.0885,
+      "theoretical_loss": 5.352700132320135,
+      "tokens_seen": 47251456
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007965015045135407,
+      "loss": 2.8375,
+      "theoretical_loss": 5.351568967354504,
+      "tokens_seen": 47316992
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007964854563691073,
+      "loss": 2.8839,
+      "theoretical_loss": 5.350439806002757,
+      "tokens_seen": 47382528
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007964694082246741,
+      "loss": 2.999,
+      "theoretical_loss": 5.349312641952025,
+      "tokens_seen": 47448064
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007964533600802407,
+      "loss": 2.9293,
+      "theoretical_loss": 5.348187468918022,
+      "tokens_seen": 47513600
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007964373119358075,
+      "loss": 2.7788,
+      "theoretical_loss": 5.347064280644861,
+      "tokens_seen": 47579136
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007964212637913741,
+      "loss": 2.8874,
+      "theoretical_loss": 5.345943070904897,
+      "tokens_seen": 47644672
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007964052156469408,
+      "loss": 2.9413,
+      "theoretical_loss": 5.344823833498557,
+      "tokens_seen": 47710208
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007963891675025075,
+      "loss": 2.9782,
+      "theoretical_loss": 5.343706562254177,
+      "tokens_seen": 47775744
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007963731193580743,
+      "loss": 2.9903,
+      "theoretical_loss": 5.342591251027833,
+      "tokens_seen": 47841280
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007963570712136409,
+      "loss": 2.8307,
+      "theoretical_loss": 5.3414778937031855,
+      "tokens_seen": 47906816
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007963410230692077,
+      "loss": 2.9094,
+      "theoretical_loss": 5.340366484191316,
+      "tokens_seen": 47972352
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007963249749247743,
+      "loss": 2.8458,
+      "theoretical_loss": 5.339257016430563,
+      "tokens_seen": 48037888
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007963089267803411,
+      "loss": 2.814,
+      "theoretical_loss": 5.338149484386367,
+      "tokens_seen": 48103424
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007962928786359077,
+      "loss": 2.9672,
+      "theoretical_loss": 5.337043882051111,
+      "tokens_seen": 48168960
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007962768304914745,
+      "loss": 2.9521,
+      "theoretical_loss": 5.3359402034439665,
+      "tokens_seen": 48234496
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007962607823470411,
+      "loss": 2.863,
+      "theoretical_loss": 5.334838442610732,
+      "tokens_seen": 48300032
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007962447342026079,
+      "loss": 2.7467,
+      "theoretical_loss": 5.333738593623686,
+      "tokens_seen": 48365568
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007962286860581745,
+      "loss": 3.0181,
+      "theoretical_loss": 5.332640650581424,
+      "tokens_seen": 48431104
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007962126379137413,
+      "loss": 2.9079,
+      "theoretical_loss": 5.331544607608718,
+      "tokens_seen": 48496640
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007961965897693079,
+      "loss": 2.9096,
+      "theoretical_loss": 5.330450458856356,
+      "tokens_seen": 48562176
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007961805416248747,
+      "loss": 2.8218,
+      "theoretical_loss": 5.329358198500994,
+      "tokens_seen": 48627712
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007961644934804413,
+      "loss": 2.7594,
+      "theoretical_loss": 5.328267820745011,
+      "tokens_seen": 48693248
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007961484453360081,
+      "loss": 2.8122,
+      "theoretical_loss": 5.327179319816354,
+      "tokens_seen": 48758784
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007961323971915747,
+      "loss": 2.8653,
+      "theoretical_loss": 5.326092689968397,
+      "tokens_seen": 48824320
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007961163490471415,
+      "loss": 2.8692,
+      "theoretical_loss": 5.3250079254797935,
+      "tokens_seen": 48889856
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007961003009027081,
+      "loss": 2.8527,
+      "theoretical_loss": 5.323925020654329,
+      "tokens_seen": 48955392
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007960842527582749,
+      "loss": 2.8064,
+      "theoretical_loss": 5.322843969820781,
+      "tokens_seen": 49020928
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007960682046138415,
+      "loss": 2.7689,
+      "theoretical_loss": 5.321764767332772,
+      "tokens_seen": 49086464
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007960521564694083,
+      "loss": 2.7609,
+      "theoretical_loss": 5.320687407568633,
+      "tokens_seen": 49152000
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007960361083249749,
+      "loss": 2.8155,
+      "theoretical_loss": 5.3196118849312555,
+      "tokens_seen": 49217536
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007960200601805417,
+      "loss": 2.911,
+      "theoretical_loss": 5.31853819384796,
+      "tokens_seen": 49283072
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007960040120361083,
+      "loss": 2.7449,
+      "theoretical_loss": 5.31746632877035,
+      "tokens_seen": 49348608
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007959879638916751,
+      "loss": 2.8588,
+      "theoretical_loss": 5.316396284174177,
+      "tokens_seen": 49414144
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007959719157472417,
+      "loss": 2.7736,
+      "theoretical_loss": 5.3153280545592025,
+      "tokens_seen": 49479680
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007959558676028085,
+      "loss": 2.966,
+      "theoretical_loss": 5.314261634449066,
+      "tokens_seen": 49545216
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007959398194583751,
+      "loss": 2.9891,
+      "theoretical_loss": 5.313197018391143,
+      "tokens_seen": 49610752
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007959237713139419,
+      "loss": 2.8049,
+      "theoretical_loss": 5.312134200956416,
+      "tokens_seen": 49676288
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007959077231695085,
+      "loss": 2.818,
+      "theoretical_loss": 5.3110731767393435,
+      "tokens_seen": 49741824
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007958916750250753,
+      "loss": 2.742,
+      "theoretical_loss": 5.31001394035772,
+      "tokens_seen": 49807360
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007958756268806419,
+      "loss": 2.8855,
+      "theoretical_loss": 5.308956486452549,
+      "tokens_seen": 49872896
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007958595787362087,
+      "loss": 2.6922,
+      "theoretical_loss": 5.307900809687913,
+      "tokens_seen": 49938432
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007958435305917753,
+      "loss": 2.7624,
+      "theoretical_loss": 5.306846904750847,
+      "tokens_seen": 50003968
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007958274824473421,
+      "loss": 2.7657,
+      "theoretical_loss": 5.305794766351202,
+      "tokens_seen": 50069504
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007958114343029087,
+      "loss": 2.6639,
+      "theoretical_loss": 5.304744389221523,
+      "tokens_seen": 50135040
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007957953861584755,
+      "loss": 2.762,
+      "theoretical_loss": 5.303695768116921,
+      "tokens_seen": 50200576
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007957793380140421,
+      "loss": 2.815,
+      "theoretical_loss": 5.302648897814946,
+      "tokens_seen": 50266112
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007957632898696089,
+      "loss": 2.7472,
+      "theoretical_loss": 5.301603773115465,
+      "tokens_seen": 50331648
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007957472417251755,
+      "loss": 2.7874,
+      "theoretical_loss": 5.300560388840532,
+      "tokens_seen": 50397184
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007957311935807423,
+      "loss": 2.776,
+      "theoretical_loss": 5.299518739834273,
+      "tokens_seen": 50462720
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007957151454363089,
+      "loss": 2.7818,
+      "theoretical_loss": 5.298478820962751,
+      "tokens_seen": 50528256
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007956990972918757,
+      "loss": 2.7797,
+      "theoretical_loss": 5.297440627113859,
+      "tokens_seen": 50593792
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007956830491474423,
+      "loss": 2.7686,
+      "theoretical_loss": 5.296404153197189,
+      "tokens_seen": 50659328
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007956670010030091,
+      "loss": 2.7342,
+      "theoretical_loss": 5.295369394143913,
+      "tokens_seen": 50724864
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007956509528585757,
+      "loss": 2.7897,
+      "theoretical_loss": 5.294336344906668,
+      "tokens_seen": 50790400
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007956349047141425,
+      "loss": 2.8069,
+      "theoretical_loss": 5.293305000459435,
+      "tokens_seen": 50855936
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007956188565697091,
+      "loss": 2.6506,
+      "theoretical_loss": 5.292275355797421,
+      "tokens_seen": 50921472
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007956028084252759,
+      "loss": 2.8034,
+      "theoretical_loss": 5.291247405936941,
+      "tokens_seen": 50987008
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007955867602808426,
+      "loss": 2.7252,
+      "theoretical_loss": 5.290221145915307,
+      "tokens_seen": 51052544
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007955707121364093,
+      "loss": 2.7591,
+      "theoretical_loss": 5.289196570790708,
+      "tokens_seen": 51118080
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.000795554663991976,
+      "loss": 2.6543,
+      "theoretical_loss": 5.288173675642094,
+      "tokens_seen": 51183616
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007955386158475427,
+      "loss": 2.6381,
+      "theoretical_loss": 5.287152455569069,
+      "tokens_seen": 51249152
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007955225677031094,
+      "loss": 2.7299,
+      "theoretical_loss": 5.286132905691774,
+      "tokens_seen": 51314688
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007955065195586761,
+      "loss": 2.4976,
+      "theoretical_loss": 5.285115021150775,
+      "tokens_seen": 51380224
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007954904714142428,
+      "loss": 2.6586,
+      "theoretical_loss": 5.284098797106948,
+      "tokens_seen": 51445760
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007954744232698095,
+      "loss": 2.7922,
+      "theoretical_loss": 5.283084228741377,
+      "tokens_seen": 51511296
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007954583751253762,
+      "loss": 2.6488,
+      "theoretical_loss": 5.282071311255235,
+      "tokens_seen": 51576832
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007954423269809429,
+      "loss": 2.8,
+      "theoretical_loss": 5.2810600398696845,
+      "tokens_seen": 51642368
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007954262788365096,
+      "loss": 2.6972,
+      "theoretical_loss": 5.2800504098257575,
+      "tokens_seen": 51707904
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007954102306920763,
+      "loss": 2.6349,
+      "theoretical_loss": 5.279042416384257,
+      "tokens_seen": 51773440
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.000795394182547643,
+      "loss": 2.6639,
+      "theoretical_loss": 5.278036054825645,
+      "tokens_seen": 51838976
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007953781344032097,
+      "loss": 2.6419,
+      "theoretical_loss": 5.277031320449941,
+      "tokens_seen": 51904512
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007953620862587764,
+      "loss": 2.7968,
+      "theoretical_loss": 5.27602820857661,
+      "tokens_seen": 51970048
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007953460381143431,
+      "loss": 2.6677,
+      "theoretical_loss": 5.275026714544463,
+      "tokens_seen": 52035584
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007953299899699098,
+      "loss": 2.6115,
+      "theoretical_loss": 5.27402683371155,
+      "tokens_seen": 52101120
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007953139418254765,
+      "loss": 2.6203,
+      "theoretical_loss": 5.273028561455056,
+      "tokens_seen": 52166656
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007952978936810432,
+      "loss": 2.7309,
+      "theoretical_loss": 5.272031893171203,
+      "tokens_seen": 52232192
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007952818455366099,
+      "loss": 2.7589,
+      "theoretical_loss": 5.27103682427514,
+      "tokens_seen": 52297728
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007952657973921766,
+      "loss": 2.7783,
+      "theoretical_loss": 5.270043350200851,
+      "tokens_seen": 52363264
+    },
+    {
+      "epoch": 0.02,
+      "objective/train/docs_used": 27101,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 2.5798048973083496,
+      "objective/train/theoretical_loss": 5.269051466401043,
+      "objective/train/tokens_used": 72888800,
+      "theoretical_loss": 5.269051466401043,
+      "tokens_seen": 52428800
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007952497492477433,
+      "loss": 2.5798,
+      "theoretical_loss": 5.269051466401043,
+      "tokens_seen": 52428800
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.00079523370110331,
+      "loss": 2.748,
+      "theoretical_loss": 5.268061168347058,
+      "tokens_seen": 52494336
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007952176529588767,
+      "loss": 2.5742,
+      "theoretical_loss": 5.267072451528764,
+      "tokens_seen": 52559872
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007952016048144434,
+      "loss": 2.4865,
+      "theoretical_loss": 5.266085311454461,
+      "tokens_seen": 52625408
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007951855566700101,
+      "loss": 2.7233,
+      "theoretical_loss": 5.265099743650779,
+      "tokens_seen": 52690944
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007951695085255768,
+      "loss": 2.6638,
+      "theoretical_loss": 5.264115743662588,
+      "tokens_seen": 52756480
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007951534603811434,
+      "loss": 2.7692,
+      "theoretical_loss": 5.26313330705289,
+      "tokens_seen": 52822016
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007951374122367102,
+      "loss": 2.5976,
+      "theoretical_loss": 5.262152429402734,
+      "tokens_seen": 52887552
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007951213640922768,
+      "loss": 2.7364,
+      "theoretical_loss": 5.261173106311116,
+      "tokens_seen": 52953088
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007951053159478436,
+      "loss": 2.5708,
+      "theoretical_loss": 5.2601953333948765,
+      "tokens_seen": 53018624
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007950892678034102,
+      "loss": 2.6982,
+      "theoretical_loss": 5.25921910628862,
+      "tokens_seen": 53084160
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.000795073219658977,
+      "loss": 2.6809,
+      "theoretical_loss": 5.258244420644611,
+      "tokens_seen": 53149696
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007950571715145436,
+      "loss": 2.7632,
+      "theoretical_loss": 5.257271272132685,
+      "tokens_seen": 53215232
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007950411233701104,
+      "loss": 2.7068,
+      "theoretical_loss": 5.256299656440154,
+      "tokens_seen": 53280768
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.000795025075225677,
+      "loss": 2.7195,
+      "theoretical_loss": 5.255329569271717,
+      "tokens_seen": 53346304
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007950090270812438,
+      "loss": 2.5318,
+      "theoretical_loss": 5.254361006349367,
+      "tokens_seen": 53411840
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007949929789368104,
+      "loss": 2.6181,
+      "theoretical_loss": 5.2533939634122975,
+      "tokens_seen": 53477376
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007949769307923772,
+      "loss": 2.7352,
+      "theoretical_loss": 5.252428436216821,
+      "tokens_seen": 53542912
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007949608826479438,
+      "loss": 2.6394,
+      "theoretical_loss": 5.251464420536269,
+      "tokens_seen": 53608448
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007949448345035106,
+      "loss": 2.668,
+      "theoretical_loss": 5.250501912160911,
+      "tokens_seen": 53673984
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007949287863590772,
+      "loss": 2.7109,
+      "theoretical_loss": 5.24954090689786,
+      "tokens_seen": 53739520
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.000794912738214644,
+      "loss": 2.5298,
+      "theoretical_loss": 5.248581400570989,
+      "tokens_seen": 53805056
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007948966900702106,
+      "loss": 2.6028,
+      "theoretical_loss": 5.247623389020841,
+      "tokens_seen": 53870592
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007948806419257774,
+      "loss": 2.5502,
+      "theoretical_loss": 5.246666868104547,
+      "tokens_seen": 53936128
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.000794864593781344,
+      "loss": 2.5574,
+      "theoretical_loss": 5.245711833695731,
+      "tokens_seen": 54001664
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007948485456369108,
+      "loss": 2.6273,
+      "theoretical_loss": 5.244758281684433,
+      "tokens_seen": 54067200
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007948324974924774,
+      "loss": 2.5492,
+      "theoretical_loss": 5.24380620797702,
+      "tokens_seen": 54132736
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007948164493480442,
+      "loss": 2.6207,
+      "theoretical_loss": 5.242855608496102,
+      "tokens_seen": 54198272
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007948004012036108,
+      "loss": 2.5862,
+      "theoretical_loss": 5.2419064791804475,
+      "tokens_seen": 54263808
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007947843530591776,
+      "loss": 2.6388,
+      "theoretical_loss": 5.240958815984901,
+      "tokens_seen": 54329344
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007947683049147442,
+      "loss": 2.5906,
+      "theoretical_loss": 5.2400126148803,
+      "tokens_seen": 54394880
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.000794752256770311,
+      "loss": 2.5325,
+      "theoretical_loss": 5.23906787185339,
+      "tokens_seen": 54460416
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007947362086258776,
+      "loss": 2.5763,
+      "theoretical_loss": 5.238124582906748,
+      "tokens_seen": 54525952
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007947201604814444,
+      "loss": 2.8481,
+      "theoretical_loss": 5.237182744058696,
+      "tokens_seen": 54591488
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.000794704112337011,
+      "loss": 2.4253,
+      "theoretical_loss": 5.236242351343224,
+      "tokens_seen": 54657024
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007946880641925778,
+      "loss": 2.5232,
+      "theoretical_loss": 5.235303400809904,
+      "tokens_seen": 54722560
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007946720160481444,
+      "loss": 2.6427,
+      "theoretical_loss": 5.2343658885238185,
+      "tokens_seen": 54788096
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007946559679037112,
+      "loss": 2.5511,
+      "theoretical_loss": 5.2334298105654735,
+      "tokens_seen": 54853632
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007946399197592778,
+      "loss": 2.5845,
+      "theoretical_loss": 5.232495163030723,
+      "tokens_seen": 54919168
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007946238716148446,
+      "loss": 2.572,
+      "theoretical_loss": 5.231561942030694,
+      "tokens_seen": 54984704
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007946078234704112,
+      "loss": 2.6078,
+      "theoretical_loss": 5.230630143691698,
+      "tokens_seen": 55050240
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.000794591775325978,
+      "loss": 2.5558,
+      "theoretical_loss": 5.22969976415517,
+      "tokens_seen": 55115776
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007945757271815446,
+      "loss": 2.6206,
+      "theoretical_loss": 5.228770799577576,
+      "tokens_seen": 55181312
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007945596790371114,
+      "loss": 2.4544,
+      "theoretical_loss": 5.227843246130343,
+      "tokens_seen": 55246848
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.000794543630892678,
+      "loss": 2.5025,
+      "theoretical_loss": 5.2269170999997865,
+      "tokens_seen": 55312384
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007945275827482448,
+      "loss": 2.5459,
+      "theoretical_loss": 5.225992357387031,
+      "tokens_seen": 55377920
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007945115346038114,
+      "loss": 2.5993,
+      "theoretical_loss": 5.225069014507933,
+      "tokens_seen": 55443456
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007944954864593782,
+      "loss": 2.5679,
+      "theoretical_loss": 5.224147067593012,
+      "tokens_seen": 55508992
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007944794383149448,
+      "loss": 2.5954,
+      "theoretical_loss": 5.223226512887375,
+      "tokens_seen": 55574528
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007944633901705116,
+      "loss": 2.5244,
+      "theoretical_loss": 5.222307346650636,
+      "tokens_seen": 55640064
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007944473420260782,
+      "loss": 2.4612,
+      "theoretical_loss": 5.221389565156855,
+      "tokens_seen": 55705600
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.000794431293881645,
+      "loss": 2.5992,
+      "theoretical_loss": 5.2204731646944555,
+      "tokens_seen": 55771136
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007944152457372116,
+      "loss": 2.4763,
+      "theoretical_loss": 5.219558141566156,
+      "tokens_seen": 55836672
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007943991975927784,
+      "loss": 2.5833,
+      "theoretical_loss": 5.218644492088899,
+      "tokens_seen": 55902208
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.000794383149448345,
+      "loss": 2.5174,
+      "theoretical_loss": 5.217732212593778,
+      "tokens_seen": 55967744
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007943671013039118,
+      "loss": 2.4826,
+      "theoretical_loss": 5.216821299425968,
+      "tokens_seen": 56033280
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007943510531594784,
+      "loss": 2.6457,
+      "theoretical_loss": 5.215911748944652,
+      "tokens_seen": 56098816
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007943350050150452,
+      "loss": 2.5952,
+      "theoretical_loss": 5.215003557522959,
+      "tokens_seen": 56164352
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007943189568706118,
+      "loss": 2.4906,
+      "theoretical_loss": 5.21409672154788,
+      "tokens_seen": 56229888
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007943029087261786,
+      "loss": 2.5894,
+      "theoretical_loss": 5.213191237420215,
+      "tokens_seen": 56295424
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007942868605817452,
+      "loss": 2.5959,
+      "theoretical_loss": 5.212287101554496,
+      "tokens_seen": 56360960
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.000794270812437312,
+      "loss": 2.5346,
+      "theoretical_loss": 5.211384310378916,
+      "tokens_seen": 56426496
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007942547642928786,
+      "loss": 2.5443,
+      "theoretical_loss": 5.210482860335269,
+      "tokens_seen": 56492032
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007942387161484454,
+      "loss": 2.5812,
+      "theoretical_loss": 5.2095827478788745,
+      "tokens_seen": 56557568
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.000794222668004012,
+      "loss": 2.4374,
+      "theoretical_loss": 5.208683969478518,
+      "tokens_seen": 56623104
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007942066198595788,
+      "loss": 2.4504,
+      "theoretical_loss": 5.2077865216163755,
+      "tokens_seen": 56688640
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007941905717151455,
+      "loss": 2.4962,
+      "theoretical_loss": 5.206890400787959,
+      "tokens_seen": 56754176
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007941745235707122,
+      "loss": 2.4319,
+      "theoretical_loss": 5.205995603502039,
+      "tokens_seen": 56819712
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007941584754262789,
+      "loss": 2.4834,
+      "theoretical_loss": 5.205102126280586,
+      "tokens_seen": 56885248
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007941424272818456,
+      "loss": 2.5599,
+      "theoretical_loss": 5.204209965658704,
+      "tokens_seen": 56950784
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007941263791374123,
+      "loss": 2.5048,
+      "theoretical_loss": 5.203319118184565,
+      "tokens_seen": 57016320
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.000794110330992979,
+      "loss": 2.604,
+      "theoretical_loss": 5.202429580419343,
+      "tokens_seen": 57081856
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007940942828485457,
+      "loss": 2.4432,
+      "theoretical_loss": 5.201541348937157,
+      "tokens_seen": 57147392
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007940782347041124,
+      "loss": 2.5292,
+      "theoretical_loss": 5.200654420324998,
+      "tokens_seen": 57212928
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007940621865596791,
+      "loss": 2.4261,
+      "theoretical_loss": 5.199768791182677,
+      "tokens_seen": 57278464
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007940461384152458,
+      "loss": 2.4407,
+      "theoretical_loss": 5.198884458122748,
+      "tokens_seen": 57344000
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007940300902708125,
+      "loss": 2.3966,
+      "theoretical_loss": 5.198001417770461,
+      "tokens_seen": 57409536
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007940140421263792,
+      "loss": 2.5058,
+      "theoretical_loss": 5.1971196667636885,
+      "tokens_seen": 57475072
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007939979939819459,
+      "loss": 2.5757,
+      "theoretical_loss": 5.196239201752871,
+      "tokens_seen": 57540608
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007939819458375126,
+      "loss": 2.4193,
+      "theoretical_loss": 5.19536001940095,
+      "tokens_seen": 57606144
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007939658976930794,
+      "loss": 2.4897,
+      "theoretical_loss": 5.1944821163833135,
+      "tokens_seen": 57671680
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.000793949849548646,
+      "loss": 2.4132,
+      "theoretical_loss": 5.19360548938773,
+      "tokens_seen": 57737216
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007939338014042128,
+      "loss": 2.4693,
+      "theoretical_loss": 5.192730135114292,
+      "tokens_seen": 57802752
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007939177532597794,
+      "loss": 2.4497,
+      "theoretical_loss": 5.191856050275353,
+      "tokens_seen": 57868288
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007939017051153462,
+      "loss": 2.4771,
+      "theoretical_loss": 5.190983231595473,
+      "tokens_seen": 57933824
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007938856569709128,
+      "loss": 2.4195,
+      "theoretical_loss": 5.1901116758113535,
+      "tokens_seen": 57999360
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007938696088264795,
+      "loss": 2.4006,
+      "theoretical_loss": 5.189241379671782,
+      "tokens_seen": 58064896
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007938535606820462,
+      "loss": 2.5972,
+      "theoretical_loss": 5.188372339937576,
+      "tokens_seen": 58130432
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007938375125376129,
+      "loss": 2.4239,
+      "theoretical_loss": 5.187504553381521,
+      "tokens_seen": 58195968
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007938214643931796,
+      "loss": 2.6121,
+      "theoretical_loss": 5.1866380167883115,
+      "tokens_seen": 58261504
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007938054162487463,
+      "loss": 2.5954,
+      "theoretical_loss": 5.1857727269545,
+      "tokens_seen": 58327040
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.000793789368104313,
+      "loss": 2.4094,
+      "theoretical_loss": 5.184908680688437,
+      "tokens_seen": 58392576
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007937733199598797,
+      "loss": 2.4097,
+      "theoretical_loss": 5.18404587481021,
+      "tokens_seen": 58458112
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007937572718154463,
+      "loss": 2.4062,
+      "theoretical_loss": 5.183184306151595,
+      "tokens_seen": 58523648
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007937412236710131,
+      "loss": 2.3971,
+      "theoretical_loss": 5.182323971555995,
+      "tokens_seen": 58589184
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007937251755265797,
+      "loss": 2.477,
+      "theoretical_loss": 5.181464867878384,
+      "tokens_seen": 58654720
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007937091273821465,
+      "loss": 2.6,
+      "theoretical_loss": 5.1806069919852575,
+      "tokens_seen": 58720256
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007936930792377131,
+      "loss": 2.5417,
+      "theoretical_loss": 5.17975034075457,
+      "tokens_seen": 58785792
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007936770310932799,
+      "loss": 2.3662,
+      "theoretical_loss": 5.1788949110756874,
+      "tokens_seen": 58851328
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007936609829488465,
+      "loss": 2.4413,
+      "theoretical_loss": 5.178040699849327,
+      "tokens_seen": 58916864
+    },
+    {
+      "epoch": 0.02,
+      "objective/train/docs_used": 29491,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 2.4602999687194824,
+      "objective/train/theoretical_loss": 5.1771877039875065,
+      "objective/train/tokens_used": 79442400,
+      "theoretical_loss": 5.1771877039875065,
+      "tokens_seen": 58982400
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007936449348044133,
+      "loss": 2.4603,
+      "theoretical_loss": 5.1771877039875065,
+      "tokens_seen": 58982400
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007936288866599799,
+      "loss": 2.4153,
+      "theoretical_loss": 5.176335920413489,
+      "tokens_seen": 59047936
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007936128385155467,
+      "loss": 2.6104,
+      "theoretical_loss": 5.175485346061732,
+      "tokens_seen": 59113472
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007935967903711133,
+      "loss": 2.4952,
+      "theoretical_loss": 5.174635977877831,
+      "tokens_seen": 59179008
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007935807422266801,
+      "loss": 2.3725,
+      "theoretical_loss": 5.173787812818468,
+      "tokens_seen": 59244544
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007935646940822467,
+      "loss": 2.2532,
+      "theoretical_loss": 5.172940847851359,
+      "tokens_seen": 59310080
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007935486459378135,
+      "loss": 2.4305,
+      "theoretical_loss": 5.172095079955204,
+      "tokens_seen": 59375616
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007935325977933801,
+      "loss": 2.4633,
+      "theoretical_loss": 5.171250506119634,
+      "tokens_seen": 59441152
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007935165496489469,
+      "loss": 2.4208,
+      "theoretical_loss": 5.170407123345155,
+      "tokens_seen": 59506688
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007935005015045135,
+      "loss": 2.4468,
+      "theoretical_loss": 5.169564928643103,
+      "tokens_seen": 59572224
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007934844533600803,
+      "loss": 2.3946,
+      "theoretical_loss": 5.168723919035591,
+      "tokens_seen": 59637760
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007934684052156469,
+      "loss": 2.4687,
+      "theoretical_loss": 5.167884091555456,
+      "tokens_seen": 59703296
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007934523570712137,
+      "loss": 2.4246,
+      "theoretical_loss": 5.167045443246213,
+      "tokens_seen": 59768832
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007934363089267803,
+      "loss": 2.4423,
+      "theoretical_loss": 5.166207971162001,
+      "tokens_seen": 59834368
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007934202607823471,
+      "loss": 2.3983,
+      "theoretical_loss": 5.1653716723675345,
+      "tokens_seen": 59899904
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007934042126379137,
+      "loss": 2.4609,
+      "theoretical_loss": 5.164536543938056,
+      "tokens_seen": 59965440
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007933881644934805,
+      "loss": 2.3967,
+      "theoretical_loss": 5.163702582959283,
+      "tokens_seen": 60030976
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007933721163490471,
+      "loss": 2.3973,
+      "theoretical_loss": 5.162869786527363,
+      "tokens_seen": 60096512
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007933560682046139,
+      "loss": 2.4105,
+      "theoretical_loss": 5.1620381517488205,
+      "tokens_seen": 60162048
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007933400200601805,
+      "loss": 2.4944,
+      "theoretical_loss": 5.161207675740515,
+      "tokens_seen": 60227584
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007933239719157473,
+      "loss": 2.3914,
+      "theoretical_loss": 5.160378355629584,
+      "tokens_seen": 60293120
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007933079237713139,
+      "loss": 2.2923,
+      "theoretical_loss": 5.159550188553405,
+      "tokens_seen": 60358656
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007932918756268807,
+      "loss": 2.4562,
+      "theoretical_loss": 5.15872317165954,
+      "tokens_seen": 60424192
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007932758274824473,
+      "loss": 2.2974,
+      "theoretical_loss": 5.1578973021056935,
+      "tokens_seen": 60489728
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007932597793380141,
+      "loss": 2.3379,
+      "theoretical_loss": 5.157072577059662,
+      "tokens_seen": 60555264
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007932437311935807,
+      "loss": 2.4035,
+      "theoretical_loss": 5.15624899369929,
+      "tokens_seen": 60620800
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007932276830491475,
+      "loss": 2.4439,
+      "theoretical_loss": 5.155426549212422,
+      "tokens_seen": 60686336
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007932116349047141,
+      "loss": 2.1819,
+      "theoretical_loss": 5.154605240796855,
+      "tokens_seen": 60751872
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007931955867602809,
+      "loss": 2.3471,
+      "theoretical_loss": 5.153785065660296,
+      "tokens_seen": 60817408
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007931795386158475,
+      "loss": 2.4269,
+      "theoretical_loss": 5.152966021020315,
+      "tokens_seen": 60882944
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007931634904714143,
+      "loss": 2.3903,
+      "theoretical_loss": 5.152148104104297,
+      "tokens_seen": 60948480
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007931474423269809,
+      "loss": 2.2739,
+      "theoretical_loss": 5.1513313121494,
+      "tokens_seen": 61014016
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007931313941825477,
+      "loss": 2.3234,
+      "theoretical_loss": 5.15051564240251,
+      "tokens_seen": 61079552
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007931153460381143,
+      "loss": 2.3041,
+      "theoretical_loss": 5.149701092120194,
+      "tokens_seen": 61145088
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007930992978936811,
+      "loss": 2.3438,
+      "theoretical_loss": 5.148887658568658,
+      "tokens_seen": 61210624
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007930832497492477,
+      "loss": 2.4146,
+      "theoretical_loss": 5.148075339023699,
+      "tokens_seen": 61276160
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007930672016048145,
+      "loss": 2.3163,
+      "theoretical_loss": 5.147264130770671,
+      "tokens_seen": 61341696
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007930511534603811,
+      "loss": 2.3089,
+      "theoretical_loss": 5.146454031104426,
+      "tokens_seen": 61407232
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007930351053159479,
+      "loss": 2.445,
+      "theoretical_loss": 5.1456450373292855,
+      "tokens_seen": 61472768
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007930190571715145,
+      "loss": 2.404,
+      "theoretical_loss": 5.144837146758987,
+      "tokens_seen": 61538304
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007930030090270813,
+      "loss": 2.4489,
+      "theoretical_loss": 5.144030356716646,
+      "tokens_seen": 61603840
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007929869608826479,
+      "loss": 2.5282,
+      "theoretical_loss": 5.1432246645347135,
+      "tokens_seen": 61669376
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007929709127382147,
+      "loss": 2.317,
+      "theoretical_loss": 5.14242006755493,
+      "tokens_seen": 61734912
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007929548645937813,
+      "loss": 2.4457,
+      "theoretical_loss": 5.141616563128288,
+      "tokens_seen": 61800448
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007929388164493481,
+      "loss": 2.2941,
+      "theoretical_loss": 5.140814148614985,
+      "tokens_seen": 61865984
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007929227683049147,
+      "loss": 2.4758,
+      "theoretical_loss": 5.1400128213843885,
+      "tokens_seen": 61931520
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007929067201604815,
+      "loss": 2.3312,
+      "theoretical_loss": 5.139212578814986,
+      "tokens_seen": 61997056
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007928906720160481,
+      "loss": 2.4919,
+      "theoretical_loss": 5.13841341829435,
+      "tokens_seen": 62062592
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007928746238716149,
+      "loss": 2.4333,
+      "theoretical_loss": 5.137615337219098,
+      "tokens_seen": 62128128
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007928585757271815,
+      "loss": 2.3636,
+      "theoretical_loss": 5.136818332994844,
+      "tokens_seen": 62193664
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007928425275827483,
+      "loss": 2.4269,
+      "theoretical_loss": 5.136022403036167,
+      "tokens_seen": 62259200
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.000792826479438315,
+      "loss": 2.3482,
+      "theoretical_loss": 5.135227544766565,
+      "tokens_seen": 62324736
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007928104312938817,
+      "loss": 2.2758,
+      "theoretical_loss": 5.134433755618414,
+      "tokens_seen": 62390272
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007927943831494484,
+      "loss": 2.3603,
+      "theoretical_loss": 5.1336410330329345,
+      "tokens_seen": 62455808
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007927783350050151,
+      "loss": 2.4681,
+      "theoretical_loss": 5.132849374460145,
+      "tokens_seen": 62521344
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007927622868605819,
+      "loss": 2.2553,
+      "theoretical_loss": 5.132058777358827,
+      "tokens_seen": 62586880
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007927462387161485,
+      "loss": 2.4522,
+      "theoretical_loss": 5.1312692391964845,
+      "tokens_seen": 62652416
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007927301905717153,
+      "loss": 2.3626,
+      "theoretical_loss": 5.1304807574493,
+      "tokens_seen": 62717952
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007927141424272819,
+      "loss": 2.3232,
+      "theoretical_loss": 5.129693329602108,
+      "tokens_seen": 62783488
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007926980942828487,
+      "loss": 2.3389,
+      "theoretical_loss": 5.128906953148342,
+      "tokens_seen": 62849024
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007926820461384153,
+      "loss": 2.487,
+      "theoretical_loss": 5.128121625590007,
+      "tokens_seen": 62914560
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007926659979939821,
+      "loss": 2.4006,
+      "theoretical_loss": 5.127337344437633,
+      "tokens_seen": 62980096
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007926499498495487,
+      "loss": 2.4503,
+      "theoretical_loss": 5.126554107210245,
+      "tokens_seen": 63045632
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007926339017051155,
+      "loss": 2.2779,
+      "theoretical_loss": 5.125771911435321,
+      "tokens_seen": 63111168
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007926178535606821,
+      "loss": 2.2988,
+      "theoretical_loss": 5.124990754648754,
+      "tokens_seen": 63176704
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007926018054162489,
+      "loss": 2.3609,
+      "theoretical_loss": 5.124210634394814,
+      "tokens_seen": 63242240
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007925857572718155,
+      "loss": 2.3058,
+      "theoretical_loss": 5.1234315482261135,
+      "tokens_seen": 63307776
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007925697091273823,
+      "loss": 2.3137,
+      "theoretical_loss": 5.122653493703571,
+      "tokens_seen": 63373312
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007925536609829489,
+      "loss": 2.3022,
+      "theoretical_loss": 5.121876468396373,
+      "tokens_seen": 63438848
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007925376128385157,
+      "loss": 2.4712,
+      "theoretical_loss": 5.121100469881933,
+      "tokens_seen": 63504384
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007925215646940823,
+      "loss": 2.2307,
+      "theoretical_loss": 5.120325495745863,
+      "tokens_seen": 63569920
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007925055165496491,
+      "loss": 2.4119,
+      "theoretical_loss": 5.119551543581933,
+      "tokens_seen": 63635456
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007924894684052157,
+      "loss": 2.296,
+      "theoretical_loss": 5.118778610992038,
+      "tokens_seen": 63700992
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007924734202607824,
+      "loss": 2.2566,
+      "theoretical_loss": 5.118006695586154,
+      "tokens_seen": 63766528
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007924573721163491,
+      "loss": 2.3311,
+      "theoretical_loss": 5.1172357949823155,
+      "tokens_seen": 63832064
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007924413239719158,
+      "loss": 2.3055,
+      "theoretical_loss": 5.116465906806567,
+      "tokens_seen": 63897600
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007924252758274825,
+      "loss": 2.313,
+      "theoretical_loss": 5.11569702869294,
+      "tokens_seen": 63963136
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007924092276830492,
+      "loss": 2.3058,
+      "theoretical_loss": 5.114929158283407,
+      "tokens_seen": 64028672
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007923931795386159,
+      "loss": 2.3471,
+      "theoretical_loss": 5.114162293227853,
+      "tokens_seen": 64094208
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007923771313941826,
+      "loss": 2.3048,
+      "theoretical_loss": 5.113396431184043,
+      "tokens_seen": 64159744
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007923610832497493,
+      "loss": 2.3641,
+      "theoretical_loss": 5.112631569817578,
+      "tokens_seen": 64225280
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.000792345035105316,
+      "loss": 2.3404,
+      "theoretical_loss": 5.111867706801872,
+      "tokens_seen": 64290816
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007923289869608826,
+      "loss": 2.4302,
+      "theoretical_loss": 5.111104839818113,
+      "tokens_seen": 64356352
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007923129388164494,
+      "loss": 2.4105,
+      "theoretical_loss": 5.1103429665552245,
+      "tokens_seen": 64421888
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.000792296890672016,
+      "loss": 2.2838,
+      "theoretical_loss": 5.109582084709841,
+      "tokens_seen": 64487424
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007922808425275828,
+      "loss": 2.3176,
+      "theoretical_loss": 5.108822191986269,
+      "tokens_seen": 64552960
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007922647943831494,
+      "loss": 2.32,
+      "theoretical_loss": 5.108063286096453,
+      "tokens_seen": 64618496
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007922487462387162,
+      "loss": 2.2686,
+      "theoretical_loss": 5.107305364759943,
+      "tokens_seen": 64684032
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007922326980942828,
+      "loss": 2.2104,
+      "theoretical_loss": 5.106548425703869,
+      "tokens_seen": 64749568
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007922166499498496,
+      "loss": 2.3018,
+      "theoretical_loss": 5.1057924666628915,
+      "tokens_seen": 64815104
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007922006018054162,
+      "loss": 2.2446,
+      "theoretical_loss": 5.105037485379189,
+      "tokens_seen": 64880640
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.000792184553660983,
+      "loss": 2.1876,
+      "theoretical_loss": 5.104283479602407,
+      "tokens_seen": 64946176
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007921685055165496,
+      "loss": 2.3042,
+      "theoretical_loss": 5.103530447089642,
+      "tokens_seen": 65011712
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007921524573721164,
+      "loss": 2.2763,
+      "theoretical_loss": 5.102778385605397,
+      "tokens_seen": 65077248
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.000792136409227683,
+      "loss": 2.3011,
+      "theoretical_loss": 5.102027292921557,
+      "tokens_seen": 65142784
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007921203610832498,
+      "loss": 2.2809,
+      "theoretical_loss": 5.101277166817351,
+      "tokens_seen": 65208320
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007921043129388164,
+      "loss": 2.4468,
+      "theoretical_loss": 5.100528005079328,
+      "tokens_seen": 65273856
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007920882647943832,
+      "loss": 2.348,
+      "theoretical_loss": 5.099779805501318,
+      "tokens_seen": 65339392
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007920722166499498,
+      "loss": 2.4444,
+      "theoretical_loss": 5.099032565884409,
+      "tokens_seen": 65404928
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007920561685055166,
+      "loss": 2.2229,
+      "theoretical_loss": 5.098286284036908,
+      "tokens_seen": 65470464
+    },
+    {
+      "epoch": 0.02,
+      "objective/train/docs_used": 32172,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 2.2884788513183594,
+      "objective/train/theoretical_loss": 5.097540957774312,
+      "objective/train/tokens_used": 85996000,
+      "theoretical_loss": 5.097540957774312,
+      "tokens_seen": 65536000
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007920401203610832,
+      "loss": 2.2885,
+      "theoretical_loss": 5.097540957774312,
+      "tokens_seen": 65536000
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.00079202407221665,
+      "loss": 2.144,
+      "theoretical_loss": 5.096796584919282,
+      "tokens_seen": 65601536
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007920080240722166,
+      "loss": 2.236,
+      "theoretical_loss": 5.096053163301608,
+      "tokens_seen": 65667072
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007919919759277834,
+      "loss": 2.2372,
+      "theoretical_loss": 5.095310690758177,
+      "tokens_seen": 65732608
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.00079197592778335,
+      "loss": 2.2302,
+      "theoretical_loss": 5.094569165132949,
+      "tokens_seen": 65798144
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007919598796389168,
+      "loss": 2.1925,
+      "theoretical_loss": 5.09382858427692,
+      "tokens_seen": 65863680
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007919438314944834,
+      "loss": 2.3218,
+      "theoretical_loss": 5.0930889460481,
+      "tokens_seen": 65929216
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007919277833500502,
+      "loss": 2.3288,
+      "theoretical_loss": 5.092350248311471,
+      "tokens_seen": 65994752
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007919117352056168,
+      "loss": 2.2272,
+      "theoretical_loss": 5.091612488938971,
+      "tokens_seen": 66060288
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007918956870611836,
+      "loss": 2.3599,
+      "theoretical_loss": 5.090875665809454,
+      "tokens_seen": 66125824
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007918796389167502,
+      "loss": 2.2867,
+      "theoretical_loss": 5.09013977680867,
+      "tokens_seen": 66191360
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.000791863590772317,
+      "loss": 2.3893,
+      "theoretical_loss": 5.089404819829225,
+      "tokens_seen": 66256896
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007918475426278836,
+      "loss": 2.3538,
+      "theoretical_loss": 5.088670792770563,
+      "tokens_seen": 66322432
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007918314944834504,
+      "loss": 2.3108,
+      "theoretical_loss": 5.087937693538927,
+      "tokens_seen": 66387968
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.000791815446339017,
+      "loss": 2.3665,
+      "theoretical_loss": 5.0872055200473385,
+      "tokens_seen": 66453504
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007917993981945838,
+      "loss": 2.3964,
+      "theoretical_loss": 5.0864742702155645,
+      "tokens_seen": 66519040
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007917833500501504,
+      "loss": 2.1746,
+      "theoretical_loss": 5.0857439419700885,
+      "tokens_seen": 66584576
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007917673019057172,
+      "loss": 2.2295,
+      "theoretical_loss": 5.085014533244088,
+      "tokens_seen": 66650112
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007917512537612838,
+      "loss": 2.4937,
+      "theoretical_loss": 5.084286041977396,
+      "tokens_seen": 66715648
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007917352056168506,
+      "loss": 2.3194,
+      "theoretical_loss": 5.083558466116487,
+      "tokens_seen": 66781184
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007917191574724172,
+      "loss": 2.1903,
+      "theoretical_loss": 5.082831803614433,
+      "tokens_seen": 66846720
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.000791703109327984,
+      "loss": 2.2532,
+      "theoretical_loss": 5.082106052430891,
+      "tokens_seen": 66912256
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007916870611835506,
+      "loss": 2.3094,
+      "theoretical_loss": 5.081381210532063,
+      "tokens_seen": 66977792
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007916710130391174,
+      "loss": 2.4095,
+      "theoretical_loss": 5.08065727589068,
+      "tokens_seen": 67043328
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.000791654964894684,
+      "loss": 2.406,
+      "theoretical_loss": 5.079934246485963,
+      "tokens_seen": 67108864
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007916389167502508,
+      "loss": 2.1762,
+      "theoretical_loss": 5.079212120303606,
+      "tokens_seen": 67174400
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007916228686058174,
+      "loss": 2.3653,
+      "theoretical_loss": 5.078490895335744,
+      "tokens_seen": 67239936
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007916068204613842,
+      "loss": 2.1304,
+      "theoretical_loss": 5.077770569580924,
+      "tokens_seen": 67305472
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.000791590772316951,
+      "loss": 2.3093,
+      "theoretical_loss": 5.077051141044086,
+      "tokens_seen": 67371008
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007915747241725176,
+      "loss": 2.2845,
+      "theoretical_loss": 5.076332607736527,
+      "tokens_seen": 67436544
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007915586760280844,
+      "loss": 2.3291,
+      "theoretical_loss": 5.0756149676758815,
+      "tokens_seen": 67502080
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.000791542627883651,
+      "loss": 2.2732,
+      "theoretical_loss": 5.0748982188860925,
+      "tokens_seen": 67567616
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007915265797392178,
+      "loss": 2.2529,
+      "theoretical_loss": 5.074182359397387,
+      "tokens_seen": 67633152
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007915105315947844,
+      "loss": 2.3085,
+      "theoretical_loss": 5.073467387246247,
+      "tokens_seen": 67698688
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007914944834503512,
+      "loss": 2.2992,
+      "theoretical_loss": 5.072753300475385,
+      "tokens_seen": 67764224
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007914784353059178,
+      "loss": 2.2994,
+      "theoretical_loss": 5.072040097133718,
+      "tokens_seen": 67829760
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007914623871614846,
+      "loss": 2.3977,
+      "theoretical_loss": 5.0713277752763455,
+      "tokens_seen": 67895296
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007914463390170512,
+      "loss": 2.2032,
+      "theoretical_loss": 5.070616332964518,
+      "tokens_seen": 67960832
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.000791430290872618,
+      "loss": 2.282,
+      "theoretical_loss": 5.069905768265615,
+      "tokens_seen": 68026368
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007914142427281846,
+      "loss": 2.2396,
+      "theoretical_loss": 5.069196079253119,
+      "tokens_seen": 68091904
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007913981945837514,
+      "loss": 2.2792,
+      "theoretical_loss": 5.068487264006592,
+      "tokens_seen": 68157440
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.000791382146439318,
+      "loss": 2.255,
+      "theoretical_loss": 5.0677793206116455,
+      "tokens_seen": 68222976
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007913660982948848,
+      "loss": 2.3527,
+      "theoretical_loss": 5.067072247159923,
+      "tokens_seen": 68288512
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007913500501504514,
+      "loss": 2.2779,
+      "theoretical_loss": 5.06636604174907,
+      "tokens_seen": 68354048
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007913340020060182,
+      "loss": 2.25,
+      "theoretical_loss": 5.065660702482711,
+      "tokens_seen": 68419584
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007913179538615848,
+      "loss": 2.3612,
+      "theoretical_loss": 5.064956227470425,
+      "tokens_seen": 68485120
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007913019057171516,
+      "loss": 2.243,
+      "theoretical_loss": 5.06425261482772,
+      "tokens_seen": 68550656
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007912858575727182,
+      "loss": 2.221,
+      "theoretical_loss": 5.063549862676011,
+      "tokens_seen": 68616192
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.000791269809428285,
+      "loss": 2.1482,
+      "theoretical_loss": 5.062847969142591,
+      "tokens_seen": 68681728
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007912537612838516,
+      "loss": 2.2824,
+      "theoretical_loss": 5.062146932360616,
+      "tokens_seen": 68747264
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007912377131394184,
+      "loss": 2.2794,
+      "theoretical_loss": 5.061446750469072,
+      "tokens_seen": 68812800
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.000791221664994985,
+      "loss": 2.3805,
+      "theoretical_loss": 5.060747421612758,
+      "tokens_seen": 68878336
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007912056168505518,
+      "loss": 2.1662,
+      "theoretical_loss": 5.060048943942254,
+      "tokens_seen": 68943872
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007911895687061184,
+      "loss": 2.1614,
+      "theoretical_loss": 5.059351315613908,
+      "tokens_seen": 69009408
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007911735205616852,
+      "loss": 2.3213,
+      "theoretical_loss": 5.058654534789804,
+      "tokens_seen": 69074944
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007911574724172518,
+      "loss": 2.3061,
+      "theoretical_loss": 5.057958599637746,
+      "tokens_seen": 69140480
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007911414242728186,
+      "loss": 2.2007,
+      "theoretical_loss": 5.057263508331225,
+      "tokens_seen": 69206016
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007911253761283852,
+      "loss": 2.2452,
+      "theoretical_loss": 5.0565692590494065,
+      "tokens_seen": 69271552
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.000791109327983952,
+      "loss": 2.1786,
+      "theoretical_loss": 5.055875849977101,
+      "tokens_seen": 69337088
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007910932798395186,
+      "loss": 2.262,
+      "theoretical_loss": 5.055183279304744,
+      "tokens_seen": 69402624
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007910772316950853,
+      "loss": 2.1715,
+      "theoretical_loss": 5.0544915452283705,
+      "tokens_seen": 69468160
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.000791061183550652,
+      "loss": 2.2834,
+      "theoretical_loss": 5.053800645949597,
+      "tokens_seen": 69533696
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007910451354062187,
+      "loss": 2.2832,
+      "theoretical_loss": 5.053110579675593,
+      "tokens_seen": 69599232
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007910290872617854,
+      "loss": 2.2676,
+      "theoretical_loss": 5.052421344619064,
+      "tokens_seen": 69664768
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007910130391173521,
+      "loss": 2.243,
+      "theoretical_loss": 5.051732938998227,
+      "tokens_seen": 69730304
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007909969909729188,
+      "loss": 2.2622,
+      "theoretical_loss": 5.051045361036787,
+      "tokens_seen": 69795840
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007909809428284855,
+      "loss": 2.1861,
+      "theoretical_loss": 5.0503586089639185,
+      "tokens_seen": 69861376
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007909648946840522,
+      "loss": 2.2282,
+      "theoretical_loss": 5.04967268101424,
+      "tokens_seen": 69926912
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007909488465396189,
+      "loss": 2.3254,
+      "theoretical_loss": 5.048987575427793,
+      "tokens_seen": 69992448
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007909327983951855,
+      "loss": 2.3074,
+      "theoretical_loss": 5.048303290450023,
+      "tokens_seen": 70057984
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007909167502507523,
+      "loss": 2.3328,
+      "theoretical_loss": 5.047619824331753,
+      "tokens_seen": 70123520
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007909007021063189,
+      "loss": 2.1674,
+      "theoretical_loss": 5.046937175329167,
+      "tokens_seen": 70189056
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007908846539618857,
+      "loss": 2.3524,
+      "theoretical_loss": 5.046255341703786,
+      "tokens_seen": 70254592
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007908686058174523,
+      "loss": 2.1512,
+      "theoretical_loss": 5.045574321722445,
+      "tokens_seen": 70320128
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007908525576730191,
+      "loss": 2.3023,
+      "theoretical_loss": 5.0448941136572785,
+      "tokens_seen": 70385664
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007908365095285857,
+      "loss": 2.1874,
+      "theoretical_loss": 5.044214715785689,
+      "tokens_seen": 70451200
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007908204613841525,
+      "loss": 2.3012,
+      "theoretical_loss": 5.043536126390334,
+      "tokens_seen": 70516736
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007908044132397191,
+      "loss": 2.265,
+      "theoretical_loss": 5.042858343759107,
+      "tokens_seen": 70582272
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007907883650952859,
+      "loss": 2.2598,
+      "theoretical_loss": 5.0421813661851065,
+      "tokens_seen": 70647808
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007907723169508525,
+      "loss": 2.2182,
+      "theoretical_loss": 5.041505191966625,
+      "tokens_seen": 70713344
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007907562688064193,
+      "loss": 2.1663,
+      "theoretical_loss": 5.040829819407123,
+      "tokens_seen": 70778880
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007907402206619859,
+      "loss": 2.2501,
+      "theoretical_loss": 5.040155246815212,
+      "tokens_seen": 70844416
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007907241725175527,
+      "loss": 2.3264,
+      "theoretical_loss": 5.039481472504633,
+      "tokens_seen": 70909952
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007907081243731193,
+      "loss": 2.257,
+      "theoretical_loss": 5.038808494794232,
+      "tokens_seen": 70975488
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007906920762286861,
+      "loss": 2.2255,
+      "theoretical_loss": 5.038136312007948,
+      "tokens_seen": 71041024
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007906760280842527,
+      "loss": 2.2399,
+      "theoretical_loss": 5.037464922474786,
+      "tokens_seen": 71106560
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007906599799398195,
+      "loss": 2.208,
+      "theoretical_loss": 5.036794324528802,
+      "tokens_seen": 71172096
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007906439317953861,
+      "loss": 2.2584,
+      "theoretical_loss": 5.036124516509076,
+      "tokens_seen": 71237632
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007906278836509529,
+      "loss": 2.1656,
+      "theoretical_loss": 5.035455496759701,
+      "tokens_seen": 71303168
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007906118355065195,
+      "loss": 2.214,
+      "theoretical_loss": 5.034787263629758,
+      "tokens_seen": 71368704
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007905957873620863,
+      "loss": 2.2408,
+      "theoretical_loss": 5.0341198154732965,
+      "tokens_seen": 71434240
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007905797392176529,
+      "loss": 2.2403,
+      "theoretical_loss": 5.033453150649318,
+      "tokens_seen": 71499776
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007905636910732197,
+      "loss": 2.299,
+      "theoretical_loss": 5.032787267521752,
+      "tokens_seen": 71565312
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007905476429287863,
+      "loss": 2.1793,
+      "theoretical_loss": 5.032122164459443,
+      "tokens_seen": 71630848
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007905315947843531,
+      "loss": 2.1469,
+      "theoretical_loss": 5.031457839836122,
+      "tokens_seen": 71696384
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007905155466399197,
+      "loss": 2.2367,
+      "theoretical_loss": 5.0307942920304,
+      "tokens_seen": 71761920
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007904994984954865,
+      "loss": 2.1601,
+      "theoretical_loss": 5.030131519425737,
+      "tokens_seen": 71827456
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007904834503510531,
+      "loss": 2.1401,
+      "theoretical_loss": 5.02946952041043,
+      "tokens_seen": 71892992
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007904674022066199,
+      "loss": 2.4711,
+      "theoretical_loss": 5.028808293377589,
+      "tokens_seen": 71958528
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007904513540621865,
+      "loss": 2.2271,
+      "theoretical_loss": 5.028147836725127,
+      "tokens_seen": 72024064
+    },
+    {
+      "epoch": 0.02,
+      "objective/train/docs_used": 34719,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 2.1790566444396973,
+      "objective/train/theoretical_loss": 5.0274881488557295,
+      "objective/train/tokens_used": 92549600,
+      "theoretical_loss": 5.0274881488557295,
+      "tokens_seen": 72089600
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007904353059177533,
+      "loss": 2.1791,
+      "theoretical_loss": 5.0274881488557295,
+      "tokens_seen": 72089600
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007904192577733199,
+      "loss": 2.295,
+      "theoretical_loss": 5.026829228176848,
+      "tokens_seen": 72155136
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007904032096288867,
+      "loss": 2.2253,
+      "theoretical_loss": 5.026171073100672,
+      "tokens_seen": 72220672
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007903871614844534,
+      "loss": 2.1531,
+      "theoretical_loss": 5.025513682044114,
+      "tokens_seen": 72286208
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007903711133400201,
+      "loss": 2.1894,
+      "theoretical_loss": 5.024857053428793,
+      "tokens_seen": 72351744
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007903550651955868,
+      "loss": 2.1588,
+      "theoretical_loss": 5.024201185681015,
+      "tokens_seen": 72417280
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007903390170511535,
+      "loss": 2.2089,
+      "theoretical_loss": 5.023546077231752,
+      "tokens_seen": 72482816
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007903229689067202,
+      "loss": 2.1823,
+      "theoretical_loss": 5.0228917265166295,
+      "tokens_seen": 72548352
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007903069207622869,
+      "loss": 2.1929,
+      "theoretical_loss": 5.022238131975905,
+      "tokens_seen": 72613888
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007902908726178537,
+      "loss": 2.1978,
+      "theoretical_loss": 5.021585292054446,
+      "tokens_seen": 72679424
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007902748244734203,
+      "loss": 2.0794,
+      "theoretical_loss": 5.020933205201725,
+      "tokens_seen": 72744960
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.000790258776328987,
+      "loss": 2.1309,
+      "theoretical_loss": 5.020281869871787,
+      "tokens_seen": 72810496
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007902427281845537,
+      "loss": 2.3304,
+      "theoretical_loss": 5.019631284523241,
+      "tokens_seen": 72876032
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007902266800401205,
+      "loss": 2.2437,
+      "theoretical_loss": 5.01898144761924,
+      "tokens_seen": 72941568
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007902106318956871,
+      "loss": 2.1908,
+      "theoretical_loss": 5.018332357627462,
+      "tokens_seen": 73007104
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007901945837512539,
+      "loss": 2.1411,
+      "theoretical_loss": 5.017684013020096,
+      "tokens_seen": 73072640
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007901785356068205,
+      "loss": 2.2339,
+      "theoretical_loss": 5.017036412273823,
+      "tokens_seen": 73138176
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007901624874623873,
+      "loss": 2.254,
+      "theoretical_loss": 5.016389553869796,
+      "tokens_seen": 73203712
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007901464393179539,
+      "loss": 2.2574,
+      "theoretical_loss": 5.0157434362936275,
+      "tokens_seen": 73269248
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007901303911735207,
+      "loss": 2.2642,
+      "theoretical_loss": 5.01509805803537,
+      "tokens_seen": 73334784
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007901143430290873,
+      "loss": 2.2578,
+      "theoretical_loss": 5.0144534175894995,
+      "tokens_seen": 73400320
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007900982948846541,
+      "loss": 2.196,
+      "theoretical_loss": 5.013809513454898,
+      "tokens_seen": 73465856
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007900822467402207,
+      "loss": 2.2533,
+      "theoretical_loss": 5.013166344134838,
+      "tokens_seen": 73531392
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007900661985957875,
+      "loss": 2.1913,
+      "theoretical_loss": 5.012523908136963,
+      "tokens_seen": 73596928
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007900501504513541,
+      "loss": 2.2703,
+      "theoretical_loss": 5.011882203973277,
+      "tokens_seen": 73662464
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007900341023069209,
+      "loss": 2.1812,
+      "theoretical_loss": 5.011241230160119,
+      "tokens_seen": 73728000
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007900180541624875,
+      "loss": 2.2361,
+      "theoretical_loss": 5.0106009852181534,
+      "tokens_seen": 73793536
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007900020060180543,
+      "loss": 2.2004,
+      "theoretical_loss": 5.009961467672353,
+      "tokens_seen": 73859072
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007899859578736209,
+      "loss": 2.1511,
+      "theoretical_loss": 5.009322676051976,
+      "tokens_seen": 73924608
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007899699097291877,
+      "loss": 2.2886,
+      "theoretical_loss": 5.008684608890562,
+      "tokens_seen": 73990144
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007899538615847543,
+      "loss": 2.2065,
+      "theoretical_loss": 5.008047264725901,
+      "tokens_seen": 74055680
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007899378134403211,
+      "loss": 2.2599,
+      "theoretical_loss": 5.007410642100032,
+      "tokens_seen": 74121216
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007899217652958877,
+      "loss": 2.0565,
+      "theoretical_loss": 5.006774739559216,
+      "tokens_seen": 74186752
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007899057171514545,
+      "loss": 2.161,
+      "theoretical_loss": 5.00613955565392,
+      "tokens_seen": 74252288
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007898896690070211,
+      "loss": 2.0608,
+      "theoretical_loss": 5.005505088938815,
+      "tokens_seen": 74317824
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007898736208625879,
+      "loss": 2.1718,
+      "theoretical_loss": 5.004871337972742,
+      "tokens_seen": 74383360
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007898575727181545,
+      "loss": 2.1034,
+      "theoretical_loss": 5.004238301318706,
+      "tokens_seen": 74448896
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007898415245737213,
+      "loss": 2.2283,
+      "theoretical_loss": 5.003605977543861,
+      "tokens_seen": 74514432
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007898254764292879,
+      "loss": 2.1599,
+      "theoretical_loss": 5.00297436521949,
+      "tokens_seen": 74579968
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007898094282848547,
+      "loss": 2.2186,
+      "theoretical_loss": 5.002343462920992,
+      "tokens_seen": 74645504
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007897933801404213,
+      "loss": 2.2726,
+      "theoretical_loss": 5.001713269227867,
+      "tokens_seen": 74711040
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007897773319959881,
+      "loss": 2.1977,
+      "theoretical_loss": 5.0010837827237005,
+      "tokens_seen": 74776576
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007897612838515547,
+      "loss": 2.2277,
+      "theoretical_loss": 5.000455001996143,
+      "tokens_seen": 74842112
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007897452357071215,
+      "loss": 2.2989,
+      "theoretical_loss": 4.999826925636905,
+      "tokens_seen": 74907648
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007897291875626881,
+      "loss": 2.2514,
+      "theoretical_loss": 4.999199552241732,
+      "tokens_seen": 74973184
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007897131394182549,
+      "loss": 2.312,
+      "theoretical_loss": 4.998572880410396,
+      "tokens_seen": 75038720
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007896970912738215,
+      "loss": 2.1228,
+      "theoretical_loss": 4.997946908746677,
+      "tokens_seen": 75104256
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007896810431293882,
+      "loss": 2.2409,
+      "theoretical_loss": 4.997321635858347,
+      "tokens_seen": 75169792
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007896649949849549,
+      "loss": 2.2001,
+      "theoretical_loss": 4.9966970603571585,
+      "tokens_seen": 75235328
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007896489468405216,
+      "loss": 2.2675,
+      "theoretical_loss": 4.996073180858831,
+      "tokens_seen": 75300864
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007896328986960883,
+      "loss": 2.1907,
+      "theoretical_loss": 4.995449995983028,
+      "tokens_seen": 75366400
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.000789616850551655,
+      "loss": 2.2062,
+      "theoretical_loss": 4.994827504353353,
+      "tokens_seen": 75431936
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007896008024072217,
+      "loss": 2.183,
+      "theoretical_loss": 4.994205704597325,
+      "tokens_seen": 75497472
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007895847542627884,
+      "loss": 2.1929,
+      "theoretical_loss": 4.993584595346373,
+      "tokens_seen": 75563008
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007895687061183551,
+      "loss": 2.2374,
+      "theoretical_loss": 4.992964175235814,
+      "tokens_seen": 75628544
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007895526579739218,
+      "loss": 2.1921,
+      "theoretical_loss": 4.992344442904843,
+      "tokens_seen": 75694080
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007895366098294884,
+      "loss": 2.1265,
+      "theoretical_loss": 4.991725396996515,
+      "tokens_seen": 75759616
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007895205616850552,
+      "loss": 2.028,
+      "theoretical_loss": 4.991107036157738,
+      "tokens_seen": 75825152
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007895045135406218,
+      "loss": 2.1131,
+      "theoretical_loss": 4.990489359039248,
+      "tokens_seen": 75890688
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007894884653961886,
+      "loss": 2.1755,
+      "theoretical_loss": 4.989872364295605,
+      "tokens_seen": 75956224
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007894724172517552,
+      "loss": 2.1793,
+      "theoretical_loss": 4.989256050585174,
+      "tokens_seen": 76021760
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.000789456369107322,
+      "loss": 2.2634,
+      "theoretical_loss": 4.9886404165701075,
+      "tokens_seen": 76087296
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007894403209628886,
+      "loss": 2.1688,
+      "theoretical_loss": 4.98802546091634,
+      "tokens_seen": 76152832
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007894242728184554,
+      "loss": 2.2535,
+      "theoretical_loss": 4.9874111822935685,
+      "tokens_seen": 76218368
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.000789408224674022,
+      "loss": 2.0708,
+      "theoretical_loss": 4.986797579375239,
+      "tokens_seen": 76283904
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007893921765295888,
+      "loss": 2.1895,
+      "theoretical_loss": 4.986184650838532,
+      "tokens_seen": 76349440
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007893761283851554,
+      "loss": 2.1738,
+      "theoretical_loss": 4.985572395364354,
+      "tokens_seen": 76414976
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007893600802407222,
+      "loss": 2.0693,
+      "theoretical_loss": 4.98496081163732,
+      "tokens_seen": 76480512
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007893440320962888,
+      "loss": 2.0786,
+      "theoretical_loss": 4.984349898345735,
+      "tokens_seen": 76546048
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007893279839518556,
+      "loss": 2.147,
+      "theoretical_loss": 4.98373965418159,
+      "tokens_seen": 76611584
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007893119358074222,
+      "loss": 2.2609,
+      "theoretical_loss": 4.983130077840543,
+      "tokens_seen": 76677120
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.000789295887662989,
+      "loss": 2.2384,
+      "theoretical_loss": 4.982521168021906,
+      "tokens_seen": 76742656
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007892798395185556,
+      "loss": 2.0189,
+      "theoretical_loss": 4.981912923428634,
+      "tokens_seen": 76808192
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007892637913741224,
+      "loss": 2.2331,
+      "theoretical_loss": 4.981305342767307,
+      "tokens_seen": 76873728
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.000789247743229689,
+      "loss": 2.0864,
+      "theoretical_loss": 4.980698424748122,
+      "tokens_seen": 76939264
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007892316950852558,
+      "loss": 2.2626,
+      "theoretical_loss": 4.980092168084878,
+      "tokens_seen": 77004800
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007892156469408224,
+      "loss": 2.1412,
+      "theoretical_loss": 4.97948657149496,
+      "tokens_seen": 77070336
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007891995987963892,
+      "loss": 2.1168,
+      "theoretical_loss": 4.978881633699331,
+      "tokens_seen": 77135872
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007891835506519559,
+      "loss": 2.2427,
+      "theoretical_loss": 4.9782773534225155,
+      "tokens_seen": 77201408
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007891675025075226,
+      "loss": 2.1032,
+      "theoretical_loss": 4.977673729392586,
+      "tokens_seen": 77266944
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007891514543630893,
+      "loss": 2.1723,
+      "theoretical_loss": 4.977070760341153,
+      "tokens_seen": 77332480
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.000789135406218656,
+      "loss": 2.0192,
+      "theoretical_loss": 4.976468445003352,
+      "tokens_seen": 77398016
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007891193580742227,
+      "loss": 2.0722,
+      "theoretical_loss": 4.975866782117828,
+      "tokens_seen": 77463552
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007891033099297894,
+      "loss": 2.2401,
+      "theoretical_loss": 4.975265770426724,
+      "tokens_seen": 77529088
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007890872617853561,
+      "loss": 2.1938,
+      "theoretical_loss": 4.974665408675667,
+      "tokens_seen": 77594624
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007890712136409228,
+      "loss": 2.2931,
+      "theoretical_loss": 4.974065695613763,
+      "tokens_seen": 77660160
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007890551654964895,
+      "loss": 2.1322,
+      "theoretical_loss": 4.973466629993574,
+      "tokens_seen": 77725696
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007890391173520562,
+      "loss": 2.2289,
+      "theoretical_loss": 4.972868210571109,
+      "tokens_seen": 77791232
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.000789023069207623,
+      "loss": 2.2627,
+      "theoretical_loss": 4.972270436105815,
+      "tokens_seen": 77856768
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007890070210631896,
+      "loss": 2.2138,
+      "theoretical_loss": 4.971673305360563,
+      "tokens_seen": 77922304
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007889909729187563,
+      "loss": 2.0407,
+      "theoretical_loss": 4.971076817101633,
+      "tokens_seen": 77987840
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.000788974924774323,
+      "loss": 2.1262,
+      "theoretical_loss": 4.970480970098703,
+      "tokens_seen": 78053376
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007889588766298897,
+      "loss": 2.1401,
+      "theoretical_loss": 4.96988576312484,
+      "tokens_seen": 78118912
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007889428284854564,
+      "loss": 2.1337,
+      "theoretical_loss": 4.969291194956485,
+      "tokens_seen": 78184448
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007889267803410232,
+      "loss": 2.2153,
+      "theoretical_loss": 4.968697264373438,
+      "tokens_seen": 78249984
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007889107321965898,
+      "loss": 2.2326,
+      "theoretical_loss": 4.968103970158854,
+      "tokens_seen": 78315520
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007888946840521566,
+      "loss": 2.1139,
+      "theoretical_loss": 4.967511311099223,
+      "tokens_seen": 78381056
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007888786359077232,
+      "loss": 2.2219,
+      "theoretical_loss": 4.96691928598436,
+      "tokens_seen": 78446592
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.00078886258776329,
+      "loss": 2.1414,
+      "theoretical_loss": 4.966327893607398,
+      "tokens_seen": 78512128
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007888465396188566,
+      "loss": 2.2232,
+      "theoretical_loss": 4.96573713276477,
+      "tokens_seen": 78577664
+    },
+    {
+      "epoch": 0.02,
+      "objective/train/docs_used": 36940,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 2.0884451866149902,
+      "objective/train/theoretical_loss": 4.965147002256201,
+      "objective/train/tokens_used": 99103200,
+      "theoretical_loss": 4.965147002256201,
+      "tokens_seen": 78643200
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007888304914744234,
+      "loss": 2.0884,
+      "theoretical_loss": 4.965147002256201,
+      "tokens_seen": 78643200
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.00078881444332999,
+      "loss": 2.0536,
+      "theoretical_loss": 4.964557500884695,
+      "tokens_seen": 78708736
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007887983951855568,
+      "loss": 2.1919,
+      "theoretical_loss": 4.963968627456523,
+      "tokens_seen": 78774272
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007887823470411234,
+      "loss": 2.2003,
+      "theoretical_loss": 4.96338038078121,
+      "tokens_seen": 78839808
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007887662988966902,
+      "loss": 2.0346,
+      "theoretical_loss": 4.962792759671528,
+      "tokens_seen": 78905344
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007887502507522568,
+      "loss": 2.1342,
+      "theoretical_loss": 4.962205762943478,
+      "tokens_seen": 78970880
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007887342026078236,
+      "loss": 2.2052,
+      "theoretical_loss": 4.961619389416288,
+      "tokens_seen": 79036416
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007887181544633902,
+      "loss": 2.2535,
+      "theoretical_loss": 4.96103363791239,
+      "tokens_seen": 79101952
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.000788702106318957,
+      "loss": 2.1753,
+      "theoretical_loss": 4.960448507257416,
+      "tokens_seen": 79167488
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007886860581745236,
+      "loss": 2.1496,
+      "theoretical_loss": 4.959863996280188,
+      "tokens_seen": 79233024
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007886700100300904,
+      "loss": 2.0766,
+      "theoretical_loss": 4.959280103812697,
+      "tokens_seen": 79298560
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.000788653961885657,
+      "loss": 2.0173,
+      "theoretical_loss": 4.958696828690105,
+      "tokens_seen": 79364096
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007886379137412238,
+      "loss": 2.0692,
+      "theoretical_loss": 4.958114169750726,
+      "tokens_seen": 79429632
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007886218655967904,
+      "loss": 2.206,
+      "theoretical_loss": 4.957532125836012,
+      "tokens_seen": 79495168
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007886058174523572,
+      "loss": 2.0888,
+      "theoretical_loss": 4.956950695790551,
+      "tokens_seen": 79560704
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007885897693079238,
+      "loss": 2.1318,
+      "theoretical_loss": 4.95636987846205,
+      "tokens_seen": 79626240
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007885737211634906,
+      "loss": 2.1276,
+      "theoretical_loss": 4.955789672701323,
+      "tokens_seen": 79691776
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007885576730190572,
+      "loss": 2.1568,
+      "theoretical_loss": 4.955210077362285,
+      "tokens_seen": 79757312
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.000788541624874624,
+      "loss": 2.2067,
+      "theoretical_loss": 4.954631091301934,
+      "tokens_seen": 79822848
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007885255767301906,
+      "loss": 2.2183,
+      "theoretical_loss": 4.954052713380349,
+      "tokens_seen": 79888384
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007885095285857574,
+      "loss": 2.2139,
+      "theoretical_loss": 4.953474942460673,
+      "tokens_seen": 79953920
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.000788493480441324,
+      "loss": 2.3113,
+      "theoretical_loss": 4.952897777409104,
+      "tokens_seen": 80019456
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007884774322968908,
+      "loss": 2.2662,
+      "theoretical_loss": 4.952321217094882,
+      "tokens_seen": 80084992
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007884613841524574,
+      "loss": 2.102,
+      "theoretical_loss": 4.951745260390284,
+      "tokens_seen": 80150528
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007884453360080242,
+      "loss": 2.2117,
+      "theoretical_loss": 4.951169906170607,
+      "tokens_seen": 80216064
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007884292878635908,
+      "loss": 2.1562,
+      "theoretical_loss": 4.950595153314161,
+      "tokens_seen": 80281600
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007884132397191576,
+      "loss": 2.164,
+      "theoretical_loss": 4.950021000702261,
+      "tokens_seen": 80347136
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007883971915747242,
+      "loss": 2.2053,
+      "theoretical_loss": 4.949447447219208,
+      "tokens_seen": 80412672
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.000788381143430291,
+      "loss": 2.2245,
+      "theoretical_loss": 4.948874491752287,
+      "tokens_seen": 80478208
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007883650952858576,
+      "loss": 2.1905,
+      "theoretical_loss": 4.9483021331917545,
+      "tokens_seen": 80543744
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007883490471414244,
+      "loss": 2.142,
+      "theoretical_loss": 4.9477303704308255,
+      "tokens_seen": 80609280
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.000788332998996991,
+      "loss": 2.1618,
+      "theoretical_loss": 4.947159202365664,
+      "tokens_seen": 80674816
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007883169508525578,
+      "loss": 2.1577,
+      "theoretical_loss": 4.946588627895375,
+      "tokens_seen": 80740352
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007883009027081244,
+      "loss": 2.0829,
+      "theoretical_loss": 4.946018645921994,
+      "tokens_seen": 80805888
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007882848545636911,
+      "loss": 2.1016,
+      "theoretical_loss": 4.945449255350473,
+      "tokens_seen": 80871424
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007882688064192578,
+      "loss": 2.1309,
+      "theoretical_loss": 4.944880455088677,
+      "tokens_seen": 80936960
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007882527582748245,
+      "loss": 2.1288,
+      "theoretical_loss": 4.944312244047365,
+      "tokens_seen": 81002496
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007882367101303912,
+      "loss": 2.1433,
+      "theoretical_loss": 4.943744621140189,
+      "tokens_seen": 81068032
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007882206619859579,
+      "loss": 2.1942,
+      "theoretical_loss": 4.94317758528368,
+      "tokens_seen": 81133568
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007882046138415246,
+      "loss": 2.1101,
+      "theoretical_loss": 4.942611135397234,
+      "tokens_seen": 81199104
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007881885656970913,
+      "loss": 2.1531,
+      "theoretical_loss": 4.942045270403112,
+      "tokens_seen": 81264640
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.000788172517552658,
+      "loss": 2.2471,
+      "theoretical_loss": 4.941479989226421,
+      "tokens_seen": 81330176
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007881564694082247,
+      "loss": 2.2154,
+      "theoretical_loss": 4.940915290795106,
+      "tokens_seen": 81395712
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007881404212637913,
+      "loss": 2.1522,
+      "theoretical_loss": 4.940351174039947,
+      "tokens_seen": 81461248
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007881243731193581,
+      "loss": 2.168,
+      "theoretical_loss": 4.939787637894537,
+      "tokens_seen": 81526784
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007881083249749247,
+      "loss": 2.1257,
+      "theoretical_loss": 4.939224681295286,
+      "tokens_seen": 81592320
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007880922768304915,
+      "loss": 2.0687,
+      "theoretical_loss": 4.938662303181399,
+      "tokens_seen": 81657856
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007880762286860581,
+      "loss": 2.0866,
+      "theoretical_loss": 4.938100502494878,
+      "tokens_seen": 81723392
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007880601805416249,
+      "loss": 2.1731,
+      "theoretical_loss": 4.9375392781805,
+      "tokens_seen": 81788928
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007880441323971915,
+      "loss": 2.1461,
+      "theoretical_loss": 4.936978629185818,
+      "tokens_seen": 81854464
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007880280842527583,
+      "loss": 2.1646,
+      "theoretical_loss": 4.936418554461147,
+      "tokens_seen": 81920000
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007880120361083249,
+      "loss": 2.0447,
+      "theoretical_loss": 4.935859052959554,
+      "tokens_seen": 81985536
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007879959879638917,
+      "loss": 2.2203,
+      "theoretical_loss": 4.935300123636851,
+      "tokens_seen": 82051072
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007879799398194584,
+      "loss": 2.1977,
+      "theoretical_loss": 4.93474176545158,
+      "tokens_seen": 82116608
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007879638916750251,
+      "loss": 2.1775,
+      "theoretical_loss": 4.934183977365015,
+      "tokens_seen": 82182144
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007879478435305918,
+      "loss": 2.1124,
+      "theoretical_loss": 4.9336267583411395,
+      "tokens_seen": 82247680
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007879317953861585,
+      "loss": 2.0796,
+      "theoretical_loss": 4.933070107346644,
+      "tokens_seen": 82313216
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007879157472417252,
+      "loss": 2.0991,
+      "theoretical_loss": 4.932514023350921,
+      "tokens_seen": 82378752
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007878996990972919,
+      "loss": 2.1563,
+      "theoretical_loss": 4.931958505326042,
+      "tokens_seen": 82444288
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007878836509528586,
+      "loss": 2.0317,
+      "theoretical_loss": 4.931403552246769,
+      "tokens_seen": 82509824
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007878676028084253,
+      "loss": 2.135,
+      "theoretical_loss": 4.930849163090521,
+      "tokens_seen": 82575360
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.000787851554663992,
+      "loss": 2.1251,
+      "theoretical_loss": 4.930295336837387,
+      "tokens_seen": 82640896
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007878355065195587,
+      "loss": 2.105,
+      "theoretical_loss": 4.9297420724701055,
+      "tokens_seen": 82706432
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007878194583751254,
+      "loss": 2.1526,
+      "theoretical_loss": 4.929189368974054,
+      "tokens_seen": 82771968
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007878034102306921,
+      "loss": 2.1333,
+      "theoretical_loss": 4.928637225337246,
+      "tokens_seen": 82837504
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007877873620862588,
+      "loss": 2.0942,
+      "theoretical_loss": 4.9280856405503215,
+      "tokens_seen": 82903040
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007877713139418255,
+      "loss": 2.0241,
+      "theoretical_loss": 4.927534613606534,
+      "tokens_seen": 82968576
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007877552657973922,
+      "loss": 2.1364,
+      "theoretical_loss": 4.926984143501745,
+      "tokens_seen": 83034112
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007877392176529589,
+      "loss": 2.2187,
+      "theoretical_loss": 4.926434229234411,
+      "tokens_seen": 83099648
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007877231695085256,
+      "loss": 2.1759,
+      "theoretical_loss": 4.925884869805586,
+      "tokens_seen": 83165184
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007877071213640923,
+      "loss": 2.1543,
+      "theoretical_loss": 4.925336064218893,
+      "tokens_seen": 83230720
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.000787691073219659,
+      "loss": 2.1695,
+      "theoretical_loss": 4.924787811480538,
+      "tokens_seen": 83296256
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007876750250752257,
+      "loss": 2.1016,
+      "theoretical_loss": 4.9242401105992855,
+      "tokens_seen": 83361792
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007876589769307924,
+      "loss": 1.9869,
+      "theoretical_loss": 4.923692960586452,
+      "tokens_seen": 83427328
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007876429287863591,
+      "loss": 1.9533,
+      "theoretical_loss": 4.923146360455906,
+      "tokens_seen": 83492864
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007876268806419258,
+      "loss": 2.0596,
+      "theoretical_loss": 4.92260030922405,
+      "tokens_seen": 83558400
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007876108324974925,
+      "loss": 2.1809,
+      "theoretical_loss": 4.922054805909816,
+      "tokens_seen": 83623936
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007875947843530592,
+      "loss": 2.0021,
+      "theoretical_loss": 4.921509849534658,
+      "tokens_seen": 83689472
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007875787362086259,
+      "loss": 2.1222,
+      "theoretical_loss": 4.920965439122538,
+      "tokens_seen": 83755008
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007875626880641927,
+      "loss": 2.2289,
+      "theoretical_loss": 4.9204215736999295,
+      "tokens_seen": 83820544
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007875466399197593,
+      "loss": 2.2235,
+      "theoretical_loss": 4.919878252295793,
+      "tokens_seen": 83886080
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.000787530591775326,
+      "loss": 2.0148,
+      "theoretical_loss": 4.919335473941581,
+      "tokens_seen": 83951616
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007875145436308927,
+      "loss": 2.1073,
+      "theoretical_loss": 4.918793237671224,
+      "tokens_seen": 84017152
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007874984954864595,
+      "loss": 2.2016,
+      "theoretical_loss": 4.918251542521121,
+      "tokens_seen": 84082688
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007874824473420261,
+      "loss": 2.1669,
+      "theoretical_loss": 4.9177103875301365,
+      "tokens_seen": 84148224
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007874663991975929,
+      "loss": 2.0046,
+      "theoretical_loss": 4.917169771739586,
+      "tokens_seen": 84213760
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007874503510531595,
+      "loss": 1.9713,
+      "theoretical_loss": 4.916629694193233,
+      "tokens_seen": 84279296
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007874343029087263,
+      "loss": 2.1506,
+      "theoretical_loss": 4.916090153937276,
+      "tokens_seen": 84344832
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007874182547642929,
+      "loss": 2.2232,
+      "theoretical_loss": 4.915551150020346,
+      "tokens_seen": 84410368
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007874022066198597,
+      "loss": 2.0884,
+      "theoretical_loss": 4.915012681493495,
+      "tokens_seen": 84475904
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007873861584754263,
+      "loss": 2.0906,
+      "theoretical_loss": 4.914474747410187,
+      "tokens_seen": 84541440
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007873701103309931,
+      "loss": 2.0496,
+      "theoretical_loss": 4.913937346826293,
+      "tokens_seen": 84606976
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007873540621865597,
+      "loss": 2.0105,
+      "theoretical_loss": 4.91340047880008,
+      "tokens_seen": 84672512
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007873380140421265,
+      "loss": 2.0424,
+      "theoretical_loss": 4.9128641423922055,
+      "tokens_seen": 84738048
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007873219658976931,
+      "loss": 1.9974,
+      "theoretical_loss": 4.912328336665711,
+      "tokens_seen": 84803584
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007873059177532599,
+      "loss": 2.1345,
+      "theoretical_loss": 4.911793060686007,
+      "tokens_seen": 84869120
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007872898696088265,
+      "loss": 2.1816,
+      "theoretical_loss": 4.9112583135208725,
+      "tokens_seen": 84934656
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007872738214643933,
+      "loss": 2.2291,
+      "theoretical_loss": 4.910724094240447,
+      "tokens_seen": 85000192
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007872577733199599,
+      "loss": 2.0991,
+      "theoretical_loss": 4.910190401917214,
+      "tokens_seen": 85065728
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007872417251755267,
+      "loss": 2.018,
+      "theoretical_loss": 4.909657235626009,
+      "tokens_seen": 85131264
+    },
+    {
+      "epoch": 0.03,
+      "objective/train/docs_used": 39240,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 2.064126491546631,
+      "objective/train/theoretical_loss": 4.909124594443993,
+      "objective/train/tokens_used": 105656800,
+      "theoretical_loss": 4.909124594443993,
+      "tokens_seen": 85196800
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007872256770310933,
+      "loss": 2.0641,
+      "theoretical_loss": 4.909124594443993,
+      "tokens_seen": 85196800
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007872096288866601,
+      "loss": 2.1136,
+      "theoretical_loss": 4.908592477450659,
+      "tokens_seen": 85262336
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007871935807422267,
+      "loss": 2.0321,
+      "theoretical_loss": 4.908060883727821,
+      "tokens_seen": 85327872
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007871775325977935,
+      "loss": 2.1324,
+      "theoretical_loss": 4.9075298123596,
+      "tokens_seen": 85393408
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007871614844533601,
+      "loss": 2.1192,
+      "theoretical_loss": 4.906999262432427,
+      "tokens_seen": 85458944
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007871454363089269,
+      "loss": 2.2101,
+      "theoretical_loss": 4.906469233035025,
+      "tokens_seen": 85524480
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007871293881644935,
+      "loss": 2.1805,
+      "theoretical_loss": 4.905939723258408,
+      "tokens_seen": 85590016
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007871133400200603,
+      "loss": 2.2154,
+      "theoretical_loss": 4.905410732195875,
+      "tokens_seen": 85655552
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007870972918756269,
+      "loss": 2.2694,
+      "theoretical_loss": 4.904882258942994,
+      "tokens_seen": 85721088
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007870812437311937,
+      "loss": 2.0957,
+      "theoretical_loss": 4.904354302597603,
+      "tokens_seen": 85786624
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007870651955867603,
+      "loss": 1.9985,
+      "theoretical_loss": 4.903826862259798,
+      "tokens_seen": 85852160
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007870491474423271,
+      "loss": 1.8715,
+      "theoretical_loss": 4.90329993703193,
+      "tokens_seen": 85917696
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007870330992978937,
+      "loss": 1.9849,
+      "theoretical_loss": 4.902773526018591,
+      "tokens_seen": 85983232
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007870170511534605,
+      "loss": 2.2081,
+      "theoretical_loss": 4.902247628326613,
+      "tokens_seen": 86048768
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007870010030090271,
+      "loss": 2.134,
+      "theoretical_loss": 4.9017222430650556,
+      "tokens_seen": 86114304
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007869849548645939,
+      "loss": 2.1433,
+      "theoretical_loss": 4.901197369345205,
+      "tokens_seen": 86179840
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007869689067201605,
+      "loss": 2.0677,
+      "theoretical_loss": 4.9006730062805595,
+      "tokens_seen": 86245376
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007869528585757273,
+      "loss": 2.1004,
+      "theoretical_loss": 4.9001491529868275,
+      "tokens_seen": 86310912
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007869368104312939,
+      "loss": 2.1944,
+      "theoretical_loss": 4.899625808581918,
+      "tokens_seen": 86376448
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007869207622868607,
+      "loss": 2.0342,
+      "theoretical_loss": 4.899102972185936,
+      "tokens_seen": 86441984
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007869047141424273,
+      "loss": 2.0419,
+      "theoretical_loss": 4.898580642921171,
+      "tokens_seen": 86507520
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007868886659979941,
+      "loss": 2.0886,
+      "theoretical_loss": 4.898058819912093,
+      "tokens_seen": 86573056
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007868726178535607,
+      "loss": 2.0689,
+      "theoretical_loss": 4.897537502285347,
+      "tokens_seen": 86638592
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007868565697091274,
+      "loss": 2.1105,
+      "theoretical_loss": 4.897016689169741,
+      "tokens_seen": 86704128
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007868405215646941,
+      "loss": 2.1364,
+      "theoretical_loss": 4.896496379696243,
+      "tokens_seen": 86769664
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007868244734202608,
+      "loss": 2.1053,
+      "theoretical_loss": 4.895976572997973,
+      "tokens_seen": 86835200
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007868084252758275,
+      "loss": 2.2204,
+      "theoretical_loss": 4.895457268210196,
+      "tokens_seen": 86900736
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007867923771313942,
+      "loss": 2.0587,
+      "theoretical_loss": 4.8949384644703136,
+      "tokens_seen": 86966272
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007867763289869609,
+      "loss": 2.1282,
+      "theoretical_loss": 4.894420160917861,
+      "tokens_seen": 87031808
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007867602808425276,
+      "loss": 2.0678,
+      "theoretical_loss": 4.893902356694495,
+      "tokens_seen": 87097344
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007867442326980943,
+      "loss": 2.1591,
+      "theoretical_loss": 4.8933850509439925,
+      "tokens_seen": 87162880
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.000786728184553661,
+      "loss": 2.0862,
+      "theoretical_loss": 4.89286824281224,
+      "tokens_seen": 87228416
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007867121364092277,
+      "loss": 2.2062,
+      "theoretical_loss": 4.892351931447224,
+      "tokens_seen": 87293952
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007866960882647944,
+      "loss": 2.0411,
+      "theoretical_loss": 4.891836115999034,
+      "tokens_seen": 87359488
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007866800401203611,
+      "loss": 2.1868,
+      "theoretical_loss": 4.891320795619847,
+      "tokens_seen": 87425024
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007866639919759278,
+      "loss": 2.0745,
+      "theoretical_loss": 4.890805969463923,
+      "tokens_seen": 87490560
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007866479438314945,
+      "loss": 2.0923,
+      "theoretical_loss": 4.8902916366876,
+      "tokens_seen": 87556096
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007866318956870612,
+      "loss": 2.0783,
+      "theoretical_loss": 4.889777796449285,
+      "tokens_seen": 87621632
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007866158475426279,
+      "loss": 2.037,
+      "theoretical_loss": 4.889264447909451,
+      "tokens_seen": 87687168
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007865997993981946,
+      "loss": 2.1086,
+      "theoretical_loss": 4.888751590230626,
+      "tokens_seen": 87752704
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007865837512537613,
+      "loss": 2.154,
+      "theoretical_loss": 4.888239222577389,
+      "tokens_seen": 87818240
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.000786567703109328,
+      "loss": 2.1473,
+      "theoretical_loss": 4.887727344116364,
+      "tokens_seen": 87883776
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007865516549648947,
+      "loss": 2.2003,
+      "theoretical_loss": 4.8872159540162095,
+      "tokens_seen": 87949312
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007865356068204614,
+      "loss": 2.0546,
+      "theoretical_loss": 4.88670505144762,
+      "tokens_seen": 88014848
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007865195586760281,
+      "loss": 2.1056,
+      "theoretical_loss": 4.8861946355833075,
+      "tokens_seen": 88080384
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007865035105315948,
+      "loss": 2.1079,
+      "theoretical_loss": 4.885684705598009,
+      "tokens_seen": 88145920
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007864874623871615,
+      "loss": 2.0428,
+      "theoretical_loss": 4.885175260668468,
+      "tokens_seen": 88211456
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007864714142427282,
+      "loss": 2.1482,
+      "theoretical_loss": 4.884666299973436,
+      "tokens_seen": 88276992
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007864553660982949,
+      "loss": 1.985,
+      "theoretical_loss": 4.884157822693663,
+      "tokens_seen": 88342528
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007864393179538616,
+      "loss": 2.1221,
+      "theoretical_loss": 4.88364982801189,
+      "tokens_seen": 88408064
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007864232698094283,
+      "loss": 2.1343,
+      "theoretical_loss": 4.8831423151128455,
+      "tokens_seen": 88473600
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.000786407221664995,
+      "loss": 1.9906,
+      "theoretical_loss": 4.8826352831832365,
+      "tokens_seen": 88539136
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007863911735205617,
+      "loss": 2.0461,
+      "theoretical_loss": 4.882128731411745,
+      "tokens_seen": 88604672
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007863751253761284,
+      "loss": 1.9914,
+      "theoretical_loss": 4.881622658989022,
+      "tokens_seen": 88670208
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007863590772316951,
+      "loss": 2.0667,
+      "theoretical_loss": 4.881117065107675,
+      "tokens_seen": 88735744
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007863430290872618,
+      "loss": 2.055,
+      "theoretical_loss": 4.880611948962271,
+      "tokens_seen": 88801280
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007863269809428285,
+      "loss": 2.2267,
+      "theoretical_loss": 4.880107309749322,
+      "tokens_seen": 88866816
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007863109327983952,
+      "loss": 2.1405,
+      "theoretical_loss": 4.879603146667284,
+      "tokens_seen": 88932352
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.000786294884653962,
+      "loss": 2.175,
+      "theoretical_loss": 4.879099458916553,
+      "tokens_seen": 88997888
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007862788365095286,
+      "loss": 2.1198,
+      "theoretical_loss": 4.878596245699449,
+      "tokens_seen": 89063424
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007862627883650953,
+      "loss": 2.1215,
+      "theoretical_loss": 4.878093506220221,
+      "tokens_seen": 89128960
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.000786246740220662,
+      "loss": 2.0613,
+      "theoretical_loss": 4.877591239685035,
+      "tokens_seen": 89194496
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007862306920762287,
+      "loss": 2.0941,
+      "theoretical_loss": 4.877089445301969,
+      "tokens_seen": 89260032
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007862146439317954,
+      "loss": 2.0956,
+      "theoretical_loss": 4.876588122281007,
+      "tokens_seen": 89325568
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007861985957873622,
+      "loss": 2.0404,
+      "theoretical_loss": 4.876087269834034,
+      "tokens_seen": 89391104
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007861825476429288,
+      "loss": 2.195,
+      "theoretical_loss": 4.875586887174828,
+      "tokens_seen": 89456640
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007861664994984956,
+      "loss": 2.0452,
+      "theoretical_loss": 4.875086973519058,
+      "tokens_seen": 89522176
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007861504513540622,
+      "loss": 2.0918,
+      "theoretical_loss": 4.874587528084271,
+      "tokens_seen": 89587712
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.000786134403209629,
+      "loss": 2.0711,
+      "theoretical_loss": 4.874088550089896,
+      "tokens_seen": 89653248
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007861183550651956,
+      "loss": 2.018,
+      "theoretical_loss": 4.873590038757229,
+      "tokens_seen": 89718784
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007861023069207624,
+      "loss": 1.9862,
+      "theoretical_loss": 4.87309199330943,
+      "tokens_seen": 89784320
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.000786086258776329,
+      "loss": 2.1764,
+      "theoretical_loss": 4.872594412971521,
+      "tokens_seen": 89849856
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007860702106318958,
+      "loss": 2.0663,
+      "theoretical_loss": 4.872097296970376,
+      "tokens_seen": 89915392
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007860541624874624,
+      "loss": 1.9736,
+      "theoretical_loss": 4.871600644534717,
+      "tokens_seen": 89980928
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007860381143430292,
+      "loss": 2.2087,
+      "theoretical_loss": 4.871104454895106,
+      "tokens_seen": 90046464
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007860220661985958,
+      "loss": 2.1285,
+      "theoretical_loss": 4.870608727283942,
+      "tokens_seen": 90112000
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007860060180541626,
+      "loss": 2.2043,
+      "theoretical_loss": 4.870113460935455,
+      "tokens_seen": 90177536
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007859899699097292,
+      "loss": 2.1125,
+      "theoretical_loss": 4.869618655085697,
+      "tokens_seen": 90243072
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.000785973921765296,
+      "loss": 2.1254,
+      "theoretical_loss": 4.869124308972543,
+      "tokens_seen": 90308608
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007859578736208626,
+      "loss": 2.1154,
+      "theoretical_loss": 4.868630421835677,
+      "tokens_seen": 90374144
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007859418254764294,
+      "loss": 2.0145,
+      "theoretical_loss": 4.868136992916593,
+      "tokens_seen": 90439680
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.000785925777331996,
+      "loss": 2.2243,
+      "theoretical_loss": 4.867644021458585,
+      "tokens_seen": 90505216
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007859097291875628,
+      "loss": 2.0343,
+      "theoretical_loss": 4.8671515067067475,
+      "tokens_seen": 90570752
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007858936810431294,
+      "loss": 2.0246,
+      "theoretical_loss": 4.866659447907962,
+      "tokens_seen": 90636288
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007858776328986962,
+      "loss": 2.0421,
+      "theoretical_loss": 4.866167844310896,
+      "tokens_seen": 90701824
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007858615847542628,
+      "loss": 2.0609,
+      "theoretical_loss": 4.865676695165996,
+      "tokens_seen": 90767360
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007858455366098296,
+      "loss": 2.0328,
+      "theoretical_loss": 4.865185999725487,
+      "tokens_seen": 90832896
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007858294884653962,
+      "loss": 2.0952,
+      "theoretical_loss": 4.864695757243359,
+      "tokens_seen": 90898432
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.000785813440320963,
+      "loss": 2.0305,
+      "theoretical_loss": 4.864205966975364,
+      "tokens_seen": 90963968
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007857973921765296,
+      "loss": 2.1004,
+      "theoretical_loss": 4.8637166281790165,
+      "tokens_seen": 91029504
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007857813440320964,
+      "loss": 2.016,
+      "theoretical_loss": 4.86322774011358,
+      "tokens_seen": 91095040
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.000785765295887663,
+      "loss": 2.1677,
+      "theoretical_loss": 4.862739302040067,
+      "tokens_seen": 91160576
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007857492477432298,
+      "loss": 2.1866,
+      "theoretical_loss": 4.862251313221231,
+      "tokens_seen": 91226112
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007857331995987964,
+      "loss": 2.0291,
+      "theoretical_loss": 4.861763772921561,
+      "tokens_seen": 91291648
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007857171514543632,
+      "loss": 2.1311,
+      "theoretical_loss": 4.861276680407279,
+      "tokens_seen": 91357184
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007857011033099298,
+      "loss": 2.0395,
+      "theoretical_loss": 4.860790034946331,
+      "tokens_seen": 91422720
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007856850551654966,
+      "loss": 2.1172,
+      "theoretical_loss": 4.860303835808384,
+      "tokens_seen": 91488256
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007856690070210632,
+      "loss": 2.0382,
+      "theoretical_loss": 4.859818082264822,
+      "tokens_seen": 91553792
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.00078565295887663,
+      "loss": 2.1008,
+      "theoretical_loss": 4.859332773588736,
+      "tokens_seen": 91619328
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007856369107321966,
+      "loss": 2.0718,
+      "theoretical_loss": 4.858847909054923,
+      "tokens_seen": 91684864
+    },
+    {
+      "epoch": 0.03,
+      "objective/train/docs_used": 41581,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 2.044309139251709,
+      "objective/train/theoretical_loss": 4.858363487939879,
+      "objective/train/tokens_used": 112210400,
+      "theoretical_loss": 4.858363487939879,
+      "tokens_seen": 91750400
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007856208625877634,
+      "loss": 2.0443,
+      "theoretical_loss": 4.858363487939879,
+      "tokens_seen": 91750400
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.00078560481444333,
+      "loss": 2.0722,
+      "theoretical_loss": 4.857879509521796,
+      "tokens_seen": 91815936
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007855887662988968,
+      "loss": 1.9915,
+      "theoretical_loss": 4.857395973080552,
+      "tokens_seen": 91881472
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007855727181544634,
+      "loss": 2.1225,
+      "theoretical_loss": 4.856912877897713,
+      "tokens_seen": 91947008
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007855566700100302,
+      "loss": 2.079,
+      "theoretical_loss": 4.85643022325652,
+      "tokens_seen": 92012544
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007855406218655968,
+      "loss": 2.1277,
+      "theoretical_loss": 4.855948008441891,
+      "tokens_seen": 92078080
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007855245737211636,
+      "loss": 2.0752,
+      "theoretical_loss": 4.85546623274041,
+      "tokens_seen": 92143616
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007855085255767302,
+      "loss": 1.9623,
+      "theoretical_loss": 4.854984895440325,
+      "tokens_seen": 92209152
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.000785492477432297,
+      "loss": 2.0694,
+      "theoretical_loss": 4.854503995831545,
+      "tokens_seen": 92274688
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007854764292878636,
+      "loss": 2.137,
+      "theoretical_loss": 4.854023533205627,
+      "tokens_seen": 92340224
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007854603811434303,
+      "loss": 1.9219,
+      "theoretical_loss": 4.853543506855784,
+      "tokens_seen": 92405760
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.000785444332998997,
+      "loss": 2.0269,
+      "theoretical_loss": 4.853063916076865,
+      "tokens_seen": 92471296
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007854282848545637,
+      "loss": 2.0388,
+      "theoretical_loss": 4.852584760165362,
+      "tokens_seen": 92536832
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007854122367101304,
+      "loss": 2.0979,
+      "theoretical_loss": 4.8521060384194,
+      "tokens_seen": 92602368
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007853961885656971,
+      "loss": 2.0948,
+      "theoretical_loss": 4.851627750138731,
+      "tokens_seen": 92667904
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007853801404212638,
+      "loss": 2.0571,
+      "theoretical_loss": 4.851149894624731,
+      "tokens_seen": 92733440
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007853640922768305,
+      "loss": 2.0777,
+      "theoretical_loss": 4.8506724711803955,
+      "tokens_seen": 92798976
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007853480441323972,
+      "loss": 2.0432,
+      "theoretical_loss": 4.850195479110333,
+      "tokens_seen": 92864512
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007853319959879639,
+      "loss": 2.0506,
+      "theoretical_loss": 4.849718917720762,
+      "tokens_seen": 92930048
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007853159478435306,
+      "loss": 2.1005,
+      "theoretical_loss": 4.849242786319503,
+      "tokens_seen": 92995584
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007852998996990973,
+      "loss": 2.1595,
+      "theoretical_loss": 4.848767084215979,
+      "tokens_seen": 93061120
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.000785283851554664,
+      "loss": 2.0331,
+      "theoretical_loss": 4.848291810721204,
+      "tokens_seen": 93126656
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007852678034102307,
+      "loss": 1.9728,
+      "theoretical_loss": 4.847816965147783,
+      "tokens_seen": 93192192
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007852517552657974,
+      "loss": 2.0968,
+      "theoretical_loss": 4.847342546809907,
+      "tokens_seen": 93257728
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007852357071213641,
+      "loss": 2.0359,
+      "theoretical_loss": 4.846868555023345,
+      "tokens_seen": 93323264
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007852196589769308,
+      "loss": 2.0387,
+      "theoretical_loss": 4.846394989105444,
+      "tokens_seen": 93388800
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007852036108324975,
+      "loss": 2.1412,
+      "theoretical_loss": 4.845921848375118,
+      "tokens_seen": 93454336
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007851875626880642,
+      "loss": 2.0493,
+      "theoretical_loss": 4.845449132152851,
+      "tokens_seen": 93519872
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007851715145436309,
+      "loss": 2.1466,
+      "theoretical_loss": 4.844976839760683,
+      "tokens_seen": 93585408
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007851554663991976,
+      "loss": 2.0749,
+      "theoretical_loss": 4.844504970522217,
+      "tokens_seen": 93650944
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007851394182547643,
+      "loss": 2.1461,
+      "theoretical_loss": 4.844033523762603,
+      "tokens_seen": 93716480
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.000785123370110331,
+      "loss": 2.0977,
+      "theoretical_loss": 4.843562498808538,
+      "tokens_seen": 93782016
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007851073219658977,
+      "loss": 1.9856,
+      "theoretical_loss": 4.843091894988265,
+      "tokens_seen": 93847552
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007850912738214644,
+      "loss": 2.1281,
+      "theoretical_loss": 4.842621711631564,
+      "tokens_seen": 93913088
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007850752256770311,
+      "loss": 1.9901,
+      "theoretical_loss": 4.842151948069745,
+      "tokens_seen": 93978624
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007850591775325978,
+      "loss": 2.1158,
+      "theoretical_loss": 4.84168260363565,
+      "tokens_seen": 94044160
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007850431293881645,
+      "loss": 1.9949,
+      "theoretical_loss": 4.841213677663648,
+      "tokens_seen": 94109696
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007850270812437312,
+      "loss": 2.1107,
+      "theoretical_loss": 4.840745169489621,
+      "tokens_seen": 94175232
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007850110330992979,
+      "loss": 2.0583,
+      "theoretical_loss": 4.840277078450969,
+      "tokens_seen": 94240768
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007849949849548646,
+      "loss": 1.8914,
+      "theoretical_loss": 4.839809403886606,
+      "tokens_seen": 94306304
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007849789368104313,
+      "loss": 2.0857,
+      "theoretical_loss": 4.839342145136947,
+      "tokens_seen": 94371840
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.000784962888665998,
+      "loss": 1.991,
+      "theoretical_loss": 4.838875301543913,
+      "tokens_seen": 94437376
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007849468405215647,
+      "loss": 2.0393,
+      "theoretical_loss": 4.838408872450919,
+      "tokens_seen": 94502912
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007849307923771314,
+      "loss": 1.9876,
+      "theoretical_loss": 4.837942857202876,
+      "tokens_seen": 94568448
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007849147442326981,
+      "loss": 2.1477,
+      "theoretical_loss": 4.837477255146178,
+      "tokens_seen": 94633984
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007848986960882648,
+      "loss": 2.0716,
+      "theoretical_loss": 4.837012065628711,
+      "tokens_seen": 94699520
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007848826479438315,
+      "loss": 2.0637,
+      "theoretical_loss": 4.836547287999833,
+      "tokens_seen": 94765056
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007848665997993982,
+      "loss": 2.0833,
+      "theoretical_loss": 4.836082921610381,
+      "tokens_seen": 94830592
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007848505516549649,
+      "loss": 2.0517,
+      "theoretical_loss": 4.835618965812664,
+      "tokens_seen": 94896128
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007848345035105317,
+      "loss": 2.0187,
+      "theoretical_loss": 4.835155419960453,
+      "tokens_seen": 94961664
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007848184553660983,
+      "loss": 1.9685,
+      "theoretical_loss": 4.8346922834089865,
+      "tokens_seen": 95027200
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.000784802407221665,
+      "loss": 2.0483,
+      "theoretical_loss": 4.834229555514956,
+      "tokens_seen": 95092736
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007847863590772317,
+      "loss": 2.021,
+      "theoretical_loss": 4.833767235636511,
+      "tokens_seen": 95158272
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007847703109327985,
+      "loss": 1.9857,
+      "theoretical_loss": 4.833305323133246,
+      "tokens_seen": 95223808
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007847542627883651,
+      "loss": 2.1295,
+      "theoretical_loss": 4.832843817366204,
+      "tokens_seen": 95289344
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007847382146439319,
+      "loss": 2.0209,
+      "theoretical_loss": 4.832382717697866,
+      "tokens_seen": 95354880
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007847221664994985,
+      "loss": 2.1253,
+      "theoretical_loss": 4.831922023492153,
+      "tokens_seen": 95420416
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007847061183550653,
+      "loss": 2.1372,
+      "theoretical_loss": 4.831461734114415,
+      "tokens_seen": 95485952
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007846900702106319,
+      "loss": 1.971,
+      "theoretical_loss": 4.831001848931429,
+      "tokens_seen": 95551488
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007846740220661987,
+      "loss": 1.9966,
+      "theoretical_loss": 4.830542367311402,
+      "tokens_seen": 95617024
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007846579739217653,
+      "loss": 2.0957,
+      "theoretical_loss": 4.830083288623955,
+      "tokens_seen": 95682560
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007846419257773321,
+      "loss": 2.0049,
+      "theoretical_loss": 4.829624612240126,
+      "tokens_seen": 95748096
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007846258776328987,
+      "loss": 2.0561,
+      "theoretical_loss": 4.829166337532367,
+      "tokens_seen": 95813632
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007846098294884655,
+      "loss": 1.9718,
+      "theoretical_loss": 4.828708463874532,
+      "tokens_seen": 95879168
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007845937813440321,
+      "loss": 2.0241,
+      "theoretical_loss": 4.828250990641886,
+      "tokens_seen": 95944704
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007845777331995989,
+      "loss": 2.1065,
+      "theoretical_loss": 4.827793917211083,
+      "tokens_seen": 96010240
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007845616850551655,
+      "loss": 2.0386,
+      "theoretical_loss": 4.827337242960181,
+      "tokens_seen": 96075776
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007845456369107323,
+      "loss": 1.9958,
+      "theoretical_loss": 4.8268809672686235,
+      "tokens_seen": 96141312
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007845295887662989,
+      "loss": 2.136,
+      "theoretical_loss": 4.826425089517245,
+      "tokens_seen": 96206848
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007845135406218657,
+      "loss": 1.9563,
+      "theoretical_loss": 4.825969609088258,
+      "tokens_seen": 96272384
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007844974924774323,
+      "loss": 1.9684,
+      "theoretical_loss": 4.825514525365258,
+      "tokens_seen": 96337920
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007844814443329991,
+      "loss": 2.0392,
+      "theoretical_loss": 4.825059837733214,
+      "tokens_seen": 96403456
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007844653961885657,
+      "loss": 2.0169,
+      "theoretical_loss": 4.824605545578463,
+      "tokens_seen": 96468992
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007844493480441325,
+      "loss": 1.9456,
+      "theoretical_loss": 4.824151648288714,
+      "tokens_seen": 96534528
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007844332998996991,
+      "loss": 2.1419,
+      "theoretical_loss": 4.823698145253035,
+      "tokens_seen": 96600064
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007844172517552659,
+      "loss": 2.0515,
+      "theoretical_loss": 4.823245035861852,
+      "tokens_seen": 96665600
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007844012036108325,
+      "loss": 2.0123,
+      "theoretical_loss": 4.822792319506952,
+      "tokens_seen": 96731136
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007843851554663993,
+      "loss": 2.0603,
+      "theoretical_loss": 4.822339995581466,
+      "tokens_seen": 96796672
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.000784369107321966,
+      "loss": 2.0218,
+      "theoretical_loss": 4.821888063479874,
+      "tokens_seen": 96862208
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007843530591775327,
+      "loss": 2.084,
+      "theoretical_loss": 4.8214365225980025,
+      "tokens_seen": 96927744
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007843370110330994,
+      "loss": 1.9555,
+      "theoretical_loss": 4.8209853723330145,
+      "tokens_seen": 96993280
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007843209628886661,
+      "loss": 2.02,
+      "theoretical_loss": 4.820534612083408,
+      "tokens_seen": 97058816
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007843049147442328,
+      "loss": 2.1019,
+      "theoretical_loss": 4.820084241249013,
+      "tokens_seen": 97124352
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007842888665997995,
+      "loss": 2.0529,
+      "theoretical_loss": 4.819634259230991,
+      "tokens_seen": 97189888
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007842728184553662,
+      "loss": 1.9951,
+      "theoretical_loss": 4.819184665431821,
+      "tokens_seen": 97255424
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007842567703109329,
+      "loss": 2.0711,
+      "theoretical_loss": 4.818735459255306,
+      "tokens_seen": 97320960
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007842407221664996,
+      "loss": 1.9041,
+      "theoretical_loss": 4.818286640106564,
+      "tokens_seen": 97386496
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007842246740220663,
+      "loss": 2.0159,
+      "theoretical_loss": 4.817838207392027,
+      "tokens_seen": 97452032
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007842086258776329,
+      "loss": 2.109,
+      "theoretical_loss": 4.817390160519435,
+      "tokens_seen": 97517568
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007841925777331997,
+      "loss": 2.0206,
+      "theoretical_loss": 4.816942498897831,
+      "tokens_seen": 97583104
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007841765295887663,
+      "loss": 1.9728,
+      "theoretical_loss": 4.816495221937563,
+      "tokens_seen": 97648640
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007841604814443331,
+      "loss": 1.9432,
+      "theoretical_loss": 4.816048329050273,
+      "tokens_seen": 97714176
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007841444332998997,
+      "loss": 2.0888,
+      "theoretical_loss": 4.815601819648897,
+      "tokens_seen": 97779712
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007841283851554665,
+      "loss": 2.1234,
+      "theoretical_loss": 4.815155693147666,
+      "tokens_seen": 97845248
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007841123370110331,
+      "loss": 2.1408,
+      "theoretical_loss": 4.81470994896209,
+      "tokens_seen": 97910784
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007840962888665999,
+      "loss": 2.0632,
+      "theoretical_loss": 4.8142645865089655,
+      "tokens_seen": 97976320
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007840802407221665,
+      "loss": 1.9535,
+      "theoretical_loss": 4.813819605206369,
+      "tokens_seen": 98041856
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007840641925777332,
+      "loss": 2.0469,
+      "theoretical_loss": 4.813375004473651,
+      "tokens_seen": 98107392
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007840481444332999,
+      "loss": 2.0781,
+      "theoretical_loss": 4.812930783731433,
+      "tokens_seen": 98172928
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007840320962888666,
+      "loss": 2.0,
+      "theoretical_loss": 4.8124869424016055,
+      "tokens_seen": 98238464
+    },
+    {
+      "epoch": 0.03,
+      "objective/train/docs_used": 43774,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 2.0564637184143066,
+      "objective/train/theoretical_loss": 4.812043479907323,
+      "objective/train/tokens_used": 118764000,
+      "theoretical_loss": 4.812043479907323,
+      "tokens_seen": 98304000
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007840160481444333,
+      "loss": 2.0565,
+      "theoretical_loss": 4.812043479907323,
+      "tokens_seen": 98304000
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.000784,
+      "loss": 2.0398,
+      "theoretical_loss": 4.8116003956730005,
+      "tokens_seen": 98369536
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007839839518555667,
+      "loss": 2.028,
+      "theoretical_loss": 4.811157689124311,
+      "tokens_seen": 98435072
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007839679037111334,
+      "loss": 2.0112,
+      "theoretical_loss": 4.8107153596881815,
+      "tokens_seen": 98500608
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007839518555667001,
+      "loss": 1.9807,
+      "theoretical_loss": 4.810273406792788,
+      "tokens_seen": 98566144
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007839358074222668,
+      "loss": 2.0177,
+      "theoretical_loss": 4.809831829867555,
+      "tokens_seen": 98631680
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007839197592778335,
+      "loss": 2.0595,
+      "theoretical_loss": 4.809390628343146,
+      "tokens_seen": 98697216
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007839037111334002,
+      "loss": 2.0136,
+      "theoretical_loss": 4.808949801651468,
+      "tokens_seen": 98762752
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007838876629889669,
+      "loss": 2.0394,
+      "theoretical_loss": 4.8085093492256625,
+      "tokens_seen": 98828288
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007838716148445336,
+      "loss": 2.0705,
+      "theoretical_loss": 4.808069270500102,
+      "tokens_seen": 98893824
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007838555667001003,
+      "loss": 2.061,
+      "theoretical_loss": 4.80762956491039,
+      "tokens_seen": 98959360
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.000783839518555667,
+      "loss": 1.9414,
+      "theoretical_loss": 4.807190231893355,
+      "tokens_seen": 99024896
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007838234704112337,
+      "loss": 1.9911,
+      "theoretical_loss": 4.806751270887048,
+      "tokens_seen": 99090432
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007838074222668004,
+      "loss": 2.0794,
+      "theoretical_loss": 4.806312681330736,
+      "tokens_seen": 99155968
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007837913741223671,
+      "loss": 2.0249,
+      "theoretical_loss": 4.805874462664903,
+      "tokens_seen": 99221504
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007837753259779338,
+      "loss": 2.051,
+      "theoretical_loss": 4.805436614331247,
+      "tokens_seen": 99287040
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007837592778335005,
+      "loss": 1.8966,
+      "theoretical_loss": 4.804999135772668,
+      "tokens_seen": 99352576
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007837432296890672,
+      "loss": 2.1574,
+      "theoretical_loss": 4.804562026433278,
+      "tokens_seen": 99418112
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007837271815446339,
+      "loss": 2.0045,
+      "theoretical_loss": 4.804125285758383,
+      "tokens_seen": 99483648
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007837111334002006,
+      "loss": 2.0258,
+      "theoretical_loss": 4.803688913194495,
+      "tokens_seen": 99549184
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007836950852557673,
+      "loss": 2.1233,
+      "theoretical_loss": 4.8032529081893145,
+      "tokens_seen": 99614720
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.000783679037111334,
+      "loss": 1.9813,
+      "theoretical_loss": 4.802817270191736,
+      "tokens_seen": 99680256
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007836629889669007,
+      "loss": 1.9358,
+      "theoretical_loss": 4.802381998651841,
+      "tokens_seen": 99745792
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007836469408224674,
+      "loss": 2.0028,
+      "theoretical_loss": 4.801947093020896,
+      "tokens_seen": 99811328
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007836308926780341,
+      "loss": 2.0859,
+      "theoretical_loss": 4.801512552751349,
+      "tokens_seen": 99876864
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007836148445336008,
+      "loss": 2.0154,
+      "theoretical_loss": 4.801078377296827,
+      "tokens_seen": 99942400
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007835987963891675,
+      "loss": 1.8705,
+      "theoretical_loss": 4.800644566112128,
+      "tokens_seen": 100007936
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007835827482447342,
+      "loss": 1.9481,
+      "theoretical_loss": 4.800211118653225,
+      "tokens_seen": 100073472
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.000783566700100301,
+      "loss": 2.0314,
+      "theoretical_loss": 4.799778034377258,
+      "tokens_seen": 100139008
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007835506519558676,
+      "loss": 1.9933,
+      "theoretical_loss": 4.799345312742531,
+      "tokens_seen": 100204544
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007835346038114343,
+      "loss": 1.993,
+      "theoretical_loss": 4.798912953208511,
+      "tokens_seen": 100270080
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.000783518555667001,
+      "loss": 2.149,
+      "theoretical_loss": 4.798480955235822,
+      "tokens_seen": 100335616
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007835025075225677,
+      "loss": 2.0972,
+      "theoretical_loss": 4.798049318286243,
+      "tokens_seen": 100401152
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007834864593781344,
+      "loss": 1.946,
+      "theoretical_loss": 4.797618041822707,
+      "tokens_seen": 100466688
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007834704112337011,
+      "loss": 1.9969,
+      "theoretical_loss": 4.797187125309293,
+      "tokens_seen": 100532224
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007834543630892678,
+      "loss": 2.0673,
+      "theoretical_loss": 4.796756568211227,
+      "tokens_seen": 100597760
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007834383149448346,
+      "loss": 1.9801,
+      "theoretical_loss": 4.796326369994876,
+      "tokens_seen": 100663296
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007834222668004012,
+      "loss": 1.964,
+      "theoretical_loss": 4.795896530127747,
+      "tokens_seen": 100728832
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.000783406218655968,
+      "loss": 1.99,
+      "theoretical_loss": 4.795467048078484,
+      "tokens_seen": 100794368
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007833901705115346,
+      "loss": 2.089,
+      "theoretical_loss": 4.79503792331686,
+      "tokens_seen": 100859904
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007833741223671014,
+      "loss": 2.0026,
+      "theoretical_loss": 4.794609155313782,
+      "tokens_seen": 100925440
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.000783358074222668,
+      "loss": 2.1178,
+      "theoretical_loss": 4.79418074354128,
+      "tokens_seen": 100990976
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007833420260782348,
+      "loss": 2.0955,
+      "theoretical_loss": 4.79375268747251,
+      "tokens_seen": 101056512
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007833259779338014,
+      "loss": 1.859,
+      "theoretical_loss": 4.793324986581746,
+      "tokens_seen": 101122048
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007833099297893682,
+      "loss": 2.0182,
+      "theoretical_loss": 4.79289764034438,
+      "tokens_seen": 101187584
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007832938816449348,
+      "loss": 2.0961,
+      "theoretical_loss": 4.792470648236918,
+      "tokens_seen": 101253120
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007832778335005016,
+      "loss": 2.0372,
+      "theoretical_loss": 4.792044009736978,
+      "tokens_seen": 101318656
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007832617853560682,
+      "loss": 1.8907,
+      "theoretical_loss": 4.791617724323283,
+      "tokens_seen": 101384192
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.000783245737211635,
+      "loss": 2.1526,
+      "theoretical_loss": 4.791191791475663,
+      "tokens_seen": 101449728
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007832296890672016,
+      "loss": 2.0196,
+      "theoretical_loss": 4.79076621067505,
+      "tokens_seen": 101515264
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007832136409227684,
+      "loss": 1.9194,
+      "theoretical_loss": 4.790340981403474,
+      "tokens_seen": 101580800
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.000783197592778335,
+      "loss": 2.0683,
+      "theoretical_loss": 4.7899161031440585,
+      "tokens_seen": 101646336
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007831815446339018,
+      "loss": 2.0078,
+      "theoretical_loss": 4.7894915753810245,
+      "tokens_seen": 101711872
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007831654964894685,
+      "loss": 2.0629,
+      "theoretical_loss": 4.789067397599677,
+      "tokens_seen": 101777408
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007831494483450352,
+      "loss": 2.1398,
+      "theoretical_loss": 4.788643569286411,
+      "tokens_seen": 101842944
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007831334002006019,
+      "loss": 1.9818,
+      "theoretical_loss": 4.788220089928705,
+      "tokens_seen": 101908480
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007831173520561686,
+      "loss": 1.9772,
+      "theoretical_loss": 4.787796959015118,
+      "tokens_seen": 101974016
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007831013039117353,
+      "loss": 2.0421,
+      "theoretical_loss": 4.787374176035287,
+      "tokens_seen": 102039552
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.000783085255767302,
+      "loss": 2.0502,
+      "theoretical_loss": 4.78695174047992,
+      "tokens_seen": 102105088
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007830692076228687,
+      "loss": 2.0433,
+      "theoretical_loss": 4.786529651840802,
+      "tokens_seen": 102170624
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007830531594784354,
+      "loss": 1.8912,
+      "theoretical_loss": 4.786107909610784,
+      "tokens_seen": 102236160
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007830371113340021,
+      "loss": 2.0267,
+      "theoretical_loss": 4.785686513283783,
+      "tokens_seen": 102301696
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007830210631895688,
+      "loss": 2.0968,
+      "theoretical_loss": 4.785265462354779,
+      "tokens_seen": 102367232
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007830050150451355,
+      "loss": 2.0695,
+      "theoretical_loss": 4.7848447563198135,
+      "tokens_seen": 102432768
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007829889669007022,
+      "loss": 1.9965,
+      "theoretical_loss": 4.784424394675983,
+      "tokens_seen": 102498304
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007829729187562689,
+      "loss": 2.0899,
+      "theoretical_loss": 4.784004376921439,
+      "tokens_seen": 102563840
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007829568706118356,
+      "loss": 1.9928,
+      "theoretical_loss": 4.783584702555387,
+      "tokens_seen": 102629376
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007829408224674023,
+      "loss": 2.0884,
+      "theoretical_loss": 4.783165371078075,
+      "tokens_seen": 102694912
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.000782924774322969,
+      "loss": 2.0653,
+      "theoretical_loss": 4.782746381990803,
+      "tokens_seen": 102760448
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007829087261785357,
+      "loss": 2.0589,
+      "theoretical_loss": 4.78232773479591,
+      "tokens_seen": 102825984
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007828926780341024,
+      "loss": 2.0893,
+      "theoretical_loss": 4.7819094289967765,
+      "tokens_seen": 102891520
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007828766298896691,
+      "loss": 2.0153,
+      "theoretical_loss": 4.781491464097819,
+      "tokens_seen": 102957056
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007828605817452358,
+      "loss": 2.0343,
+      "theoretical_loss": 4.781073839604488,
+      "tokens_seen": 103022592
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007828445336008025,
+      "loss": 1.9041,
+      "theoretical_loss": 4.780656555023268,
+      "tokens_seen": 103088128
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007828284854563692,
+      "loss": 2.0899,
+      "theoretical_loss": 4.780239609861669,
+      "tokens_seen": 103153664
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007828124373119358,
+      "loss": 2.0505,
+      "theoretical_loss": 4.77982300362823,
+      "tokens_seen": 103219200
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007827963891675026,
+      "loss": 1.9479,
+      "theoretical_loss": 4.779406735832509,
+      "tokens_seen": 103284736
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007827803410230692,
+      "loss": 2.0336,
+      "theoretical_loss": 4.778990805985089,
+      "tokens_seen": 103350272
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.000782764292878636,
+      "loss": 1.9833,
+      "theoretical_loss": 4.778575213597568,
+      "tokens_seen": 103415808
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007827482447342026,
+      "loss": 1.949,
+      "theoretical_loss": 4.778159958182557,
+      "tokens_seen": 103481344
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007827321965897694,
+      "loss": 1.9087,
+      "theoretical_loss": 4.777745039253682,
+      "tokens_seen": 103546880
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.000782716148445336,
+      "loss": 2.1781,
+      "theoretical_loss": 4.777330456325579,
+      "tokens_seen": 103612416
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007827001003009028,
+      "loss": 1.9407,
+      "theoretical_loss": 4.776916208913887,
+      "tokens_seen": 103677952
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007826840521564694,
+      "loss": 1.9856,
+      "theoretical_loss": 4.776502296535252,
+      "tokens_seen": 103743488
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007826680040120361,
+      "loss": 2.1527,
+      "theoretical_loss": 4.776088718707317,
+      "tokens_seen": 103809024
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007826519558676028,
+      "loss": 1.965,
+      "theoretical_loss": 4.77567547494873,
+      "tokens_seen": 103874560
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007826359077231695,
+      "loss": 1.9659,
+      "theoretical_loss": 4.775262564779128,
+      "tokens_seen": 103940096
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007826198595787362,
+      "loss": 1.9975,
+      "theoretical_loss": 4.774849987719145,
+      "tokens_seen": 104005632
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007826038114343029,
+      "loss": 1.8441,
+      "theoretical_loss": 4.774437743290404,
+      "tokens_seen": 104071168
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007825877632898696,
+      "loss": 1.9205,
+      "theoretical_loss": 4.774025831015517,
+      "tokens_seen": 104136704
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007825717151454363,
+      "loss": 2.0538,
+      "theoretical_loss": 4.7736142504180785,
+      "tokens_seen": 104202240
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.000782555667001003,
+      "loss": 2.0038,
+      "theoretical_loss": 4.773203001022668,
+      "tokens_seen": 104267776
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007825396188565697,
+      "loss": 2.0319,
+      "theoretical_loss": 4.772792082354843,
+      "tokens_seen": 104333312
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007825235707121364,
+      "loss": 1.9619,
+      "theoretical_loss": 4.77238149394114,
+      "tokens_seen": 104398848
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007825075225677031,
+      "loss": 2.1084,
+      "theoretical_loss": 4.771971235309065,
+      "tokens_seen": 104464384
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007824914744232698,
+      "loss": 1.9957,
+      "theoretical_loss": 4.771561305987104,
+      "tokens_seen": 104529920
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007824754262788365,
+      "loss": 1.953,
+      "theoretical_loss": 4.771151705504707,
+      "tokens_seen": 104595456
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007824593781344032,
+      "loss": 2.0034,
+      "theoretical_loss": 4.770742433392289,
+      "tokens_seen": 104660992
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007824433299899699,
+      "loss": 2.0302,
+      "theoretical_loss": 4.770333489181233,
+      "tokens_seen": 104726528
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007824272818455366,
+      "loss": 2.0035,
+      "theoretical_loss": 4.769924872403883,
+      "tokens_seen": 104792064
+    },
+    {
+      "epoch": 0.03,
+      "objective/train/docs_used": 46458,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.9015421867370605,
+      "objective/train/theoretical_loss": 4.769516582593539,
+      "objective/train/tokens_used": 125317600,
+      "theoretical_loss": 4.769516582593539,
+      "tokens_seen": 104857600
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007824112337011033,
+      "loss": 1.9015,
+      "theoretical_loss": 4.769516582593539,
+      "tokens_seen": 104857600
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.00078239518555667,
+      "loss": 1.8893,
+      "theoretical_loss": 4.769108619284461,
+      "tokens_seen": 104923136
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007823791374122367,
+      "loss": 1.978,
+      "theoretical_loss": 4.768700982011859,
+      "tokens_seen": 104988672
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007823630892678034,
+      "loss": 1.8873,
+      "theoretical_loss": 4.768293670311899,
+      "tokens_seen": 105054208
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007823470411233701,
+      "loss": 2.0086,
+      "theoretical_loss": 4.7678866837216916,
+      "tokens_seen": 105119744
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007823309929789368,
+      "loss": 1.969,
+      "theoretical_loss": 4.767480021779294,
+      "tokens_seen": 105185280
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007823149448345035,
+      "loss": 2.1185,
+      "theoretical_loss": 4.76707368402371,
+      "tokens_seen": 105250816
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007822988966900702,
+      "loss": 2.0132,
+      "theoretical_loss": 4.766667669994881,
+      "tokens_seen": 105316352
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007822828485456369,
+      "loss": 2.0492,
+      "theoretical_loss": 4.76626197923369,
+      "tokens_seen": 105381888
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007822668004012036,
+      "loss": 2.0807,
+      "theoretical_loss": 4.765856611281953,
+      "tokens_seen": 105447424
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007822507522567703,
+      "loss": 1.9707,
+      "theoretical_loss": 4.765451565682423,
+      "tokens_seen": 105512960
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.000782234704112337,
+      "loss": 1.9635,
+      "theoretical_loss": 4.765046841978783,
+      "tokens_seen": 105578496
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007822186559679037,
+      "loss": 2.0984,
+      "theoretical_loss": 4.764642439715642,
+      "tokens_seen": 105644032
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007822026078234704,
+      "loss": 2.1008,
+      "theoretical_loss": 4.764238358438539,
+      "tokens_seen": 105709568
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007821865596790371,
+      "loss": 1.9651,
+      "theoretical_loss": 4.763834597693935,
+      "tokens_seen": 105775104
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007821705115346038,
+      "loss": 1.9968,
+      "theoretical_loss": 4.763431157029213,
+      "tokens_seen": 105840640
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007821544633901705,
+      "loss": 1.9752,
+      "theoretical_loss": 4.763028035992675,
+      "tokens_seen": 105906176
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007821384152457372,
+      "loss": 2.1189,
+      "theoretical_loss": 4.762625234133537,
+      "tokens_seen": 105971712
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007821223671013039,
+      "loss": 2.0155,
+      "theoretical_loss": 4.762222751001933,
+      "tokens_seen": 106037248
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007821063189568706,
+      "loss": 1.9496,
+      "theoretical_loss": 4.761820586148906,
+      "tokens_seen": 106102784
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007820902708124373,
+      "loss": 1.951,
+      "theoretical_loss": 4.761418739126407,
+      "tokens_seen": 106168320
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.000782074222668004,
+      "loss": 1.8936,
+      "theoretical_loss": 4.761017209487298,
+      "tokens_seen": 106233856
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007820581745235707,
+      "loss": 1.8837,
+      "theoretical_loss": 4.760615996785342,
+      "tokens_seen": 106299392
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007820421263791375,
+      "loss": 2.0782,
+      "theoretical_loss": 4.760215100575204,
+      "tokens_seen": 106364928
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007820260782347041,
+      "loss": 1.9477,
+      "theoretical_loss": 4.759814520412451,
+      "tokens_seen": 106430464
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007820100300902709,
+      "loss": 2.0783,
+      "theoretical_loss": 4.759414255853547,
+      "tokens_seen": 106496000
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007819939819458375,
+      "loss": 2.0302,
+      "theoretical_loss": 4.759014306455848,
+      "tokens_seen": 106561536
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007819779338014043,
+      "loss": 1.9217,
+      "theoretical_loss": 4.758614671777604,
+      "tokens_seen": 106627072
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.000781961885656971,
+      "loss": 2.0256,
+      "theoretical_loss": 4.758215351377956,
+      "tokens_seen": 106692608
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007819458375125377,
+      "loss": 2.076,
+      "theoretical_loss": 4.757816344816933,
+      "tokens_seen": 106758144
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007819297893681044,
+      "loss": 1.9322,
+      "theoretical_loss": 4.75741765165545,
+      "tokens_seen": 106823680
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007819137412236711,
+      "loss": 1.942,
+      "theoretical_loss": 4.7570192714553015,
+      "tokens_seen": 106889216
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007818976930792378,
+      "loss": 1.8512,
+      "theoretical_loss": 4.756621203779169,
+      "tokens_seen": 106954752
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007818816449348045,
+      "loss": 2.0647,
+      "theoretical_loss": 4.756223448190605,
+      "tokens_seen": 107020288
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007818655967903712,
+      "loss": 1.9946,
+      "theoretical_loss": 4.755826004254046,
+      "tokens_seen": 107085824
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007818495486459379,
+      "loss": 1.9706,
+      "theoretical_loss": 4.755428871534798,
+      "tokens_seen": 107151360
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007818335005015046,
+      "loss": 1.98,
+      "theoretical_loss": 4.755032049599039,
+      "tokens_seen": 107216896
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007818174523570713,
+      "loss": 1.994,
+      "theoretical_loss": 4.754635538013817,
+      "tokens_seen": 107282432
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.000781801404212638,
+      "loss": 1.8982,
+      "theoretical_loss": 4.754239336347048,
+      "tokens_seen": 107347968
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007817853560682047,
+      "loss": 2.0406,
+      "theoretical_loss": 4.753843444167513,
+      "tokens_seen": 107413504
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007817693079237714,
+      "loss": 1.9653,
+      "theoretical_loss": 4.753447861044853,
+      "tokens_seen": 107479040
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007817532597793381,
+      "loss": 1.8986,
+      "theoretical_loss": 4.753052586549572,
+      "tokens_seen": 107544576
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007817372116349048,
+      "loss": 2.018,
+      "theoretical_loss": 4.75265762025303,
+      "tokens_seen": 107610112
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007817211634904715,
+      "loss": 2.1007,
+      "theoretical_loss": 4.752262961727446,
+      "tokens_seen": 107675648
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007817051153460382,
+      "loss": 1.9031,
+      "theoretical_loss": 4.75186861054589,
+      "tokens_seen": 107741184
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007816890672016049,
+      "loss": 1.9357,
+      "theoretical_loss": 4.751474566282285,
+      "tokens_seen": 107806720
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007816730190571716,
+      "loss": 1.9696,
+      "theoretical_loss": 4.751080828511402,
+      "tokens_seen": 107872256
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007816569709127383,
+      "loss": 1.9687,
+      "theoretical_loss": 4.750687396808859,
+      "tokens_seen": 107937792
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.000781640922768305,
+      "loss": 2.0962,
+      "theoretical_loss": 4.7502942707511195,
+      "tokens_seen": 108003328
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007816248746238717,
+      "loss": 2.0112,
+      "theoretical_loss": 4.749901449915489,
+      "tokens_seen": 108068864
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007816088264794384,
+      "loss": 1.9628,
+      "theoretical_loss": 4.749508933880117,
+      "tokens_seen": 108134400
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007815927783350051,
+      "loss": 2.0048,
+      "theoretical_loss": 4.7491167222239845,
+      "tokens_seen": 108199936
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007815767301905718,
+      "loss": 1.9553,
+      "theoretical_loss": 4.7487248145269145,
+      "tokens_seen": 108265472
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007815606820461385,
+      "loss": 1.9705,
+      "theoretical_loss": 4.7483332103695615,
+      "tokens_seen": 108331008
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007815446339017052,
+      "loss": 1.9298,
+      "theoretical_loss": 4.747941909333411,
+      "tokens_seen": 108396544
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007815285857572719,
+      "loss": 1.9251,
+      "theoretical_loss": 4.747550911000781,
+      "tokens_seen": 108462080
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007815125376128386,
+      "loss": 1.9817,
+      "theoretical_loss": 4.747160214954813,
+      "tokens_seen": 108527616
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007814964894684053,
+      "loss": 2.0413,
+      "theoretical_loss": 4.746769820779477,
+      "tokens_seen": 108593152
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.000781480441323972,
+      "loss": 1.931,
+      "theoretical_loss": 4.746379728059566,
+      "tokens_seen": 108658688
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007814643931795387,
+      "loss": 1.9666,
+      "theoretical_loss": 4.745989936380692,
+      "tokens_seen": 108724224
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007814483450351054,
+      "loss": 1.9137,
+      "theoretical_loss": 4.745600445329288,
+      "tokens_seen": 108789760
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007814322968906721,
+      "loss": 2.0553,
+      "theoretical_loss": 4.745211254492602,
+      "tokens_seen": 108855296
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007814162487462388,
+      "loss": 2.0251,
+      "theoretical_loss": 4.744822363458699,
+      "tokens_seen": 108920832
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007814002006018055,
+      "loss": 1.8646,
+      "theoretical_loss": 4.744433771816455,
+      "tokens_seen": 108986368
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007813841524573721,
+      "loss": 1.913,
+      "theoretical_loss": 4.744045479155557,
+      "tokens_seen": 109051904
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007813681043129389,
+      "loss": 1.8624,
+      "theoretical_loss": 4.7436574850665005,
+      "tokens_seen": 109117440
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007813520561685055,
+      "loss": 2.0835,
+      "theoretical_loss": 4.743269789140587,
+      "tokens_seen": 109182976
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007813360080240723,
+      "loss": 1.8438,
+      "theoretical_loss": 4.742882390969925,
+      "tokens_seen": 109248512
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007813199598796389,
+      "loss": 2.0004,
+      "theoretical_loss": 4.742495290147421,
+      "tokens_seen": 109314048
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007813039117352057,
+      "loss": 1.9366,
+      "theoretical_loss": 4.742108486266783,
+      "tokens_seen": 109379584
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007812878635907723,
+      "loss": 2.0586,
+      "theoretical_loss": 4.74172197892252,
+      "tokens_seen": 109445120
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.000781271815446339,
+      "loss": 1.9156,
+      "theoretical_loss": 4.741335767709934,
+      "tokens_seen": 109510656
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007812557673019057,
+      "loss": 1.9783,
+      "theoretical_loss": 4.740949852225122,
+      "tokens_seen": 109576192
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007812397191574724,
+      "loss": 1.9158,
+      "theoretical_loss": 4.740564232064973,
+      "tokens_seen": 109641728
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007812236710130391,
+      "loss": 2.059,
+      "theoretical_loss": 4.7401789068271665,
+      "tokens_seen": 109707264
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007812076228686058,
+      "loss": 2.0115,
+      "theoretical_loss": 4.73979387611017,
+      "tokens_seen": 109772800
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007811915747241725,
+      "loss": 1.8627,
+      "theoretical_loss": 4.739409139513235,
+      "tokens_seen": 109838336
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007811755265797392,
+      "loss": 1.9969,
+      "theoretical_loss": 4.739024696636401,
+      "tokens_seen": 109903872
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007811594784353059,
+      "loss": 2.0016,
+      "theoretical_loss": 4.738640547080485,
+      "tokens_seen": 109969408
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007811434302908726,
+      "loss": 1.9306,
+      "theoretical_loss": 4.738256690447088,
+      "tokens_seen": 110034944
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007811273821464393,
+      "loss": 1.9452,
+      "theoretical_loss": 4.7378731263385845,
+      "tokens_seen": 110100480
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.000781111334002006,
+      "loss": 2.0136,
+      "theoretical_loss": 4.737489854358129,
+      "tokens_seen": 110166016
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007810952858575727,
+      "loss": 2.14,
+      "theoretical_loss": 4.737106874109649,
+      "tokens_seen": 110231552
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007810792377131394,
+      "loss": 1.9898,
+      "theoretical_loss": 4.736724185197844,
+      "tokens_seen": 110297088
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007810631895687061,
+      "loss": 1.8838,
+      "theoretical_loss": 4.7363417872281826,
+      "tokens_seen": 110362624
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007810471414242728,
+      "loss": 1.9304,
+      "theoretical_loss": 4.735959679806903,
+      "tokens_seen": 110428160
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007810310932798395,
+      "loss": 1.8655,
+      "theoretical_loss": 4.735577862541007,
+      "tokens_seen": 110493696
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007810150451354062,
+      "loss": 1.8707,
+      "theoretical_loss": 4.735196335038264,
+      "tokens_seen": 110559232
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007809989969909729,
+      "loss": 1.905,
+      "theoretical_loss": 4.734815096907203,
+      "tokens_seen": 110624768
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007809829488465396,
+      "loss": 1.9189,
+      "theoretical_loss": 4.734434147757117,
+      "tokens_seen": 110690304
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007809669007021063,
+      "loss": 1.991,
+      "theoretical_loss": 4.734053487198054,
+      "tokens_seen": 110755840
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.000780950852557673,
+      "loss": 1.8235,
+      "theoretical_loss": 4.733673114840817,
+      "tokens_seen": 110821376
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007809348044132397,
+      "loss": 1.8794,
+      "theoretical_loss": 4.73329303029697,
+      "tokens_seen": 110886912
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007809187562688064,
+      "loss": 1.9225,
+      "theoretical_loss": 4.732913233178825,
+      "tokens_seen": 110952448
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007809027081243731,
+      "loss": 2.0501,
+      "theoretical_loss": 4.732533723099444,
+      "tokens_seen": 111017984
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007808866599799398,
+      "loss": 1.9909,
+      "theoretical_loss": 4.732154499672642,
+      "tokens_seen": 111083520
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007808706118355065,
+      "loss": 1.9695,
+      "theoretical_loss": 4.731775562512977,
+      "tokens_seen": 111149056
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007808545636910732,
+      "loss": 2.0358,
+      "theoretical_loss": 4.7313969112357555,
+      "tokens_seen": 111214592
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007808385155466399,
+      "loss": 2.052,
+      "theoretical_loss": 4.731018545457024,
+      "tokens_seen": 111280128
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007808224674022066,
+      "loss": 1.9492,
+      "theoretical_loss": 4.730640464793574,
+      "tokens_seen": 111345664
+    },
+    {
+      "epoch": 0.03,
+      "objective/train/docs_used": 48676,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 2.0233168601989746,
+      "objective/train/theoretical_loss": 4.730262668862931,
+      "objective/train/tokens_used": 131871200,
+      "theoretical_loss": 4.730262668862931,
+      "tokens_seen": 111411200
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007808064192577733,
+      "loss": 2.0233,
+      "theoretical_loss": 4.730262668862931,
+      "tokens_seen": 111411200
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.00078079037111334,
+      "loss": 1.9551,
+      "theoretical_loss": 4.729885157283366,
+      "tokens_seen": 111476736
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007807743229689067,
+      "loss": 1.9509,
+      "theoretical_loss": 4.729507929673879,
+      "tokens_seen": 111542272
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007807582748244735,
+      "loss": 2.0183,
+      "theoretical_loss": 4.729130985654208,
+      "tokens_seen": 111607808
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007807422266800401,
+      "loss": 1.9095,
+      "theoretical_loss": 4.72875432484482,
+      "tokens_seen": 111673344
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007807261785356069,
+      "loss": 2.0049,
+      "theoretical_loss": 4.728377946866915,
+      "tokens_seen": 111738880
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007807101303911736,
+      "loss": 1.9661,
+      "theoretical_loss": 4.72800185134242,
+      "tokens_seen": 111804416
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007806940822467403,
+      "loss": 1.9616,
+      "theoretical_loss": 4.727626037893989,
+      "tokens_seen": 111869952
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.000780678034102307,
+      "loss": 1.9889,
+      "theoretical_loss": 4.727250506145001,
+      "tokens_seen": 111935488
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007806619859578737,
+      "loss": 1.9736,
+      "theoretical_loss": 4.726875255719557,
+      "tokens_seen": 112001024
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007806459378134404,
+      "loss": 1.7773,
+      "theoretical_loss": 4.726500286242482,
+      "tokens_seen": 112066560
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007806298896690071,
+      "loss": 1.9239,
+      "theoretical_loss": 4.7261255973393155,
+      "tokens_seen": 112132096
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007806138415245738,
+      "loss": 1.9951,
+      "theoretical_loss": 4.725751188636319,
+      "tokens_seen": 112197632
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007805977933801405,
+      "loss": 1.957,
+      "theoretical_loss": 4.725377059760468,
+      "tokens_seen": 112263168
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007805817452357072,
+      "loss": 1.9312,
+      "theoretical_loss": 4.725003210339452,
+      "tokens_seen": 112328704
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007805656970912739,
+      "loss": 1.9269,
+      "theoretical_loss": 4.724629640001672,
+      "tokens_seen": 112394240
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007805496489468406,
+      "loss": 1.9711,
+      "theoretical_loss": 4.724256348376242,
+      "tokens_seen": 112459776
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007805336008024073,
+      "loss": 1.9213,
+      "theoretical_loss": 4.723883335092982,
+      "tokens_seen": 112525312
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.000780517552657974,
+      "loss": 1.9006,
+      "theoretical_loss": 4.723510599782419,
+      "tokens_seen": 112590848
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007805015045135407,
+      "loss": 1.955,
+      "theoretical_loss": 4.723138142075788,
+      "tokens_seen": 112656384
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007804854563691074,
+      "loss": 1.995,
+      "theoretical_loss": 4.722765961605024,
+      "tokens_seen": 112721920
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007804694082246741,
+      "loss": 1.8787,
+      "theoretical_loss": 4.722394058002765,
+      "tokens_seen": 112787456
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007804533600802408,
+      "loss": 1.9327,
+      "theoretical_loss": 4.7220224309023475,
+      "tokens_seen": 112852992
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007804373119358075,
+      "loss": 1.9779,
+      "theoretical_loss": 4.721651079937811,
+      "tokens_seen": 112918528
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007804212637913742,
+      "loss": 1.9749,
+      "theoretical_loss": 4.7212800047438845,
+      "tokens_seen": 112984064
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007804052156469409,
+      "loss": 2.0586,
+      "theoretical_loss": 4.720909204955996,
+      "tokens_seen": 113049600
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007803891675025076,
+      "loss": 2.066,
+      "theoretical_loss": 4.720538680210268,
+      "tokens_seen": 113115136
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007803731193580743,
+      "loss": 2.0196,
+      "theoretical_loss": 4.720168430143508,
+      "tokens_seen": 113180672
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.000780357071213641,
+      "loss": 1.9345,
+      "theoretical_loss": 4.719798454393219,
+      "tokens_seen": 113246208
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007803410230692077,
+      "loss": 1.973,
+      "theoretical_loss": 4.719428752597588,
+      "tokens_seen": 113311744
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007803249749247744,
+      "loss": 1.9379,
+      "theoretical_loss": 4.719059324395493,
+      "tokens_seen": 113377280
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007803089267803411,
+      "loss": 1.9317,
+      "theoretical_loss": 4.71869016942649,
+      "tokens_seen": 113442816
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007802928786359078,
+      "loss": 2.0539,
+      "theoretical_loss": 4.718321287330823,
+      "tokens_seen": 113508352
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007802768304914745,
+      "loss": 2.0419,
+      "theoretical_loss": 4.7179526777494125,
+      "tokens_seen": 113573888
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007802607823470412,
+      "loss": 1.9361,
+      "theoretical_loss": 4.717584340323864,
+      "tokens_seen": 113639424
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007802447342026079,
+      "loss": 1.9626,
+      "theoretical_loss": 4.7172162746964545,
+      "tokens_seen": 113704960
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007802286860581746,
+      "loss": 2.1289,
+      "theoretical_loss": 4.716848480510142,
+      "tokens_seen": 113770496
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007802126379137413,
+      "loss": 1.8971,
+      "theoretical_loss": 4.716480957408556,
+      "tokens_seen": 113836032
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.000780196589769308,
+      "loss": 1.9621,
+      "theoretical_loss": 4.716113705035999,
+      "tokens_seen": 113901568
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007801805416248747,
+      "loss": 1.9462,
+      "theoretical_loss": 4.715746723037448,
+      "tokens_seen": 113967104
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007801644934804414,
+      "loss": 1.9668,
+      "theoretical_loss": 4.715380011058542,
+      "tokens_seen": 114032640
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007801484453360081,
+      "loss": 2.0023,
+      "theoretical_loss": 4.715013568745597,
+      "tokens_seen": 114098176
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007801323971915748,
+      "loss": 1.967,
+      "theoretical_loss": 4.714647395745587,
+      "tokens_seen": 114163712
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007801163490471415,
+      "loss": 1.9684,
+      "theoretical_loss": 4.714281491706154,
+      "tokens_seen": 114229248
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007801003009027082,
+      "loss": 2.0977,
+      "theoretical_loss": 4.713915856275603,
+      "tokens_seen": 114294784
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007800842527582749,
+      "loss": 1.8655,
+      "theoretical_loss": 4.713550489102901,
+      "tokens_seen": 114360320
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007800682046138416,
+      "loss": 1.8451,
+      "theoretical_loss": 4.7131853898376725,
+      "tokens_seen": 114425856
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007800521564694083,
+      "loss": 1.8491,
+      "theoretical_loss": 4.7128205581302005,
+      "tokens_seen": 114491392
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.000780036108324975,
+      "loss": 1.8477,
+      "theoretical_loss": 4.712455993631425,
+      "tokens_seen": 114556928
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007800200601805417,
+      "loss": 2.0076,
+      "theoretical_loss": 4.712091695992941,
+      "tokens_seen": 114622464
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007800040120361084,
+      "loss": 1.9494,
+      "theoretical_loss": 4.711727664866995,
+      "tokens_seen": 114688000
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.000779987963891675,
+      "loss": 1.9673,
+      "theoretical_loss": 4.7113638999064875,
+      "tokens_seen": 114753536
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007799719157472418,
+      "loss": 2.0366,
+      "theoretical_loss": 4.711000400764967,
+      "tokens_seen": 114819072
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007799558676028084,
+      "loss": 2.0538,
+      "theoretical_loss": 4.71063716709663,
+      "tokens_seen": 114884608
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007799398194583752,
+      "loss": 2.0796,
+      "theoretical_loss": 4.7102741985563235,
+      "tokens_seen": 114950144
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007799237713139418,
+      "loss": 1.9856,
+      "theoretical_loss": 4.709911494799533,
+      "tokens_seen": 115015680
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007799077231695086,
+      "loss": 1.9479,
+      "theoretical_loss": 4.709549055482395,
+      "tokens_seen": 115081216
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007798916750250752,
+      "loss": 1.8631,
+      "theoretical_loss": 4.709186880261683,
+      "tokens_seen": 115146752
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.000779875626880642,
+      "loss": 1.8449,
+      "theoretical_loss": 4.708824968794813,
+      "tokens_seen": 115212288
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007798595787362086,
+      "loss": 1.8909,
+      "theoretical_loss": 4.70846332073984,
+      "tokens_seen": 115277824
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007798435305917753,
+      "loss": 2.0191,
+      "theoretical_loss": 4.708101935755455,
+      "tokens_seen": 115343360
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.000779827482447342,
+      "loss": 1.9506,
+      "theoretical_loss": 4.707740813500988,
+      "tokens_seen": 115408896
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007798114343029087,
+      "loss": 1.9725,
+      "theoretical_loss": 4.707379953636399,
+      "tokens_seen": 115474432
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007797953861584754,
+      "loss": 1.9368,
+      "theoretical_loss": 4.7070193558222835,
+      "tokens_seen": 115539968
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007797793380140421,
+      "loss": 1.8776,
+      "theoretical_loss": 4.7066590197198686,
+      "tokens_seen": 115605504
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007797632898696088,
+      "loss": 1.9702,
+      "theoretical_loss": 4.706298944991008,
+      "tokens_seen": 115671040
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007797472417251755,
+      "loss": 2.0099,
+      "theoretical_loss": 4.705939131298189,
+      "tokens_seen": 115736576
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007797311935807422,
+      "loss": 1.9403,
+      "theoretical_loss": 4.705579578304519,
+      "tokens_seen": 115802112
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007797151454363089,
+      "loss": 2.0244,
+      "theoretical_loss": 4.705220285673736,
+      "tokens_seen": 115867648
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007796990972918756,
+      "loss": 1.9738,
+      "theoretical_loss": 4.704861253070198,
+      "tokens_seen": 115933184
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007796830491474423,
+      "loss": 1.8945,
+      "theoretical_loss": 4.704502480158887,
+      "tokens_seen": 115998720
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.000779667001003009,
+      "loss": 1.8195,
+      "theoretical_loss": 4.704143966605406,
+      "tokens_seen": 116064256
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007796509528585757,
+      "loss": 1.9645,
+      "theoretical_loss": 4.703785712075973,
+      "tokens_seen": 116129792
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007796349047141424,
+      "loss": 1.9542,
+      "theoretical_loss": 4.703427716237432,
+      "tokens_seen": 116195328
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007796188565697091,
+      "loss": 1.9322,
+      "theoretical_loss": 4.703069978757233,
+      "tokens_seen": 116260864
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007796028084252758,
+      "loss": 1.82,
+      "theoretical_loss": 4.702712499303448,
+      "tokens_seen": 116326400
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007795867602808425,
+      "loss": 1.9836,
+      "theoretical_loss": 4.7023552775447595,
+      "tokens_seen": 116391936
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007795707121364092,
+      "loss": 1.939,
+      "theoretical_loss": 4.701998313150462,
+      "tokens_seen": 116457472
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.000779554663991976,
+      "loss": 2.086,
+      "theoretical_loss": 4.701641605790459,
+      "tokens_seen": 116523008
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007795386158475426,
+      "loss": 1.7689,
+      "theoretical_loss": 4.701285155135265,
+      "tokens_seen": 116588544
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007795225677031094,
+      "loss": 2.0352,
+      "theoretical_loss": 4.700928960856,
+      "tokens_seen": 116654080
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.000779506519558676,
+      "loss": 2.0243,
+      "theoretical_loss": 4.70057302262439,
+      "tokens_seen": 116719616
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007794904714142428,
+      "loss": 1.945,
+      "theoretical_loss": 4.700217340112767,
+      "tokens_seen": 116785152
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007794744232698094,
+      "loss": 1.9355,
+      "theoretical_loss": 4.699861912994063,
+      "tokens_seen": 116850688
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007794583751253762,
+      "loss": 1.7788,
+      "theoretical_loss": 4.699506740941813,
+      "tokens_seen": 116916224
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007794423269809428,
+      "loss": 2.1069,
+      "theoretical_loss": 4.699151823630155,
+      "tokens_seen": 116981760
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007794262788365096,
+      "loss": 1.904,
+      "theoretical_loss": 4.698797160733818,
+      "tokens_seen": 117047296
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007794102306920762,
+      "loss": 1.9836,
+      "theoretical_loss": 4.6984427519281375,
+      "tokens_seen": 117112832
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.000779394182547643,
+      "loss": 1.9304,
+      "theoretical_loss": 4.698088596889037,
+      "tokens_seen": 117178368
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007793781344032096,
+      "loss": 2.0516,
+      "theoretical_loss": 4.697734695293039,
+      "tokens_seen": 117243904
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007793620862587764,
+      "loss": 1.9003,
+      "theoretical_loss": 4.697381046817256,
+      "tokens_seen": 117309440
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.000779346038114343,
+      "loss": 1.8785,
+      "theoretical_loss": 4.697027651139393,
+      "tokens_seen": 117374976
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007793299899699098,
+      "loss": 2.0907,
+      "theoretical_loss": 4.696674507937746,
+      "tokens_seen": 117440512
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007793139418254765,
+      "loss": 1.8739,
+      "theoretical_loss": 4.696321616891199,
+      "tokens_seen": 117506048
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007792978936810432,
+      "loss": 1.9546,
+      "theoretical_loss": 4.695968977679221,
+      "tokens_seen": 117571584
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007792818455366099,
+      "loss": 1.9477,
+      "theoretical_loss": 4.695616589981871,
+      "tokens_seen": 117637120
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007792657973921766,
+      "loss": 2.028,
+      "theoretical_loss": 4.695264453479789,
+      "tokens_seen": 117702656
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007792497492477433,
+      "loss": 1.9423,
+      "theoretical_loss": 4.6949125678542,
+      "tokens_seen": 117768192
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00077923370110331,
+      "loss": 1.9417,
+      "theoretical_loss": 4.69456093278691,
+      "tokens_seen": 117833728
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007792176529588767,
+      "loss": 1.9821,
+      "theoretical_loss": 4.694209547960304,
+      "tokens_seen": 117899264
+    },
+    {
+      "epoch": 0.04,
+      "objective/train/docs_used": 50996,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.9989361763000488,
+      "objective/train/theoretical_loss": 4.693858413057347,
+      "objective/train/tokens_used": 138424800,
+      "theoretical_loss": 4.693858413057347,
+      "tokens_seen": 117964800
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007792016048144434,
+      "loss": 1.9989,
+      "theoretical_loss": 4.693858413057347,
+      "tokens_seen": 117964800
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007791855566700101,
+      "loss": 1.8493,
+      "theoretical_loss": 4.693507527761584,
+      "tokens_seen": 118030336
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007791695085255768,
+      "loss": 1.9184,
+      "theoretical_loss": 4.6931568917571305,
+      "tokens_seen": 118095872
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007791534603811435,
+      "loss": 2.0155,
+      "theoretical_loss": 4.692806504728683,
+      "tokens_seen": 118161408
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007791374122367102,
+      "loss": 2.0563,
+      "theoretical_loss": 4.692456366361506,
+      "tokens_seen": 118226944
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007791213640922769,
+      "loss": 1.853,
+      "theoretical_loss": 4.692106476341438,
+      "tokens_seen": 118292480
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007791053159478436,
+      "loss": 2.0302,
+      "theoretical_loss": 4.691756834354891,
+      "tokens_seen": 118358016
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007790892678034103,
+      "loss": 1.9261,
+      "theoretical_loss": 4.691407440088843,
+      "tokens_seen": 118423552
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.000779073219658977,
+      "loss": 1.8879,
+      "theoretical_loss": 4.69105829323084,
+      "tokens_seen": 118489088
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007790571715145437,
+      "loss": 2.0205,
+      "theoretical_loss": 4.690709393468995,
+      "tokens_seen": 118554624
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007790411233701104,
+      "loss": 1.976,
+      "theoretical_loss": 4.690360740491989,
+      "tokens_seen": 118620160
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007790250752256771,
+      "loss": 1.9956,
+      "theoretical_loss": 4.690012333989063,
+      "tokens_seen": 118685696
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007790090270812438,
+      "loss": 2.083,
+      "theoretical_loss": 4.689664173650023,
+      "tokens_seen": 118751232
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007789929789368105,
+      "loss": 1.83,
+      "theoretical_loss": 4.689316259165235,
+      "tokens_seen": 118816768
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007789769307923772,
+      "loss": 1.9149,
+      "theoretical_loss": 4.6889685902256275,
+      "tokens_seen": 118882304
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007789608826479439,
+      "loss": 1.9617,
+      "theoretical_loss": 4.688621166522684,
+      "tokens_seen": 118947840
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007789448345035106,
+      "loss": 2.0881,
+      "theoretical_loss": 4.688273987748448,
+      "tokens_seen": 119013376
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007789287863590773,
+      "loss": 1.9944,
+      "theoretical_loss": 4.687927053595518,
+      "tokens_seen": 119078912
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.000778912738214644,
+      "loss": 1.9076,
+      "theoretical_loss": 4.687580363757048,
+      "tokens_seen": 119144448
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007788966900702107,
+      "loss": 1.8912,
+      "theoretical_loss": 4.6872339179267435,
+      "tokens_seen": 119209984
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007788806419257774,
+      "loss": 1.9355,
+      "theoretical_loss": 4.686887715798864,
+      "tokens_seen": 119275520
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007788645937813441,
+      "loss": 2.0869,
+      "theoretical_loss": 4.6865417570682215,
+      "tokens_seen": 119341056
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007788485456369108,
+      "loss": 1.9936,
+      "theoretical_loss": 4.686196041430172,
+      "tokens_seen": 119406592
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007788324974924775,
+      "loss": 1.8853,
+      "theoretical_loss": 4.685850568580624,
+      "tokens_seen": 119472128
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007788164493480442,
+      "loss": 1.9053,
+      "theoretical_loss": 4.685505338216032,
+      "tokens_seen": 119537664
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007788004012036109,
+      "loss": 2.103,
+      "theoretical_loss": 4.685160350033398,
+      "tokens_seen": 119603200
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007787843530591776,
+      "loss": 2.0069,
+      "theoretical_loss": 4.684815603730263,
+      "tokens_seen": 119668736
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007787683049147443,
+      "loss": 1.9858,
+      "theoretical_loss": 4.684471099004718,
+      "tokens_seen": 119734272
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.000778752256770311,
+      "loss": 2.022,
+      "theoretical_loss": 4.684126835555389,
+      "tokens_seen": 119799808
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007787362086258777,
+      "loss": 2.0067,
+      "theoretical_loss": 4.683782813081448,
+      "tokens_seen": 119865344
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007787201604814444,
+      "loss": 1.9334,
+      "theoretical_loss": 4.683439031282606,
+      "tokens_seen": 119930880
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007787041123370111,
+      "loss": 1.8767,
+      "theoretical_loss": 4.683095489859107,
+      "tokens_seen": 119996416
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007786880641925778,
+      "loss": 1.937,
+      "theoretical_loss": 4.682752188511737,
+      "tokens_seen": 120061952
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007786720160481445,
+      "loss": 1.9756,
+      "theoretical_loss": 4.682409126941815,
+      "tokens_seen": 120127488
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007786559679037112,
+      "loss": 2.0309,
+      "theoretical_loss": 4.682066304851195,
+      "tokens_seen": 120193024
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007786399197592779,
+      "loss": 1.9196,
+      "theoretical_loss": 4.681723721942264,
+      "tokens_seen": 120258560
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007786238716148446,
+      "loss": 1.9544,
+      "theoretical_loss": 4.6813813779179405,
+      "tokens_seen": 120324096
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007786078234704113,
+      "loss": 2.0535,
+      "theoretical_loss": 4.681039272481674,
+      "tokens_seen": 120389632
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007785917753259779,
+      "loss": 2.1047,
+      "theoretical_loss": 4.680697405337443,
+      "tokens_seen": 120455168
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007785757271815447,
+      "loss": 1.9193,
+      "theoretical_loss": 4.680355776189755,
+      "tokens_seen": 120520704
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007785596790371113,
+      "loss": 1.7992,
+      "theoretical_loss": 4.680014384743641,
+      "tokens_seen": 120586240
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007785436308926781,
+      "loss": 1.9377,
+      "theoretical_loss": 4.679673230704664,
+      "tokens_seen": 120651776
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007785275827482447,
+      "loss": 1.8548,
+      "theoretical_loss": 4.679332313778904,
+      "tokens_seen": 120717312
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007785115346038115,
+      "loss": 2.0185,
+      "theoretical_loss": 4.678991633672968,
+      "tokens_seen": 120782848
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007784954864593781,
+      "loss": 1.8658,
+      "theoretical_loss": 4.678651190093988,
+      "tokens_seen": 120848384
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007784794383149449,
+      "loss": 1.9428,
+      "theoretical_loss": 4.67831098274961,
+      "tokens_seen": 120913920
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007784633901705115,
+      "loss": 1.9332,
+      "theoretical_loss": 4.677971011348005,
+      "tokens_seen": 120979456
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007784473420260782,
+      "loss": 1.9761,
+      "theoretical_loss": 4.677631275597859,
+      "tokens_seen": 121044992
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007784312938816449,
+      "loss": 1.9033,
+      "theoretical_loss": 4.677291775208378,
+      "tokens_seen": 121110528
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007784152457372116,
+      "loss": 1.9176,
+      "theoretical_loss": 4.676952509889281,
+      "tokens_seen": 121176064
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007783991975927783,
+      "loss": 1.9518,
+      "theoretical_loss": 4.676613479350804,
+      "tokens_seen": 121241600
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.000778383149448345,
+      "loss": 1.8744,
+      "theoretical_loss": 4.6762746833036966,
+      "tokens_seen": 121307136
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007783671013039117,
+      "loss": 1.9279,
+      "theoretical_loss": 4.675936121459218,
+      "tokens_seen": 121372672
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007783510531594785,
+      "loss": 1.8694,
+      "theoretical_loss": 4.675597793529143,
+      "tokens_seen": 121438208
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007783350050150451,
+      "loss": 1.8999,
+      "theoretical_loss": 4.675259699225752,
+      "tokens_seen": 121503744
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007783189568706119,
+      "loss": 1.8602,
+      "theoretical_loss": 4.674921838261836,
+      "tokens_seen": 121569280
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007783029087261785,
+      "loss": 1.9503,
+      "theoretical_loss": 4.674584210350695,
+      "tokens_seen": 121634816
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007782868605817453,
+      "loss": 1.8664,
+      "theoretical_loss": 4.674246815206136,
+      "tokens_seen": 121700352
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007782708124373119,
+      "loss": 1.8057,
+      "theoretical_loss": 4.673909652542466,
+      "tokens_seen": 121765888
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007782547642928787,
+      "loss": 1.981,
+      "theoretical_loss": 4.673572722074502,
+      "tokens_seen": 121831424
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007782387161484453,
+      "loss": 2.042,
+      "theoretical_loss": 4.673236023517562,
+      "tokens_seen": 121896960
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007782226680040121,
+      "loss": 1.8831,
+      "theoretical_loss": 4.672899556587465,
+      "tokens_seen": 121962496
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007782066198595787,
+      "loss": 1.9124,
+      "theoretical_loss": 4.672563321000533,
+      "tokens_seen": 122028032
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007781905717151455,
+      "loss": 1.9745,
+      "theoretical_loss": 4.672227316473584,
+      "tokens_seen": 122093568
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007781745235707121,
+      "loss": 1.9028,
+      "theoretical_loss": 4.671891542723937,
+      "tokens_seen": 122159104
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007781584754262789,
+      "loss": 2.0556,
+      "theoretical_loss": 4.671555999469408,
+      "tokens_seen": 122224640
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007781424272818455,
+      "loss": 1.988,
+      "theoretical_loss": 4.671220686428308,
+      "tokens_seen": 122290176
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007781263791374123,
+      "loss": 1.9089,
+      "theoretical_loss": 4.670885603319445,
+      "tokens_seen": 122355712
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007781103309929789,
+      "loss": 1.9169,
+      "theoretical_loss": 4.670550749862119,
+      "tokens_seen": 122421248
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007780942828485457,
+      "loss": 1.8623,
+      "theoretical_loss": 4.670216125776122,
+      "tokens_seen": 122486784
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007780782347041123,
+      "loss": 1.8273,
+      "theoretical_loss": 4.669881730781741,
+      "tokens_seen": 122552320
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007780621865596791,
+      "loss": 1.9365,
+      "theoretical_loss": 4.66954756459975,
+      "tokens_seen": 122617856
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007780461384152457,
+      "loss": 1.9484,
+      "theoretical_loss": 4.669213626951413,
+      "tokens_seen": 122683392
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007780300902708125,
+      "loss": 2.0991,
+      "theoretical_loss": 4.668879917558484,
+      "tokens_seen": 122748928
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007780140421263791,
+      "loss": 1.9432,
+      "theoretical_loss": 4.6685464361432025,
+      "tokens_seen": 122814464
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007779979939819459,
+      "loss": 1.9753,
+      "theoretical_loss": 4.668213182428295,
+      "tokens_seen": 122880000
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007779819458375126,
+      "loss": 1.8774,
+      "theoretical_loss": 4.667880156136971,
+      "tokens_seen": 122945536
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007779658976930793,
+      "loss": 1.9131,
+      "theoretical_loss": 4.667547356992927,
+      "tokens_seen": 123011072
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.000777949849548646,
+      "loss": 1.9159,
+      "theoretical_loss": 4.667214784720338,
+      "tokens_seen": 123076608
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007779338014042127,
+      "loss": 1.9548,
+      "theoretical_loss": 4.666882439043865,
+      "tokens_seen": 123142144
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007779177532597794,
+      "loss": 1.9843,
+      "theoretical_loss": 4.666550319688646,
+      "tokens_seen": 123207680
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007779017051153461,
+      "loss": 1.8915,
+      "theoretical_loss": 4.6662184263803,
+      "tokens_seen": 123273216
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007778856569709128,
+      "loss": 1.966,
+      "theoretical_loss": 4.665886758844923,
+      "tokens_seen": 123338752
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007778696088264795,
+      "loss": 1.9598,
+      "theoretical_loss": 4.66555531680909,
+      "tokens_seen": 123404288
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007778535606820462,
+      "loss": 1.8788,
+      "theoretical_loss": 4.665224099999852,
+      "tokens_seen": 123469824
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007778375125376129,
+      "loss": 1.8986,
+      "theoretical_loss": 4.664893108144735,
+      "tokens_seen": 123535360
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007778214643931796,
+      "loss": 1.7917,
+      "theoretical_loss": 4.664562340971735,
+      "tokens_seen": 123600896
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007778054162487463,
+      "loss": 2.1133,
+      "theoretical_loss": 4.664231798209327,
+      "tokens_seen": 123666432
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.000777789368104313,
+      "loss": 1.7635,
+      "theoretical_loss": 4.663901479586454,
+      "tokens_seen": 123731968
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007777733199598797,
+      "loss": 2.0078,
+      "theoretical_loss": 4.663571384832531,
+      "tokens_seen": 123797504
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007777572718154464,
+      "loss": 1.9385,
+      "theoretical_loss": 4.663241513677443,
+      "tokens_seen": 123863040
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007777412236710131,
+      "loss": 1.8945,
+      "theoretical_loss": 4.662911865851541,
+      "tokens_seen": 123928576
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007777251755265798,
+      "loss": 2.0032,
+      "theoretical_loss": 4.662582441085648,
+      "tokens_seen": 123994112
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007777091273821465,
+      "loss": 1.9572,
+      "theoretical_loss": 4.662253239111049,
+      "tokens_seen": 124059648
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007776930792377132,
+      "loss": 1.9845,
+      "theoretical_loss": 4.661924259659497,
+      "tokens_seen": 124125184
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007776770310932799,
+      "loss": 1.9505,
+      "theoretical_loss": 4.661595502463211,
+      "tokens_seen": 124190720
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007776609829488466,
+      "loss": 1.9152,
+      "theoretical_loss": 4.661266967254868,
+      "tokens_seen": 124256256
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007776449348044133,
+      "loss": 1.9791,
+      "theoretical_loss": 4.660938653767614,
+      "tokens_seen": 124321792
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00077762888665998,
+      "loss": 1.8655,
+      "theoretical_loss": 4.660610561735049,
+      "tokens_seen": 124387328
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007776128385155467,
+      "loss": 2.0847,
+      "theoretical_loss": 4.66028269089124,
+      "tokens_seen": 124452864
+    },
+    {
+      "epoch": 0.04,
+      "objective/train/docs_used": 53273,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 2.0250937938690186,
+      "objective/train/theoretical_loss": 4.659955040970709,
+      "objective/train/tokens_used": 144978400,
+      "theoretical_loss": 4.659955040970709,
+      "tokens_seen": 124518400
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007775967903711134,
+      "loss": 2.0251,
+      "theoretical_loss": 4.659955040970709,
+      "tokens_seen": 124518400
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007775807422266801,
+      "loss": 1.8376,
+      "theoretical_loss": 4.659627611708437,
+      "tokens_seen": 124583936
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007775646940822468,
+      "loss": 1.9538,
+      "theoretical_loss": 4.659300402839863,
+      "tokens_seen": 124649472
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007775486459378135,
+      "loss": 1.8889,
+      "theoretical_loss": 4.658973414100881,
+      "tokens_seen": 124715008
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007775325977933802,
+      "loss": 2.039,
+      "theoretical_loss": 4.658646645227842,
+      "tokens_seen": 124780544
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007775165496489469,
+      "loss": 1.8919,
+      "theoretical_loss": 4.6583200959575475,
+      "tokens_seen": 124846080
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007775005015045136,
+      "loss": 2.011,
+      "theoretical_loss": 4.657993766027255,
+      "tokens_seen": 124911616
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007774844533600803,
+      "loss": 1.9573,
+      "theoretical_loss": 4.657667655174674,
+      "tokens_seen": 124977152
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.000777468405215647,
+      "loss": 1.8056,
+      "theoretical_loss": 4.657341763137963,
+      "tokens_seen": 125042688
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007774523570712137,
+      "loss": 1.942,
+      "theoretical_loss": 4.657016089655732,
+      "tokens_seen": 125108224
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007774363089267804,
+      "loss": 1.9043,
+      "theoretical_loss": 4.65669063446704,
+      "tokens_seen": 125173760
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007774202607823471,
+      "loss": 1.7944,
+      "theoretical_loss": 4.656365397311393,
+      "tokens_seen": 125239296
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007774042126379138,
+      "loss": 1.8253,
+      "theoretical_loss": 4.656040377928743,
+      "tokens_seen": 125304832
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007773881644934805,
+      "loss": 1.7155,
+      "theoretical_loss": 4.655715576059493,
+      "tokens_seen": 125370368
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007773721163490472,
+      "loss": 1.9427,
+      "theoretical_loss": 4.655390991444484,
+      "tokens_seen": 125435904
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007773560682046139,
+      "loss": 1.9742,
+      "theoretical_loss": 4.6550666238250065,
+      "tokens_seen": 125501440
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007773400200601806,
+      "loss": 1.8937,
+      "theoretical_loss": 4.65474247294279,
+      "tokens_seen": 125566976
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007773239719157473,
+      "loss": 1.9277,
+      "theoretical_loss": 4.654418538540009,
+      "tokens_seen": 125632512
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.000777307923771314,
+      "loss": 2.006,
+      "theoretical_loss": 4.654094820359278,
+      "tokens_seen": 125698048
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007772918756268807,
+      "loss": 2.005,
+      "theoretical_loss": 4.653771318143651,
+      "tokens_seen": 125763584
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007772758274824474,
+      "loss": 1.7953,
+      "theoretical_loss": 4.65344803163662,
+      "tokens_seen": 125829120
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007772597793380141,
+      "loss": 1.863,
+      "theoretical_loss": 4.653124960582118,
+      "tokens_seen": 125894656
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007772437311935808,
+      "loss": 2.0596,
+      "theoretical_loss": 4.652802104724513,
+      "tokens_seen": 125960192
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007772276830491475,
+      "loss": 1.96,
+      "theoretical_loss": 4.652479463808609,
+      "tokens_seen": 126025728
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007772116349047142,
+      "loss": 1.9092,
+      "theoretical_loss": 4.652157037579647,
+      "tokens_seen": 126091264
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007771955867602808,
+      "loss": 1.9429,
+      "theoretical_loss": 4.6518348257833,
+      "tokens_seen": 126156800
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007771795386158476,
+      "loss": 1.962,
+      "theoretical_loss": 4.651512828165675,
+      "tokens_seen": 126222336
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007771634904714142,
+      "loss": 1.9165,
+      "theoretical_loss": 4.651191044473311,
+      "tokens_seen": 126287872
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.000777147442326981,
+      "loss": 2.0334,
+      "theoretical_loss": 4.65086947445318,
+      "tokens_seen": 126353408
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007771313941825476,
+      "loss": 1.9604,
+      "theoretical_loss": 4.650548117852681,
+      "tokens_seen": 126418944
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007771153460381144,
+      "loss": 1.8603,
+      "theoretical_loss": 4.650226974419646,
+      "tokens_seen": 126484480
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.000777099297893681,
+      "loss": 1.9798,
+      "theoretical_loss": 4.649906043902332,
+      "tokens_seen": 126550016
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007770832497492478,
+      "loss": 2.0097,
+      "theoretical_loss": 4.6495853260494275,
+      "tokens_seen": 126615552
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007770672016048144,
+      "loss": 1.9388,
+      "theoretical_loss": 4.649264820610043,
+      "tokens_seen": 126681088
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007770511534603812,
+      "loss": 2.0125,
+      "theoretical_loss": 4.648944527333717,
+      "tokens_seen": 126746624
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007770351053159478,
+      "loss": 1.8029,
+      "theoretical_loss": 4.648624445970414,
+      "tokens_seen": 126812160
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007770190571715146,
+      "loss": 1.9509,
+      "theoretical_loss": 4.648304576270519,
+      "tokens_seen": 126877696
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007770030090270812,
+      "loss": 1.9193,
+      "theoretical_loss": 4.647984917984841,
+      "tokens_seen": 126943232
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.000776986960882648,
+      "loss": 1.9608,
+      "theoretical_loss": 4.647665470864613,
+      "tokens_seen": 127008768
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007769709127382146,
+      "loss": 1.9046,
+      "theoretical_loss": 4.6473462346614856,
+      "tokens_seen": 127074304
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007769548645937814,
+      "loss": 1.8915,
+      "theoretical_loss": 4.647027209127531,
+      "tokens_seen": 127139840
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.000776938816449348,
+      "loss": 1.9559,
+      "theoretical_loss": 4.646708394015242,
+      "tokens_seen": 127205376
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007769227683049148,
+      "loss": 2.0008,
+      "theoretical_loss": 4.646389789077524,
+      "tokens_seen": 127270912
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007769067201604814,
+      "loss": 2.0062,
+      "theoretical_loss": 4.646071394067707,
+      "tokens_seen": 127336448
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007768906720160482,
+      "loss": 1.9537,
+      "theoretical_loss": 4.645753208739533,
+      "tokens_seen": 127401984
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007768746238716148,
+      "loss": 1.9244,
+      "theoretical_loss": 4.645435232847159,
+      "tokens_seen": 127467520
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007768585757271816,
+      "loss": 1.8396,
+      "theoretical_loss": 4.645117466145157,
+      "tokens_seen": 127533056
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007768425275827482,
+      "loss": 1.7292,
+      "theoretical_loss": 4.6447999083885145,
+      "tokens_seen": 127598592
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.000776826479438315,
+      "loss": 1.732,
+      "theoretical_loss": 4.644482559332629,
+      "tokens_seen": 127664128
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007768104312938816,
+      "loss": 1.9047,
+      "theoretical_loss": 4.644165418733312,
+      "tokens_seen": 127729664
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007767943831494484,
+      "loss": 1.8288,
+      "theoretical_loss": 4.6438484863467835,
+      "tokens_seen": 127795200
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.000776778335005015,
+      "loss": 1.986,
+      "theoretical_loss": 4.643531761929674,
+      "tokens_seen": 127860736
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007767622868605818,
+      "loss": 1.8109,
+      "theoretical_loss": 4.643215245239025,
+      "tokens_seen": 127926272
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007767462387161484,
+      "loss": 1.8901,
+      "theoretical_loss": 4.642898936032285,
+      "tokens_seen": 127991808
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007767301905717152,
+      "loss": 1.8135,
+      "theoretical_loss": 4.6425828340673085,
+      "tokens_seen": 128057344
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007767141424272818,
+      "loss": 1.9875,
+      "theoretical_loss": 4.642266939102357,
+      "tokens_seen": 128122880
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007766980942828486,
+      "loss": 1.9042,
+      "theoretical_loss": 4.641951250896098,
+      "tokens_seen": 128188416
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007766820461384152,
+      "loss": 2.1037,
+      "theoretical_loss": 4.641635769207603,
+      "tokens_seen": 128253952
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.000776665997993982,
+      "loss": 1.8608,
+      "theoretical_loss": 4.641320493796349,
+      "tokens_seen": 128319488
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007766499498495486,
+      "loss": 1.9688,
+      "theoretical_loss": 4.641005424422213,
+      "tokens_seen": 128385024
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007766339017051154,
+      "loss": 1.8788,
+      "theoretical_loss": 4.640690560845474,
+      "tokens_seen": 128450560
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.000776617853560682,
+      "loss": 1.9964,
+      "theoretical_loss": 4.640375902826815,
+      "tokens_seen": 128516096
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007766018054162488,
+      "loss": 1.9816,
+      "theoretical_loss": 4.640061450127316,
+      "tokens_seen": 128581632
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007765857572718155,
+      "loss": 1.9421,
+      "theoretical_loss": 4.639747202508459,
+      "tokens_seen": 128647168
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007765697091273822,
+      "loss": 1.9786,
+      "theoretical_loss": 4.639433159732121,
+      "tokens_seen": 128712704
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007765536609829489,
+      "loss": 1.9394,
+      "theoretical_loss": 4.639119321560581,
+      "tokens_seen": 128778240
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007765376128385156,
+      "loss": 1.8522,
+      "theoretical_loss": 4.638805687756511,
+      "tokens_seen": 128843776
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007765215646940823,
+      "loss": 2.0082,
+      "theoretical_loss": 4.63849225808298,
+      "tokens_seen": 128909312
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.000776505516549649,
+      "loss": 1.8465,
+      "theoretical_loss": 4.6381790323034515,
+      "tokens_seen": 128974848
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007764894684052157,
+      "loss": 1.8826,
+      "theoretical_loss": 4.637866010181784,
+      "tokens_seen": 129040384
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007764734202607824,
+      "loss": 1.9264,
+      "theoretical_loss": 4.63755319148223,
+      "tokens_seen": 129105920
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007764573721163491,
+      "loss": 1.917,
+      "theoretical_loss": 4.637240575969432,
+      "tokens_seen": 129171456
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007764413239719158,
+      "loss": 1.8722,
+      "theoretical_loss": 4.636928163408426,
+      "tokens_seen": 129236992
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007764252758274825,
+      "loss": 1.9663,
+      "theoretical_loss": 4.636615953564637,
+      "tokens_seen": 129302528
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007764092276830492,
+      "loss": 1.8886,
+      "theoretical_loss": 4.636303946203881,
+      "tokens_seen": 129368064
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007763931795386159,
+      "loss": 1.9165,
+      "theoretical_loss": 4.635992141092364,
+      "tokens_seen": 129433600
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007763771313941826,
+      "loss": 1.8921,
+      "theoretical_loss": 4.635680537996679,
+      "tokens_seen": 129499136
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007763610832497493,
+      "loss": 1.9513,
+      "theoretical_loss": 4.635369136683805,
+      "tokens_seen": 129564672
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.000776345035105316,
+      "loss": 1.9293,
+      "theoretical_loss": 4.6350579369211085,
+      "tokens_seen": 129630208
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007763289869608827,
+      "loss": 1.8728,
+      "theoretical_loss": 4.634746938476344,
+      "tokens_seen": 129695744
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007763129388164494,
+      "loss": 1.9293,
+      "theoretical_loss": 4.634436141117645,
+      "tokens_seen": 129761280
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007762968906720161,
+      "loss": 1.9717,
+      "theoretical_loss": 4.634125544613536,
+      "tokens_seen": 129826816
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007762808425275828,
+      "loss": 1.9701,
+      "theoretical_loss": 4.63381514873292,
+      "tokens_seen": 129892352
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007762647943831495,
+      "loss": 1.886,
+      "theoretical_loss": 4.633504953245085,
+      "tokens_seen": 129957888
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007762487462387162,
+      "loss": 1.8814,
+      "theoretical_loss": 4.633194957919697,
+      "tokens_seen": 130023424
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007762326980942829,
+      "loss": 1.9399,
+      "theoretical_loss": 4.632885162526806,
+      "tokens_seen": 130088960
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007762166499498496,
+      "loss": 1.8572,
+      "theoretical_loss": 4.63257556683684,
+      "tokens_seen": 130154496
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007762006018054163,
+      "loss": 1.8296,
+      "theoretical_loss": 4.632266170620607,
+      "tokens_seen": 130220032
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.000776184553660983,
+      "loss": 1.8973,
+      "theoretical_loss": 4.631956973649293,
+      "tokens_seen": 130285568
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007761685055165497,
+      "loss": 1.9464,
+      "theoretical_loss": 4.631647975694461,
+      "tokens_seen": 130351104
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007761524573721164,
+      "loss": 1.9565,
+      "theoretical_loss": 4.631339176528051,
+      "tokens_seen": 130416640
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007761364092276831,
+      "loss": 1.9187,
+      "theoretical_loss": 4.631030575922379,
+      "tokens_seen": 130482176
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007761203610832498,
+      "loss": 2.0334,
+      "theoretical_loss": 4.630722173650135,
+      "tokens_seen": 130547712
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007761043129388165,
+      "loss": 1.8334,
+      "theoretical_loss": 4.630413969484383,
+      "tokens_seen": 130613248
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007760882647943832,
+      "loss": 2.0124,
+      "theoretical_loss": 4.630105963198563,
+      "tokens_seen": 130678784
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007760722166499499,
+      "loss": 1.997,
+      "theoretical_loss": 4.629798154566485,
+      "tokens_seen": 130744320
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007760561685055166,
+      "loss": 1.9289,
+      "theoretical_loss": 4.62949054336233,
+      "tokens_seen": 130809856
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007760401203610833,
+      "loss": 1.889,
+      "theoretical_loss": 4.629183129360653,
+      "tokens_seen": 130875392
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00077602407221665,
+      "loss": 1.9031,
+      "theoretical_loss": 4.628875912336378,
+      "tokens_seen": 130940928
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007760080240722167,
+      "loss": 1.9547,
+      "theoretical_loss": 4.628568892064797,
+      "tokens_seen": 131006464
+    },
+    {
+      "epoch": 0.04,
+      "objective/train/docs_used": 55660,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.9429093599319458,
+      "objective/train/theoretical_loss": 4.62826206832157,
+      "objective/train/tokens_used": 151532000,
+      "theoretical_loss": 4.62826206832157,
+      "tokens_seen": 131072000
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007759919759277834,
+      "loss": 1.9429,
+      "theoretical_loss": 4.62826206832157,
+      "tokens_seen": 131072000
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007759759277833502,
+      "loss": 1.951,
+      "theoretical_loss": 4.6279554408827295,
+      "tokens_seen": 131137536
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007759598796389168,
+      "loss": 1.8921,
+      "theoretical_loss": 4.62764900952467,
+      "tokens_seen": 131203072
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007759438314944836,
+      "loss": 1.9007,
+      "theoretical_loss": 4.627342774024155,
+      "tokens_seen": 131268608
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007759277833500502,
+      "loss": 2.032,
+      "theoretical_loss": 4.6270367341583105,
+      "tokens_seen": 131334144
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007759117352056169,
+      "loss": 1.9691,
+      "theoretical_loss": 4.626730889704632,
+      "tokens_seen": 131399680
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007758956870611836,
+      "loss": 1.911,
+      "theoretical_loss": 4.626425240440973,
+      "tokens_seen": 131465216
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007758796389167503,
+      "loss": 1.9496,
+      "theoretical_loss": 4.626119786145557,
+      "tokens_seen": 131530752
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.000775863590772317,
+      "loss": 1.9619,
+      "theoretical_loss": 4.625814526596962,
+      "tokens_seen": 131596288
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007758475426278837,
+      "loss": 1.9118,
+      "theoretical_loss": 4.625509461574135,
+      "tokens_seen": 131661824
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007758314944834504,
+      "loss": 1.8598,
+      "theoretical_loss": 4.625204590856377,
+      "tokens_seen": 131727360
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007758154463390171,
+      "loss": 1.9388,
+      "theoretical_loss": 4.624899914223355,
+      "tokens_seen": 131792896
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007757993981945837,
+      "loss": 1.7842,
+      "theoretical_loss": 4.624595431455091,
+      "tokens_seen": 131858432
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007757833500501505,
+      "loss": 1.9605,
+      "theoretical_loss": 4.62429114233197,
+      "tokens_seen": 131923968
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007757673019057171,
+      "loss": 2.0652,
+      "theoretical_loss": 4.623987046634729,
+      "tokens_seen": 131989504
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007757512537612839,
+      "loss": 1.8835,
+      "theoretical_loss": 4.623683144144465,
+      "tokens_seen": 132055040
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007757352056168505,
+      "loss": 1.9783,
+      "theoretical_loss": 4.623379434642632,
+      "tokens_seen": 132120576
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007757191574724173,
+      "loss": 1.8827,
+      "theoretical_loss": 4.623075917911039,
+      "tokens_seen": 132186112
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007757031093279839,
+      "loss": 1.8191,
+      "theoretical_loss": 4.622772593731848,
+      "tokens_seen": 132251648
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007756870611835507,
+      "loss": 1.8235,
+      "theoretical_loss": 4.622469461887577,
+      "tokens_seen": 132317184
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007756710130391173,
+      "loss": 1.9826,
+      "theoretical_loss": 4.622166522161096,
+      "tokens_seen": 132382720
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007756549648946841,
+      "loss": 2.0123,
+      "theoretical_loss": 4.621863774335628,
+      "tokens_seen": 132448256
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007756389167502507,
+      "loss": 1.8155,
+      "theoretical_loss": 4.621561218194747,
+      "tokens_seen": 132513792
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007756228686058175,
+      "loss": 1.9544,
+      "theoretical_loss": 4.62125885352238,
+      "tokens_seen": 132579328
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007756068204613841,
+      "loss": 1.9323,
+      "theoretical_loss": 4.620956680102801,
+      "tokens_seen": 132644864
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007755907723169509,
+      "loss": 1.8998,
+      "theoretical_loss": 4.620654697720637,
+      "tokens_seen": 132710400
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007755747241725175,
+      "loss": 1.845,
+      "theoretical_loss": 4.620352906160861,
+      "tokens_seen": 132775936
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007755586760280843,
+      "loss": 1.8736,
+      "theoretical_loss": 4.6200513052087935,
+      "tokens_seen": 132841472
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007755426278836509,
+      "loss": 1.9324,
+      "theoretical_loss": 4.6197498946501065,
+      "tokens_seen": 132907008
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007755265797392177,
+      "loss": 1.8713,
+      "theoretical_loss": 4.6194486742708145,
+      "tokens_seen": 132972544
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007755105315947843,
+      "loss": 2.0236,
+      "theoretical_loss": 4.6191476438572785,
+      "tokens_seen": 133038080
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007754944834503511,
+      "loss": 1.9157,
+      "theoretical_loss": 4.618846803196206,
+      "tokens_seen": 133103616
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007754784353059177,
+      "loss": 1.8248,
+      "theoretical_loss": 4.618546152074648,
+      "tokens_seen": 133169152
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007754623871614845,
+      "loss": 1.8383,
+      "theoretical_loss": 4.61824569028,
+      "tokens_seen": 133234688
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007754463390170511,
+      "loss": 1.8905,
+      "theoretical_loss": 4.617945417599998,
+      "tokens_seen": 133300224
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007754302908726179,
+      "loss": 1.8923,
+      "theoretical_loss": 4.617645333822722,
+      "tokens_seen": 133365760
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007754142427281845,
+      "loss": 1.8889,
+      "theoretical_loss": 4.617345438736594,
+      "tokens_seen": 133431296
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007753981945837513,
+      "loss": 1.9894,
+      "theoretical_loss": 4.617045732130375,
+      "tokens_seen": 133496832
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007753821464393179,
+      "loss": 2.1688,
+      "theoretical_loss": 4.616746213793167,
+      "tokens_seen": 133562368
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007753660982948847,
+      "loss": 1.9595,
+      "theoretical_loss": 4.616446883514412,
+      "tokens_seen": 133627904
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007753500501504513,
+      "loss": 1.8683,
+      "theoretical_loss": 4.616147741083891,
+      "tokens_seen": 133693440
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007753340020060181,
+      "loss": 1.8758,
+      "theoretical_loss": 4.6158487862917195,
+      "tokens_seen": 133758976
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007753179538615847,
+      "loss": 1.9103,
+      "theoretical_loss": 4.615550018928353,
+      "tokens_seen": 133824512
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007753019057171515,
+      "loss": 1.8714,
+      "theoretical_loss": 4.615251438784584,
+      "tokens_seen": 133890048
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007752858575727181,
+      "loss": 1.8895,
+      "theoretical_loss": 4.61495304565154,
+      "tokens_seen": 133955584
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007752698094282849,
+      "loss": 1.892,
+      "theoretical_loss": 4.614654839320682,
+      "tokens_seen": 134021120
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007752537612838515,
+      "loss": 2.0136,
+      "theoretical_loss": 4.614356819583808,
+      "tokens_seen": 134086656
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007752377131394183,
+      "loss": 1.9087,
+      "theoretical_loss": 4.614058986233047,
+      "tokens_seen": 134152192
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.000775221664994985,
+      "loss": 1.9696,
+      "theoretical_loss": 4.613761339060863,
+      "tokens_seen": 134217728
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007752056168505517,
+      "loss": 1.9087,
+      "theoretical_loss": 4.613463877860051,
+      "tokens_seen": 134283264
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007751895687061184,
+      "loss": 2.06,
+      "theoretical_loss": 4.61316660242374,
+      "tokens_seen": 134348800
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007751735205616851,
+      "loss": 2.0039,
+      "theoretical_loss": 4.6128695125453865,
+      "tokens_seen": 134414336
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007751574724172518,
+      "loss": 1.9339,
+      "theoretical_loss": 4.612572608018779,
+      "tokens_seen": 134479872
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007751414242728185,
+      "loss": 2.2059,
+      "theoretical_loss": 4.612275888638036,
+      "tokens_seen": 134545408
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007751253761283852,
+      "loss": 2.2559,
+      "theoretical_loss": 4.611979354197603,
+      "tokens_seen": 134610944
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007751093279839519,
+      "loss": 1.9838,
+      "theoretical_loss": 4.6116830044922565,
+      "tokens_seen": 134676480
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007750932798395186,
+      "loss": 2.212,
+      "theoretical_loss": 4.611386839317097,
+      "tokens_seen": 134742016
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007750772316950853,
+      "loss": 2.0549,
+      "theoretical_loss": 4.611090858467554,
+      "tokens_seen": 134807552
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.000775061183550652,
+      "loss": 2.469,
+      "theoretical_loss": 4.610795061739384,
+      "tokens_seen": 134873088
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007750451354062187,
+      "loss": 2.051,
+      "theoretical_loss": 4.610499448928666,
+      "tokens_seen": 134938624
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007750290872617854,
+      "loss": 2.0906,
+      "theoretical_loss": 4.610204019831805,
+      "tokens_seen": 135004160
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007750130391173521,
+      "loss": 2.0623,
+      "theoretical_loss": 4.609908774245532,
+      "tokens_seen": 135069696
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007749969909729188,
+      "loss": 2.146,
+      "theoretical_loss": 4.609613711966898,
+      "tokens_seen": 135135232
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007749809428284855,
+      "loss": 2.0754,
+      "theoretical_loss": 4.60931883279328,
+      "tokens_seen": 135200768
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007749648946840522,
+      "loss": 2.0812,
+      "theoretical_loss": 4.6090241365223745,
+      "tokens_seen": 135266304
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007749488465396189,
+      "loss": 2.1798,
+      "theoretical_loss": 4.6087296229522,
+      "tokens_seen": 135331840
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007749327983951856,
+      "loss": 2.2384,
+      "theoretical_loss": 4.608435291881099,
+      "tokens_seen": 135397376
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007749167502507523,
+      "loss": 2.0114,
+      "theoretical_loss": 4.608141143107728,
+      "tokens_seen": 135462912
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.000774900702106319,
+      "loss": 2.032,
+      "theoretical_loss": 4.607847176431067,
+      "tokens_seen": 135528448
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007748846539618857,
+      "loss": 2.1132,
+      "theoretical_loss": 4.6075533916504146,
+      "tokens_seen": 135593984
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007748686058174524,
+      "loss": 2.045,
+      "theoretical_loss": 4.607259788565385,
+      "tokens_seen": 135659520
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007748525576730191,
+      "loss": 1.8809,
+      "theoretical_loss": 4.606966366975914,
+      "tokens_seen": 135725056
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007748365095285858,
+      "loss": 2.0153,
+      "theoretical_loss": 4.606673126682249,
+      "tokens_seen": 135790592
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007748204613841525,
+      "loss": 1.9686,
+      "theoretical_loss": 4.6063800674849595,
+      "tokens_seen": 135856128
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007748044132397192,
+      "loss": 2.1016,
+      "theoretical_loss": 4.606087189184924,
+      "tokens_seen": 135921664
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007747883650952859,
+      "loss": 1.929,
+      "theoretical_loss": 4.6057944915833415,
+      "tokens_seen": 135987200
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007747723169508527,
+      "loss": 1.9082,
+      "theoretical_loss": 4.6055019744817205,
+      "tokens_seen": 136052736
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007747562688064193,
+      "loss": 2.0086,
+      "theoretical_loss": 4.605209637681886,
+      "tokens_seen": 136118272
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007747402206619861,
+      "loss": 1.8842,
+      "theoretical_loss": 4.604917480985975,
+      "tokens_seen": 136183808
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007747241725175527,
+      "loss": 1.9044,
+      "theoretical_loss": 4.604625504196438,
+      "tokens_seen": 136249344
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007747081243731195,
+      "loss": 1.9976,
+      "theoretical_loss": 4.604333707116033,
+      "tokens_seen": 136314880
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007746920762286861,
+      "loss": 1.87,
+      "theoretical_loss": 4.604042089547834,
+      "tokens_seen": 136380416
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007746760280842529,
+      "loss": 2.0676,
+      "theoretical_loss": 4.603750651295222,
+      "tokens_seen": 136445952
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007746599799398195,
+      "loss": 2.0567,
+      "theoretical_loss": 4.603459392161888,
+      "tokens_seen": 136511488
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007746439317953863,
+      "loss": 1.9025,
+      "theoretical_loss": 4.603168311951835,
+      "tokens_seen": 136577024
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007746278836509529,
+      "loss": 1.9117,
+      "theoretical_loss": 4.60287741046937,
+      "tokens_seen": 136642560
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007746118355065197,
+      "loss": 2.0292,
+      "theoretical_loss": 4.602586687519111,
+      "tokens_seen": 136708096
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007745957873620863,
+      "loss": 2.0501,
+      "theoretical_loss": 4.602296142905981,
+      "tokens_seen": 136773632
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007745797392176531,
+      "loss": 1.9745,
+      "theoretical_loss": 4.602005776435211,
+      "tokens_seen": 136839168
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007745636910732197,
+      "loss": 1.9776,
+      "theoretical_loss": 4.601715587912338,
+      "tokens_seen": 136904704
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007745476429287865,
+      "loss": 2.0835,
+      "theoretical_loss": 4.601425577143203,
+      "tokens_seen": 136970240
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007745315947843531,
+      "loss": 2.0722,
+      "theoretical_loss": 4.601135743933952,
+      "tokens_seen": 137035776
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007745155466399198,
+      "loss": 2.1287,
+      "theoretical_loss": 4.600846088091036,
+      "tokens_seen": 137101312
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007744994984954865,
+      "loss": 1.9373,
+      "theoretical_loss": 4.600556609421206,
+      "tokens_seen": 137166848
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007744834503510532,
+      "loss": 2.0686,
+      "theoretical_loss": 4.600267307731521,
+      "tokens_seen": 137232384
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007744674022066199,
+      "loss": 2.0877,
+      "theoretical_loss": 4.5999781828293385,
+      "tokens_seen": 137297920
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007744513540621866,
+      "loss": 1.9723,
+      "theoretical_loss": 4.5996892345223195,
+      "tokens_seen": 137363456
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007744353059177533,
+      "loss": 2.0593,
+      "theoretical_loss": 4.599400462618423,
+      "tokens_seen": 137428992
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00077441925777332,
+      "loss": 1.9818,
+      "theoretical_loss": 4.59911186692591,
+      "tokens_seen": 137494528
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007744032096288867,
+      "loss": 1.9381,
+      "theoretical_loss": 4.598823447253343,
+      "tokens_seen": 137560064
+    },
+    {
+      "epoch": 0.04,
+      "objective/train/docs_used": 58515,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.963025689125061,
+      "objective/train/theoretical_loss": 4.598535203409582,
+      "objective/train/tokens_used": 158085600,
+      "theoretical_loss": 4.598535203409582,
+      "tokens_seen": 137625600
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007743871614844534,
+      "loss": 1.963,
+      "theoretical_loss": 4.598535203409582,
+      "tokens_seen": 137625600
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00077437111334002,
+      "loss": 2.0808,
+      "theoretical_loss": 4.598247135203783,
+      "tokens_seen": 137691136
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007743550651955868,
+      "loss": 2.07,
+      "theoretical_loss": 4.5979592424454045,
+      "tokens_seen": 137756672
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007743390170511534,
+      "loss": 2.0288,
+      "theoretical_loss": 4.597671524944198,
+      "tokens_seen": 137822208
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007743229689067202,
+      "loss": 2.1158,
+      "theoretical_loss": 4.597383982510216,
+      "tokens_seen": 137887744
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007743069207622868,
+      "loss": 1.8745,
+      "theoretical_loss": 4.597096614953802,
+      "tokens_seen": 137953280
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007742908726178536,
+      "loss": 2.2142,
+      "theoretical_loss": 4.596809422085599,
+      "tokens_seen": 138018816
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007742748244734202,
+      "loss": 1.9296,
+      "theoretical_loss": 4.596522403716542,
+      "tokens_seen": 138084352
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.000774258776328987,
+      "loss": 2.0163,
+      "theoretical_loss": 4.596235559657863,
+      "tokens_seen": 138149888
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007742427281845536,
+      "loss": 2.0199,
+      "theoretical_loss": 4.595948889721086,
+      "tokens_seen": 138215424
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007742266800401204,
+      "loss": 2.0205,
+      "theoretical_loss": 4.595662393718026,
+      "tokens_seen": 138280960
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.000774210631895687,
+      "loss": 2.0998,
+      "theoretical_loss": 4.595376071460796,
+      "tokens_seen": 138346496
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007741945837512538,
+      "loss": 2.0387,
+      "theoretical_loss": 4.595089922761796,
+      "tokens_seen": 138412032
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007741785356068204,
+      "loss": 2.1723,
+      "theoretical_loss": 4.594803947433717,
+      "tokens_seen": 138477568
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007741624874623872,
+      "loss": 1.961,
+      "theoretical_loss": 4.594518145289545,
+      "tokens_seen": 138543104
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007741464393179538,
+      "loss": 2.0687,
+      "theoretical_loss": 4.594232516142553,
+      "tokens_seen": 138608640
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007741303911735206,
+      "loss": 1.9523,
+      "theoretical_loss": 4.593947059806302,
+      "tokens_seen": 138674176
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007741143430290872,
+      "loss": 2.2408,
+      "theoretical_loss": 4.593661776094647,
+      "tokens_seen": 138739712
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.000774098294884654,
+      "loss": 2.0517,
+      "theoretical_loss": 4.593376664821726,
+      "tokens_seen": 138805248
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007740822467402206,
+      "loss": 2.0091,
+      "theoretical_loss": 4.593091725801967,
+      "tokens_seen": 138870784
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007740661985957874,
+      "loss": 1.9329,
+      "theoretical_loss": 4.592806958850087,
+      "tokens_seen": 138936320
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.000774050150451354,
+      "loss": 2.0677,
+      "theoretical_loss": 4.592522363781086,
+      "tokens_seen": 139001856
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007740341023069208,
+      "loss": 1.8435,
+      "theoretical_loss": 4.592237940410252,
+      "tokens_seen": 139067392
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007740180541624874,
+      "loss": 2.0432,
+      "theoretical_loss": 4.591953688553159,
+      "tokens_seen": 139132928
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007740020060180542,
+      "loss": 2.1285,
+      "theoretical_loss": 4.591669608025663,
+      "tokens_seen": 139198464
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007739859578736208,
+      "loss": 2.0844,
+      "theoretical_loss": 4.591385698643908,
+      "tokens_seen": 139264000
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007739699097291876,
+      "loss": 1.9827,
+      "theoretical_loss": 4.591101960224321,
+      "tokens_seen": 139329536
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007739538615847542,
+      "loss": 1.9665,
+      "theoretical_loss": 4.590818392583608,
+      "tokens_seen": 139395072
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.000773937813440321,
+      "loss": 2.0399,
+      "theoretical_loss": 4.590534995538762,
+      "tokens_seen": 139460608
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007739217652958876,
+      "loss": 1.9692,
+      "theoretical_loss": 4.590251768907057,
+      "tokens_seen": 139526144
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007739057171514544,
+      "loss": 1.9305,
+      "theoretical_loss": 4.5899687125060495,
+      "tokens_seen": 139591680
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.000773889669007021,
+      "loss": 2.0723,
+      "theoretical_loss": 4.589685826153573,
+      "tokens_seen": 139657216
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007738736208625878,
+      "loss": 1.9322,
+      "theoretical_loss": 4.589403109667746,
+      "tokens_seen": 139722752
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007738575727181545,
+      "loss": 1.917,
+      "theoretical_loss": 4.589120562866961,
+      "tokens_seen": 139788288
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007738415245737212,
+      "loss": 2.0315,
+      "theoretical_loss": 4.588838185569897,
+      "tokens_seen": 139853824
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007738254764292879,
+      "loss": 2.0167,
+      "theoretical_loss": 4.588555977595506,
+      "tokens_seen": 139919360
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007738094282848546,
+      "loss": 2.0811,
+      "theoretical_loss": 4.58827393876302,
+      "tokens_seen": 139984896
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007737933801404213,
+      "loss": 1.9253,
+      "theoretical_loss": 4.587992068891948,
+      "tokens_seen": 140050432
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.000773777331995988,
+      "loss": 1.9362,
+      "theoretical_loss": 4.587710367802076,
+      "tokens_seen": 140115968
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007737612838515547,
+      "loss": 2.0221,
+      "theoretical_loss": 4.587428835313469,
+      "tokens_seen": 140181504
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007737452357071214,
+      "loss": 2.0653,
+      "theoretical_loss": 4.587147471246463,
+      "tokens_seen": 140247040
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007737291875626881,
+      "loss": 1.927,
+      "theoretical_loss": 4.586866275421673,
+      "tokens_seen": 140312576
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007737131394182548,
+      "loss": 1.9966,
+      "theoretical_loss": 4.586585247659986,
+      "tokens_seen": 140378112
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007736970912738215,
+      "loss": 1.9139,
+      "theoretical_loss": 4.586304387782567,
+      "tokens_seen": 140443648
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007736810431293882,
+      "loss": 1.973,
+      "theoretical_loss": 4.5860236956108515,
+      "tokens_seen": 140509184
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007736649949849549,
+      "loss": 1.8935,
+      "theoretical_loss": 4.585743170966548,
+      "tokens_seen": 140574720
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007736489468405216,
+      "loss": 2.0073,
+      "theoretical_loss": 4.58546281367164,
+      "tokens_seen": 140640256
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007736328986960883,
+      "loss": 1.968,
+      "theoretical_loss": 4.585182623548381,
+      "tokens_seen": 140705792
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.000773616850551655,
+      "loss": 2.0452,
+      "theoretical_loss": 4.584902600419295,
+      "tokens_seen": 140771328
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007736008024072217,
+      "loss": 1.8525,
+      "theoretical_loss": 4.584622744107181,
+      "tokens_seen": 140836864
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007735847542627884,
+      "loss": 1.9866,
+      "theoretical_loss": 4.584343054435103,
+      "tokens_seen": 140902400
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007735687061183552,
+      "loss": 1.9483,
+      "theoretical_loss": 4.5840635312264,
+      "tokens_seen": 140967936
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007735526579739218,
+      "loss": 1.9608,
+      "theoretical_loss": 4.583784174304677,
+      "tokens_seen": 141033472
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007735366098294886,
+      "loss": 1.8675,
+      "theoretical_loss": 4.583504983493809,
+      "tokens_seen": 141099008
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007735205616850552,
+      "loss": 1.8178,
+      "theoretical_loss": 4.583225958617938,
+      "tokens_seen": 141164544
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.000773504513540622,
+      "loss": 1.8741,
+      "theoretical_loss": 4.582947099501475,
+      "tokens_seen": 141230080
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007734884653961886,
+      "loss": 1.9559,
+      "theoretical_loss": 4.5826684059691,
+      "tokens_seen": 141295616
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007734724172517554,
+      "loss": 1.9883,
+      "theoretical_loss": 4.582389877845754,
+      "tokens_seen": 141361152
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.000773456369107322,
+      "loss": 1.9825,
+      "theoretical_loss": 4.58211151495665,
+      "tokens_seen": 141426688
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007734403209628888,
+      "loss": 1.7169,
+      "theoretical_loss": 4.581833317127263,
+      "tokens_seen": 141492224
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007734242728184554,
+      "loss": 1.9222,
+      "theoretical_loss": 4.581555284183335,
+      "tokens_seen": 141557760
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007734082246740222,
+      "loss": 1.9835,
+      "theoretical_loss": 4.581277415950872,
+      "tokens_seen": 141623296
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007733921765295888,
+      "loss": 2.1773,
+      "theoretical_loss": 4.580999712256142,
+      "tokens_seen": 141688832
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007733761283851556,
+      "loss": 1.8872,
+      "theoretical_loss": 4.5807221729256815,
+      "tokens_seen": 141754368
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007733600802407222,
+      "loss": 1.8931,
+      "theoretical_loss": 4.580444797786283,
+      "tokens_seen": 141819904
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.000773344032096289,
+      "loss": 1.9273,
+      "theoretical_loss": 4.5801675866650085,
+      "tokens_seen": 141885440
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007733279839518556,
+      "loss": 2.0059,
+      "theoretical_loss": 4.579890539389177,
+      "tokens_seen": 141950976
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007733119358074224,
+      "loss": 2.1157,
+      "theoretical_loss": 4.579613655786369,
+      "tokens_seen": 142016512
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.000773295887662989,
+      "loss": 1.873,
+      "theoretical_loss": 4.579336935684431,
+      "tokens_seen": 142082048
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007732798395185558,
+      "loss": 2.0334,
+      "theoretical_loss": 4.579060378911464,
+      "tokens_seen": 142147584
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007732637913741224,
+      "loss": 1.9416,
+      "theoretical_loss": 4.578783985295832,
+      "tokens_seen": 142213120
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007732477432296892,
+      "loss": 2.013,
+      "theoretical_loss": 4.578507754666157,
+      "tokens_seen": 142278656
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007732316950852558,
+      "loss": 1.9329,
+      "theoretical_loss": 4.57823168685132,
+      "tokens_seen": 142344192
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007732156469408226,
+      "loss": 1.9049,
+      "theoretical_loss": 4.577955781680462,
+      "tokens_seen": 142409728
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007731995987963892,
+      "loss": 1.9542,
+      "theoretical_loss": 4.57768003898298,
+      "tokens_seen": 142475264
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.000773183550651956,
+      "loss": 2.1182,
+      "theoretical_loss": 4.577404458588529,
+      "tokens_seen": 142540800
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007731675025075226,
+      "loss": 1.9606,
+      "theoretical_loss": 4.577129040327021,
+      "tokens_seen": 142606336
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007731514543630894,
+      "loss": 1.851,
+      "theoretical_loss": 4.576853784028622,
+      "tokens_seen": 142671872
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.000773135406218656,
+      "loss": 1.8816,
+      "theoretical_loss": 4.576578689523759,
+      "tokens_seen": 142737408
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007731193580742227,
+      "loss": 1.9601,
+      "theoretical_loss": 4.576303756643108,
+      "tokens_seen": 142802944
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007731033099297894,
+      "loss": 1.7964,
+      "theoretical_loss": 4.576028985217606,
+      "tokens_seen": 142868480
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007730872617853561,
+      "loss": 2.0734,
+      "theoretical_loss": 4.5757543750784375,
+      "tokens_seen": 142934016
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007730712136409228,
+      "loss": 1.9743,
+      "theoretical_loss": 4.5754799260570485,
+      "tokens_seen": 142999552
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007730551654964895,
+      "loss": 1.979,
+      "theoretical_loss": 4.575205637985132,
+      "tokens_seen": 143065088
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007730391173520562,
+      "loss": 1.945,
+      "theoretical_loss": 4.574931510694635,
+      "tokens_seen": 143130624
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007730230692076229,
+      "loss": 1.9358,
+      "theoretical_loss": 4.574657544017761,
+      "tokens_seen": 143196160
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007730070210631896,
+      "loss": 2.0871,
+      "theoretical_loss": 4.57438373778696,
+      "tokens_seen": 143261696
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007729909729187563,
+      "loss": 1.8903,
+      "theoretical_loss": 4.574110091834936,
+      "tokens_seen": 143327232
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007729749247743229,
+      "loss": 2.0474,
+      "theoretical_loss": 4.573836605994645,
+      "tokens_seen": 143392768
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007729588766298897,
+      "loss": 2.0572,
+      "theoretical_loss": 4.573563280099288,
+      "tokens_seen": 143458304
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007729428284854563,
+      "loss": 1.8948,
+      "theoretical_loss": 4.573290113982324,
+      "tokens_seen": 143523840
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007729267803410231,
+      "loss": 1.9941,
+      "theoretical_loss": 4.573017107477455,
+      "tokens_seen": 143589376
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007729107321965897,
+      "loss": 1.9212,
+      "theoretical_loss": 4.572744260418633,
+      "tokens_seen": 143654912
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007728946840521565,
+      "loss": 1.918,
+      "theoretical_loss": 4.572471572640059,
+      "tokens_seen": 143720448
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007728786359077231,
+      "loss": 2.0237,
+      "theoretical_loss": 4.572199043976185,
+      "tokens_seen": 143785984
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007728625877632899,
+      "loss": 1.8364,
+      "theoretical_loss": 4.571926674261705,
+      "tokens_seen": 143851520
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007728465396188565,
+      "loss": 1.9238,
+      "theoretical_loss": 4.571654463331563,
+      "tokens_seen": 143917056
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007728304914744233,
+      "loss": 1.9978,
+      "theoretical_loss": 4.571382411020949,
+      "tokens_seen": 143982592
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007728144433299899,
+      "loss": 1.8705,
+      "theoretical_loss": 4.5711105171653,
+      "tokens_seen": 144048128
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007727983951855567,
+      "loss": 1.8143,
+      "theoretical_loss": 4.570838781600296,
+      "tokens_seen": 144113664
+    },
+    {
+      "epoch": 0.04,
+      "objective/train/docs_used": 60961,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.9238232374191284,
+      "objective/train/theoretical_loss": 4.570567204161865,
+      "objective/train/tokens_used": 164639200,
+      "theoretical_loss": 4.570567204161865,
+      "tokens_seen": 144179200
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007727823470411233,
+      "loss": 1.9238,
+      "theoretical_loss": 4.570567204161865,
+      "tokens_seen": 144179200
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007727662988966901,
+      "loss": 1.8309,
+      "theoretical_loss": 4.570295784686176,
+      "tokens_seen": 144244736
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007727502507522567,
+      "loss": 1.9888,
+      "theoretical_loss": 4.570024523009646,
+      "tokens_seen": 144310272
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007727342026078235,
+      "loss": 1.9715,
+      "theoretical_loss": 4.569753418968933,
+      "tokens_seen": 144375808
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007727181544633901,
+      "loss": 1.9952,
+      "theoretical_loss": 4.569482472400939,
+      "tokens_seen": 144441344
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007727021063189569,
+      "loss": 1.8836,
+      "theoretical_loss": 4.5692116831428065,
+      "tokens_seen": 144506880
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007726860581745235,
+      "loss": 2.0555,
+      "theoretical_loss": 4.568941051031925,
+      "tokens_seen": 144572416
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007726700100300903,
+      "loss": 1.951,
+      "theoretical_loss": 4.568670575905921,
+      "tokens_seen": 144637952
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007726539618856569,
+      "loss": 1.8794,
+      "theoretical_loss": 4.568400257602666,
+      "tokens_seen": 144703488
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007726379137412237,
+      "loss": 1.9484,
+      "theoretical_loss": 4.568130095960268,
+      "tokens_seen": 144769024
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007726218655967903,
+      "loss": 1.9693,
+      "theoretical_loss": 4.56786009081708,
+      "tokens_seen": 144834560
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007726058174523571,
+      "loss": 1.8785,
+      "theoretical_loss": 4.567590242011692,
+      "tokens_seen": 144900096
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007725897693079237,
+      "loss": 1.9614,
+      "theoretical_loss": 4.567320549382931,
+      "tokens_seen": 144965632
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007725737211634905,
+      "loss": 1.9111,
+      "theoretical_loss": 4.56705101276987,
+      "tokens_seen": 145031168
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007725576730190571,
+      "loss": 1.9446,
+      "theoretical_loss": 4.566781632011814,
+      "tokens_seen": 145096704
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007725416248746239,
+      "loss": 2.0029,
+      "theoretical_loss": 4.566512406948309,
+      "tokens_seen": 145162240
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007725255767301905,
+      "loss": 1.7963,
+      "theoretical_loss": 4.566243337419137,
+      "tokens_seen": 145227776
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007725095285857573,
+      "loss": 2.0048,
+      "theoretical_loss": 4.5659744232643185,
+      "tokens_seen": 145293312
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.000772493480441324,
+      "loss": 2.0227,
+      "theoretical_loss": 4.565705664324111,
+      "tokens_seen": 145358848
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007724774322968907,
+      "loss": 1.9297,
+      "theoretical_loss": 4.565437060439006,
+      "tokens_seen": 145424384
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007724613841524574,
+      "loss": 1.9079,
+      "theoretical_loss": 4.565168611449732,
+      "tokens_seen": 145489920
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007724453360080241,
+      "loss": 1.8954,
+      "theoretical_loss": 4.564900317197251,
+      "tokens_seen": 145555456
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007724292878635908,
+      "loss": 1.8184,
+      "theoretical_loss": 4.564632177522766,
+      "tokens_seen": 145620992
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007724132397191575,
+      "loss": 1.9318,
+      "theoretical_loss": 4.564364192267705,
+      "tokens_seen": 145686528
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007723971915747242,
+      "loss": 1.8754,
+      "theoretical_loss": 4.564096361273736,
+      "tokens_seen": 145752064
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007723811434302909,
+      "loss": 1.993,
+      "theoretical_loss": 4.563828684382761,
+      "tokens_seen": 145817600
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007723650952858577,
+      "loss": 1.9158,
+      "theoretical_loss": 4.563561161436912,
+      "tokens_seen": 145883136
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007723490471414243,
+      "loss": 1.9014,
+      "theoretical_loss": 4.563293792278554,
+      "tokens_seen": 145948672
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007723329989969911,
+      "loss": 2.0462,
+      "theoretical_loss": 4.563026576750286,
+      "tokens_seen": 146014208
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007723169508525577,
+      "loss": 1.8942,
+      "theoretical_loss": 4.562759514694935,
+      "tokens_seen": 146079744
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007723009027081245,
+      "loss": 1.9372,
+      "theoretical_loss": 4.5624926059555655,
+      "tokens_seen": 146145280
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007722848545636911,
+      "loss": 1.8486,
+      "theoretical_loss": 4.562225850375466,
+      "tokens_seen": 146210816
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007722688064192579,
+      "loss": 2.0066,
+      "theoretical_loss": 4.561959247798159,
+      "tokens_seen": 146276352
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007722527582748245,
+      "loss": 1.8802,
+      "theoretical_loss": 4.561692798067396,
+      "tokens_seen": 146341888
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007722367101303913,
+      "loss": 1.9621,
+      "theoretical_loss": 4.561426501027158,
+      "tokens_seen": 146407424
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007722206619859579,
+      "loss": 1.9914,
+      "theoretical_loss": 4.561160356521656,
+      "tokens_seen": 146472960
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007722046138415247,
+      "loss": 1.9165,
+      "theoretical_loss": 4.5608943643953275,
+      "tokens_seen": 146538496
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007721885656970913,
+      "loss": 1.9507,
+      "theoretical_loss": 4.56062852449284,
+      "tokens_seen": 146604032
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007721725175526581,
+      "loss": 1.876,
+      "theoretical_loss": 4.5603628366590865,
+      "tokens_seen": 146669568
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007721564694082247,
+      "loss": 1.8305,
+      "theoretical_loss": 4.560097300739191,
+      "tokens_seen": 146735104
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007721404212637915,
+      "loss": 2.0827,
+      "theoretical_loss": 4.5598319165785,
+      "tokens_seen": 146800640
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007721243731193581,
+      "loss": 1.7736,
+      "theoretical_loss": 4.55956668402259,
+      "tokens_seen": 146866176
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007721083249749249,
+      "loss": 1.904,
+      "theoretical_loss": 4.559301602917261,
+      "tokens_seen": 146931712
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007720922768304915,
+      "loss": 1.9652,
+      "theoretical_loss": 4.55903667310854,
+      "tokens_seen": 146997248
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007720762286860583,
+      "loss": 1.8459,
+      "theoretical_loss": 4.558771894442677,
+      "tokens_seen": 147062784
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007720601805416249,
+      "loss": 1.8826,
+      "theoretical_loss": 4.558507266766151,
+      "tokens_seen": 147128320
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007720441323971917,
+      "loss": 1.9618,
+      "theoretical_loss": 4.558242789925659,
+      "tokens_seen": 147193856
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007720280842527583,
+      "loss": 1.867,
+      "theoretical_loss": 4.557978463768127,
+      "tokens_seen": 147259392
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007720120361083251,
+      "loss": 1.8419,
+      "theoretical_loss": 4.557714288140703,
+      "tokens_seen": 147324928
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007719959879638917,
+      "loss": 1.8418,
+      "theoretical_loss": 4.557450262890757,
+      "tokens_seen": 147390464
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007719799398194585,
+      "loss": 1.8463,
+      "theoretical_loss": 4.557186387865883,
+      "tokens_seen": 147456000
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007719638916750251,
+      "loss": 1.9147,
+      "theoretical_loss": 4.556922662913895,
+      "tokens_seen": 147521536
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007719478435305919,
+      "loss": 1.8544,
+      "theoretical_loss": 4.55665908788283,
+      "tokens_seen": 147587072
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007719317953861585,
+      "loss": 1.8465,
+      "theoretical_loss": 4.556395662620945,
+      "tokens_seen": 147652608
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007719157472417253,
+      "loss": 1.7813,
+      "theoretical_loss": 4.5561323869767225,
+      "tokens_seen": 147718144
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007718996990972919,
+      "loss": 2.1008,
+      "theoretical_loss": 4.555869260798859,
+      "tokens_seen": 147783680
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007718836509528587,
+      "loss": 1.9972,
+      "theoretical_loss": 4.5556062839362745,
+      "tokens_seen": 147849216
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007718676028084253,
+      "loss": 1.9089,
+      "theoretical_loss": 4.555343456238109,
+      "tokens_seen": 147914752
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007718515546639921,
+      "loss": 1.9051,
+      "theoretical_loss": 4.55508077755372,
+      "tokens_seen": 147980288
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007718355065195587,
+      "loss": 1.9815,
+      "theoretical_loss": 4.554818247732685,
+      "tokens_seen": 148045824
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007718194583751255,
+      "loss": 1.8505,
+      "theoretical_loss": 4.554555866624798,
+      "tokens_seen": 148111360
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007718034102306921,
+      "loss": 1.8926,
+      "theoretical_loss": 4.554293634080074,
+      "tokens_seen": 148176896
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007717873620862589,
+      "loss": 1.8781,
+      "theoretical_loss": 4.554031549948743,
+      "tokens_seen": 148242432
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007717713139418255,
+      "loss": 1.8525,
+      "theoretical_loss": 4.5537696140812525,
+      "tokens_seen": 148307968
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007717552657973923,
+      "loss": 1.8953,
+      "theoretical_loss": 4.553507826328268,
+      "tokens_seen": 148373504
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007717392176529589,
+      "loss": 1.9935,
+      "theoretical_loss": 4.553246186540671,
+      "tokens_seen": 148439040
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007717231695085256,
+      "loss": 2.0621,
+      "theoretical_loss": 4.552984694569555,
+      "tokens_seen": 148504576
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007717071213640923,
+      "loss": 1.8323,
+      "theoretical_loss": 4.552723350266236,
+      "tokens_seen": 148570112
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.000771691073219659,
+      "loss": 1.8755,
+      "theoretical_loss": 4.552462153482239,
+      "tokens_seen": 148635648
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007716750250752257,
+      "loss": 1.9182,
+      "theoretical_loss": 4.552201104069307,
+      "tokens_seen": 148701184
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007716589769307924,
+      "loss": 1.9216,
+      "theoretical_loss": 4.551940201879396,
+      "tokens_seen": 148766720
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007716429287863591,
+      "loss": 1.8752,
+      "theoretical_loss": 4.551679446764677,
+      "tokens_seen": 148832256
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007716268806419258,
+      "loss": 1.936,
+      "theoretical_loss": 4.551418838577531,
+      "tokens_seen": 148897792
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007716108324974925,
+      "loss": 1.8786,
+      "theoretical_loss": 4.551158377170557,
+      "tokens_seen": 148963328
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007715947843530592,
+      "loss": 1.9279,
+      "theoretical_loss": 4.5508980623965645,
+      "tokens_seen": 149028864
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007715787362086258,
+      "loss": 1.8514,
+      "theoretical_loss": 4.550637894108572,
+      "tokens_seen": 149094400
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007715626880641926,
+      "loss": 1.8439,
+      "theoretical_loss": 4.5503778721598165,
+      "tokens_seen": 149159936
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007715466399197592,
+      "loss": 2.1313,
+      "theoretical_loss": 4.55011799640374,
+      "tokens_seen": 149225472
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.000771530591775326,
+      "loss": 1.9196,
+      "theoretical_loss": 4.549858266694001,
+      "tokens_seen": 149291008
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007715145436308926,
+      "loss": 1.8477,
+      "theoretical_loss": 4.549598682884463,
+      "tokens_seen": 149356544
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007714984954864594,
+      "loss": 1.9181,
+      "theoretical_loss": 4.549339244829204,
+      "tokens_seen": 149422080
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.000771482447342026,
+      "loss": 1.9003,
+      "theoretical_loss": 4.549079952382511,
+      "tokens_seen": 149487616
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007714663991975928,
+      "loss": 1.8811,
+      "theoretical_loss": 4.548820805398881,
+      "tokens_seen": 149553152
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007714503510531594,
+      "loss": 1.8629,
+      "theoretical_loss": 4.548561803733017,
+      "tokens_seen": 149618688
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007714343029087262,
+      "loss": 1.9023,
+      "theoretical_loss": 4.548302947239834,
+      "tokens_seen": 149684224
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007714182547642928,
+      "loss": 1.8714,
+      "theoretical_loss": 4.548044235774454,
+      "tokens_seen": 149749760
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007714022066198596,
+      "loss": 1.9741,
+      "theoretical_loss": 4.547785669192207,
+      "tokens_seen": 149815296
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007713861584754262,
+      "loss": 1.804,
+      "theoretical_loss": 4.547527247348631,
+      "tokens_seen": 149880832
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.000771370110330993,
+      "loss": 1.7849,
+      "theoretical_loss": 4.547268970099471,
+      "tokens_seen": 149946368
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007713540621865596,
+      "loss": 1.8523,
+      "theoretical_loss": 4.547010837300677,
+      "tokens_seen": 150011904
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007713380140421264,
+      "loss": 1.8931,
+      "theoretical_loss": 4.546752848808408,
+      "tokens_seen": 150077440
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.000771321965897693,
+      "loss": 1.9961,
+      "theoretical_loss": 4.546495004479027,
+      "tokens_seen": 150142976
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007713059177532598,
+      "loss": 1.9581,
+      "theoretical_loss": 4.546237304169104,
+      "tokens_seen": 150208512
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007712898696088264,
+      "loss": 1.944,
+      "theoretical_loss": 4.545979747735413,
+      "tokens_seen": 150274048
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007712738214643932,
+      "loss": 1.8109,
+      "theoretical_loss": 4.5457223350349345,
+      "tokens_seen": 150339584
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007712577733199598,
+      "loss": 1.8735,
+      "theoretical_loss": 4.545465065924851,
+      "tokens_seen": 150405120
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007712417251755266,
+      "loss": 1.8665,
+      "theoretical_loss": 4.54520794026255,
+      "tokens_seen": 150470656
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007712256770310932,
+      "loss": 1.9122,
+      "theoretical_loss": 4.544950957905625,
+      "tokens_seen": 150536192
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00077120962888666,
+      "loss": 1.9116,
+      "theoretical_loss": 4.54469411871187,
+      "tokens_seen": 150601728
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007711935807422266,
+      "loss": 1.946,
+      "theoretical_loss": 4.544437422539282,
+      "tokens_seen": 150667264
+    },
+    {
+      "epoch": 0.05,
+      "objective/train/docs_used": 63294,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.7721097469329834,
+      "objective/train/theoretical_loss": 4.544180869246063,
+      "objective/train/tokens_used": 171192800,
+      "theoretical_loss": 4.544180869246063,
+      "tokens_seen": 150732800
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007711775325977934,
+      "loss": 1.7721,
+      "theoretical_loss": 4.544180869246063,
+      "tokens_seen": 150732800
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007711614844533602,
+      "loss": 1.9401,
+      "theoretical_loss": 4.543924458690612,
+      "tokens_seen": 150798336
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007711454363089268,
+      "loss": 1.9516,
+      "theoretical_loss": 4.5436681907315375,
+      "tokens_seen": 150863872
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007711293881644936,
+      "loss": 1.834,
+      "theoretical_loss": 4.543412065227642,
+      "tokens_seen": 150929408
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007711133400200602,
+      "loss": 1.8954,
+      "theoretical_loss": 4.5431560820379335,
+      "tokens_seen": 150994944
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.000771097291875627,
+      "loss": 1.9046,
+      "theoretical_loss": 4.54290024102162,
+      "tokens_seen": 151060480
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007710812437311936,
+      "loss": 1.7943,
+      "theoretical_loss": 4.542644542038108,
+      "tokens_seen": 151126016
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007710651955867604,
+      "loss": 2.0143,
+      "theoretical_loss": 4.542388984947005,
+      "tokens_seen": 151191552
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.000771049147442327,
+      "loss": 1.9111,
+      "theoretical_loss": 4.5421335696081195,
+      "tokens_seen": 151257088
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007710330992978938,
+      "loss": 1.8796,
+      "theoretical_loss": 4.541878295881456,
+      "tokens_seen": 151322624
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007710170511534604,
+      "loss": 1.9437,
+      "theoretical_loss": 4.541623163627221,
+      "tokens_seen": 151388160
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007710010030090272,
+      "loss": 1.9952,
+      "theoretical_loss": 4.541368172705817,
+      "tokens_seen": 151453696
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007709849548645938,
+      "loss": 1.9175,
+      "theoretical_loss": 4.541113322977846,
+      "tokens_seen": 151519232
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007709689067201606,
+      "loss": 1.9967,
+      "theoretical_loss": 4.540858614304108,
+      "tokens_seen": 151584768
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007709528585757272,
+      "loss": 1.8233,
+      "theoretical_loss": 4.5406040465455995,
+      "tokens_seen": 151650304
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.000770936810431294,
+      "loss": 1.9011,
+      "theoretical_loss": 4.5403496195635125,
+      "tokens_seen": 151715840
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007709207622868606,
+      "loss": 2.0133,
+      "theoretical_loss": 4.54009533321924,
+      "tokens_seen": 151781376
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007709047141424274,
+      "loss": 1.892,
+      "theoretical_loss": 4.539841187374366,
+      "tokens_seen": 151846912
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.000770888665997994,
+      "loss": 1.8707,
+      "theoretical_loss": 4.539587181890674,
+      "tokens_seen": 151912448
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007708726178535608,
+      "loss": 1.9512,
+      "theoretical_loss": 4.539333316630143,
+      "tokens_seen": 151977984
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007708565697091274,
+      "loss": 1.8424,
+      "theoretical_loss": 4.539079591454943,
+      "tokens_seen": 152043520
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007708405215646942,
+      "loss": 1.8447,
+      "theoretical_loss": 4.5388260062274455,
+      "tokens_seen": 152109056
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007708244734202608,
+      "loss": 1.9066,
+      "theoretical_loss": 4.538572560810211,
+      "tokens_seen": 152174592
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007708084252758276,
+      "loss": 1.8271,
+      "theoretical_loss": 4.538319255065995,
+      "tokens_seen": 152240128
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007707923771313942,
+      "loss": 1.9183,
+      "theoretical_loss": 4.53806608885775,
+      "tokens_seen": 152305664
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.000770776328986961,
+      "loss": 1.8153,
+      "theoretical_loss": 4.537813062048618,
+      "tokens_seen": 152371200
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007707602808425276,
+      "loss": 1.9209,
+      "theoretical_loss": 4.537560174501936,
+      "tokens_seen": 152436736
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007707442326980944,
+      "loss": 1.8581,
+      "theoretical_loss": 4.537307426081232,
+      "tokens_seen": 152502272
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.000770728184553661,
+      "loss": 1.9426,
+      "theoretical_loss": 4.53705481665023,
+      "tokens_seen": 152567808
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007707121364092278,
+      "loss": 1.8507,
+      "theoretical_loss": 4.536802346072841,
+      "tokens_seen": 152633344
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007706960882647944,
+      "loss": 1.8256,
+      "theoretical_loss": 4.536550014213171,
+      "tokens_seen": 152698880
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007706800401203612,
+      "loss": 1.9372,
+      "theoretical_loss": 4.536297820935517,
+      "tokens_seen": 152764416
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007706639919759278,
+      "loss": 2.0163,
+      "theoretical_loss": 4.536045766104366,
+      "tokens_seen": 152829952
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007706479438314946,
+      "loss": 1.8985,
+      "theoretical_loss": 4.535793849584396,
+      "tokens_seen": 152895488
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007706318956870612,
+      "loss": 1.9097,
+      "theoretical_loss": 4.535542071240473,
+      "tokens_seen": 152961024
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.000770615847542628,
+      "loss": 1.9219,
+      "theoretical_loss": 4.535290430937657,
+      "tokens_seen": 153026560
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007705997993981946,
+      "loss": 1.9989,
+      "theoretical_loss": 4.535038928541194,
+      "tokens_seen": 153092096
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007705837512537614,
+      "loss": 1.8063,
+      "theoretical_loss": 4.534787563916522,
+      "tokens_seen": 153157632
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.000770567703109328,
+      "loss": 1.9977,
+      "theoretical_loss": 4.534536336929264,
+      "tokens_seen": 153223168
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007705516549648948,
+      "loss": 1.8801,
+      "theoretical_loss": 4.534285247445236,
+      "tokens_seen": 153288704
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007705356068204614,
+      "loss": 1.843,
+      "theoretical_loss": 4.534034295330439,
+      "tokens_seen": 153354240
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007705195586760282,
+      "loss": 1.8133,
+      "theoretical_loss": 4.5337834804510635,
+      "tokens_seen": 153419776
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007705035105315948,
+      "loss": 1.8428,
+      "theoretical_loss": 4.533532802673484,
+      "tokens_seen": 153485312
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007704874623871616,
+      "loss": 1.7653,
+      "theoretical_loss": 4.533282261864267,
+      "tokens_seen": 153550848
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007704714142427282,
+      "loss": 1.7916,
+      "theoretical_loss": 4.533031857890164,
+      "tokens_seen": 153616384
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.000770455366098295,
+      "loss": 1.8959,
+      "theoretical_loss": 4.532781590618109,
+      "tokens_seen": 153681920
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007704393179538616,
+      "loss": 1.8476,
+      "theoretical_loss": 4.532531459915229,
+      "tokens_seen": 153747456
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007704232698094284,
+      "loss": 1.9944,
+      "theoretical_loss": 4.532281465648831,
+      "tokens_seen": 153812992
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.000770407221664995,
+      "loss": 1.8107,
+      "theoretical_loss": 4.532031607686411,
+      "tokens_seen": 153878528
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007703911735205618,
+      "loss": 1.8925,
+      "theoretical_loss": 4.531781885895647,
+      "tokens_seen": 153944064
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007703751253761284,
+      "loss": 1.8457,
+      "theoretical_loss": 4.531532300144404,
+      "tokens_seen": 154009600
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007703590772316952,
+      "loss": 1.9029,
+      "theoretical_loss": 4.531282850300731,
+      "tokens_seen": 154075136
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007703430290872618,
+      "loss": 1.8317,
+      "theoretical_loss": 4.53103353623286,
+      "tokens_seen": 154140672
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007703269809428285,
+      "loss": 1.8707,
+      "theoretical_loss": 4.530784357809209,
+      "tokens_seen": 154206208
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007703109327983952,
+      "loss": 1.8097,
+      "theoretical_loss": 4.5305353148983745,
+      "tokens_seen": 154271744
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007702948846539619,
+      "loss": 1.7925,
+      "theoretical_loss": 4.530286407369143,
+      "tokens_seen": 154337280
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007702788365095286,
+      "loss": 1.7986,
+      "theoretical_loss": 4.530037635090476,
+      "tokens_seen": 154402816
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007702627883650953,
+      "loss": 1.7938,
+      "theoretical_loss": 4.529788997931526,
+      "tokens_seen": 154468352
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.000770246740220662,
+      "loss": 1.8068,
+      "theoretical_loss": 4.52954049576162,
+      "tokens_seen": 154533888
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007702306920762287,
+      "loss": 1.9102,
+      "theoretical_loss": 4.529292128450271,
+      "tokens_seen": 154599424
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007702146439317954,
+      "loss": 1.917,
+      "theoretical_loss": 4.52904389586717,
+      "tokens_seen": 154664960
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007701985957873621,
+      "loss": 1.8685,
+      "theoretical_loss": 4.528795797882193,
+      "tokens_seen": 154730496
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007701825476429287,
+      "loss": 1.9085,
+      "theoretical_loss": 4.528547834365394,
+      "tokens_seen": 154796032
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007701664994984955,
+      "loss": 1.8218,
+      "theoretical_loss": 4.52830000518701,
+      "tokens_seen": 154861568
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007701504513540621,
+      "loss": 1.7771,
+      "theoretical_loss": 4.528052310217453,
+      "tokens_seen": 154927104
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007701344032096289,
+      "loss": 1.9205,
+      "theoretical_loss": 4.527804749327321,
+      "tokens_seen": 154992640
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007701183550651955,
+      "loss": 1.7947,
+      "theoretical_loss": 4.527557322387387,
+      "tokens_seen": 155058176
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007701023069207623,
+      "loss": 1.8499,
+      "theoretical_loss": 4.527310029268604,
+      "tokens_seen": 155123712
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007700862587763289,
+      "loss": 2.0212,
+      "theoretical_loss": 4.527062869842107,
+      "tokens_seen": 155189248
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007700702106318957,
+      "loss": 1.8781,
+      "theoretical_loss": 4.526815843979204,
+      "tokens_seen": 155254784
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007700541624874623,
+      "loss": 1.7566,
+      "theoretical_loss": 4.5265689515513845,
+      "tokens_seen": 155320320
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007700381143430291,
+      "loss": 1.7649,
+      "theoretical_loss": 4.526322192430316,
+      "tokens_seen": 155385856
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007700220661985957,
+      "loss": 1.8284,
+      "theoretical_loss": 4.526075566487842,
+      "tokens_seen": 155451392
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007700060180541625,
+      "loss": 1.9143,
+      "theoretical_loss": 4.525829073595984,
+      "tokens_seen": 155516928
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007699899699097291,
+      "loss": 1.8158,
+      "theoretical_loss": 4.5255827136269415,
+      "tokens_seen": 155582464
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007699739217652959,
+      "loss": 1.7568,
+      "theoretical_loss": 4.525336486453088,
+      "tokens_seen": 155648000
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007699578736208627,
+      "loss": 1.763,
+      "theoretical_loss": 4.525090391946974,
+      "tokens_seen": 155713536
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007699418254764293,
+      "loss": 1.8462,
+      "theoretical_loss": 4.524844429981327,
+      "tokens_seen": 155779072
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.000769925777331996,
+      "loss": 1.8713,
+      "theoretical_loss": 4.52459860042905,
+      "tokens_seen": 155844608
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007699097291875627,
+      "loss": 1.9221,
+      "theoretical_loss": 4.524352903163219,
+      "tokens_seen": 155910144
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007698936810431295,
+      "loss": 1.9221,
+      "theoretical_loss": 4.524107338057089,
+      "tokens_seen": 155975680
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007698776328986961,
+      "loss": 1.7941,
+      "theoretical_loss": 4.523861904984086,
+      "tokens_seen": 156041216
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007698615847542629,
+      "loss": 1.7618,
+      "theoretical_loss": 4.5236166038178105,
+      "tokens_seen": 156106752
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007698455366098295,
+      "loss": 1.9284,
+      "theoretical_loss": 4.52337143443204,
+      "tokens_seen": 156172288
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007698294884653963,
+      "loss": 1.8633,
+      "theoretical_loss": 4.523126396700722,
+      "tokens_seen": 156237824
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007698134403209629,
+      "loss": 2.0108,
+      "theoretical_loss": 4.522881490497981,
+      "tokens_seen": 156303360
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007697973921765297,
+      "loss": 1.8716,
+      "theoretical_loss": 4.522636715698111,
+      "tokens_seen": 156368896
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007697813440320963,
+      "loss": 1.9278,
+      "theoretical_loss": 4.522392072175582,
+      "tokens_seen": 156434432
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007697652958876631,
+      "loss": 1.8369,
+      "theoretical_loss": 4.522147559805034,
+      "tokens_seen": 156499968
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007697492477432297,
+      "loss": 1.8796,
+      "theoretical_loss": 4.5219031784612795,
+      "tokens_seen": 156565504
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007697331995987965,
+      "loss": 1.853,
+      "theoretical_loss": 4.521658928019305,
+      "tokens_seen": 156631040
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007697171514543631,
+      "loss": 1.7639,
+      "theoretical_loss": 4.521414808354266,
+      "tokens_seen": 156696576
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007697011033099299,
+      "loss": 1.9447,
+      "theoretical_loss": 4.52117081934149,
+      "tokens_seen": 156762112
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007696850551654965,
+      "loss": 1.8389,
+      "theoretical_loss": 4.520926960856476,
+      "tokens_seen": 156827648
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007696690070210633,
+      "loss": 1.9491,
+      "theoretical_loss": 4.520683232774893,
+      "tokens_seen": 156893184
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007696529588766299,
+      "loss": 1.9055,
+      "theoretical_loss": 4.520439634972579,
+      "tokens_seen": 156958720
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007696369107321967,
+      "loss": 1.9063,
+      "theoretical_loss": 4.520196167325546,
+      "tokens_seen": 157024256
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007696208625877633,
+      "loss": 1.8827,
+      "theoretical_loss": 4.519952829709972,
+      "tokens_seen": 157089792
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007696048144433301,
+      "loss": 1.7906,
+      "theoretical_loss": 4.519709622002205,
+      "tokens_seen": 157155328
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007695887662988967,
+      "loss": 1.808,
+      "theoretical_loss": 4.519466544078762,
+      "tokens_seen": 157220864
+    },
+    {
+      "epoch": 0.05,
+      "objective/train/docs_used": 65757,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.8944233655929565,
+      "objective/train/theoretical_loss": 4.519223595816332,
+      "objective/train/tokens_used": 177746400,
+      "theoretical_loss": 4.519223595816332,
+      "tokens_seen": 157286400
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007695727181544635,
+      "loss": 1.8944,
+      "theoretical_loss": 4.519223595816332,
+      "tokens_seen": 157286400
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007695566700100301,
+      "loss": 1.8895,
+      "theoretical_loss": 4.518980777091768,
+      "tokens_seen": 157351936
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007695406218655969,
+      "loss": 1.8229,
+      "theoretical_loss": 4.518738087782093,
+      "tokens_seen": 157417472
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007695245737211635,
+      "loss": 1.8728,
+      "theoretical_loss": 4.5184955277644985,
+      "tokens_seen": 157483008
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007695085255767303,
+      "loss": 2.0235,
+      "theoretical_loss": 4.518253096916341,
+      "tokens_seen": 157548544
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007694924774322969,
+      "loss": 1.869,
+      "theoretical_loss": 4.518010795115151,
+      "tokens_seen": 157614080
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007694764292878637,
+      "loss": 1.8485,
+      "theoretical_loss": 4.517768622238616,
+      "tokens_seen": 157679616
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007694603811434303,
+      "loss": 1.6952,
+      "theoretical_loss": 4.5175265781646,
+      "tokens_seen": 157745152
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007694443329989971,
+      "loss": 1.8528,
+      "theoretical_loss": 4.517284662771125,
+      "tokens_seen": 157810688
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007694282848545637,
+      "loss": 1.8879,
+      "theoretical_loss": 4.5170428759363865,
+      "tokens_seen": 157876224
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007694122367101305,
+      "loss": 2.0272,
+      "theoretical_loss": 4.516801217538739,
+      "tokens_seen": 157941760
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007693961885656971,
+      "loss": 1.9991,
+      "theoretical_loss": 4.516559687456708,
+      "tokens_seen": 158007296
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007693801404212639,
+      "loss": 1.8421,
+      "theoretical_loss": 4.516318285568982,
+      "tokens_seen": 158072832
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007693640922768305,
+      "loss": 1.7922,
+      "theoretical_loss": 4.516077011754414,
+      "tokens_seen": 158138368
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007693480441323973,
+      "loss": 2.0169,
+      "theoretical_loss": 4.51583586589202,
+      "tokens_seen": 158203904
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007693319959879639,
+      "loss": 1.864,
+      "theoretical_loss": 4.515594847860985,
+      "tokens_seen": 158269440
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007693159478435307,
+      "loss": 1.9233,
+      "theoretical_loss": 4.515353957540656,
+      "tokens_seen": 158334976
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007692998996990973,
+      "loss": 1.9782,
+      "theoretical_loss": 4.515113194810541,
+      "tokens_seen": 158400512
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007692838515546641,
+      "loss": 1.9078,
+      "theoretical_loss": 4.514872559550316,
+      "tokens_seen": 158466048
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007692678034102307,
+      "loss": 1.8511,
+      "theoretical_loss": 4.514632051639815,
+      "tokens_seen": 158531584
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007692517552657975,
+      "loss": 1.8825,
+      "theoretical_loss": 4.51439167095904,
+      "tokens_seen": 158597120
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007692357071213641,
+      "loss": 1.827,
+      "theoretical_loss": 4.5141514173881525,
+      "tokens_seen": 158662656
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007692196589769309,
+      "loss": 1.8861,
+      "theoretical_loss": 4.513911290807478,
+      "tokens_seen": 158728192
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007692036108324975,
+      "loss": 1.7447,
+      "theoretical_loss": 4.513671291097502,
+      "tokens_seen": 158793728
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007691875626880643,
+      "loss": 1.916,
+      "theoretical_loss": 4.513431418138874,
+      "tokens_seen": 158859264
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007691715145436309,
+      "loss": 1.9695,
+      "theoretical_loss": 4.513191671812404,
+      "tokens_seen": 158924800
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007691554663991977,
+      "loss": 1.9074,
+      "theoretical_loss": 4.512952051999061,
+      "tokens_seen": 158990336
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007691394182547643,
+      "loss": 1.8326,
+      "theoretical_loss": 4.512712558579979,
+      "tokens_seen": 159055872
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007691233701103311,
+      "loss": 1.8922,
+      "theoretical_loss": 4.512473191436451,
+      "tokens_seen": 159121408
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007691073219658977,
+      "loss": 1.9429,
+      "theoretical_loss": 4.512233950449927,
+      "tokens_seen": 159186944
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007690912738214645,
+      "loss": 1.7524,
+      "theoretical_loss": 4.511994835502023,
+      "tokens_seen": 159252480
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007690752256770311,
+      "loss": 1.879,
+      "theoretical_loss": 4.511755846474511,
+      "tokens_seen": 159318016
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007690591775325979,
+      "loss": 2.0729,
+      "theoretical_loss": 4.511516983249322,
+      "tokens_seen": 159383552
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007690431293881645,
+      "loss": 1.867,
+      "theoretical_loss": 4.511278245708548,
+      "tokens_seen": 159449088
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007690270812437313,
+      "loss": 1.9098,
+      "theoretical_loss": 4.511039633734439,
+      "tokens_seen": 159514624
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007690110330992979,
+      "loss": 1.8243,
+      "theoretical_loss": 4.5108011472094045,
+      "tokens_seen": 159580160
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007689949849548647,
+      "loss": 1.8814,
+      "theoretical_loss": 4.510562786016012,
+      "tokens_seen": 159645696
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007689789368104313,
+      "loss": 1.8576,
+      "theoretical_loss": 4.5103245500369855,
+      "tokens_seen": 159711232
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007689628886659981,
+      "loss": 1.8567,
+      "theoretical_loss": 4.510086439155209,
+      "tokens_seen": 159776768
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007689468405215647,
+      "loss": 1.9073,
+      "theoretical_loss": 4.509848453253722,
+      "tokens_seen": 159842304
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007689307923771315,
+      "loss": 1.9951,
+      "theoretical_loss": 4.509610592215726,
+      "tokens_seen": 159907840
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007689147442326981,
+      "loss": 1.8949,
+      "theoretical_loss": 4.509372855924571,
+      "tokens_seen": 159973376
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007688986960882648,
+      "loss": 1.7646,
+      "theoretical_loss": 4.5091352442637715,
+      "tokens_seen": 160038912
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007688826479438315,
+      "loss": 1.8691,
+      "theoretical_loss": 4.508897757116995,
+      "tokens_seen": 160104448
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007688665997993982,
+      "loss": 1.8943,
+      "theoretical_loss": 4.508660394368066,
+      "tokens_seen": 160169984
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007688505516549649,
+      "loss": 1.9748,
+      "theoretical_loss": 4.508423155900964,
+      "tokens_seen": 160235520
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007688345035105316,
+      "loss": 1.7981,
+      "theoretical_loss": 4.508186041599823,
+      "tokens_seen": 160301056
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007688184553660983,
+      "loss": 1.7237,
+      "theoretical_loss": 4.5079490513489375,
+      "tokens_seen": 160366592
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.000768802407221665,
+      "loss": 1.8996,
+      "theoretical_loss": 4.50771218503275,
+      "tokens_seen": 160432128
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007687863590772316,
+      "loss": 2.0224,
+      "theoretical_loss": 4.507475442535863,
+      "tokens_seen": 160497664
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007687703109327984,
+      "loss": 1.9483,
+      "theoretical_loss": 4.507238823743033,
+      "tokens_seen": 160563200
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007687542627883651,
+      "loss": 1.8003,
+      "theoretical_loss": 4.507002328539167,
+      "tokens_seen": 160628736
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007687382146439318,
+      "loss": 1.7646,
+      "theoretical_loss": 4.506765956809331,
+      "tokens_seen": 160694272
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007687221664994985,
+      "loss": 1.8767,
+      "theoretical_loss": 4.50652970843874,
+      "tokens_seen": 160759808
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007687061183550652,
+      "loss": 1.8529,
+      "theoretical_loss": 4.506293583312766,
+      "tokens_seen": 160825344
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.000768690070210632,
+      "loss": 1.6486,
+      "theoretical_loss": 4.506057581316933,
+      "tokens_seen": 160890880
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007686740220661986,
+      "loss": 1.81,
+      "theoretical_loss": 4.505821702336917,
+      "tokens_seen": 160956416
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007686579739217653,
+      "loss": 1.8689,
+      "theoretical_loss": 4.505585946258547,
+      "tokens_seen": 161021952
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.000768641925777332,
+      "loss": 1.842,
+      "theoretical_loss": 4.505350312967805,
+      "tokens_seen": 161087488
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007686258776328987,
+      "loss": 1.7522,
+      "theoretical_loss": 4.5051148023508265,
+      "tokens_seen": 161153024
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007686098294884654,
+      "loss": 1.9213,
+      "theoretical_loss": 4.504879414293896,
+      "tokens_seen": 161218560
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007685937813440322,
+      "loss": 1.9442,
+      "theoretical_loss": 4.50464414868345,
+      "tokens_seen": 161284096
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007685777331995988,
+      "loss": 1.8515,
+      "theoretical_loss": 4.504409005406078,
+      "tokens_seen": 161349632
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007685616850551656,
+      "loss": 1.8907,
+      "theoretical_loss": 4.504173984348521,
+      "tokens_seen": 161415168
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007685456369107322,
+      "loss": 1.8962,
+      "theoretical_loss": 4.503939085397667,
+      "tokens_seen": 161480704
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.000768529588766299,
+      "loss": 1.888,
+      "theoretical_loss": 4.5037043084405575,
+      "tokens_seen": 161546240
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007685135406218656,
+      "loss": 1.8349,
+      "theoretical_loss": 4.503469653364386,
+      "tokens_seen": 161611776
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007684974924774324,
+      "loss": 1.9325,
+      "theoretical_loss": 4.503235120056491,
+      "tokens_seen": 161677312
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.000768481444332999,
+      "loss": 1.9023,
+      "theoretical_loss": 4.503000708404365,
+      "tokens_seen": 161742848
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007684653961885658,
+      "loss": 1.7732,
+      "theoretical_loss": 4.502766418295647,
+      "tokens_seen": 161808384
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007684493480441324,
+      "loss": 1.8676,
+      "theoretical_loss": 4.502532249618129,
+      "tokens_seen": 161873920
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007684332998996992,
+      "loss": 1.6695,
+      "theoretical_loss": 4.502298202259748,
+      "tokens_seen": 161939456
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007684172517552658,
+      "loss": 1.8236,
+      "theoretical_loss": 4.502064276108593,
+      "tokens_seen": 162004992
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007684012036108326,
+      "loss": 1.8506,
+      "theoretical_loss": 4.501830471052898,
+      "tokens_seen": 162070528
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007683851554663992,
+      "loss": 1.8639,
+      "theoretical_loss": 4.501596786981047,
+      "tokens_seen": 162136064
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.000768369107321966,
+      "loss": 1.8099,
+      "theoretical_loss": 4.501363223781574,
+      "tokens_seen": 162201600
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007683530591775326,
+      "loss": 1.8228,
+      "theoretical_loss": 4.501129781343157,
+      "tokens_seen": 162267136
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007683370110330994,
+      "loss": 1.818,
+      "theoretical_loss": 4.500896459554624,
+      "tokens_seen": 162332672
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.000768320962888666,
+      "loss": 1.8882,
+      "theoretical_loss": 4.500663258304948,
+      "tokens_seen": 162398208
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007683049147442328,
+      "loss": 1.8502,
+      "theoretical_loss": 4.500430177483253,
+      "tokens_seen": 162463744
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007682888665997994,
+      "loss": 1.7494,
+      "theoretical_loss": 4.500197216978804,
+      "tokens_seen": 162529280
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007682728184553662,
+      "loss": 1.7342,
+      "theoretical_loss": 4.499964376681016,
+      "tokens_seen": 162594816
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007682567703109328,
+      "loss": 1.8732,
+      "theoretical_loss": 4.49973165647945,
+      "tokens_seen": 162660352
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007682407221664996,
+      "loss": 1.8919,
+      "theoretical_loss": 4.4994990562638115,
+      "tokens_seen": 162725888
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007682246740220662,
+      "loss": 1.9732,
+      "theoretical_loss": 4.499266575923954,
+      "tokens_seen": 162791424
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.000768208625877633,
+      "loss": 1.8033,
+      "theoretical_loss": 4.499034215349875,
+      "tokens_seen": 162856960
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007681925777331996,
+      "loss": 1.754,
+      "theoretical_loss": 4.498801974431714,
+      "tokens_seen": 162922496
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007681765295887664,
+      "loss": 1.8017,
+      "theoretical_loss": 4.498569853059762,
+      "tokens_seen": 162988032
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.000768160481444333,
+      "loss": 1.8504,
+      "theoretical_loss": 4.49833785112445,
+      "tokens_seen": 163053568
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007681444332998998,
+      "loss": 1.8278,
+      "theoretical_loss": 4.498105968516354,
+      "tokens_seen": 163119104
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007681283851554664,
+      "loss": 1.8672,
+      "theoretical_loss": 4.497874205126195,
+      "tokens_seen": 163184640
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007681123370110332,
+      "loss": 1.8862,
+      "theoretical_loss": 4.497642560844838,
+      "tokens_seen": 163250176
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007680962888665998,
+      "loss": 2.042,
+      "theoretical_loss": 4.497411035563291,
+      "tokens_seen": 163315712
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007680802407221666,
+      "loss": 1.8039,
+      "theoretical_loss": 4.497179629172704,
+      "tokens_seen": 163381248
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007680641925777332,
+      "loss": 1.8527,
+      "theoretical_loss": 4.4969483415643765,
+      "tokens_seen": 163446784
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007680481444333,
+      "loss": 1.8565,
+      "theoretical_loss": 4.496717172629742,
+      "tokens_seen": 163512320
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007680320962888666,
+      "loss": 1.9447,
+      "theoretical_loss": 4.496486122260382,
+      "tokens_seen": 163577856
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007680160481444334,
+      "loss": 1.7381,
+      "theoretical_loss": 4.4962551903480215,
+      "tokens_seen": 163643392
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.000768,
+      "loss": 1.8522,
+      "theoretical_loss": 4.496024376784522,
+      "tokens_seen": 163708928
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007679839518555668,
+      "loss": 1.7952,
+      "theoretical_loss": 4.495793681461894,
+      "tokens_seen": 163774464
+    },
+    {
+      "epoch": 0.05,
+      "objective/train/docs_used": 68034,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.8746311664581299,
+      "objective/train/theoretical_loss": 4.495563104272286,
+      "objective/train/tokens_used": 184300000,
+      "theoretical_loss": 4.495563104272286,
+      "tokens_seen": 163840000
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007679679037111334,
+      "loss": 1.8746,
+      "theoretical_loss": 4.495563104272286,
+      "tokens_seen": 163840000
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007679518555667002,
+      "loss": 1.7779,
+      "theoretical_loss": 4.495332645107985,
+      "tokens_seen": 163905536
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007679358074222668,
+      "loss": 1.8592,
+      "theoretical_loss": 4.495102303861426,
+      "tokens_seen": 163971072
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007679197592778336,
+      "loss": 1.8209,
+      "theoretical_loss": 4.4948720804251785,
+      "tokens_seen": 164036608
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007679037111334002,
+      "loss": 1.7162,
+      "theoretical_loss": 4.494641974691959,
+      "tokens_seen": 164102144
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.000767887662988967,
+      "loss": 1.8027,
+      "theoretical_loss": 4.494411986554619,
+      "tokens_seen": 164167680
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007678716148445336,
+      "loss": 1.8185,
+      "theoretical_loss": 4.494182115906153,
+      "tokens_seen": 164233216
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007678555667001004,
+      "loss": 1.713,
+      "theoretical_loss": 4.493952362639694,
+      "tokens_seen": 164298752
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.000767839518555667,
+      "loss": 1.9033,
+      "theoretical_loss": 4.493722726648519,
+      "tokens_seen": 164364288
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007678234704112338,
+      "loss": 1.9633,
+      "theoretical_loss": 4.493493207826037,
+      "tokens_seen": 164429824
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007678074222668004,
+      "loss": 1.7328,
+      "theoretical_loss": 4.493263806065803,
+      "tokens_seen": 164495360
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007677913741223672,
+      "loss": 1.7692,
+      "theoretical_loss": 4.493034521261509,
+      "tokens_seen": 164560896
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007677753259779338,
+      "loss": 1.9106,
+      "theoretical_loss": 4.492805353306984,
+      "tokens_seen": 164626432
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007677592778335006,
+      "loss": 1.8737,
+      "theoretical_loss": 4.492576302096199,
+      "tokens_seen": 164691968
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007677432296890672,
+      "loss": 1.7918,
+      "theoretical_loss": 4.49234736752326,
+      "tokens_seen": 164757504
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.000767727181544634,
+      "loss": 1.871,
+      "theoretical_loss": 4.492118549482413,
+      "tokens_seen": 164823040
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007677111334002006,
+      "loss": 1.8635,
+      "theoretical_loss": 4.491889847868041,
+      "tokens_seen": 164888576
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007676950852557674,
+      "loss": 1.8765,
+      "theoretical_loss": 4.491661262574665,
+      "tokens_seen": 164954112
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.000767679037111334,
+      "loss": 1.895,
+      "theoretical_loss": 4.491432793496945,
+      "tokens_seen": 165019648
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007676629889669008,
+      "loss": 1.8275,
+      "theoretical_loss": 4.4912044405296765,
+      "tokens_seen": 165085184
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007676469408224674,
+      "loss": 1.8488,
+      "theoretical_loss": 4.49097620356779,
+      "tokens_seen": 165150720
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007676308926780342,
+      "loss": 1.8425,
+      "theoretical_loss": 4.490748082506356,
+      "tokens_seen": 165216256
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007676148445336008,
+      "loss": 1.8183,
+      "theoretical_loss": 4.490520077240582,
+      "tokens_seen": 165281792
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007675987963891676,
+      "loss": 1.7612,
+      "theoretical_loss": 4.490292187665808,
+      "tokens_seen": 165347328
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007675827482447342,
+      "loss": 1.8528,
+      "theoretical_loss": 4.490064413677514,
+      "tokens_seen": 165412864
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.000767566700100301,
+      "loss": 1.9439,
+      "theoretical_loss": 4.4898367551713125,
+      "tokens_seen": 165478400
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007675506519558676,
+      "loss": 1.8569,
+      "theoretical_loss": 4.489609212042954,
+      "tokens_seen": 165543936
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007675346038114344,
+      "loss": 1.9445,
+      "theoretical_loss": 4.489381784188322,
+      "tokens_seen": 165609472
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.000767518555667001,
+      "loss": 1.9607,
+      "theoretical_loss": 4.489154471503438,
+      "tokens_seen": 165675008
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007675025075225677,
+      "loss": 1.9154,
+      "theoretical_loss": 4.488927273884456,
+      "tokens_seen": 165740544
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007674864593781344,
+      "loss": 1.793,
+      "theoretical_loss": 4.488700191227665,
+      "tokens_seen": 165806080
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007674704112337011,
+      "loss": 1.77,
+      "theoretical_loss": 4.488473223429491,
+      "tokens_seen": 165871616
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007674543630892678,
+      "loss": 1.7784,
+      "theoretical_loss": 4.48824637038649,
+      "tokens_seen": 165937152
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007674383149448345,
+      "loss": 1.757,
+      "theoretical_loss": 4.488019631995354,
+      "tokens_seen": 166002688
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007674222668004012,
+      "loss": 1.9577,
+      "theoretical_loss": 4.48779300815291,
+      "tokens_seen": 166068224
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007674062186559679,
+      "loss": 1.7849,
+      "theoretical_loss": 4.4875664987561175,
+      "tokens_seen": 166133760
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007673901705115346,
+      "loss": 1.9536,
+      "theoretical_loss": 4.487340103702068,
+      "tokens_seen": 166199296
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007673741223671013,
+      "loss": 1.7424,
+      "theoretical_loss": 4.487113822887988,
+      "tokens_seen": 166264832
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.000767358074222668,
+      "loss": 1.7993,
+      "theoretical_loss": 4.486887656211234,
+      "tokens_seen": 166330368
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007673420260782347,
+      "loss": 1.7571,
+      "theoretical_loss": 4.486661603569301,
+      "tokens_seen": 166395904
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007673259779338014,
+      "loss": 1.7829,
+      "theoretical_loss": 4.486435664859809,
+      "tokens_seen": 166461440
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007673099297893681,
+      "loss": 1.809,
+      "theoretical_loss": 4.486209839980516,
+      "tokens_seen": 166526976
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007672938816449348,
+      "loss": 1.8188,
+      "theoretical_loss": 4.485984128829307,
+      "tokens_seen": 166592512
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007672778335005015,
+      "loss": 1.8781,
+      "theoretical_loss": 4.485758531304205,
+      "tokens_seen": 166658048
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007672617853560682,
+      "loss": 1.9506,
+      "theoretical_loss": 4.485533047303358,
+      "tokens_seen": 166723584
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007672457372116349,
+      "loss": 1.9245,
+      "theoretical_loss": 4.485307676725048,
+      "tokens_seen": 166789120
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007672296890672016,
+      "loss": 1.8347,
+      "theoretical_loss": 4.485082419467691,
+      "tokens_seen": 166854656
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007672136409227683,
+      "loss": 1.8,
+      "theoretical_loss": 4.484857275429828,
+      "tokens_seen": 166920192
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.000767197592778335,
+      "loss": 1.7967,
+      "theoretical_loss": 4.484632244510136,
+      "tokens_seen": 166985728
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007671815446339017,
+      "loss": 1.7989,
+      "theoretical_loss": 4.4844073266074185,
+      "tokens_seen": 167051264
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007671654964894685,
+      "loss": 1.8184,
+      "theoretical_loss": 4.484182521620611,
+      "tokens_seen": 167116800
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007671494483450351,
+      "loss": 1.7635,
+      "theoretical_loss": 4.48395782944878,
+      "tokens_seen": 167182336
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007671334002006019,
+      "loss": 1.8308,
+      "theoretical_loss": 4.483733249991118,
+      "tokens_seen": 167247872
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007671173520561685,
+      "loss": 1.8299,
+      "theoretical_loss": 4.483508783146952,
+      "tokens_seen": 167313408
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007671013039117353,
+      "loss": 1.8609,
+      "theoretical_loss": 4.483284428815735,
+      "tokens_seen": 167378944
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007670852557673019,
+      "loss": 1.8029,
+      "theoretical_loss": 4.483060186897049,
+      "tokens_seen": 167444480
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007670692076228687,
+      "loss": 1.8604,
+      "theoretical_loss": 4.482836057290607,
+      "tokens_seen": 167510016
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007670531594784353,
+      "loss": 1.8782,
+      "theoretical_loss": 4.4826120398962495,
+      "tokens_seen": 167575552
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007670371113340021,
+      "loss": 1.7944,
+      "theoretical_loss": 4.482388134613943,
+      "tokens_seen": 167641088
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007670210631895687,
+      "loss": 1.8592,
+      "theoretical_loss": 4.482164341343788,
+      "tokens_seen": 167706624
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007670050150451355,
+      "loss": 1.7592,
+      "theoretical_loss": 4.481940659986008,
+      "tokens_seen": 167772160
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007669889669007021,
+      "loss": 1.7657,
+      "theoretical_loss": 4.481717090440956,
+      "tokens_seen": 167837696
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007669729187562689,
+      "loss": 1.7377,
+      "theoretical_loss": 4.481493632609112,
+      "tokens_seen": 167903232
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007669568706118355,
+      "loss": 1.8152,
+      "theoretical_loss": 4.481270286391084,
+      "tokens_seen": 167968768
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007669408224674023,
+      "loss": 1.7121,
+      "theoretical_loss": 4.481047051687609,
+      "tokens_seen": 168034304
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007669247743229689,
+      "loss": 1.9053,
+      "theoretical_loss": 4.480823928399547,
+      "tokens_seen": 168099840
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007669087261785357,
+      "loss": 1.81,
+      "theoretical_loss": 4.480600916427887,
+      "tokens_seen": 168165376
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007668926780341023,
+      "loss": 1.8638,
+      "theoretical_loss": 4.480378015673745,
+      "tokens_seen": 168230912
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007668766298896691,
+      "loss": 1.8954,
+      "theoretical_loss": 4.480155226038362,
+      "tokens_seen": 168296448
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007668605817452357,
+      "loss": 1.7743,
+      "theoretical_loss": 4.479932547423107,
+      "tokens_seen": 168361984
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007668445336008025,
+      "loss": 1.8612,
+      "theoretical_loss": 4.479709979729472,
+      "tokens_seen": 168427520
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007668284854563691,
+      "loss": 1.8205,
+      "theoretical_loss": 4.479487522859078,
+      "tokens_seen": 168493056
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007668124373119359,
+      "loss": 1.768,
+      "theoretical_loss": 4.4792651767136675,
+      "tokens_seen": 168558592
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007667963891675025,
+      "loss": 1.796,
+      "theoretical_loss": 4.479042941195113,
+      "tokens_seen": 168624128
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007667803410230693,
+      "loss": 1.8316,
+      "theoretical_loss": 4.478820816205408,
+      "tokens_seen": 168689664
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007667642928786359,
+      "loss": 1.7718,
+      "theoretical_loss": 4.4785988016466725,
+      "tokens_seen": 168755200
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007667482447342027,
+      "loss": 1.8341,
+      "theoretical_loss": 4.4783768974211515,
+      "tokens_seen": 168820736
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007667321965897693,
+      "loss": 1.9364,
+      "theoretical_loss": 4.478155103431215,
+      "tokens_seen": 168886272
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007667161484453361,
+      "loss": 1.7932,
+      "theoretical_loss": 4.477933419579356,
+      "tokens_seen": 168951808
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007667001003009027,
+      "loss": 1.8589,
+      "theoretical_loss": 4.477711845768191,
+      "tokens_seen": 169017344
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007666840521564695,
+      "loss": 1.8692,
+      "theoretical_loss": 4.477490381900461,
+      "tokens_seen": 169082880
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007666680040120361,
+      "loss": 1.8408,
+      "theoretical_loss": 4.477269027879032,
+      "tokens_seen": 169148416
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007666519558676029,
+      "loss": 1.8432,
+      "theoretical_loss": 4.477047783606892,
+      "tokens_seen": 169213952
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007666359077231695,
+      "loss": 1.9238,
+      "theoretical_loss": 4.476826648987151,
+      "tokens_seen": 169279488
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007666198595787363,
+      "loss": 1.8107,
+      "theoretical_loss": 4.476605623923044,
+      "tokens_seen": 169345024
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007666038114343029,
+      "loss": 1.7872,
+      "theoretical_loss": 4.47638470831793,
+      "tokens_seen": 169410560
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007665877632898697,
+      "loss": 1.9428,
+      "theoretical_loss": 4.476163902075286,
+      "tokens_seen": 169476096
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007665717151454363,
+      "loss": 1.8615,
+      "theoretical_loss": 4.475943205098716,
+      "tokens_seen": 169541632
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007665556670010031,
+      "loss": 1.8295,
+      "theoretical_loss": 4.475722617291944,
+      "tokens_seen": 169607168
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007665396188565697,
+      "loss": 1.9984,
+      "theoretical_loss": 4.4755021385588165,
+      "tokens_seen": 169672704
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007665235707121365,
+      "loss": 1.8964,
+      "theoretical_loss": 4.475281768803301,
+      "tokens_seen": 169738240
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007665075225677031,
+      "loss": 1.7493,
+      "theoretical_loss": 4.475061507929487,
+      "tokens_seen": 169803776
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007664914744232699,
+      "loss": 1.9898,
+      "theoretical_loss": 4.474841355841587,
+      "tokens_seen": 169869312
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007664754262788365,
+      "loss": 1.822,
+      "theoretical_loss": 4.474621312443932,
+      "tokens_seen": 169934848
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007664593781344033,
+      "loss": 1.9682,
+      "theoretical_loss": 4.474401377640977,
+      "tokens_seen": 170000384
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007664433299899699,
+      "loss": 1.8189,
+      "theoretical_loss": 4.474181551337294,
+      "tokens_seen": 170065920
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007664272818455367,
+      "loss": 1.8791,
+      "theoretical_loss": 4.473961833437579,
+      "tokens_seen": 170131456
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007664112337011033,
+      "loss": 1.9552,
+      "theoretical_loss": 4.473742223846646,
+      "tokens_seen": 170196992
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007663951855566701,
+      "loss": 1.8052,
+      "theoretical_loss": 4.473522722469431,
+      "tokens_seen": 170262528
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007663791374122367,
+      "loss": 1.9086,
+      "theoretical_loss": 4.47330332921099,
+      "tokens_seen": 170328064
+    },
+    {
+      "epoch": 0.05,
+      "objective/train/docs_used": 70789,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.8844119310379028,
+      "objective/train/theoretical_loss": 4.473084043976495,
+      "objective/train/tokens_used": 190853600,
+      "theoretical_loss": 4.473084043976495,
+      "tokens_seen": 170393600
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007663630892678035,
+      "loss": 1.8844,
+      "theoretical_loss": 4.473084043976495,
+      "tokens_seen": 170393600
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007663470411233702,
+      "loss": 1.9333,
+      "theoretical_loss": 4.472864866671245,
+      "tokens_seen": 170459136
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007663309929789369,
+      "loss": 1.8575,
+      "theoretical_loss": 4.472645797200651,
+      "tokens_seen": 170524672
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007663149448345036,
+      "loss": 1.8499,
+      "theoretical_loss": 4.472426835470247,
+      "tokens_seen": 170590208
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007662988966900703,
+      "loss": 1.881,
+      "theoretical_loss": 4.472207981385686,
+      "tokens_seen": 170655744
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.000766282848545637,
+      "loss": 1.8488,
+      "theoretical_loss": 4.471989234852737,
+      "tokens_seen": 170721280
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007662668004012037,
+      "loss": 1.8342,
+      "theoretical_loss": 4.471770595777292,
+      "tokens_seen": 170786816
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007662507522567703,
+      "loss": 1.8127,
+      "theoretical_loss": 4.4715520640653565,
+      "tokens_seen": 170852352
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007662347041123371,
+      "loss": 1.8033,
+      "theoretical_loss": 4.471333639623059,
+      "tokens_seen": 170917888
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007662186559679037,
+      "loss": 1.892,
+      "theoretical_loss": 4.471115322356643,
+      "tokens_seen": 170983424
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007662026078234705,
+      "loss": 1.7726,
+      "theoretical_loss": 4.470897112172471,
+      "tokens_seen": 171048960
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007661865596790371,
+      "loss": 2.0568,
+      "theoretical_loss": 4.470679008977022,
+      "tokens_seen": 171114496
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007661705115346039,
+      "loss": 1.7931,
+      "theoretical_loss": 4.470461012676894,
+      "tokens_seen": 171180032
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007661544633901705,
+      "loss": 1.6596,
+      "theoretical_loss": 4.470243123178802,
+      "tokens_seen": 171245568
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007661384152457373,
+      "loss": 1.8412,
+      "theoretical_loss": 4.4700253403895776,
+      "tokens_seen": 171311104
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007661223671013039,
+      "loss": 1.8774,
+      "theoretical_loss": 4.469807664216168,
+      "tokens_seen": 171376640
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007661063189568706,
+      "loss": 1.8796,
+      "theoretical_loss": 4.469590094565638,
+      "tokens_seen": 171442176
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007660902708124373,
+      "loss": 1.9763,
+      "theoretical_loss": 4.469372631345172,
+      "tokens_seen": 171507712
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.000766074222668004,
+      "loss": 1.737,
+      "theoretical_loss": 4.469155274462064,
+      "tokens_seen": 171573248
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007660581745235707,
+      "loss": 1.7833,
+      "theoretical_loss": 4.468938023823732,
+      "tokens_seen": 171638784
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007660421263791374,
+      "loss": 1.8698,
+      "theoretical_loss": 4.468720879337704,
+      "tokens_seen": 171704320
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007660260782347041,
+      "loss": 1.8787,
+      "theoretical_loss": 4.468503840911627,
+      "tokens_seen": 171769856
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007660100300902708,
+      "loss": 1.8694,
+      "theoretical_loss": 4.46828690845326,
+      "tokens_seen": 171835392
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007659939819458375,
+      "loss": 1.9094,
+      "theoretical_loss": 4.468070081870483,
+      "tokens_seen": 171900928
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007659779338014042,
+      "loss": 1.7438,
+      "theoretical_loss": 4.467853361071284,
+      "tokens_seen": 171966464
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.000765961885656971,
+      "loss": 1.8563,
+      "theoretical_loss": 4.467636745963772,
+      "tokens_seen": 172032000
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007659458375125376,
+      "loss": 1.7819,
+      "theoretical_loss": 4.467420236456169,
+      "tokens_seen": 172097536
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007659297893681043,
+      "loss": 1.6783,
+      "theoretical_loss": 4.46720383245681,
+      "tokens_seen": 172163072
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.000765913741223671,
+      "loss": 1.9597,
+      "theoretical_loss": 4.466987533874145,
+      "tokens_seen": 172228608
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007658976930792377,
+      "loss": 1.8961,
+      "theoretical_loss": 4.4667713406167415,
+      "tokens_seen": 172294144
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007658816449348044,
+      "loss": 1.8189,
+      "theoretical_loss": 4.466555252593277,
+      "tokens_seen": 172359680
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007658655967903711,
+      "loss": 1.9351,
+      "theoretical_loss": 4.4663392697125435,
+      "tokens_seen": 172425216
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007658495486459378,
+      "loss": 1.7525,
+      "theoretical_loss": 4.466123391883448,
+      "tokens_seen": 172490752
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007658335005015046,
+      "loss": 1.864,
+      "theoretical_loss": 4.4659076190150095,
+      "tokens_seen": 172556288
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007658174523570712,
+      "loss": 1.8967,
+      "theoretical_loss": 4.465691951016364,
+      "tokens_seen": 172621824
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.000765801404212638,
+      "loss": 1.766,
+      "theoretical_loss": 4.465476387796755,
+      "tokens_seen": 172687360
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007657853560682046,
+      "loss": 1.7432,
+      "theoretical_loss": 4.465260929265543,
+      "tokens_seen": 172752896
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007657693079237714,
+      "loss": 1.8648,
+      "theoretical_loss": 4.4650455753322005,
+      "tokens_seen": 172818432
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.000765753259779338,
+      "loss": 1.7019,
+      "theoretical_loss": 4.464830325906312,
+      "tokens_seen": 172883968
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007657372116349048,
+      "loss": 1.7988,
+      "theoretical_loss": 4.464615180897574,
+      "tokens_seen": 172949504
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007657211634904714,
+      "loss": 1.6652,
+      "theoretical_loss": 4.464400140215796,
+      "tokens_seen": 173015040
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007657051153460382,
+      "loss": 1.775,
+      "theoretical_loss": 4.4641852037708984,
+      "tokens_seen": 173080576
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007656890672016048,
+      "loss": 1.7437,
+      "theoretical_loss": 4.463970371472916,
+      "tokens_seen": 173146112
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007656730190571716,
+      "loss": 1.8064,
+      "theoretical_loss": 4.463755643231992,
+      "tokens_seen": 173211648
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007656569709127382,
+      "loss": 1.8278,
+      "theoretical_loss": 4.463541018958384,
+      "tokens_seen": 173277184
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.000765640922768305,
+      "loss": 1.9779,
+      "theoretical_loss": 4.463326498562459,
+      "tokens_seen": 173342720
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007656248746238716,
+      "loss": 1.8041,
+      "theoretical_loss": 4.463112081954696,
+      "tokens_seen": 173408256
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007656088264794384,
+      "loss": 1.8698,
+      "theoretical_loss": 4.462897769045684,
+      "tokens_seen": 173473792
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.000765592778335005,
+      "loss": 1.7609,
+      "theoretical_loss": 4.462683559746123,
+      "tokens_seen": 173539328
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007655767301905718,
+      "loss": 1.8934,
+      "theoretical_loss": 4.4624694539668255,
+      "tokens_seen": 173604864
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007655606820461384,
+      "loss": 1.79,
+      "theoretical_loss": 4.462255451618713,
+      "tokens_seen": 173670400
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007655446339017052,
+      "loss": 1.7272,
+      "theoretical_loss": 4.462041552612815,
+      "tokens_seen": 173735936
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007655285857572718,
+      "loss": 1.8072,
+      "theoretical_loss": 4.461827756860275,
+      "tokens_seen": 173801472
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007655125376128386,
+      "loss": 1.8277,
+      "theoretical_loss": 4.461614064272345,
+      "tokens_seen": 173867008
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007654964894684052,
+      "loss": 1.8084,
+      "theoretical_loss": 4.461400474760383,
+      "tokens_seen": 173932544
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.000765480441323972,
+      "loss": 1.7361,
+      "theoretical_loss": 4.461186988235864,
+      "tokens_seen": 173998080
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007654643931795386,
+      "loss": 1.7529,
+      "theoretical_loss": 4.460973604610366,
+      "tokens_seen": 174063616
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007654483450351054,
+      "loss": 1.8618,
+      "theoretical_loss": 4.4607603237955775,
+      "tokens_seen": 174129152
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.000765432296890672,
+      "loss": 1.7613,
+      "theoretical_loss": 4.460547145703298,
+      "tokens_seen": 174194688
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007654162487462388,
+      "loss": 1.8339,
+      "theoretical_loss": 4.460334070245436,
+      "tokens_seen": 174260224
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007654002006018054,
+      "loss": 1.9052,
+      "theoretical_loss": 4.460121097334004,
+      "tokens_seen": 174325760
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007653841524573722,
+      "loss": 1.843,
+      "theoretical_loss": 4.4599082268811285,
+      "tokens_seen": 174391296
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007653681043129388,
+      "loss": 1.8093,
+      "theoretical_loss": 4.459695458799041,
+      "tokens_seen": 174456832
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007653520561685056,
+      "loss": 1.9478,
+      "theoretical_loss": 4.459482793000083,
+      "tokens_seen": 174522368
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007653360080240722,
+      "loss": 1.8297,
+      "theoretical_loss": 4.459270229396701,
+      "tokens_seen": 174587904
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.000765319959879639,
+      "loss": 1.8508,
+      "theoretical_loss": 4.459057767901454,
+      "tokens_seen": 174653440
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007653039117352056,
+      "loss": 1.8418,
+      "theoretical_loss": 4.458845408427004,
+      "tokens_seen": 174718976
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007652878635907724,
+      "loss": 1.722,
+      "theoretical_loss": 4.458633150886124,
+      "tokens_seen": 174784512
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.000765271815446339,
+      "loss": 1.7616,
+      "theoretical_loss": 4.458420995191691,
+      "tokens_seen": 174850048
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007652557673019058,
+      "loss": 1.6859,
+      "theoretical_loss": 4.458208941256691,
+      "tokens_seen": 174915584
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007652397191574724,
+      "loss": 1.8105,
+      "theoretical_loss": 4.457996988994218,
+      "tokens_seen": 174981120
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007652236710130392,
+      "loss": 1.762,
+      "theoretical_loss": 4.457785138317469,
+      "tokens_seen": 175046656
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007652076228686058,
+      "loss": 1.8061,
+      "theoretical_loss": 4.4575733891397515,
+      "tokens_seen": 175112192
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007651915747241726,
+      "loss": 1.7592,
+      "theoretical_loss": 4.457361741374477,
+      "tokens_seen": 175177728
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007651755265797392,
+      "loss": 1.7275,
+      "theoretical_loss": 4.457150194935165,
+      "tokens_seen": 175243264
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.000765159478435306,
+      "loss": 1.9058,
+      "theoretical_loss": 4.456938749735439,
+      "tokens_seen": 175308800
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007651434302908727,
+      "loss": 1.8365,
+      "theoretical_loss": 4.45672740568903,
+      "tokens_seen": 175374336
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007651273821464394,
+      "loss": 1.8521,
+      "theoretical_loss": 4.456516162709774,
+      "tokens_seen": 175439872
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007651113340020061,
+      "loss": 1.8275,
+      "theoretical_loss": 4.456305020711613,
+      "tokens_seen": 175505408
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007650952858575728,
+      "loss": 1.8127,
+      "theoretical_loss": 4.456093979608594,
+      "tokens_seen": 175570944
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007650792377131395,
+      "loss": 1.7682,
+      "theoretical_loss": 4.455883039314869,
+      "tokens_seen": 175636480
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007650631895687062,
+      "loss": 1.8201,
+      "theoretical_loss": 4.455672199744695,
+      "tokens_seen": 175702016
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007650471414242729,
+      "loss": 1.8676,
+      "theoretical_loss": 4.455461460812435,
+      "tokens_seen": 175767552
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007650310932798396,
+      "loss": 1.8545,
+      "theoretical_loss": 4.455250822432556,
+      "tokens_seen": 175833088
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007650150451354063,
+      "loss": 1.8366,
+      "theoretical_loss": 4.4550402845196295,
+      "tokens_seen": 175898624
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.000764998996990973,
+      "loss": 1.7392,
+      "theoretical_loss": 4.45482984698833,
+      "tokens_seen": 175964160
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007649829488465397,
+      "loss": 1.777,
+      "theoretical_loss": 4.454619509753439,
+      "tokens_seen": 176029696
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007649669007021064,
+      "loss": 1.7912,
+      "theoretical_loss": 4.45440927272984,
+      "tokens_seen": 176095232
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007649508525576731,
+      "loss": 1.7614,
+      "theoretical_loss": 4.454199135832521,
+      "tokens_seen": 176160768
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007649348044132398,
+      "loss": 1.793,
+      "theoretical_loss": 4.4539890989765745,
+      "tokens_seen": 176226304
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007649187562688065,
+      "loss": 1.7368,
+      "theoretical_loss": 4.453779162077193,
+      "tokens_seen": 176291840
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007649027081243732,
+      "loss": 1.719,
+      "theoretical_loss": 4.453569325049679,
+      "tokens_seen": 176357376
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007648866599799399,
+      "loss": 1.9351,
+      "theoretical_loss": 4.453359587809431,
+      "tokens_seen": 176422912
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007648706118355066,
+      "loss": 1.818,
+      "theoretical_loss": 4.453149950271956,
+      "tokens_seen": 176488448
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007648545636910732,
+      "loss": 1.8517,
+      "theoretical_loss": 4.452940412352861,
+      "tokens_seen": 176553984
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00076483851554664,
+      "loss": 1.7958,
+      "theoretical_loss": 4.452730973967856,
+      "tokens_seen": 176619520
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007648224674022066,
+      "loss": 1.9261,
+      "theoretical_loss": 4.452521635032754,
+      "tokens_seen": 176685056
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007648064192577734,
+      "loss": 1.861,
+      "theoretical_loss": 4.452312395463473,
+      "tokens_seen": 176750592
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00076479037111334,
+      "loss": 1.6451,
+      "theoretical_loss": 4.452103255176027,
+      "tokens_seen": 176816128
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007647743229689068,
+      "loss": 1.8513,
+      "theoretical_loss": 4.451894214086537,
+      "tokens_seen": 176881664
+    },
+    {
+      "epoch": 0.05,
+      "objective/train/docs_used": 73013,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.7999730110168457,
+      "objective/train/theoretical_loss": 4.4516852721112254,
+      "objective/train/tokens_used": 197407200,
+      "theoretical_loss": 4.4516852721112254,
+      "tokens_seen": 176947200
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007647582748244734,
+      "loss": 1.8,
+      "theoretical_loss": 4.4516852721112254,
+      "tokens_seen": 176947200
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007647422266800402,
+      "loss": 1.6797,
+      "theoretical_loss": 4.451476429166416,
+      "tokens_seen": 177012736
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007647261785356068,
+      "loss": 1.7962,
+      "theoretical_loss": 4.451267685168532,
+      "tokens_seen": 177078272
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007647101303911735,
+      "loss": 1.9107,
+      "theoretical_loss": 4.451059040034102,
+      "tokens_seen": 177143808
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007646940822467402,
+      "loss": 1.8403,
+      "theoretical_loss": 4.450850493679752,
+      "tokens_seen": 177209344
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007646780341023069,
+      "loss": 1.7199,
+      "theoretical_loss": 4.450642046022211,
+      "tokens_seen": 177274880
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007646619859578736,
+      "loss": 1.8501,
+      "theoretical_loss": 4.45043369697831,
+      "tokens_seen": 177340416
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007646459378134403,
+      "loss": 1.7564,
+      "theoretical_loss": 4.450225446464978,
+      "tokens_seen": 177405952
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.000764629889669007,
+      "loss": 1.8507,
+      "theoretical_loss": 4.450017294399247,
+      "tokens_seen": 177471488
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007646138415245737,
+      "loss": 1.8915,
+      "theoretical_loss": 4.449809240698249,
+      "tokens_seen": 177537024
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007645977933801404,
+      "loss": 1.8548,
+      "theoretical_loss": 4.449601285279215,
+      "tokens_seen": 177602560
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007645817452357071,
+      "loss": 1.7469,
+      "theoretical_loss": 4.449393428059478,
+      "tokens_seen": 177668096
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007645656970912738,
+      "loss": 1.7879,
+      "theoretical_loss": 4.449185668956469,
+      "tokens_seen": 177733632
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007645496489468405,
+      "loss": 1.7461,
+      "theoretical_loss": 4.448978007887721,
+      "tokens_seen": 177799168
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007645336008024072,
+      "loss": 1.8203,
+      "theoretical_loss": 4.448770444770865,
+      "tokens_seen": 177864704
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007645175526579739,
+      "loss": 1.7898,
+      "theoretical_loss": 4.448562979523633,
+      "tokens_seen": 177930240
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007645015045135406,
+      "loss": 1.8204,
+      "theoretical_loss": 4.448355612063855,
+      "tokens_seen": 177995776
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007644854563691073,
+      "loss": 1.6813,
+      "theoretical_loss": 4.4481483423094605,
+      "tokens_seen": 178061312
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.000764469408224674,
+      "loss": 1.8498,
+      "theoretical_loss": 4.447941170178478,
+      "tokens_seen": 178126848
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007644533600802407,
+      "loss": 1.7561,
+      "theoretical_loss": 4.447734095589036,
+      "tokens_seen": 178192384
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007644373119358075,
+      "loss": 1.8036,
+      "theoretical_loss": 4.4475271184593606,
+      "tokens_seen": 178257920
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007644212637913741,
+      "loss": 1.7504,
+      "theoretical_loss": 4.447320238707777,
+      "tokens_seen": 178323456
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007644052156469409,
+      "loss": 1.9131,
+      "theoretical_loss": 4.447113456252707,
+      "tokens_seen": 178388992
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007643891675025075,
+      "loss": 1.7592,
+      "theoretical_loss": 4.446906771012674,
+      "tokens_seen": 178454528
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007643731193580743,
+      "loss": 1.8133,
+      "theoretical_loss": 4.446700182906298,
+      "tokens_seen": 178520064
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007643570712136409,
+      "loss": 1.7284,
+      "theoretical_loss": 4.446493691852297,
+      "tokens_seen": 178585600
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007643410230692077,
+      "loss": 1.8666,
+      "theoretical_loss": 4.4462872977694845,
+      "tokens_seen": 178651136
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007643249749247743,
+      "loss": 1.7361,
+      "theoretical_loss": 4.4460810005767755,
+      "tokens_seen": 178716672
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007643089267803411,
+      "loss": 1.8114,
+      "theoretical_loss": 4.445874800193181,
+      "tokens_seen": 178782208
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007642928786359077,
+      "loss": 1.8329,
+      "theoretical_loss": 4.445668696537808,
+      "tokens_seen": 178847744
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007642768304914745,
+      "loss": 1.8223,
+      "theoretical_loss": 4.4454626895298635,
+      "tokens_seen": 178913280
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007642607823470411,
+      "loss": 1.7408,
+      "theoretical_loss": 4.4452567790886475,
+      "tokens_seen": 178978816
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007642447342026079,
+      "loss": 1.7559,
+      "theoretical_loss": 4.445050965133561,
+      "tokens_seen": 179044352
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007642286860581745,
+      "loss": 1.6963,
+      "theoretical_loss": 4.4448452475841,
+      "tokens_seen": 179109888
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007642126379137413,
+      "loss": 1.7475,
+      "theoretical_loss": 4.444639626359856,
+      "tokens_seen": 179175424
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007641965897693079,
+      "loss": 1.8811,
+      "theoretical_loss": 4.444434101380519,
+      "tokens_seen": 179240960
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007641805416248747,
+      "loss": 1.8442,
+      "theoretical_loss": 4.444228672565874,
+      "tokens_seen": 179306496
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007641644934804413,
+      "loss": 1.8744,
+      "theoretical_loss": 4.444023339835803,
+      "tokens_seen": 179372032
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007641484453360081,
+      "loss": 1.7885,
+      "theoretical_loss": 4.443818103110283,
+      "tokens_seen": 179437568
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007641323971915747,
+      "loss": 1.81,
+      "theoretical_loss": 4.443612962309387,
+      "tokens_seen": 179503104
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007641163490471415,
+      "loss": 1.7993,
+      "theoretical_loss": 4.4434079173532846,
+      "tokens_seen": 179568640
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007641003009027081,
+      "loss": 1.8098,
+      "theoretical_loss": 4.4432029681622405,
+      "tokens_seen": 179634176
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007640842527582749,
+      "loss": 1.7731,
+      "theoretical_loss": 4.442998114656614,
+      "tokens_seen": 179699712
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007640682046138415,
+      "loss": 1.6623,
+      "theoretical_loss": 4.442793356756861,
+      "tokens_seen": 179765248
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007640521564694083,
+      "loss": 1.8102,
+      "theoretical_loss": 4.442588694383532,
+      "tokens_seen": 179830784
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007640361083249749,
+      "loss": 1.759,
+      "theoretical_loss": 4.44238412745727,
+      "tokens_seen": 179896320
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007640200601805417,
+      "loss": 1.7714,
+      "theoretical_loss": 4.442179655898819,
+      "tokens_seen": 179961856
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007640040120361083,
+      "loss": 1.8469,
+      "theoretical_loss": 4.44197527962901,
+      "tokens_seen": 180027392
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007639879638916751,
+      "loss": 1.7636,
+      "theoretical_loss": 4.441770998568773,
+      "tokens_seen": 180092928
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007639719157472417,
+      "loss": 1.7409,
+      "theoretical_loss": 4.441566812639133,
+      "tokens_seen": 180158464
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007639558676028085,
+      "loss": 1.7341,
+      "theoretical_loss": 4.441362721761207,
+      "tokens_seen": 180224000
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007639398194583752,
+      "loss": 1.8142,
+      "theoretical_loss": 4.441158725856207,
+      "tokens_seen": 180289536
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007639237713139419,
+      "loss": 1.8437,
+      "theoretical_loss": 4.440954824845438,
+      "tokens_seen": 180355072
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007639077231695086,
+      "loss": 1.9326,
+      "theoretical_loss": 4.4407510186503,
+      "tokens_seen": 180420608
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007638916750250753,
+      "loss": 1.809,
+      "theoretical_loss": 4.440547307192286,
+      "tokens_seen": 180486144
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.000763875626880642,
+      "loss": 1.9295,
+      "theoretical_loss": 4.440343690392984,
+      "tokens_seen": 180551680
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007638595787362087,
+      "loss": 1.8522,
+      "theoretical_loss": 4.4401401681740715,
+      "tokens_seen": 180617216
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007638435305917754,
+      "loss": 1.7237,
+      "theoretical_loss": 4.439936740457323,
+      "tokens_seen": 180682752
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007638274824473421,
+      "loss": 1.789,
+      "theoretical_loss": 4.439733407164605,
+      "tokens_seen": 180748288
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007638114343029088,
+      "loss": 1.9288,
+      "theoretical_loss": 4.439530168217875,
+      "tokens_seen": 180813824
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007637953861584755,
+      "loss": 1.8781,
+      "theoretical_loss": 4.439327023539187,
+      "tokens_seen": 180879360
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007637793380140422,
+      "loss": 1.7994,
+      "theoretical_loss": 4.439123973050684,
+      "tokens_seen": 180944896
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007637632898696089,
+      "loss": 1.7935,
+      "theoretical_loss": 4.438921016674603,
+      "tokens_seen": 181010432
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007637472417251756,
+      "loss": 1.7576,
+      "theoretical_loss": 4.438718154333274,
+      "tokens_seen": 181075968
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007637311935807423,
+      "loss": 1.7834,
+      "theoretical_loss": 4.438515385949118,
+      "tokens_seen": 181141504
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.000763715145436309,
+      "loss": 1.7537,
+      "theoretical_loss": 4.438312711444647,
+      "tokens_seen": 181207040
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007636990972918757,
+      "loss": 1.8203,
+      "theoretical_loss": 4.438110130742469,
+      "tokens_seen": 181272576
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007636830491474424,
+      "loss": 1.7772,
+      "theoretical_loss": 4.437907643765281,
+      "tokens_seen": 181338112
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007636670010030091,
+      "loss": 1.7507,
+      "theoretical_loss": 4.43770525043587,
+      "tokens_seen": 181403648
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007636509528585758,
+      "loss": 1.8583,
+      "theoretical_loss": 4.437502950677116,
+      "tokens_seen": 181469184
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007636349047141425,
+      "loss": 1.7524,
+      "theoretical_loss": 4.437300744411992,
+      "tokens_seen": 181534720
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007636188565697092,
+      "loss": 1.6643,
+      "theoretical_loss": 4.43709863156356,
+      "tokens_seen": 181600256
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007636028084252759,
+      "loss": 1.8245,
+      "theoretical_loss": 4.436896612054975,
+      "tokens_seen": 181665792
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007635867602808426,
+      "loss": 1.8651,
+      "theoretical_loss": 4.43669468580948,
+      "tokens_seen": 181731328
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007635707121364093,
+      "loss": 1.8246,
+      "theoretical_loss": 4.43649285275041,
+      "tokens_seen": 181796864
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.000763554663991976,
+      "loss": 1.9091,
+      "theoretical_loss": 4.436291112801193,
+      "tokens_seen": 181862400
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007635386158475427,
+      "loss": 1.8584,
+      "theoretical_loss": 4.436089465885345,
+      "tokens_seen": 181927936
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007635225677031094,
+      "loss": 1.724,
+      "theoretical_loss": 4.435887911926472,
+      "tokens_seen": 181993472
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007635065195586761,
+      "loss": 1.6499,
+      "theoretical_loss": 4.435686450848273,
+      "tokens_seen": 182059008
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007634904714142428,
+      "loss": 1.7099,
+      "theoretical_loss": 4.435485082574533,
+      "tokens_seen": 182124544
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007634744232698095,
+      "loss": 1.699,
+      "theoretical_loss": 4.43528380702913,
+      "tokens_seen": 182190080
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007634583751253762,
+      "loss": 1.7324,
+      "theoretical_loss": 4.435082624136031,
+      "tokens_seen": 182255616
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007634423269809429,
+      "loss": 1.747,
+      "theoretical_loss": 4.4348815338192935,
+      "tokens_seen": 182321152
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007634262788365095,
+      "loss": 1.8122,
+      "theoretical_loss": 4.434680536003062,
+      "tokens_seen": 182386688
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007634102306920763,
+      "loss": 1.7842,
+      "theoretical_loss": 4.434479630611573,
+      "tokens_seen": 182452224
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007633941825476429,
+      "loss": 1.8103,
+      "theoretical_loss": 4.43427881756915,
+      "tokens_seen": 182517760
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007633781344032097,
+      "loss": 1.8227,
+      "theoretical_loss": 4.434078096800209,
+      "tokens_seen": 182583296
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007633620862587763,
+      "loss": 1.7468,
+      "theoretical_loss": 4.433877468229251,
+      "tokens_seen": 182648832
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007633460381143431,
+      "loss": 1.764,
+      "theoretical_loss": 4.433676931780868,
+      "tokens_seen": 182714368
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007633299899699097,
+      "loss": 1.6694,
+      "theoretical_loss": 4.43347648737974,
+      "tokens_seen": 182779904
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007633139418254764,
+      "loss": 1.8151,
+      "theoretical_loss": 4.433276134950637,
+      "tokens_seen": 182845440
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007632978936810431,
+      "loss": 1.7672,
+      "theoretical_loss": 4.4330758744184156,
+      "tokens_seen": 182910976
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007632818455366098,
+      "loss": 1.952,
+      "theoretical_loss": 4.432875705708021,
+      "tokens_seen": 182976512
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007632657973921765,
+      "loss": 1.7285,
+      "theoretical_loss": 4.432675628744488,
+      "tokens_seen": 183042048
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007632497492477432,
+      "loss": 1.8432,
+      "theoretical_loss": 4.432475643452937,
+      "tokens_seen": 183107584
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007632337011033099,
+      "loss": 1.7405,
+      "theoretical_loss": 4.432275749758579,
+      "tokens_seen": 183173120
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007632176529588766,
+      "loss": 1.8195,
+      "theoretical_loss": 4.432075947586711,
+      "tokens_seen": 183238656
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007632016048144433,
+      "loss": 1.8257,
+      "theoretical_loss": 4.431876236862717,
+      "tokens_seen": 183304192
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.00076318555667001,
+      "loss": 1.6803,
+      "theoretical_loss": 4.43167661751207,
+      "tokens_seen": 183369728
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007631695085255767,
+      "loss": 1.905,
+      "theoretical_loss": 4.431477089460331,
+      "tokens_seen": 183435264
+    },
+    {
+      "epoch": 0.06,
+      "objective/train/docs_used": 75209,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.7620269060134888,
+      "objective/train/theoretical_loss": 4.431277652633145,
+      "objective/train/tokens_used": 203960800,
+      "theoretical_loss": 4.431277652633145,
+      "tokens_seen": 183500800
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007631534603811434,
+      "loss": 1.762,
+      "theoretical_loss": 4.431277652633145,
+      "tokens_seen": 183500800
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007631374122367101,
+      "loss": 1.8916,
+      "theoretical_loss": 4.431078306956248,
+      "tokens_seen": 183566336
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007631213640922768,
+      "loss": 1.9074,
+      "theoretical_loss": 4.43087905235546,
+      "tokens_seen": 183631872
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007631053159478436,
+      "loss": 1.7626,
+      "theoretical_loss": 4.43067988875669,
+      "tokens_seen": 183697408
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007630892678034102,
+      "loss": 1.828,
+      "theoretical_loss": 4.430480816085931,
+      "tokens_seen": 183762944
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.000763073219658977,
+      "loss": 1.8253,
+      "theoretical_loss": 4.430281834269265,
+      "tokens_seen": 183828480
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007630571715145436,
+      "loss": 1.7543,
+      "theoretical_loss": 4.4300829432328594,
+      "tokens_seen": 183894016
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007630411233701104,
+      "loss": 1.7862,
+      "theoretical_loss": 4.429884142902968,
+      "tokens_seen": 183959552
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.000763025075225677,
+      "loss": 1.6709,
+      "theoretical_loss": 4.429685433205931,
+      "tokens_seen": 184025088
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007630090270812438,
+      "loss": 1.798,
+      "theoretical_loss": 4.429486814068174,
+      "tokens_seen": 184090624
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007629929789368104,
+      "loss": 1.8637,
+      "theoretical_loss": 4.4292882854162094,
+      "tokens_seen": 184156160
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007629769307923772,
+      "loss": 1.7704,
+      "theoretical_loss": 4.429089847176634,
+      "tokens_seen": 184221696
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007629608826479438,
+      "loss": 1.7683,
+      "theoretical_loss": 4.428891499276132,
+      "tokens_seen": 184287232
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007629448345035106,
+      "loss": 1.8815,
+      "theoretical_loss": 4.42869324164147,
+      "tokens_seen": 184352768
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007629287863590772,
+      "loss": 1.7091,
+      "theoretical_loss": 4.428495074199505,
+      "tokens_seen": 184418304
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.000762912738214644,
+      "loss": 1.6927,
+      "theoretical_loss": 4.428296996877176,
+      "tokens_seen": 184483840
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007628966900702106,
+      "loss": 1.7418,
+      "theoretical_loss": 4.428099009601506,
+      "tokens_seen": 184549376
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007628806419257774,
+      "loss": 1.8684,
+      "theoretical_loss": 4.4279011122996055,
+      "tokens_seen": 184614912
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.000762864593781344,
+      "loss": 1.7133,
+      "theoretical_loss": 4.427703304898668,
+      "tokens_seen": 184680448
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007628485456369108,
+      "loss": 1.7586,
+      "theoretical_loss": 4.427505587325974,
+      "tokens_seen": 184745984
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007628324974924774,
+      "loss": 1.7689,
+      "theoretical_loss": 4.427307959508885,
+      "tokens_seen": 184811520
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007628164493480442,
+      "loss": 1.8452,
+      "theoretical_loss": 4.4271104213748504,
+      "tokens_seen": 184877056
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007628004012036108,
+      "loss": 1.7734,
+      "theoretical_loss": 4.426912972851404,
+      "tokens_seen": 184942592
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007627843530591776,
+      "loss": 1.9269,
+      "theoretical_loss": 4.426715613866159,
+      "tokens_seen": 185008128
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007627683049147442,
+      "loss": 1.783,
+      "theoretical_loss": 4.426518344346819,
+      "tokens_seen": 185073664
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.000762752256770311,
+      "loss": 1.7716,
+      "theoretical_loss": 4.426321164221168,
+      "tokens_seen": 185139200
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007627362086258777,
+      "loss": 1.7277,
+      "theoretical_loss": 4.426124073417074,
+      "tokens_seen": 185204736
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007627201604814444,
+      "loss": 1.8053,
+      "theoretical_loss": 4.42592707186249,
+      "tokens_seen": 185270272
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007627041123370111,
+      "loss": 1.6595,
+      "theoretical_loss": 4.42573015948545,
+      "tokens_seen": 185335808
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007626880641925778,
+      "loss": 1.8182,
+      "theoretical_loss": 4.425533336214076,
+      "tokens_seen": 185401344
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007626720160481445,
+      "loss": 1.7558,
+      "theoretical_loss": 4.425336601976568,
+      "tokens_seen": 185466880
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007626559679037112,
+      "loss": 1.7747,
+      "theoretical_loss": 4.425139956701214,
+      "tokens_seen": 185532416
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007626399197592779,
+      "loss": 1.7903,
+      "theoretical_loss": 4.42494340031638,
+      "tokens_seen": 185597952
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007626238716148446,
+      "loss": 1.7132,
+      "theoretical_loss": 4.424746932750521,
+      "tokens_seen": 185663488
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007626078234704113,
+      "loss": 1.9077,
+      "theoretical_loss": 4.4245505539321694,
+      "tokens_seen": 185729024
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.000762591775325978,
+      "loss": 1.9028,
+      "theoretical_loss": 4.424354263789943,
+      "tokens_seen": 185794560
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007625757271815447,
+      "loss": 1.8168,
+      "theoretical_loss": 4.424158062252541,
+      "tokens_seen": 185860096
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007625596790371114,
+      "loss": 1.8349,
+      "theoretical_loss": 4.423961949248748,
+      "tokens_seen": 185925632
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007625436308926781,
+      "loss": 1.8333,
+      "theoretical_loss": 4.423765924707427,
+      "tokens_seen": 185991168
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007625275827482448,
+      "loss": 1.739,
+      "theoretical_loss": 4.4235699885575235,
+      "tokens_seen": 186056704
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007625115346038115,
+      "loss": 1.6889,
+      "theoretical_loss": 4.423374140728068,
+      "tokens_seen": 186122240
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007624954864593782,
+      "loss": 1.746,
+      "theoretical_loss": 4.423178381148172,
+      "tokens_seen": 186187776
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007624794383149449,
+      "loss": 1.8359,
+      "theoretical_loss": 4.422982709747026,
+      "tokens_seen": 186253312
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007624633901705116,
+      "loss": 1.7539,
+      "theoretical_loss": 4.4227871264539065,
+      "tokens_seen": 186318848
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007624473420260783,
+      "loss": 1.6667,
+      "theoretical_loss": 4.422591631198167,
+      "tokens_seen": 186384384
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.000762431293881645,
+      "loss": 1.7127,
+      "theoretical_loss": 4.422396223909248,
+      "tokens_seen": 186449920
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007624152457372117,
+      "loss": 1.8472,
+      "theoretical_loss": 4.422200904516666,
+      "tokens_seen": 186515456
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007623991975927784,
+      "loss": 1.7118,
+      "theoretical_loss": 4.4220056729500214,
+      "tokens_seen": 186580992
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007623831494483451,
+      "loss": 1.7744,
+      "theoretical_loss": 4.4218105291389955,
+      "tokens_seen": 186646528
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007623671013039118,
+      "loss": 1.6852,
+      "theoretical_loss": 4.4216154730133495,
+      "tokens_seen": 186712064
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007623510531594785,
+      "loss": 1.9056,
+      "theoretical_loss": 4.421420504502928,
+      "tokens_seen": 186777600
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007623350050150452,
+      "loss": 1.7766,
+      "theoretical_loss": 4.421225623537652,
+      "tokens_seen": 186843136
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007623189568706119,
+      "loss": 1.6958,
+      "theoretical_loss": 4.421030830047528,
+      "tokens_seen": 186908672
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007623029087261786,
+      "loss": 1.8023,
+      "theoretical_loss": 4.420836123962639,
+      "tokens_seen": 186974208
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007622868605817453,
+      "loss": 1.7609,
+      "theoretical_loss": 4.420641505213151,
+      "tokens_seen": 187039744
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.000762270812437312,
+      "loss": 1.8666,
+      "theoretical_loss": 4.42044697372931,
+      "tokens_seen": 187105280
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007622547642928787,
+      "loss": 1.6924,
+      "theoretical_loss": 4.420252529441439,
+      "tokens_seen": 187170816
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007622387161484454,
+      "loss": 1.7689,
+      "theoretical_loss": 4.420058172279945,
+      "tokens_seen": 187236352
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007622226680040121,
+      "loss": 1.7036,
+      "theoretical_loss": 4.419863902175313,
+      "tokens_seen": 187301888
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007622066198595788,
+      "loss": 1.8167,
+      "theoretical_loss": 4.419669719058108,
+      "tokens_seen": 187367424
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007621905717151455,
+      "loss": 1.7143,
+      "theoretical_loss": 4.419475622858974,
+      "tokens_seen": 187432960
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007621745235707122,
+      "loss": 1.7541,
+      "theoretical_loss": 4.419281613508637,
+      "tokens_seen": 187498496
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007621584754262789,
+      "loss": 1.7815,
+      "theoretical_loss": 4.419087690937898,
+      "tokens_seen": 187564032
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007621424272818456,
+      "loss": 1.7691,
+      "theoretical_loss": 4.418893855077641,
+      "tokens_seen": 187629568
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007621263791374123,
+      "loss": 1.7913,
+      "theoretical_loss": 4.4187001058588296,
+      "tokens_seen": 187695104
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.000762110330992979,
+      "loss": 1.7181,
+      "theoretical_loss": 4.418506443212503,
+      "tokens_seen": 187760640
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007620942828485457,
+      "loss": 1.7297,
+      "theoretical_loss": 4.4183128670697815,
+      "tokens_seen": 187826176
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007620782347041124,
+      "loss": 1.7215,
+      "theoretical_loss": 4.418119377361863,
+      "tokens_seen": 187891712
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007620621865596791,
+      "loss": 1.7652,
+      "theoretical_loss": 4.417925974020028,
+      "tokens_seen": 187957248
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007620461384152458,
+      "loss": 1.8948,
+      "theoretical_loss": 4.4177326569756294,
+      "tokens_seen": 188022784
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007620300902708124,
+      "loss": 1.8231,
+      "theoretical_loss": 4.417539426160104,
+      "tokens_seen": 188088320
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007620140421263792,
+      "loss": 1.7161,
+      "theoretical_loss": 4.417346281504963,
+      "tokens_seen": 188153856
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007619979939819458,
+      "loss": 1.8138,
+      "theoretical_loss": 4.417153222941798,
+      "tokens_seen": 188219392
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007619819458375126,
+      "loss": 1.9123,
+      "theoretical_loss": 4.416960250402279,
+      "tokens_seen": 188284928
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007619658976930792,
+      "loss": 1.7367,
+      "theoretical_loss": 4.416767363818153,
+      "tokens_seen": 188350464
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.000761949849548646,
+      "loss": 1.8215,
+      "theoretical_loss": 4.4165745631212445,
+      "tokens_seen": 188416000
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007619338014042126,
+      "loss": 1.7358,
+      "theoretical_loss": 4.416381848243456,
+      "tokens_seen": 188481536
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007619177532597794,
+      "loss": 1.8557,
+      "theoretical_loss": 4.41618921911677,
+      "tokens_seen": 188547072
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.000761901705115346,
+      "loss": 1.7209,
+      "theoretical_loss": 4.415996675673242,
+      "tokens_seen": 188612608
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007618856569709127,
+      "loss": 1.8605,
+      "theoretical_loss": 4.415804217845009,
+      "tokens_seen": 188678144
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007618696088264794,
+      "loss": 1.8496,
+      "theoretical_loss": 4.415611845564284,
+      "tokens_seen": 188743680
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007618535606820461,
+      "loss": 1.775,
+      "theoretical_loss": 4.415419558763356,
+      "tokens_seen": 188809216
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007618375125376128,
+      "loss": 1.7735,
+      "theoretical_loss": 4.415227357374592,
+      "tokens_seen": 188874752
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007618214643931795,
+      "loss": 1.8272,
+      "theoretical_loss": 4.415035241330436,
+      "tokens_seen": 188940288
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007618054162487462,
+      "loss": 1.7899,
+      "theoretical_loss": 4.41484321056341,
+      "tokens_seen": 189005824
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007617893681043129,
+      "loss": 1.7192,
+      "theoretical_loss": 4.4146512650061105,
+      "tokens_seen": 189071360
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007617733199598796,
+      "loss": 1.7036,
+      "theoretical_loss": 4.414459404591211,
+      "tokens_seen": 189136896
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007617572718154463,
+      "loss": 1.775,
+      "theoretical_loss": 4.414267629251463,
+      "tokens_seen": 189202432
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.000761741223671013,
+      "loss": 1.7385,
+      "theoretical_loss": 4.414075938919694,
+      "tokens_seen": 189267968
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007617251755265797,
+      "loss": 1.5763,
+      "theoretical_loss": 4.413884333528807,
+      "tokens_seen": 189333504
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007617091273821465,
+      "loss": 1.7538,
+      "theoretical_loss": 4.413692813011781,
+      "tokens_seen": 189399040
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007616930792377131,
+      "loss": 1.7807,
+      "theoretical_loss": 4.413501377301671,
+      "tokens_seen": 189464576
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007616770310932799,
+      "loss": 1.8343,
+      "theoretical_loss": 4.41331002633161,
+      "tokens_seen": 189530112
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007616609829488465,
+      "loss": 1.7837,
+      "theoretical_loss": 4.413118760034804,
+      "tokens_seen": 189595648
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007616449348044133,
+      "loss": 1.8891,
+      "theoretical_loss": 4.4129275783445365,
+      "tokens_seen": 189661184
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007616288866599799,
+      "loss": 1.7112,
+      "theoretical_loss": 4.412736481194166,
+      "tokens_seen": 189726720
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007616128385155467,
+      "loss": 1.7991,
+      "theoretical_loss": 4.412545468517127,
+      "tokens_seen": 189792256
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007615967903711133,
+      "loss": 1.8096,
+      "theoretical_loss": 4.412354540246929,
+      "tokens_seen": 189857792
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007615807422266801,
+      "loss": 1.7634,
+      "theoretical_loss": 4.4121636963171555,
+      "tokens_seen": 189923328
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007615646940822467,
+      "loss": 1.7403,
+      "theoretical_loss": 4.411972936661467,
+      "tokens_seen": 189988864
+    },
+    {
+      "epoch": 0.06,
+      "objective/train/docs_used": 77536,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.6837046146392822,
+      "objective/train/theoretical_loss": 4.411782261213599,
+      "objective/train/tokens_used": 210514400,
+      "theoretical_loss": 4.411782261213599,
+      "tokens_seen": 190054400
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007615486459378135,
+      "loss": 1.6837,
+      "theoretical_loss": 4.411782261213599,
+      "tokens_seen": 190054400
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007615325977933802,
+      "loss": 1.7882,
+      "theoretical_loss": 4.41159166990736,
+      "tokens_seen": 190119936
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007615165496489469,
+      "loss": 1.7688,
+      "theoretical_loss": 4.411401162676636,
+      "tokens_seen": 190185472
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007615005015045136,
+      "loss": 1.6323,
+      "theoretical_loss": 4.411210739455386,
+      "tokens_seen": 190251008
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007614844533600803,
+      "loss": 1.7742,
+      "theoretical_loss": 4.4110204001776445,
+      "tokens_seen": 190316544
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.000761468405215647,
+      "loss": 1.7357,
+      "theoretical_loss": 4.410830144777519,
+      "tokens_seen": 190382080
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007614523570712137,
+      "loss": 1.6707,
+      "theoretical_loss": 4.410639973189192,
+      "tokens_seen": 190447616
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007614363089267804,
+      "loss": 1.859,
+      "theoretical_loss": 4.410449885346922,
+      "tokens_seen": 190513152
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007614202607823471,
+      "loss": 1.7445,
+      "theoretical_loss": 4.4102598811850395,
+      "tokens_seen": 190578688
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007614042126379138,
+      "loss": 1.78,
+      "theoretical_loss": 4.410069960637949,
+      "tokens_seen": 190644224
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007613881644934805,
+      "loss": 1.7216,
+      "theoretical_loss": 4.409880123640131,
+      "tokens_seen": 190709760
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007613721163490472,
+      "loss": 1.8239,
+      "theoretical_loss": 4.409690370126139,
+      "tokens_seen": 190775296
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007613560682046139,
+      "loss": 1.6882,
+      "theoretical_loss": 4.409500700030598,
+      "tokens_seen": 190840832
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007613400200601806,
+      "loss": 1.7236,
+      "theoretical_loss": 4.409311113288211,
+      "tokens_seen": 190906368
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007613239719157473,
+      "loss": 1.8261,
+      "theoretical_loss": 4.409121609833749,
+      "tokens_seen": 190971904
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.000761307923771314,
+      "loss": 1.7441,
+      "theoretical_loss": 4.408932189602061,
+      "tokens_seen": 191037440
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007612918756268807,
+      "loss": 1.7432,
+      "theoretical_loss": 4.408742852528068,
+      "tokens_seen": 191102976
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007612758274824474,
+      "loss": 1.8753,
+      "theoretical_loss": 4.408553598546764,
+      "tokens_seen": 191168512
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007612597793380141,
+      "loss": 1.7225,
+      "theoretical_loss": 4.408364427593215,
+      "tokens_seen": 191234048
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007612437311935808,
+      "loss": 1.7486,
+      "theoretical_loss": 4.40817533960256,
+      "tokens_seen": 191299584
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007612276830491475,
+      "loss": 1.7881,
+      "theoretical_loss": 4.407986334510014,
+      "tokens_seen": 191365120
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007612116349047142,
+      "loss": 1.7842,
+      "theoretical_loss": 4.407797412250861,
+      "tokens_seen": 191430656
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007611955867602809,
+      "loss": 1.8266,
+      "theoretical_loss": 4.40760857276046,
+      "tokens_seen": 191496192
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007611795386158476,
+      "loss": 1.7804,
+      "theoretical_loss": 4.407419815974242,
+      "tokens_seen": 191561728
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007611634904714143,
+      "loss": 1.7048,
+      "theoretical_loss": 4.407231141827708,
+      "tokens_seen": 191627264
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.000761147442326981,
+      "loss": 1.6665,
+      "theoretical_loss": 4.407042550256436,
+      "tokens_seen": 191692800
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007611313941825477,
+      "loss": 1.7333,
+      "theoretical_loss": 4.406854041196072,
+      "tokens_seen": 191758336
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007611153460381144,
+      "loss": 1.8141,
+      "theoretical_loss": 4.406665614582338,
+      "tokens_seen": 191823872
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007610992978936811,
+      "loss": 1.806,
+      "theoretical_loss": 4.406477270351024,
+      "tokens_seen": 191889408
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007610832497492478,
+      "loss": 1.7918,
+      "theoretical_loss": 4.4062890084379935,
+      "tokens_seen": 191954944
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007610672016048145,
+      "loss": 1.7733,
+      "theoretical_loss": 4.406100828779183,
+      "tokens_seen": 192020480
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007610511534603812,
+      "loss": 1.878,
+      "theoretical_loss": 4.4059127313106,
+      "tokens_seen": 192086016
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007610351053159479,
+      "loss": 1.7692,
+      "theoretical_loss": 4.405724715968323,
+      "tokens_seen": 192151552
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007610190571715146,
+      "loss": 1.7859,
+      "theoretical_loss": 4.405536782688502,
+      "tokens_seen": 192217088
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007610030090270813,
+      "loss": 1.7671,
+      "theoretical_loss": 4.405348931407359,
+      "tokens_seen": 192282624
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.000760986960882648,
+      "loss": 1.8184,
+      "theoretical_loss": 4.405161162061187,
+      "tokens_seen": 192348160
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007609709127382147,
+      "loss": 1.7967,
+      "theoretical_loss": 4.404973474586351,
+      "tokens_seen": 192413696
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007609548645937814,
+      "loss": 1.7232,
+      "theoretical_loss": 4.404785868919285,
+      "tokens_seen": 192479232
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007609388164493481,
+      "loss": 1.8081,
+      "theoretical_loss": 4.404598344996497,
+      "tokens_seen": 192544768
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007609227683049148,
+      "loss": 1.724,
+      "theoretical_loss": 4.404410902754562,
+      "tokens_seen": 192610304
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007609067201604815,
+      "loss": 1.7337,
+      "theoretical_loss": 4.404223542130129,
+      "tokens_seen": 192675840
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007608906720160482,
+      "loss": 1.5902,
+      "theoretical_loss": 4.404036263059917,
+      "tokens_seen": 192741376
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007608746238716149,
+      "loss": 1.7671,
+      "theoretical_loss": 4.403849065480714,
+      "tokens_seen": 192806912
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007608585757271816,
+      "loss": 1.7116,
+      "theoretical_loss": 4.403661949329381,
+      "tokens_seen": 192872448
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007608425275827483,
+      "loss": 1.7572,
+      "theoretical_loss": 4.403474914542848,
+      "tokens_seen": 192937984
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.000760826479438315,
+      "loss": 1.8424,
+      "theoretical_loss": 4.403287961058112,
+      "tokens_seen": 193003520
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007608104312938817,
+      "loss": 1.9331,
+      "theoretical_loss": 4.403101088812248,
+      "tokens_seen": 193069056
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007607943831494484,
+      "loss": 1.7388,
+      "theoretical_loss": 4.4029142977423925,
+      "tokens_seen": 193134592
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007607783350050151,
+      "loss": 1.7969,
+      "theoretical_loss": 4.402727587785758,
+      "tokens_seen": 193200128
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007607622868605818,
+      "loss": 1.6588,
+      "theoretical_loss": 4.402540958879625,
+      "tokens_seen": 193265664
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007607462387161485,
+      "loss": 1.7005,
+      "theoretical_loss": 4.402354410961341,
+      "tokens_seen": 193331200
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007607301905717152,
+      "loss": 1.8124,
+      "theoretical_loss": 4.402167943968328,
+      "tokens_seen": 193396736
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007607141424272819,
+      "loss": 1.8199,
+      "theoretical_loss": 4.4019815578380745,
+      "tokens_seen": 193462272
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007606980942828486,
+      "loss": 1.7504,
+      "theoretical_loss": 4.401795252508139,
+      "tokens_seen": 193527808
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007606820461384153,
+      "loss": 1.7858,
+      "theoretical_loss": 4.401609027916149,
+      "tokens_seen": 193593344
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.000760665997993982,
+      "loss": 1.7531,
+      "theoretical_loss": 4.4014228839998015,
+      "tokens_seen": 193658880
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007606499498495487,
+      "loss": 1.7594,
+      "theoretical_loss": 4.401236820696863,
+      "tokens_seen": 193724416
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007606339017051153,
+      "loss": 1.6804,
+      "theoretical_loss": 4.40105083794517,
+      "tokens_seen": 193789952
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007606178535606821,
+      "loss": 1.7331,
+      "theoretical_loss": 4.400864935682625,
+      "tokens_seen": 193855488
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007606018054162487,
+      "loss": 1.7447,
+      "theoretical_loss": 4.400679113847202,
+      "tokens_seen": 193921024
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007605857572718155,
+      "loss": 1.7684,
+      "theoretical_loss": 4.400493372376943,
+      "tokens_seen": 193986560
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007605697091273821,
+      "loss": 1.7692,
+      "theoretical_loss": 4.400307711209957,
+      "tokens_seen": 194052096
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007605536609829489,
+      "loss": 1.7224,
+      "theoretical_loss": 4.400122130284426,
+      "tokens_seen": 194117632
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007605376128385155,
+      "loss": 1.7067,
+      "theoretical_loss": 4.399936629538595,
+      "tokens_seen": 194183168
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007605215646940823,
+      "loss": 1.7667,
+      "theoretical_loss": 4.3997512089107795,
+      "tokens_seen": 194248704
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007605055165496489,
+      "loss": 1.72,
+      "theoretical_loss": 4.399565868339366,
+      "tokens_seen": 194314240
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007604894684052156,
+      "loss": 1.7202,
+      "theoretical_loss": 4.399380607762804,
+      "tokens_seen": 194379776
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007604734202607823,
+      "loss": 1.6807,
+      "theoretical_loss": 4.399195427119616,
+      "tokens_seen": 194445312
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.000760457372116349,
+      "loss": 1.7481,
+      "theoretical_loss": 4.399010326348388,
+      "tokens_seen": 194510848
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007604413239719157,
+      "loss": 1.7909,
+      "theoretical_loss": 4.398825305387778,
+      "tokens_seen": 194576384
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007604252758274824,
+      "loss": 1.8548,
+      "theoretical_loss": 4.39864036417651,
+      "tokens_seen": 194641920
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007604092276830491,
+      "loss": 1.8049,
+      "theoretical_loss": 4.398455502653373,
+      "tokens_seen": 194707456
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007603931795386158,
+      "loss": 1.5683,
+      "theoretical_loss": 4.398270720757228,
+      "tokens_seen": 194772992
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007603771313941825,
+      "loss": 1.7704,
+      "theoretical_loss": 4.398086018427001,
+      "tokens_seen": 194838528
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007603610832497492,
+      "loss": 1.8903,
+      "theoretical_loss": 4.397901395601687,
+      "tokens_seen": 194904064
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.000760345035105316,
+      "loss": 1.8512,
+      "theoretical_loss": 4.397716852220345,
+      "tokens_seen": 194969600
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007603289869608827,
+      "loss": 1.7635,
+      "theoretical_loss": 4.397532388222105,
+      "tokens_seen": 195035136
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007603129388164494,
+      "loss": 1.7018,
+      "theoretical_loss": 4.397348003546161,
+      "tokens_seen": 195100672
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007602968906720161,
+      "loss": 1.7344,
+      "theoretical_loss": 4.397163698131776,
+      "tokens_seen": 195166208
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007602808425275828,
+      "loss": 1.831,
+      "theoretical_loss": 4.39697947191828,
+      "tokens_seen": 195231744
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007602647943831495,
+      "loss": 1.7557,
+      "theoretical_loss": 4.396795324845069,
+      "tokens_seen": 195297280
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007602487462387162,
+      "loss": 1.6197,
+      "theoretical_loss": 4.396611256851604,
+      "tokens_seen": 195362816
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007602326980942829,
+      "loss": 1.6612,
+      "theoretical_loss": 4.3964272678774154,
+      "tokens_seen": 195428352
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007602166499498496,
+      "loss": 1.7702,
+      "theoretical_loss": 4.396243357862099,
+      "tokens_seen": 195493888
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007602006018054163,
+      "loss": 1.7023,
+      "theoretical_loss": 4.396059526745318,
+      "tokens_seen": 195559424
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.000760184553660983,
+      "loss": 1.736,
+      "theoretical_loss": 4.395875774466799,
+      "tokens_seen": 195624960
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007601685055165497,
+      "loss": 1.7532,
+      "theoretical_loss": 4.395692100966338,
+      "tokens_seen": 195690496
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007601524573721164,
+      "loss": 1.6109,
+      "theoretical_loss": 4.395508506183796,
+      "tokens_seen": 195756032
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007601364092276831,
+      "loss": 1.7201,
+      "theoretical_loss": 4.395324990059098,
+      "tokens_seen": 195821568
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007601203610832498,
+      "loss": 1.6878,
+      "theoretical_loss": 4.395141552532239,
+      "tokens_seen": 195887104
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007601043129388165,
+      "loss": 1.9185,
+      "theoretical_loss": 4.394958193543277,
+      "tokens_seen": 195952640
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007600882647943832,
+      "loss": 1.6981,
+      "theoretical_loss": 4.394774913032337,
+      "tokens_seen": 196018176
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007600722166499499,
+      "loss": 1.8504,
+      "theoretical_loss": 4.394591710939608,
+      "tokens_seen": 196083712
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007600561685055166,
+      "loss": 1.6923,
+      "theoretical_loss": 4.394408587205344,
+      "tokens_seen": 196149248
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007600401203610833,
+      "loss": 1.6863,
+      "theoretical_loss": 4.39422554176987,
+      "tokens_seen": 196214784
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.00076002407221665,
+      "loss": 1.6867,
+      "theoretical_loss": 4.39404257457357,
+      "tokens_seen": 196280320
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007600080240722167,
+      "loss": 1.8805,
+      "theoretical_loss": 4.393859685556895,
+      "tokens_seen": 196345856
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007599919759277834,
+      "loss": 1.7739,
+      "theoretical_loss": 4.393676874660364,
+      "tokens_seen": 196411392
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007599759277833501,
+      "loss": 1.6739,
+      "theoretical_loss": 4.393494141824557,
+      "tokens_seen": 196476928
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007599598796389168,
+      "loss": 1.7102,
+      "theoretical_loss": 4.393311486990122,
+      "tokens_seen": 196542464
+    },
+    {
+      "epoch": 0.06,
+      "objective/train/docs_used": 79941,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.7070708274841309,
+      "objective/train/theoretical_loss": 4.39312891009777,
+      "objective/train/tokens_used": 217068000,
+      "theoretical_loss": 4.39312891009777,
+      "tokens_seen": 196608000
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007599438314944835,
+      "loss": 1.7071,
+      "theoretical_loss": 4.39312891009777,
+      "tokens_seen": 196608000
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007599277833500502,
+      "loss": 1.7248,
+      "theoretical_loss": 4.392946411088279,
+      "tokens_seen": 196673536
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007599117352056169,
+      "loss": 1.767,
+      "theoretical_loss": 4.392763989902489,
+      "tokens_seen": 196739072
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007598956870611836,
+      "loss": 1.7185,
+      "theoretical_loss": 4.3925816464813074,
+      "tokens_seen": 196804608
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007598796389167503,
+      "loss": 1.8089,
+      "theoretical_loss": 4.392399380765704,
+      "tokens_seen": 196870144
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.000759863590772317,
+      "loss": 1.6937,
+      "theoretical_loss": 4.392217192696712,
+      "tokens_seen": 196935680
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007598475426278837,
+      "loss": 1.5537,
+      "theoretical_loss": 4.392035082215434,
+      "tokens_seen": 197001216
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007598314944834504,
+      "loss": 1.696,
+      "theoretical_loss": 4.391853049263032,
+      "tokens_seen": 197066752
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007598154463390171,
+      "loss": 1.8234,
+      "theoretical_loss": 4.391671093780733,
+      "tokens_seen": 197132288
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007597993981945838,
+      "loss": 1.8006,
+      "theoretical_loss": 4.39148921570983,
+      "tokens_seen": 197197824
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007597833500501505,
+      "loss": 1.773,
+      "theoretical_loss": 4.391307414991678,
+      "tokens_seen": 197263360
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007597673019057172,
+      "loss": 1.721,
+      "theoretical_loss": 4.391125691567697,
+      "tokens_seen": 197328896
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007597512537612839,
+      "loss": 1.7165,
+      "theoretical_loss": 4.390944045379371,
+      "tokens_seen": 197394432
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007597352056168506,
+      "loss": 1.7357,
+      "theoretical_loss": 4.390762476368245,
+      "tokens_seen": 197459968
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007597191574724173,
+      "loss": 1.8413,
+      "theoretical_loss": 4.390580984475932,
+      "tokens_seen": 197525504
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.000759703109327984,
+      "loss": 1.796,
+      "theoretical_loss": 4.3903995696441065,
+      "tokens_seen": 197591040
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007596870611835507,
+      "loss": 1.7677,
+      "theoretical_loss": 4.390218231814504,
+      "tokens_seen": 197656576
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007596710130391174,
+      "loss": 1.6409,
+      "theoretical_loss": 4.390036970928929,
+      "tokens_seen": 197722112
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007596549648946841,
+      "loss": 1.6503,
+      "theoretical_loss": 4.3898557869292425,
+      "tokens_seen": 197787648
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007596389167502508,
+      "loss": 1.8157,
+      "theoretical_loss": 4.389674679757374,
+      "tokens_seen": 197853184
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007596228686058175,
+      "loss": 1.6408,
+      "theoretical_loss": 4.389493649355314,
+      "tokens_seen": 197918720
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007596068204613842,
+      "loss": 1.7022,
+      "theoretical_loss": 4.389312695665115,
+      "tokens_seen": 197984256
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007595907723169509,
+      "loss": 1.7141,
+      "theoretical_loss": 4.389131818628895,
+      "tokens_seen": 198049792
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007595747241725176,
+      "loss": 1.7267,
+      "theoretical_loss": 4.388951018188832,
+      "tokens_seen": 198115328
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007595586760280843,
+      "loss": 1.7026,
+      "theoretical_loss": 4.388770294287169,
+      "tokens_seen": 198180864
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.000759542627883651,
+      "loss": 1.9569,
+      "theoretical_loss": 4.38858964686621,
+      "tokens_seen": 198246400
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007595265797392177,
+      "loss": 1.7857,
+      "theoretical_loss": 4.388409075868323,
+      "tokens_seen": 198311936
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007595105315947844,
+      "loss": 1.7623,
+      "theoretical_loss": 4.388228581235937,
+      "tokens_seen": 198377472
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007594944834503511,
+      "loss": 1.7654,
+      "theoretical_loss": 4.388048162911545,
+      "tokens_seen": 198443008
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007594784353059178,
+      "loss": 1.834,
+      "theoretical_loss": 4.3878678208377,
+      "tokens_seen": 198508544
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007594623871614845,
+      "loss": 1.6927,
+      "theoretical_loss": 4.38768755495702,
+      "tokens_seen": 198574080
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007594463390170512,
+      "loss": 1.7201,
+      "theoretical_loss": 4.387507365212182,
+      "tokens_seen": 198639616
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007594302908726179,
+      "loss": 1.8844,
+      "theoretical_loss": 4.387327251545927,
+      "tokens_seen": 198705152
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007594142427281846,
+      "loss": 1.7649,
+      "theoretical_loss": 4.387147213901059,
+      "tokens_seen": 198770688
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007593981945837513,
+      "loss": 1.7685,
+      "theoretical_loss": 4.386967252220441,
+      "tokens_seen": 198836224
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.000759382146439318,
+      "loss": 1.6499,
+      "theoretical_loss": 4.386787366447,
+      "tokens_seen": 198901760
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007593660982948847,
+      "loss": 1.6753,
+      "theoretical_loss": 4.386607556523723,
+      "tokens_seen": 198967296
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007593500501504514,
+      "loss": 1.6749,
+      "theoretical_loss": 4.386427822393661,
+      "tokens_seen": 199032832
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007593340020060181,
+      "loss": 1.843,
+      "theoretical_loss": 4.386248163999923,
+      "tokens_seen": 199098368
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007593179538615848,
+      "loss": 1.8036,
+      "theoretical_loss": 4.3860685812856826,
+      "tokens_seen": 199163904
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007593019057171515,
+      "loss": 1.77,
+      "theoretical_loss": 4.385889074194173,
+      "tokens_seen": 199229440
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007592858575727182,
+      "loss": 1.7111,
+      "theoretical_loss": 4.385709642668688,
+      "tokens_seen": 199294976
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.000759269809428285,
+      "loss": 1.7295,
+      "theoretical_loss": 4.385530286652586,
+      "tokens_seen": 199360512
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007592537612838516,
+      "loss": 1.7991,
+      "theoretical_loss": 4.385351006089283,
+      "tokens_seen": 199426048
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007592377131394182,
+      "loss": 1.7579,
+      "theoretical_loss": 4.385171800922257,
+      "tokens_seen": 199491584
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.000759221664994985,
+      "loss": 1.7003,
+      "theoretical_loss": 4.384992671095047,
+      "tokens_seen": 199557120
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007592056168505516,
+      "loss": 1.7583,
+      "theoretical_loss": 4.384813616551252,
+      "tokens_seen": 199622656
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007591895687061184,
+      "loss": 1.8127,
+      "theoretical_loss": 4.384634637234534,
+      "tokens_seen": 199688192
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.000759173520561685,
+      "loss": 1.7827,
+      "theoretical_loss": 4.384455733088612,
+      "tokens_seen": 199753728
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007591574724172518,
+      "loss": 1.7641,
+      "theoretical_loss": 4.38427690405727,
+      "tokens_seen": 199819264
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007591414242728184,
+      "loss": 1.7277,
+      "theoretical_loss": 4.384098150084349,
+      "tokens_seen": 199884800
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007591253761283852,
+      "loss": 1.7378,
+      "theoretical_loss": 4.383919471113751,
+      "tokens_seen": 199950336
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007591093279839518,
+      "loss": 1.7409,
+      "theoretical_loss": 4.383740867089441,
+      "tokens_seen": 200015872
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007590932798395186,
+      "loss": 1.8297,
+      "theoretical_loss": 4.383562337955439,
+      "tokens_seen": 200081408
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007590772316950852,
+      "loss": 1.804,
+      "theoretical_loss": 4.3833838836558305,
+      "tokens_seen": 200146944
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.000759061183550652,
+      "loss": 1.7346,
+      "theoretical_loss": 4.383205504134758,
+      "tokens_seen": 200212480
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007590451354062186,
+      "loss": 1.7105,
+      "theoretical_loss": 4.383027199336424,
+      "tokens_seen": 200278016
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007590290872617854,
+      "loss": 1.8286,
+      "theoretical_loss": 4.382848969205092,
+      "tokens_seen": 200343552
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.000759013039117352,
+      "loss": 1.7716,
+      "theoretical_loss": 4.3826708136850865,
+      "tokens_seen": 200409088
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007589969909729188,
+      "loss": 1.7295,
+      "theoretical_loss": 4.382492732720786,
+      "tokens_seen": 200474624
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007589809428284855,
+      "loss": 1.7726,
+      "theoretical_loss": 4.382314726256637,
+      "tokens_seen": 200540160
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007589648946840522,
+      "loss": 1.7471,
+      "theoretical_loss": 4.382136794237138,
+      "tokens_seen": 200605696
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007589488465396189,
+      "loss": 1.8189,
+      "theoretical_loss": 4.3819589366068525,
+      "tokens_seen": 200671232
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007589327983951856,
+      "loss": 1.7286,
+      "theoretical_loss": 4.381781153310399,
+      "tokens_seen": 200736768
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007589167502507523,
+      "loss": 1.7286,
+      "theoretical_loss": 4.381603444292459,
+      "tokens_seen": 200802304
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.000758900702106319,
+      "loss": 1.769,
+      "theoretical_loss": 4.38142580949777,
+      "tokens_seen": 200867840
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007588846539618857,
+      "loss": 1.7038,
+      "theoretical_loss": 4.381248248871131,
+      "tokens_seen": 200933376
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007588686058174524,
+      "loss": 1.5798,
+      "theoretical_loss": 4.3810707623574,
+      "tokens_seen": 200998912
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007588525576730191,
+      "loss": 1.7143,
+      "theoretical_loss": 4.380893349901491,
+      "tokens_seen": 201064448
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007588365095285858,
+      "loss": 1.7735,
+      "theoretical_loss": 4.380716011448381,
+      "tokens_seen": 201129984
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007588204613841525,
+      "loss": 1.902,
+      "theoretical_loss": 4.380538746943103,
+      "tokens_seen": 201195520
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007588044132397192,
+      "loss": 1.8396,
+      "theoretical_loss": 4.380361556330749,
+      "tokens_seen": 201261056
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007587883650952859,
+      "loss": 1.801,
+      "theoretical_loss": 4.380184439556472,
+      "tokens_seen": 201326592
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007587723169508526,
+      "loss": 1.6954,
+      "theoretical_loss": 4.38000739656548,
+      "tokens_seen": 201392128
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007587562688064193,
+      "loss": 1.7825,
+      "theoretical_loss": 4.379830427303041,
+      "tokens_seen": 201457664
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.000758740220661986,
+      "loss": 1.8515,
+      "theoretical_loss": 4.379653531714483,
+      "tokens_seen": 201523200
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007587241725175527,
+      "loss": 1.7923,
+      "theoretical_loss": 4.37947670974519,
+      "tokens_seen": 201588736
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007587081243731194,
+      "loss": 1.7959,
+      "theoretical_loss": 4.379299961340605,
+      "tokens_seen": 201654272
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007586920762286861,
+      "loss": 1.829,
+      "theoretical_loss": 4.37912328644623,
+      "tokens_seen": 201719808
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007586760280842528,
+      "loss": 1.659,
+      "theoretical_loss": 4.378946685007623,
+      "tokens_seen": 201785344
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007586599799398195,
+      "loss": 1.7904,
+      "theoretical_loss": 4.378770156970403,
+      "tokens_seen": 201850880
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007586439317953862,
+      "loss": 1.7975,
+      "theoretical_loss": 4.378593702280243,
+      "tokens_seen": 201916416
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007586278836509529,
+      "loss": 1.6993,
+      "theoretical_loss": 4.378417320882879,
+      "tokens_seen": 201981952
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007586118355065196,
+      "loss": 1.761,
+      "theoretical_loss": 4.378241012724099,
+      "tokens_seen": 202047488
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007585957873620863,
+      "loss": 1.7186,
+      "theoretical_loss": 4.378064777749753,
+      "tokens_seen": 202113024
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.000758579739217653,
+      "loss": 1.7332,
+      "theoretical_loss": 4.377888615905746,
+      "tokens_seen": 202178560
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007585636910732197,
+      "loss": 1.7773,
+      "theoretical_loss": 4.377712527138042,
+      "tokens_seen": 202244096
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007585476429287864,
+      "loss": 1.6866,
+      "theoretical_loss": 4.3775365113926625,
+      "tokens_seen": 202309632
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007585315947843531,
+      "loss": 1.6575,
+      "theoretical_loss": 4.377360568615685,
+      "tokens_seen": 202375168
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007585155466399198,
+      "loss": 1.7092,
+      "theoretical_loss": 4.377184698753244,
+      "tokens_seen": 202440704
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007584994984954865,
+      "loss": 1.7431,
+      "theoretical_loss": 4.377008901751536,
+      "tokens_seen": 202506240
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007584834503510532,
+      "loss": 1.8526,
+      "theoretical_loss": 4.376833177556806,
+      "tokens_seen": 202571776
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007584674022066199,
+      "loss": 1.7593,
+      "theoretical_loss": 4.3766575261153635,
+      "tokens_seen": 202637312
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007584513540621866,
+      "loss": 1.6966,
+      "theoretical_loss": 4.3764819473735725,
+      "tokens_seen": 202702848
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007584353059177533,
+      "loss": 1.7525,
+      "theoretical_loss": 4.376306441277852,
+      "tokens_seen": 202768384
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.00075841925777332,
+      "loss": 1.6903,
+      "theoretical_loss": 4.376131007774681,
+      "tokens_seen": 202833920
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007584032096288867,
+      "loss": 1.6677,
+      "theoretical_loss": 4.375955646810594,
+      "tokens_seen": 202899456
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007583871614844534,
+      "loss": 1.732,
+      "theoretical_loss": 4.37578035833218,
+      "tokens_seen": 202964992
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007583711133400201,
+      "loss": 1.849,
+      "theoretical_loss": 4.375605142286087,
+      "tokens_seen": 203030528
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007583550651955868,
+      "loss": 1.8048,
+      "theoretical_loss": 4.37542999861902,
+      "tokens_seen": 203096064
+    },
+    {
+      "epoch": 0.06,
+      "objective/train/docs_used": 82204,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.6902105808258057,
+      "objective/train/theoretical_loss": 4.375254927277737,
+      "objective/train/tokens_used": 223621600,
+      "theoretical_loss": 4.375254927277737,
+      "tokens_seen": 203161600
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007583390170511535,
+      "loss": 1.6902,
+      "theoretical_loss": 4.375254927277737,
+      "tokens_seen": 203161600
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007583229689067202,
+      "loss": 1.7447,
+      "theoretical_loss": 4.375079928209058,
+      "tokens_seen": 203227136
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007583069207622869,
+      "loss": 1.8088,
+      "theoretical_loss": 4.374905001359851,
+      "tokens_seen": 203292672
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007582908726178536,
+      "loss": 1.7274,
+      "theoretical_loss": 4.374730146677049,
+      "tokens_seen": 203358208
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007582748244734203,
+      "loss": 1.745,
+      "theoretical_loss": 4.374555364107635,
+      "tokens_seen": 203423744
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.000758258776328987,
+      "loss": 1.6991,
+      "theoretical_loss": 4.37438065359865,
+      "tokens_seen": 203489280
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007582427281845537,
+      "loss": 1.6572,
+      "theoretical_loss": 4.374206015097193,
+      "tokens_seen": 203554816
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007582266800401204,
+      "loss": 1.6793,
+      "theoretical_loss": 4.374031448550414,
+      "tokens_seen": 203620352
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007582106318956871,
+      "loss": 1.7597,
+      "theoretical_loss": 4.373856953905523,
+      "tokens_seen": 203685888
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007581945837512538,
+      "loss": 1.7406,
+      "theoretical_loss": 4.373682531109785,
+      "tokens_seen": 203751424
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007581785356068205,
+      "loss": 1.7525,
+      "theoretical_loss": 4.37350818011052,
+      "tokens_seen": 203816960
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007581624874623872,
+      "loss": 1.7741,
+      "theoretical_loss": 4.3733339008551,
+      "tokens_seen": 203882496
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007581464393179539,
+      "loss": 1.5948,
+      "theoretical_loss": 4.373159693290961,
+      "tokens_seen": 203948032
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007581303911735206,
+      "loss": 1.5639,
+      "theoretical_loss": 4.372985557365586,
+      "tokens_seen": 204013568
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007581143430290873,
+      "loss": 1.6878,
+      "theoretical_loss": 4.372811493026518,
+      "tokens_seen": 204079104
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.000758098294884654,
+      "loss": 1.715,
+      "theoretical_loss": 4.3726375002213524,
+      "tokens_seen": 204144640
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007580822467402207,
+      "loss": 1.7636,
+      "theoretical_loss": 4.372463578897743,
+      "tokens_seen": 204210176
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007580661985957874,
+      "loss": 1.8306,
+      "theoretical_loss": 4.372289729003397,
+      "tokens_seen": 204275712
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007580501504513541,
+      "loss": 1.7944,
+      "theoretical_loss": 4.372115950486075,
+      "tokens_seen": 204341248
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007580341023069208,
+      "loss": 1.7497,
+      "theoretical_loss": 4.371942243293595,
+      "tokens_seen": 204406784
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007580180541624875,
+      "loss": 1.7654,
+      "theoretical_loss": 4.371768607373829,
+      "tokens_seen": 204472320
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007580020060180542,
+      "loss": 1.7179,
+      "theoretical_loss": 4.371595042674704,
+      "tokens_seen": 204537856
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007579859578736209,
+      "loss": 1.8487,
+      "theoretical_loss": 4.371421549144202,
+      "tokens_seen": 204603392
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007579699097291876,
+      "loss": 1.6625,
+      "theoretical_loss": 4.371248126730357,
+      "tokens_seen": 204668928
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007579538615847543,
+      "loss": 1.7389,
+      "theoretical_loss": 4.3710747753812615,
+      "tokens_seen": 204734464
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.000757937813440321,
+      "loss": 1.6541,
+      "theoretical_loss": 4.370901495045061,
+      "tokens_seen": 204800000
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007579217652958877,
+      "loss": 1.7418,
+      "theoretical_loss": 4.370728285669954,
+      "tokens_seen": 204865536
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007579057171514544,
+      "loss": 1.7529,
+      "theoretical_loss": 4.370555147204195,
+      "tokens_seen": 204931072
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007578896690070211,
+      "loss": 1.7492,
+      "theoretical_loss": 4.370382079596092,
+      "tokens_seen": 204996608
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007578736208625878,
+      "loss": 1.6434,
+      "theoretical_loss": 4.370209082794006,
+      "tokens_seen": 205062144
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007578575727181545,
+      "loss": 1.6672,
+      "theoretical_loss": 4.370036156746357,
+      "tokens_seen": 205127680
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007578415245737211,
+      "loss": 1.8261,
+      "theoretical_loss": 4.369863301401612,
+      "tokens_seen": 205193216
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007578254764292879,
+      "loss": 1.6499,
+      "theoretical_loss": 4.369690516708298,
+      "tokens_seen": 205258752
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007578094282848545,
+      "loss": 1.7659,
+      "theoretical_loss": 4.3695178026149915,
+      "tokens_seen": 205324288
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007577933801404213,
+      "loss": 1.7906,
+      "theoretical_loss": 4.369345159070327,
+      "tokens_seen": 205389824
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007577773319959879,
+      "loss": 1.765,
+      "theoretical_loss": 4.369172586022989,
+      "tokens_seen": 205455360
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007577612838515547,
+      "loss": 1.6486,
+      "theoretical_loss": 4.369000083421716,
+      "tokens_seen": 205520896
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007577452357071213,
+      "loss": 1.6298,
+      "theoretical_loss": 4.368827651215304,
+      "tokens_seen": 205586432
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007577291875626881,
+      "loss": 1.7474,
+      "theoretical_loss": 4.368655289352598,
+      "tokens_seen": 205651968
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007577131394182547,
+      "loss": 1.7744,
+      "theoretical_loss": 4.3684829977824995,
+      "tokens_seen": 205717504
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007576970912738215,
+      "loss": 1.7353,
+      "theoretical_loss": 4.368310776453961,
+      "tokens_seen": 205783040
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007576810431293881,
+      "loss": 1.6938,
+      "theoretical_loss": 4.368138625315989,
+      "tokens_seen": 205848576
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007576649949849549,
+      "loss": 1.7471,
+      "theoretical_loss": 4.367966544317646,
+      "tokens_seen": 205914112
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007576489468405215,
+      "loss": 1.7376,
+      "theoretical_loss": 4.367794533408043,
+      "tokens_seen": 205979648
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007576328986960883,
+      "loss": 1.7466,
+      "theoretical_loss": 4.367622592536348,
+      "tokens_seen": 206045184
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.000757616850551655,
+      "loss": 1.6954,
+      "theoretical_loss": 4.36745072165178,
+      "tokens_seen": 206110720
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007576008024072217,
+      "loss": 1.8167,
+      "theoretical_loss": 4.36727892070361,
+      "tokens_seen": 206176256
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007575847542627884,
+      "loss": 1.7401,
+      "theoretical_loss": 4.367107189641167,
+      "tokens_seen": 206241792
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007575687061183551,
+      "loss": 1.6569,
+      "theoretical_loss": 4.3669355284138245,
+      "tokens_seen": 206307328
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007575526579739218,
+      "loss": 1.7005,
+      "theoretical_loss": 4.366763936971017,
+      "tokens_seen": 206372864
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007575366098294885,
+      "loss": 1.8048,
+      "theoretical_loss": 4.366592415262226,
+      "tokens_seen": 206438400
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007575205616850552,
+      "loss": 1.821,
+      "theoretical_loss": 4.366420963236989,
+      "tokens_seen": 206503936
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007575045135406219,
+      "loss": 1.751,
+      "theoretical_loss": 4.366249580844894,
+      "tokens_seen": 206569472
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007574884653961886,
+      "loss": 1.6118,
+      "theoretical_loss": 4.366078268035581,
+      "tokens_seen": 206635008
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007574724172517553,
+      "loss": 1.8002,
+      "theoretical_loss": 4.365907024758744,
+      "tokens_seen": 206700544
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.000757456369107322,
+      "loss": 1.6942,
+      "theoretical_loss": 4.365735850964131,
+      "tokens_seen": 206766080
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007574403209628887,
+      "loss": 1.717,
+      "theoretical_loss": 4.365564746601537,
+      "tokens_seen": 206831616
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007574242728184554,
+      "loss": 1.7408,
+      "theoretical_loss": 4.365393711620814,
+      "tokens_seen": 206897152
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007574082246740221,
+      "loss": 1.6824,
+      "theoretical_loss": 4.365222745971863,
+      "tokens_seen": 206962688
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007573921765295888,
+      "loss": 1.8375,
+      "theoretical_loss": 4.365051849604638,
+      "tokens_seen": 207028224
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007573761283851555,
+      "loss": 1.7472,
+      "theoretical_loss": 4.364881022469148,
+      "tokens_seen": 207093760
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007573600802407222,
+      "loss": 1.7056,
+      "theoretical_loss": 4.364710264515448,
+      "tokens_seen": 207159296
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007573440320962889,
+      "loss": 1.8439,
+      "theoretical_loss": 4.36453957569365,
+      "tokens_seen": 207224832
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007573279839518556,
+      "loss": 1.7218,
+      "theoretical_loss": 4.364368955953913,
+      "tokens_seen": 207290368
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007573119358074223,
+      "loss": 1.7204,
+      "theoretical_loss": 4.364198405246453,
+      "tokens_seen": 207355904
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.000757295887662989,
+      "loss": 1.7773,
+      "theoretical_loss": 4.364027923521535,
+      "tokens_seen": 207421440
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007572798395185557,
+      "loss": 1.6877,
+      "theoretical_loss": 4.363857510729473,
+      "tokens_seen": 207486976
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007572637913741224,
+      "loss": 1.7402,
+      "theoretical_loss": 4.363687166820637,
+      "tokens_seen": 207552512
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007572477432296891,
+      "loss": 1.7111,
+      "theoretical_loss": 4.363516891745445,
+      "tokens_seen": 207618048
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007572316950852558,
+      "loss": 1.7395,
+      "theoretical_loss": 4.36334668545437,
+      "tokens_seen": 207683584
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007572156469408225,
+      "loss": 1.847,
+      "theoretical_loss": 4.363176547897931,
+      "tokens_seen": 207749120
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007571995987963892,
+      "loss": 1.662,
+      "theoretical_loss": 4.363006479026702,
+      "tokens_seen": 207814656
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007571835506519559,
+      "loss": 1.6905,
+      "theoretical_loss": 4.3628364787913085,
+      "tokens_seen": 207880192
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007571675025075226,
+      "loss": 1.8538,
+      "theoretical_loss": 4.362666547142424,
+      "tokens_seen": 207945728
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007571514543630893,
+      "loss": 1.7348,
+      "theoretical_loss": 4.362496684030775,
+      "tokens_seen": 208011264
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.000757135406218656,
+      "loss": 1.8124,
+      "theoretical_loss": 4.36232688940714,
+      "tokens_seen": 208076800
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007571193580742227,
+      "loss": 1.7501,
+      "theoretical_loss": 4.362157163222346,
+      "tokens_seen": 208142336
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007571033099297894,
+      "loss": 1.7579,
+      "theoretical_loss": 4.361987505427272,
+      "tokens_seen": 208207872
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007570872617853561,
+      "loss": 1.751,
+      "theoretical_loss": 4.361817915972846,
+      "tokens_seen": 208273408
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007570712136409228,
+      "loss": 1.7616,
+      "theoretical_loss": 4.36164839481005,
+      "tokens_seen": 208338944
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007570551654964895,
+      "loss": 1.8296,
+      "theoretical_loss": 4.361478941889915,
+      "tokens_seen": 208404480
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007570391173520562,
+      "loss": 1.7435,
+      "theoretical_loss": 4.36130955716352,
+      "tokens_seen": 208470016
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007570230692076229,
+      "loss": 1.6989,
+      "theoretical_loss": 4.361140240581998,
+      "tokens_seen": 208535552
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007570070210631896,
+      "loss": 1.8035,
+      "theoretical_loss": 4.36097099209653,
+      "tokens_seen": 208601088
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007569909729187563,
+      "loss": 1.7573,
+      "theoretical_loss": 4.36080181165835,
+      "tokens_seen": 208666624
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.000756974924774323,
+      "loss": 1.6855,
+      "theoretical_loss": 4.360632699218739,
+      "tokens_seen": 208732160
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007569588766298897,
+      "loss": 1.812,
+      "theoretical_loss": 4.360463654729029,
+      "tokens_seen": 208797696
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007569428284854564,
+      "loss": 1.7353,
+      "theoretical_loss": 4.360294678140605,
+      "tokens_seen": 208863232
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007569267803410231,
+      "loss": 1.6493,
+      "theoretical_loss": 4.360125769404897,
+      "tokens_seen": 208928768
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007569107321965898,
+      "loss": 1.6855,
+      "theoretical_loss": 4.35995692847339,
+      "tokens_seen": 208994304
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007568946840521565,
+      "loss": 1.7107,
+      "theoretical_loss": 4.359788155297615,
+      "tokens_seen": 209059840
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007568786359077232,
+      "loss": 1.7624,
+      "theoretical_loss": 4.359619449829156,
+      "tokens_seen": 209125376
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007568625877632899,
+      "loss": 1.8295,
+      "theoretical_loss": 4.359450812019643,
+      "tokens_seen": 209190912
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007568465396188566,
+      "loss": 1.6752,
+      "theoretical_loss": 4.359282241820761,
+      "tokens_seen": 209256448
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007568304914744233,
+      "loss": 1.8464,
+      "theoretical_loss": 4.359113739184238,
+      "tokens_seen": 209321984
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.00075681444332999,
+      "loss": 1.8006,
+      "theoretical_loss": 4.358945304061858,
+      "tokens_seen": 209387520
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007567983951855567,
+      "loss": 1.8592,
+      "theoretical_loss": 4.35877693640545,
+      "tokens_seen": 209453056
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007567823470411234,
+      "loss": 1.7009,
+      "theoretical_loss": 4.358608636166894,
+      "tokens_seen": 209518592
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007567662988966901,
+      "loss": 1.8587,
+      "theoretical_loss": 4.358440403298119,
+      "tokens_seen": 209584128
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007567502507522569,
+      "loss": 1.669,
+      "theoretical_loss": 4.3582722377511045,
+      "tokens_seen": 209649664
+    },
+    {
+      "epoch": 0.06,
+      "objective/train/docs_used": 84957,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.7557694911956787,
+      "objective/train/theoretical_loss": 4.358104139477878,
+      "objective/train/tokens_used": 230175200,
+      "theoretical_loss": 4.358104139477878,
+      "tokens_seen": 209715200
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007567342026078235,
+      "loss": 1.7558,
+      "theoretical_loss": 4.358104139477878,
+      "tokens_seen": 209715200
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007567181544633903,
+      "loss": 1.7169,
+      "theoretical_loss": 4.357936108430517,
+      "tokens_seen": 209780736
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007567021063189569,
+      "loss": 1.6734,
+      "theoretical_loss": 4.357768144561146,
+      "tokens_seen": 209846272
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007566860581745237,
+      "loss": 1.8187,
+      "theoretical_loss": 4.357600247821942,
+      "tokens_seen": 209911808
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007566700100300903,
+      "loss": 1.6683,
+      "theoretical_loss": 4.3574324181651285,
+      "tokens_seen": 209977344
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007566539618856571,
+      "loss": 1.7935,
+      "theoretical_loss": 4.357264655542978,
+      "tokens_seen": 210042880
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007566379137412237,
+      "loss": 1.8475,
+      "theoretical_loss": 4.357096959907813,
+      "tokens_seen": 210108416
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007566218655967905,
+      "loss": 1.7322,
+      "theoretical_loss": 4.356929331212004,
+      "tokens_seen": 210173952
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007566058174523571,
+      "loss": 1.7798,
+      "theoretical_loss": 4.356761769407969,
+      "tokens_seen": 210239488
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007565897693079239,
+      "loss": 1.8022,
+      "theoretical_loss": 4.356594274448178,
+      "tokens_seen": 210305024
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007565737211634905,
+      "loss": 1.6858,
+      "theoretical_loss": 4.356426846285147,
+      "tokens_seen": 210370560
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007565576730190572,
+      "loss": 1.7772,
+      "theoretical_loss": 4.3562594848714395,
+      "tokens_seen": 210436096
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.000756541624874624,
+      "loss": 1.7201,
+      "theoretical_loss": 4.3560921901596705,
+      "tokens_seen": 210501632
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007565255767301906,
+      "loss": 1.677,
+      "theoretical_loss": 4.355924962102502,
+      "tokens_seen": 210567168
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007565095285857573,
+      "loss": 1.6862,
+      "theoretical_loss": 4.3557578006526425,
+      "tokens_seen": 210632704
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.000756493480441324,
+      "loss": 1.7718,
+      "theoretical_loss": 4.355590705762852,
+      "tokens_seen": 210698240
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007564774322968907,
+      "loss": 1.7889,
+      "theoretical_loss": 4.355423677385937,
+      "tokens_seen": 210763776
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007564613841524574,
+      "loss": 1.8241,
+      "theoretical_loss": 4.355256715474751,
+      "tokens_seen": 210829312
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007564453360080242,
+      "loss": 1.815,
+      "theoretical_loss": 4.355089819982198,
+      "tokens_seen": 210894848
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007564292878635908,
+      "loss": 1.7633,
+      "theoretical_loss": 4.354922990861229,
+      "tokens_seen": 210960384
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007564132397191574,
+      "loss": 1.7236,
+      "theoretical_loss": 4.354756228064842,
+      "tokens_seen": 211025920
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007563971915747242,
+      "loss": 1.6991,
+      "theoretical_loss": 4.354589531546083,
+      "tokens_seen": 211091456
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007563811434302908,
+      "loss": 1.7559,
+      "theoretical_loss": 4.354422901258048,
+      "tokens_seen": 211156992
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007563650952858576,
+      "loss": 1.6282,
+      "theoretical_loss": 4.354256337153878,
+      "tokens_seen": 211222528
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007563490471414242,
+      "loss": 1.7975,
+      "theoretical_loss": 4.354089839186763,
+      "tokens_seen": 211288064
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.000756332998996991,
+      "loss": 1.6835,
+      "theoretical_loss": 4.35392340730994,
+      "tokens_seen": 211353600
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007563169508525576,
+      "loss": 1.6923,
+      "theoretical_loss": 4.353757041476695,
+      "tokens_seen": 211419136
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007563009027081244,
+      "loss": 1.748,
+      "theoretical_loss": 4.35359074164036,
+      "tokens_seen": 211484672
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.000756284854563691,
+      "loss": 1.7377,
+      "theoretical_loss": 4.3534245077543146,
+      "tokens_seen": 211550208
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007562688064192578,
+      "loss": 1.6532,
+      "theoretical_loss": 4.353258339771986,
+      "tokens_seen": 211615744
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007562527582748245,
+      "loss": 1.6354,
+      "theoretical_loss": 4.353092237646849,
+      "tokens_seen": 211681280
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007562367101303912,
+      "loss": 1.7168,
+      "theoretical_loss": 4.3529262013324255,
+      "tokens_seen": 211746816
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007562206619859579,
+      "loss": 1.6738,
+      "theoretical_loss": 4.352760230782284,
+      "tokens_seen": 211812352
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007562046138415246,
+      "loss": 1.6526,
+      "theoretical_loss": 4.352594325950041,
+      "tokens_seen": 211877888
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007561885656970913,
+      "loss": 1.7619,
+      "theoretical_loss": 4.35242848678936,
+      "tokens_seen": 211943424
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.000756172517552658,
+      "loss": 1.6957,
+      "theoretical_loss": 4.35226271325395,
+      "tokens_seen": 212008960
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007561564694082247,
+      "loss": 1.7532,
+      "theoretical_loss": 4.352097005297569,
+      "tokens_seen": 212074496
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007561404212637914,
+      "loss": 1.5988,
+      "theoretical_loss": 4.351931362874019,
+      "tokens_seen": 212140032
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007561243731193581,
+      "loss": 1.7567,
+      "theoretical_loss": 4.351765785937154,
+      "tokens_seen": 212205568
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007561083249749248,
+      "loss": 1.7913,
+      "theoretical_loss": 4.3516002744408695,
+      "tokens_seen": 212271104
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007560922768304915,
+      "loss": 1.7309,
+      "theoretical_loss": 4.351434828339109,
+      "tokens_seen": 212336640
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007560762286860582,
+      "loss": 1.8523,
+      "theoretical_loss": 4.351269447585864,
+      "tokens_seen": 212402176
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007560601805416249,
+      "loss": 1.8322,
+      "theoretical_loss": 4.3511041321351716,
+      "tokens_seen": 212467712
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007560441323971916,
+      "loss": 1.723,
+      "theoretical_loss": 4.350938881941116,
+      "tokens_seen": 212533248
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007560280842527583,
+      "loss": 1.7828,
+      "theoretical_loss": 4.350773696957827,
+      "tokens_seen": 212598784
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.000756012036108325,
+      "loss": 1.7081,
+      "theoretical_loss": 4.350608577139481,
+      "tokens_seen": 212664320
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007559959879638917,
+      "loss": 1.7537,
+      "theoretical_loss": 4.350443522440301,
+      "tokens_seen": 212729856
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007559799398194584,
+      "loss": 1.6697,
+      "theoretical_loss": 4.350278532814556,
+      "tokens_seen": 212795392
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007559638916750251,
+      "loss": 1.9349,
+      "theoretical_loss": 4.350113608216563,
+      "tokens_seen": 212860928
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007559478435305918,
+      "loss": 1.7607,
+      "theoretical_loss": 4.3499487486006805,
+      "tokens_seen": 212926464
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007559317953861585,
+      "loss": 1.7092,
+      "theoretical_loss": 4.349783953921318,
+      "tokens_seen": 212992000
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007559157472417252,
+      "loss": 1.852,
+      "theoretical_loss": 4.349619224132929,
+      "tokens_seen": 213057536
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007558996990972919,
+      "loss": 1.7047,
+      "theoretical_loss": 4.349454559190011,
+      "tokens_seen": 213123072
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007558836509528586,
+      "loss": 1.7372,
+      "theoretical_loss": 4.349289959047112,
+      "tokens_seen": 213188608
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007558676028084253,
+      "loss": 1.7609,
+      "theoretical_loss": 4.349125423658821,
+      "tokens_seen": 213254144
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.000755851554663992,
+      "loss": 1.7236,
+      "theoretical_loss": 4.348960952979776,
+      "tokens_seen": 213319680
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007558355065195587,
+      "loss": 1.8124,
+      "theoretical_loss": 4.3487965469646594,
+      "tokens_seen": 213385216
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007558194583751254,
+      "loss": 1.7797,
+      "theoretical_loss": 4.3486322055682,
+      "tokens_seen": 213450752
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007558034102306921,
+      "loss": 1.6791,
+      "theoretical_loss": 4.348467928745171,
+      "tokens_seen": 213516288
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007557873620862588,
+      "loss": 1.6517,
+      "theoretical_loss": 4.348303716450391,
+      "tokens_seen": 213581824
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007557713139418255,
+      "loss": 1.7573,
+      "theoretical_loss": 4.348139568638727,
+      "tokens_seen": 213647360
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007557552657973922,
+      "loss": 1.8218,
+      "theoretical_loss": 4.347975485265087,
+      "tokens_seen": 213712896
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007557392176529589,
+      "loss": 1.8017,
+      "theoretical_loss": 4.347811466284428,
+      "tokens_seen": 213778432
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007557231695085256,
+      "loss": 1.7611,
+      "theoretical_loss": 4.34764751165175,
+      "tokens_seen": 213843968
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007557071213640923,
+      "loss": 1.8309,
+      "theoretical_loss": 4.347483621322099,
+      "tokens_seen": 213909504
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.000755691073219659,
+      "loss": 1.759,
+      "theoretical_loss": 4.347319795250567,
+      "tokens_seen": 213975040
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007556750250752257,
+      "loss": 1.7367,
+      "theoretical_loss": 4.347156033392291,
+      "tokens_seen": 214040576
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007556589769307924,
+      "loss": 1.7225,
+      "theoretical_loss": 4.346992335702451,
+      "tokens_seen": 214106112
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007556429287863591,
+      "loss": 1.7777,
+      "theoretical_loss": 4.346828702136273,
+      "tokens_seen": 214171648
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007556268806419258,
+      "loss": 1.7765,
+      "theoretical_loss": 4.34666513264903,
+      "tokens_seen": 214237184
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007556108324974925,
+      "loss": 1.6971,
+      "theoretical_loss": 4.346501627196037,
+      "tokens_seen": 214302720
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007555947843530592,
+      "loss": 1.7381,
+      "theoretical_loss": 4.346338185732655,
+      "tokens_seen": 214368256
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007555787362086259,
+      "loss": 1.7924,
+      "theoretical_loss": 4.3461748082142915,
+      "tokens_seen": 214433792
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007555626880641926,
+      "loss": 1.8189,
+      "theoretical_loss": 4.346011494596395,
+      "tokens_seen": 214499328
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007555466399197594,
+      "loss": 1.7618,
+      "theoretical_loss": 4.34584824483446,
+      "tokens_seen": 214564864
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.000755530591775326,
+      "loss": 1.775,
+      "theoretical_loss": 4.345685058884029,
+      "tokens_seen": 214630400
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007555145436308928,
+      "loss": 1.698,
+      "theoretical_loss": 4.345521936700684,
+      "tokens_seen": 214695936
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007554984954864594,
+      "loss": 1.7327,
+      "theoretical_loss": 4.345358878240053,
+      "tokens_seen": 214761472
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007554824473420262,
+      "loss": 1.7759,
+      "theoretical_loss": 4.345195883457811,
+      "tokens_seen": 214827008
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007554663991975928,
+      "loss": 1.6508,
+      "theoretical_loss": 4.3450329523096745,
+      "tokens_seen": 214892544
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007554503510531596,
+      "loss": 1.8172,
+      "theoretical_loss": 4.344870084751406,
+      "tokens_seen": 214958080
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007554343029087262,
+      "loss": 1.8609,
+      "theoretical_loss": 4.34470728073881,
+      "tokens_seen": 215023616
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.000755418254764293,
+      "loss": 1.7311,
+      "theoretical_loss": 4.344544540227737,
+      "tokens_seen": 215089152
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007554022066198596,
+      "loss": 1.8145,
+      "theoretical_loss": 4.344381863174082,
+      "tokens_seen": 215154688
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007553861584754264,
+      "loss": 1.6288,
+      "theoretical_loss": 4.344219249533783,
+      "tokens_seen": 215220224
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.000755370110330993,
+      "loss": 1.8157,
+      "theoretical_loss": 4.34405669926282,
+      "tokens_seen": 215285760
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007553540621865598,
+      "loss": 1.7173,
+      "theoretical_loss": 4.343894212317224,
+      "tokens_seen": 215351296
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007553380140421264,
+      "loss": 1.8164,
+      "theoretical_loss": 4.34373178865306,
+      "tokens_seen": 215416832
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007553219658976932,
+      "loss": 1.7982,
+      "theoretical_loss": 4.343569428226445,
+      "tokens_seen": 215482368
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007553059177532598,
+      "loss": 1.7587,
+      "theoretical_loss": 4.343407130993536,
+      "tokens_seen": 215547904
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007552898696088266,
+      "loss": 1.6608,
+      "theoretical_loss": 4.343244896910534,
+      "tokens_seen": 215613440
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007552738214643932,
+      "loss": 1.5795,
+      "theoretical_loss": 4.343082725933684,
+      "tokens_seen": 215678976
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00075525777331996,
+      "loss": 1.6416,
+      "theoretical_loss": 4.342920618019275,
+      "tokens_seen": 215744512
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007552417251755266,
+      "loss": 1.6894,
+      "theoretical_loss": 4.342758573123639,
+      "tokens_seen": 215810048
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007552256770310934,
+      "loss": 1.703,
+      "theoretical_loss": 4.342596591203153,
+      "tokens_seen": 215875584
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00075520962888666,
+      "loss": 1.7567,
+      "theoretical_loss": 4.3424346722142335,
+      "tokens_seen": 215941120
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007551935807422268,
+      "loss": 1.7788,
+      "theoretical_loss": 4.342272816113344,
+      "tokens_seen": 216006656
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007551775325977934,
+      "loss": 1.693,
+      "theoretical_loss": 4.342111022856992,
+      "tokens_seen": 216072192
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007551614844533601,
+      "loss": 1.7425,
+      "theoretical_loss": 4.341949292401725,
+      "tokens_seen": 216137728
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007551454363089268,
+      "loss": 1.7275,
+      "theoretical_loss": 4.341787624704135,
+      "tokens_seen": 216203264
+    },
+    {
+      "epoch": 0.07,
+      "objective/train/docs_used": 87357,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.7541401386260986,
+      "objective/train/theoretical_loss": 4.341626019720858,
+      "objective/train/tokens_used": 236728800,
+      "theoretical_loss": 4.341626019720858,
+      "tokens_seen": 216268800
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007551293881644935,
+      "loss": 1.7541,
+      "theoretical_loss": 4.341626019720858,
+      "tokens_seen": 216268800
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007551133400200602,
+      "loss": 1.8899,
+      "theoretical_loss": 4.341464477408572,
+      "tokens_seen": 216334336
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007550972918756269,
+      "loss": 1.6601,
+      "theoretical_loss": 4.341302997724,
+      "tokens_seen": 216399872
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007550812437311937,
+      "loss": 1.7421,
+      "theoretical_loss": 4.341141580623905,
+      "tokens_seen": 216465408
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007550651955867603,
+      "loss": 1.8931,
+      "theoretical_loss": 4.340980226065096,
+      "tokens_seen": 216530944
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.000755049147442327,
+      "loss": 1.6918,
+      "theoretical_loss": 4.340818934004421,
+      "tokens_seen": 216596480
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007550330992978937,
+      "loss": 1.754,
+      "theoretical_loss": 4.340657704398774,
+      "tokens_seen": 216662016
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007550170511534603,
+      "loss": 1.6693,
+      "theoretical_loss": 4.340496537205093,
+      "tokens_seen": 216727552
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007550010030090271,
+      "loss": 1.6629,
+      "theoretical_loss": 4.340335432380353,
+      "tokens_seen": 216793088
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007549849548645937,
+      "loss": 1.7004,
+      "theoretical_loss": 4.340174389881579,
+      "tokens_seen": 216858624
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007549689067201605,
+      "loss": 1.764,
+      "theoretical_loss": 4.340013409665833,
+      "tokens_seen": 216924160
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007549528585757271,
+      "loss": 1.7839,
+      "theoretical_loss": 4.339852491690221,
+      "tokens_seen": 216989696
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007549368104312939,
+      "loss": 1.6797,
+      "theoretical_loss": 4.339691635911892,
+      "tokens_seen": 217055232
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007549207622868605,
+      "loss": 1.7425,
+      "theoretical_loss": 4.339530842288037,
+      "tokens_seen": 217120768
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007549047141424273,
+      "loss": 1.7092,
+      "theoretical_loss": 4.339370110775892,
+      "tokens_seen": 217186304
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.000754888665997994,
+      "loss": 1.7533,
+      "theoretical_loss": 4.33920944133273,
+      "tokens_seen": 217251840
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007548726178535607,
+      "loss": 1.7142,
+      "theoretical_loss": 4.3390488339158715,
+      "tokens_seen": 217317376
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007548565697091274,
+      "loss": 1.7488,
+      "theoretical_loss": 4.338888288482675,
+      "tokens_seen": 217382912
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007548405215646941,
+      "loss": 1.7231,
+      "theoretical_loss": 4.338727804990544,
+      "tokens_seen": 217448448
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007548244734202608,
+      "loss": 1.6839,
+      "theoretical_loss": 4.338567383396924,
+      "tokens_seen": 217513984
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007548084252758275,
+      "loss": 1.7208,
+      "theoretical_loss": 4.338407023659302,
+      "tokens_seen": 217579520
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007547923771313942,
+      "loss": 1.6647,
+      "theoretical_loss": 4.338246725735205,
+      "tokens_seen": 217645056
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007547763289869609,
+      "loss": 1.6131,
+      "theoretical_loss": 4.338086489582205,
+      "tokens_seen": 217710592
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007547602808425276,
+      "loss": 1.6504,
+      "theoretical_loss": 4.3379263151579135,
+      "tokens_seen": 217776128
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007547442326980943,
+      "loss": 1.6979,
+      "theoretical_loss": 4.337766202419986,
+      "tokens_seen": 217841664
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.000754728184553661,
+      "loss": 1.7399,
+      "theoretical_loss": 4.337606151326119,
+      "tokens_seen": 217907200
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007547121364092277,
+      "loss": 1.6977,
+      "theoretical_loss": 4.337446161834048,
+      "tokens_seen": 217972736
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007546960882647944,
+      "loss": 1.7266,
+      "theoretical_loss": 4.3372862339015565,
+      "tokens_seen": 218038272
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007546800401203611,
+      "loss": 1.7607,
+      "theoretical_loss": 4.337126367486462,
+      "tokens_seen": 218103808
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007546639919759278,
+      "loss": 1.8613,
+      "theoretical_loss": 4.33696656254663,
+      "tokens_seen": 218169344
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007546479438314945,
+      "loss": 1.7391,
+      "theoretical_loss": 4.336806819039962,
+      "tokens_seen": 218234880
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007546318956870612,
+      "loss": 1.7434,
+      "theoretical_loss": 4.336647136924407,
+      "tokens_seen": 218300416
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007546158475426279,
+      "loss": 1.7183,
+      "theoretical_loss": 4.33648751615795,
+      "tokens_seen": 218365952
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007545997993981946,
+      "loss": 1.709,
+      "theoretical_loss": 4.33632795669862,
+      "tokens_seen": 218431488
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007545837512537613,
+      "loss": 1.8274,
+      "theoretical_loss": 4.336168458504486,
+      "tokens_seen": 218497024
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.000754567703109328,
+      "loss": 1.5751,
+      "theoretical_loss": 4.336009021533661,
+      "tokens_seen": 218562560
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007545516549648947,
+      "loss": 1.8043,
+      "theoretical_loss": 4.335849645744295,
+      "tokens_seen": 218628096
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007545356068204614,
+      "loss": 1.6846,
+      "theoretical_loss": 4.3356903310945825,
+      "tokens_seen": 218693632
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007545195586760281,
+      "loss": 1.7287,
+      "theoretical_loss": 4.335531077542758,
+      "tokens_seen": 218759168
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007545035105315948,
+      "loss": 1.7706,
+      "theoretical_loss": 4.335371885047096,
+      "tokens_seen": 218824704
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007544874623871615,
+      "loss": 1.7234,
+      "theoretical_loss": 4.335212753565914,
+      "tokens_seen": 218890240
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007544714142427282,
+      "loss": 1.7824,
+      "theoretical_loss": 4.33505368305757,
+      "tokens_seen": 218955776
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007544553660982949,
+      "loss": 1.7169,
+      "theoretical_loss": 4.33489467348046,
+      "tokens_seen": 219021312
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007544393179538616,
+      "loss": 1.7009,
+      "theoretical_loss": 4.334735724793026,
+      "tokens_seen": 219086848
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007544232698094283,
+      "loss": 1.7544,
+      "theoretical_loss": 4.334576836953744,
+      "tokens_seen": 219152384
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.000754407221664995,
+      "loss": 1.745,
+      "theoretical_loss": 4.334418009921139,
+      "tokens_seen": 219217920
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007543911735205617,
+      "loss": 1.8339,
+      "theoretical_loss": 4.334259243653769,
+      "tokens_seen": 219283456
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007543751253761284,
+      "loss": 1.7365,
+      "theoretical_loss": 4.334100538110237,
+      "tokens_seen": 219348992
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007543590772316951,
+      "loss": 1.7756,
+      "theoretical_loss": 4.333941893249184,
+      "tokens_seen": 219414528
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007543430290872619,
+      "loss": 1.6889,
+      "theoretical_loss": 4.333783309029296,
+      "tokens_seen": 219480064
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007543269809428285,
+      "loss": 1.5964,
+      "theoretical_loss": 4.3336247854092935,
+      "tokens_seen": 219545600
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007543109327983953,
+      "loss": 1.8013,
+      "theoretical_loss": 4.333466322347942,
+      "tokens_seen": 219611136
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007542948846539619,
+      "loss": 1.7981,
+      "theoretical_loss": 4.333307919804045,
+      "tokens_seen": 219676672
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007542788365095287,
+      "loss": 1.754,
+      "theoretical_loss": 4.333149577736446,
+      "tokens_seen": 219742208
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007542627883650953,
+      "loss": 1.6121,
+      "theoretical_loss": 4.332991296104032,
+      "tokens_seen": 219807744
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007542467402206621,
+      "loss": 1.8843,
+      "theoretical_loss": 4.332833074865725,
+      "tokens_seen": 219873280
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007542306920762287,
+      "loss": 1.7242,
+      "theoretical_loss": 4.332674913980492,
+      "tokens_seen": 219938816
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007542146439317955,
+      "loss": 1.7173,
+      "theoretical_loss": 4.332516813407337,
+      "tokens_seen": 220004352
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007541985957873621,
+      "loss": 1.8197,
+      "theoretical_loss": 4.332358773105307,
+      "tokens_seen": 220069888
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007541825476429289,
+      "loss": 1.8409,
+      "theoretical_loss": 4.332200793033485,
+      "tokens_seen": 220135424
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007541664994984955,
+      "loss": 1.7557,
+      "theoretical_loss": 4.332042873150997,
+      "tokens_seen": 220200960
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007541504513540623,
+      "loss": 1.7084,
+      "theoretical_loss": 4.3318850134170095,
+      "tokens_seen": 220266496
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007541344032096289,
+      "loss": 1.7834,
+      "theoretical_loss": 4.331727213790725,
+      "tokens_seen": 220332032
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007541183550651957,
+      "loss": 1.705,
+      "theoretical_loss": 4.331569474231388,
+      "tokens_seen": 220397568
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007541023069207623,
+      "loss": 1.736,
+      "theoretical_loss": 4.331411794698286,
+      "tokens_seen": 220463104
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007540862587763291,
+      "loss": 1.7398,
+      "theoretical_loss": 4.33125417515074,
+      "tokens_seen": 220528640
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007540702106318957,
+      "loss": 1.6683,
+      "theoretical_loss": 4.331096615548115,
+      "tokens_seen": 220594176
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007540541624874625,
+      "loss": 1.6628,
+      "theoretical_loss": 4.330939115849814,
+      "tokens_seen": 220659712
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007540381143430291,
+      "loss": 1.675,
+      "theoretical_loss": 4.33078167601528,
+      "tokens_seen": 220725248
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007540220661985959,
+      "loss": 1.6805,
+      "theoretical_loss": 4.330624296003996,
+      "tokens_seen": 220790784
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007540060180541625,
+      "loss": 1.8567,
+      "theoretical_loss": 4.330466975775483,
+      "tokens_seen": 220856320
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007539899699097293,
+      "loss": 1.6977,
+      "theoretical_loss": 4.330309715289301,
+      "tokens_seen": 220921856
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007539739217652959,
+      "loss": 1.686,
+      "theoretical_loss": 4.330152514505053,
+      "tokens_seen": 220987392
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007539578736208627,
+      "loss": 1.7442,
+      "theoretical_loss": 4.329995373382378,
+      "tokens_seen": 221052928
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007539418254764293,
+      "loss": 1.7602,
+      "theoretical_loss": 4.329838291880954,
+      "tokens_seen": 221118464
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007539257773319961,
+      "loss": 1.8866,
+      "theoretical_loss": 4.329681269960499,
+      "tokens_seen": 221184000
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007539097291875627,
+      "loss": 1.8567,
+      "theoretical_loss": 4.329524307580773,
+      "tokens_seen": 221249536
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007538936810431295,
+      "loss": 1.7735,
+      "theoretical_loss": 4.3293674047015696,
+      "tokens_seen": 221315072
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007538776328986961,
+      "loss": 1.7584,
+      "theoretical_loss": 4.329210561282726,
+      "tokens_seen": 221380608
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007538615847542629,
+      "loss": 1.6923,
+      "theoretical_loss": 4.329053777284115,
+      "tokens_seen": 221446144
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007538455366098295,
+      "loss": 1.7616,
+      "theoretical_loss": 4.328897052665653,
+      "tokens_seen": 221511680
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007538294884653963,
+      "loss": 1.7035,
+      "theoretical_loss": 4.328740387387289,
+      "tokens_seen": 221577216
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.000753813440320963,
+      "loss": 1.825,
+      "theoretical_loss": 4.328583781409016,
+      "tokens_seen": 221642752
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007537973921765297,
+      "loss": 1.7146,
+      "theoretical_loss": 4.328427234690864,
+      "tokens_seen": 221708288
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007537813440320963,
+      "loss": 1.5788,
+      "theoretical_loss": 4.328270747192901,
+      "tokens_seen": 221773824
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.000753765295887663,
+      "loss": 1.6196,
+      "theoretical_loss": 4.328114318875236,
+      "tokens_seen": 221839360
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007537492477432297,
+      "loss": 1.694,
+      "theoretical_loss": 4.327957949698012,
+      "tokens_seen": 221904896
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007537331995987964,
+      "loss": 1.6947,
+      "theoretical_loss": 4.327801639621417,
+      "tokens_seen": 221970432
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007537171514543632,
+      "loss": 1.7786,
+      "theoretical_loss": 4.3276453886056725,
+      "tokens_seen": 222035968
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007537011033099298,
+      "loss": 1.6427,
+      "theoretical_loss": 4.32748919661104,
+      "tokens_seen": 222101504
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007536850551654966,
+      "loss": 1.7688,
+      "theoretical_loss": 4.327333063597821,
+      "tokens_seen": 222167040
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007536690070210632,
+      "loss": 1.7061,
+      "theoretical_loss": 4.327176989526352,
+      "tokens_seen": 222232576
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00075365295887663,
+      "loss": 1.7345,
+      "theoretical_loss": 4.327020974357012,
+      "tokens_seen": 222298112
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007536369107321966,
+      "loss": 1.8073,
+      "theoretical_loss": 4.326865018050215,
+      "tokens_seen": 222363648
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007536208625877632,
+      "loss": 1.7301,
+      "theoretical_loss": 4.326709120566415,
+      "tokens_seen": 222429184
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00075360481444333,
+      "loss": 1.7917,
+      "theoretical_loss": 4.326553281866103,
+      "tokens_seen": 222494720
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007535887662988966,
+      "loss": 1.6926,
+      "theoretical_loss": 4.326397501909811,
+      "tokens_seen": 222560256
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007535727181544634,
+      "loss": 1.6728,
+      "theoretical_loss": 4.326241780658104,
+      "tokens_seen": 222625792
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00075355667001003,
+      "loss": 1.7051,
+      "theoretical_loss": 4.32608611807159,
+      "tokens_seen": 222691328
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007535406218655968,
+      "loss": 1.6952,
+      "theoretical_loss": 4.325930514110912,
+      "tokens_seen": 222756864
+    },
+    {
+      "epoch": 0.07,
+      "objective/train/docs_used": 89737,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.6492441892623901,
+      "objective/train/theoretical_loss": 4.325774968736753,
+      "objective/train/tokens_used": 243282400,
+      "theoretical_loss": 4.325774968736753,
+      "tokens_seen": 222822400
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007535245737211634,
+      "loss": 1.6492,
+      "theoretical_loss": 4.325774968736753,
+      "tokens_seen": 222822400
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007535085255767302,
+      "loss": 1.7581,
+      "theoretical_loss": 4.325619481909833,
+      "tokens_seen": 222887936
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007534924774322969,
+      "loss": 1.7129,
+      "theoretical_loss": 4.325464053590908,
+      "tokens_seen": 222953472
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007534764292878636,
+      "loss": 1.7744,
+      "theoretical_loss": 4.325308683740776,
+      "tokens_seen": 223019008
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007534603811434303,
+      "loss": 1.7024,
+      "theoretical_loss": 4.325153372320269,
+      "tokens_seen": 223084544
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.000753444332998997,
+      "loss": 1.6814,
+      "theoretical_loss": 4.32499811929026,
+      "tokens_seen": 223150080
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007534282848545637,
+      "loss": 1.7055,
+      "theoretical_loss": 4.324842924611655,
+      "tokens_seen": 223215616
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007534122367101304,
+      "loss": 1.841,
+      "theoretical_loss": 4.324687788245403,
+      "tokens_seen": 223281152
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007533961885656971,
+      "loss": 1.7495,
+      "theoretical_loss": 4.3245327101524875,
+      "tokens_seen": 223346688
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007533801404212638,
+      "loss": 1.6664,
+      "theoretical_loss": 4.324377690293929,
+      "tokens_seen": 223412224
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007533640922768305,
+      "loss": 1.7203,
+      "theoretical_loss": 4.324222728630787,
+      "tokens_seen": 223477760
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007533480441323972,
+      "loss": 1.7453,
+      "theoretical_loss": 4.324067825124159,
+      "tokens_seen": 223543296
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007533319959879639,
+      "loss": 1.6492,
+      "theoretical_loss": 4.323912979735178,
+      "tokens_seen": 223608832
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007533159478435306,
+      "loss": 1.7522,
+      "theoretical_loss": 4.323758192425016,
+      "tokens_seen": 223674368
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007532998996990973,
+      "loss": 1.7921,
+      "theoretical_loss": 4.323603463154882,
+      "tokens_seen": 223739904
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.000753283851554664,
+      "loss": 1.58,
+      "theoretical_loss": 4.323448791886022,
+      "tokens_seen": 223805440
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007532678034102307,
+      "loss": 1.6923,
+      "theoretical_loss": 4.323294178579717,
+      "tokens_seen": 223870976
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007532517552657974,
+      "loss": 1.7731,
+      "theoretical_loss": 4.323139623197289,
+      "tokens_seen": 223936512
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007532357071213641,
+      "loss": 1.8041,
+      "theoretical_loss": 4.322985125700096,
+      "tokens_seen": 224002048
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007532196589769308,
+      "loss": 1.6624,
+      "theoretical_loss": 4.322830686049532,
+      "tokens_seen": 224067584
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007532036108324975,
+      "loss": 1.8031,
+      "theoretical_loss": 4.322676304207027,
+      "tokens_seen": 224133120
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007531875626880642,
+      "loss": 1.6445,
+      "theoretical_loss": 4.32252198013405,
+      "tokens_seen": 224198656
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007531715145436309,
+      "loss": 1.6782,
+      "theoretical_loss": 4.32236771379211,
+      "tokens_seen": 224264192
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007531554663991976,
+      "loss": 1.7225,
+      "theoretical_loss": 4.322213505142744,
+      "tokens_seen": 224329728
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007531394182547644,
+      "loss": 1.775,
+      "theoretical_loss": 4.322059354147535,
+      "tokens_seen": 224395264
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.000753123370110331,
+      "loss": 1.7043,
+      "theoretical_loss": 4.321905260768097,
+      "tokens_seen": 224460800
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007531073219658978,
+      "loss": 1.7811,
+      "theoretical_loss": 4.321751224966083,
+      "tokens_seen": 224526336
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007530912738214644,
+      "loss": 1.7081,
+      "theoretical_loss": 4.321597246703183,
+      "tokens_seen": 224591872
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007530752256770312,
+      "loss": 1.685,
+      "theoretical_loss": 4.321443325941123,
+      "tokens_seen": 224657408
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007530591775325978,
+      "loss": 1.6451,
+      "theoretical_loss": 4.3212894626416665,
+      "tokens_seen": 224722944
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007530431293881646,
+      "loss": 1.7142,
+      "theoretical_loss": 4.321135656766611,
+      "tokens_seen": 224788480
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007530270812437312,
+      "loss": 1.755,
+      "theoretical_loss": 4.320981908277793,
+      "tokens_seen": 224854016
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.000753011033099298,
+      "loss": 1.626,
+      "theoretical_loss": 4.320828217137086,
+      "tokens_seen": 224919552
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007529949849548646,
+      "loss": 1.6604,
+      "theoretical_loss": 4.320674583306396,
+      "tokens_seen": 224985088
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007529789368104314,
+      "loss": 1.6649,
+      "theoretical_loss": 4.32052100674767,
+      "tokens_seen": 225050624
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.000752962888665998,
+      "loss": 1.7843,
+      "theoretical_loss": 4.320367487422891,
+      "tokens_seen": 225116160
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007529468405215648,
+      "loss": 1.6409,
+      "theoretical_loss": 4.3202140252940735,
+      "tokens_seen": 225181696
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007529307923771314,
+      "loss": 1.6356,
+      "theoretical_loss": 4.3200606203232725,
+      "tokens_seen": 225247232
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007529147442326982,
+      "loss": 1.7784,
+      "theoretical_loss": 4.319907272472579,
+      "tokens_seen": 225312768
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007528986960882648,
+      "loss": 1.6853,
+      "theoretical_loss": 4.319753981704119,
+      "tokens_seen": 225378304
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007528826479438316,
+      "loss": 1.7475,
+      "theoretical_loss": 4.319600747980056,
+      "tokens_seen": 225443840
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007528665997993982,
+      "loss": 1.6991,
+      "theoretical_loss": 4.319447571262586,
+      "tokens_seen": 225509376
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.000752850551654965,
+      "loss": 1.6661,
+      "theoretical_loss": 4.3192944515139455,
+      "tokens_seen": 225574912
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007528345035105316,
+      "loss": 1.6993,
+      "theoretical_loss": 4.319141388696404,
+      "tokens_seen": 225640448
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007528184553660984,
+      "loss": 1.7316,
+      "theoretical_loss": 4.318988382772269,
+      "tokens_seen": 225705984
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.000752802407221665,
+      "loss": 1.7177,
+      "theoretical_loss": 4.318835433703883,
+      "tokens_seen": 225771520
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007527863590772318,
+      "loss": 1.6656,
+      "theoretical_loss": 4.318682541453622,
+      "tokens_seen": 225837056
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007527703109327984,
+      "loss": 1.654,
+      "theoretical_loss": 4.318529705983903,
+      "tokens_seen": 225902592
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007527542627883652,
+      "loss": 1.7454,
+      "theoretical_loss": 4.318376927257174,
+      "tokens_seen": 225968128
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007527382146439318,
+      "loss": 1.6859,
+      "theoretical_loss": 4.3182242052359205,
+      "tokens_seen": 226033664
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007527221664994986,
+      "loss": 1.7374,
+      "theoretical_loss": 4.318071539882665,
+      "tokens_seen": 226099200
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007527061183550652,
+      "loss": 1.6958,
+      "theoretical_loss": 4.317918931159963,
+      "tokens_seen": 226164736
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.000752690070210632,
+      "loss": 1.7292,
+      "theoretical_loss": 4.317766379030407,
+      "tokens_seen": 226230272
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007526740220661986,
+      "loss": 1.641,
+      "theoretical_loss": 4.317613883456625,
+      "tokens_seen": 226295808
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007526579739217654,
+      "loss": 1.7295,
+      "theoretical_loss": 4.317461444401281,
+      "tokens_seen": 226361344
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.000752641925777332,
+      "loss": 1.7554,
+      "theoretical_loss": 4.317309061827073,
+      "tokens_seen": 226426880
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007526258776328988,
+      "loss": 1.7057,
+      "theoretical_loss": 4.317156735696736,
+      "tokens_seen": 226492416
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007526098294884654,
+      "loss": 1.7868,
+      "theoretical_loss": 4.3170044659730396,
+      "tokens_seen": 226557952
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007525937813440322,
+      "loss": 1.7038,
+      "theoretical_loss": 4.316852252618788,
+      "tokens_seen": 226623488
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007525777331995988,
+      "loss": 1.5657,
+      "theoretical_loss": 4.316700095596822,
+      "tokens_seen": 226689024
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007525616850551656,
+      "loss": 1.7543,
+      "theoretical_loss": 4.3165479948700165,
+      "tokens_seen": 226754560
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007525456369107322,
+      "loss": 1.7003,
+      "theoretical_loss": 4.316395950401283,
+      "tokens_seen": 226820096
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.000752529588766299,
+      "loss": 1.6757,
+      "theoretical_loss": 4.316243962153566,
+      "tokens_seen": 226885632
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007525135406218656,
+      "loss": 1.7257,
+      "theoretical_loss": 4.316092030089847,
+      "tokens_seen": 226951168
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007524974924774324,
+      "loss": 1.6205,
+      "theoretical_loss": 4.315940154173143,
+      "tokens_seen": 227016704
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.000752481444332999,
+      "loss": 1.735,
+      "theoretical_loss": 4.315788334366504,
+      "tokens_seen": 227082240
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007524653961885658,
+      "loss": 1.7005,
+      "theoretical_loss": 4.315636570633015,
+      "tokens_seen": 227147776
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007524493480441324,
+      "loss": 1.7894,
+      "theoretical_loss": 4.315484862935799,
+      "tokens_seen": 227213312
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007524332998996992,
+      "loss": 1.6059,
+      "theoretical_loss": 4.315333211238009,
+      "tokens_seen": 227278848
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007524172517552658,
+      "loss": 1.739,
+      "theoretical_loss": 4.315181615502838,
+      "tokens_seen": 227344384
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007524012036108326,
+      "loss": 1.5948,
+      "theoretical_loss": 4.31503007569351,
+      "tokens_seen": 227409920
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007523851554663992,
+      "loss": 1.6074,
+      "theoretical_loss": 4.314878591773287,
+      "tokens_seen": 227475456
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007523691073219659,
+      "loss": 1.6405,
+      "theoretical_loss": 4.31472716370546,
+      "tokens_seen": 227540992
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007523530591775327,
+      "loss": 1.6388,
+      "theoretical_loss": 4.314575791453362,
+      "tokens_seen": 227606528
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007523370110330993,
+      "loss": 1.6625,
+      "theoretical_loss": 4.314424474980355,
+      "tokens_seen": 227672064
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.000752320962888666,
+      "loss": 1.5953,
+      "theoretical_loss": 4.31427321424984,
+      "tokens_seen": 227737600
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007523049147442327,
+      "loss": 1.736,
+      "theoretical_loss": 4.314122009225248,
+      "tokens_seen": 227803136
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007522888665997995,
+      "loss": 1.7958,
+      "theoretical_loss": 4.313970859870048,
+      "tokens_seen": 227868672
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007522728184553661,
+      "loss": 1.7519,
+      "theoretical_loss": 4.313819766147741,
+      "tokens_seen": 227934208
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007522567703109329,
+      "loss": 1.7792,
+      "theoretical_loss": 4.313668728021866,
+      "tokens_seen": 227999744
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007522407221664995,
+      "loss": 1.7721,
+      "theoretical_loss": 4.313517745455991,
+      "tokens_seen": 228065280
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007522246740220661,
+      "loss": 1.635,
+      "theoretical_loss": 4.313366818413724,
+      "tokens_seen": 228130816
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007522086258776329,
+      "loss": 1.7641,
+      "theoretical_loss": 4.313215946858702,
+      "tokens_seen": 228196352
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007521925777331995,
+      "loss": 1.6951,
+      "theoretical_loss": 4.313065130754601,
+      "tokens_seen": 228261888
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007521765295887663,
+      "loss": 1.6935,
+      "theoretical_loss": 4.312914370065128,
+      "tokens_seen": 228327424
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.000752160481444333,
+      "loss": 1.7565,
+      "theoretical_loss": 4.312763664754025,
+      "tokens_seen": 228392960
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007521444332998997,
+      "loss": 1.6678,
+      "theoretical_loss": 4.312613014785069,
+      "tokens_seen": 228458496
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007521283851554664,
+      "loss": 1.7712,
+      "theoretical_loss": 4.3124624201220705,
+      "tokens_seen": 228524032
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007521123370110331,
+      "loss": 1.5884,
+      "theoretical_loss": 4.312311880728873,
+      "tokens_seen": 228589568
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007520962888665998,
+      "loss": 1.7449,
+      "theoretical_loss": 4.312161396569356,
+      "tokens_seen": 228655104
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007520802407221665,
+      "loss": 1.774,
+      "theoretical_loss": 4.312010967607431,
+      "tokens_seen": 228720640
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007520641925777332,
+      "loss": 1.7603,
+      "theoretical_loss": 4.311860593807045,
+      "tokens_seen": 228786176
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007520481444332999,
+      "loss": 1.7659,
+      "theoretical_loss": 4.311710275132178,
+      "tokens_seen": 228851712
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007520320962888666,
+      "loss": 1.7087,
+      "theoretical_loss": 4.3115600115468435,
+      "tokens_seen": 228917248
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007520160481444333,
+      "loss": 1.7327,
+      "theoretical_loss": 4.311409803015089,
+      "tokens_seen": 228982784
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.000752,
+      "loss": 1.7193,
+      "theoretical_loss": 4.3112596495009985,
+      "tokens_seen": 229048320
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007519839518555667,
+      "loss": 1.8119,
+      "theoretical_loss": 4.311109550968684,
+      "tokens_seen": 229113856
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007519679037111334,
+      "loss": 1.8058,
+      "theoretical_loss": 4.310959507382296,
+      "tokens_seen": 229179392
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007519518555667001,
+      "loss": 1.6538,
+      "theoretical_loss": 4.310809518706017,
+      "tokens_seen": 229244928
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007519358074222669,
+      "loss": 1.7477,
+      "theoretical_loss": 4.310659584904062,
+      "tokens_seen": 229310464
+    },
+    {
+      "epoch": 0.07,
+      "objective/train/docs_used": 92008,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.6449694633483887,
+      "objective/train/theoretical_loss": 4.310509705940682,
+      "objective/train/tokens_used": 249836000,
+      "theoretical_loss": 4.310509705940682,
+      "tokens_seen": 229376000
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007519197592778335,
+      "loss": 1.645,
+      "theoretical_loss": 4.310509705940682,
+      "tokens_seen": 229376000
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007519037111334003,
+      "loss": 1.74,
+      "theoretical_loss": 4.310359881780159,
+      "tokens_seen": 229441536
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007518876629889669,
+      "loss": 1.761,
+      "theoretical_loss": 4.310210112386811,
+      "tokens_seen": 229507072
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007518716148445337,
+      "loss": 1.7839,
+      "theoretical_loss": 4.310060397724987,
+      "tokens_seen": 229572608
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007518555667001003,
+      "loss": 1.7724,
+      "theoretical_loss": 4.30991073775907,
+      "tokens_seen": 229638144
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007518395185556671,
+      "loss": 1.7281,
+      "theoretical_loss": 4.3097611324534775,
+      "tokens_seen": 229703680
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007518234704112337,
+      "loss": 1.7916,
+      "theoretical_loss": 4.309611581772659,
+      "tokens_seen": 229769216
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007518074222668005,
+      "loss": 1.7421,
+      "theoretical_loss": 4.309462085681098,
+      "tokens_seen": 229834752
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007517913741223671,
+      "loss": 1.6439,
+      "theoretical_loss": 4.3093126441433105,
+      "tokens_seen": 229900288
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007517753259779339,
+      "loss": 1.6821,
+      "theoretical_loss": 4.3091632571238465,
+      "tokens_seen": 229965824
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007517592778335005,
+      "loss": 1.7777,
+      "theoretical_loss": 4.30901392458729,
+      "tokens_seen": 230031360
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007517432296890673,
+      "loss": 1.7295,
+      "theoretical_loss": 4.308864646498254,
+      "tokens_seen": 230096896
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007517271815446339,
+      "loss": 1.505,
+      "theoretical_loss": 4.30871542282139,
+      "tokens_seen": 230162432
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007517111334002007,
+      "loss": 1.7193,
+      "theoretical_loss": 4.308566253521378,
+      "tokens_seen": 230227968
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007516950852557673,
+      "loss": 1.8099,
+      "theoretical_loss": 4.308417138562934,
+      "tokens_seen": 230293504
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007516790371113341,
+      "loss": 1.693,
+      "theoretical_loss": 4.308268077910806,
+      "tokens_seen": 230359040
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007516629889669007,
+      "loss": 1.7438,
+      "theoretical_loss": 4.308119071529774,
+      "tokens_seen": 230424576
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007516469408224675,
+      "loss": 1.7061,
+      "theoretical_loss": 4.3079701193846525,
+      "tokens_seen": 230490112
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007516308926780341,
+      "loss": 1.7257,
+      "theoretical_loss": 4.307821221440286,
+      "tokens_seen": 230555648
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007516148445336009,
+      "loss": 1.7924,
+      "theoretical_loss": 4.3076723776615555,
+      "tokens_seen": 230621184
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007515987963891675,
+      "loss": 1.791,
+      "theoretical_loss": 4.307523588013371,
+      "tokens_seen": 230686720
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007515827482447343,
+      "loss": 1.733,
+      "theoretical_loss": 4.307374852460679,
+      "tokens_seen": 230752256
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007515667001003009,
+      "loss": 1.6646,
+      "theoretical_loss": 4.307226170968456,
+      "tokens_seen": 230817792
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007515506519558677,
+      "loss": 1.7016,
+      "theoretical_loss": 4.307077543501713,
+      "tokens_seen": 230883328
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007515346038114343,
+      "loss": 1.5857,
+      "theoretical_loss": 4.3069289700254885,
+      "tokens_seen": 230948864
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007515185556670011,
+      "loss": 1.6563,
+      "theoretical_loss": 4.306780450504861,
+      "tokens_seen": 231014400
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007515025075225677,
+      "loss": 1.6707,
+      "theoretical_loss": 4.3066319849049375,
+      "tokens_seen": 231079936
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007514864593781345,
+      "loss": 1.7898,
+      "theoretical_loss": 4.306483573190857,
+      "tokens_seen": 231145472
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007514704112337011,
+      "loss": 1.932,
+      "theoretical_loss": 4.306335215327791,
+      "tokens_seen": 231211008
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007514543630892679,
+      "loss": 1.7709,
+      "theoretical_loss": 4.306186911280945,
+      "tokens_seen": 231276544
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007514383149448345,
+      "loss": 1.6175,
+      "theoretical_loss": 4.306038661015557,
+      "tokens_seen": 231342080
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007514222668004013,
+      "loss": 1.7677,
+      "theoretical_loss": 4.305890464496895,
+      "tokens_seen": 231407616
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007514062186559679,
+      "loss": 1.8163,
+      "theoretical_loss": 4.305742321690261,
+      "tokens_seen": 231473152
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007513901705115347,
+      "loss": 1.7925,
+      "theoretical_loss": 4.305594232560988,
+      "tokens_seen": 231538688
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007513741223671013,
+      "loss": 1.7761,
+      "theoretical_loss": 4.305446197074443,
+      "tokens_seen": 231604224
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007513580742226681,
+      "loss": 1.6571,
+      "theoretical_loss": 4.305298215196023,
+      "tokens_seen": 231669760
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007513420260782347,
+      "loss": 1.7638,
+      "theoretical_loss": 4.305150286891159,
+      "tokens_seen": 231735296
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007513259779338015,
+      "loss": 1.7255,
+      "theoretical_loss": 4.3050024121253125,
+      "tokens_seen": 231800832
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007513099297893681,
+      "loss": 1.7697,
+      "theoretical_loss": 4.304854590863979,
+      "tokens_seen": 231866368
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007512938816449349,
+      "loss": 1.6632,
+      "theoretical_loss": 4.304706823072683,
+      "tokens_seen": 231931904
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007512778335005015,
+      "loss": 1.8148,
+      "theoretical_loss": 4.304559108716984,
+      "tokens_seen": 231997440
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007512617853560683,
+      "loss": 1.5755,
+      "theoretical_loss": 4.304411447762472,
+      "tokens_seen": 232062976
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007512457372116349,
+      "loss": 1.7349,
+      "theoretical_loss": 4.304263840174768,
+      "tokens_seen": 232128512
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007512296890672017,
+      "loss": 1.8418,
+      "theoretical_loss": 4.304116285919527,
+      "tokens_seen": 232194048
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007512136409227683,
+      "loss": 1.7876,
+      "theoretical_loss": 4.303968784962433,
+      "tokens_seen": 232259584
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007511975927783351,
+      "loss": 1.7659,
+      "theoretical_loss": 4.303821337269205,
+      "tokens_seen": 232325120
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007511815446339017,
+      "loss": 1.5599,
+      "theoretical_loss": 4.303673942805591,
+      "tokens_seen": 232390656
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007511654964894685,
+      "loss": 1.7451,
+      "theoretical_loss": 4.303526601537371,
+      "tokens_seen": 232456192
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007511494483450351,
+      "loss": 1.7257,
+      "theoretical_loss": 4.303379313430359,
+      "tokens_seen": 232521728
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007511334002006019,
+      "loss": 1.6267,
+      "theoretical_loss": 4.303232078450399,
+      "tokens_seen": 232587264
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007511173520561685,
+      "loss": 1.5978,
+      "theoretical_loss": 4.303084896563364,
+      "tokens_seen": 232652800
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007511013039117353,
+      "loss": 1.7081,
+      "theoretical_loss": 4.302937767735164,
+      "tokens_seen": 232718336
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.000751085255767302,
+      "loss": 1.7253,
+      "theoretical_loss": 4.302790691931736,
+      "tokens_seen": 232783872
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007510692076228687,
+      "loss": 1.7024,
+      "theoretical_loss": 4.30264366911905,
+      "tokens_seen": 232849408
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007510531594784353,
+      "loss": 1.8108,
+      "theoretical_loss": 4.3024966992631075,
+      "tokens_seen": 232914944
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007510371113340021,
+      "loss": 1.758,
+      "theoretical_loss": 4.302349782329942,
+      "tokens_seen": 232980480
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007510210631895687,
+      "loss": 1.7161,
+      "theoretical_loss": 4.302202918285616,
+      "tokens_seen": 233046016
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007510050150451355,
+      "loss": 1.6465,
+      "theoretical_loss": 4.302056107096226,
+      "tokens_seen": 233111552
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007509889669007021,
+      "loss": 1.6963,
+      "theoretical_loss": 4.301909348727899,
+      "tokens_seen": 233177088
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007509729187562688,
+      "loss": 1.7785,
+      "theoretical_loss": 4.301762643146792,
+      "tokens_seen": 233242624
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007509568706118356,
+      "loss": 1.7259,
+      "theoretical_loss": 4.301615990319094,
+      "tokens_seen": 233308160
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007509408224674022,
+      "loss": 1.6942,
+      "theoretical_loss": 4.301469390211025,
+      "tokens_seen": 233373696
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.000750924774322969,
+      "loss": 1.6251,
+      "theoretical_loss": 4.301322842788839,
+      "tokens_seen": 233439232
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007509087261785356,
+      "loss": 1.6583,
+      "theoretical_loss": 4.301176348018814,
+      "tokens_seen": 233504768
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007508926780341024,
+      "loss": 1.7607,
+      "theoretical_loss": 4.301029905867265,
+      "tokens_seen": 233570304
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.000750876629889669,
+      "loss": 1.716,
+      "theoretical_loss": 4.300883516300539,
+      "tokens_seen": 233635840
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007508605817452358,
+      "loss": 1.6477,
+      "theoretical_loss": 4.300737179285008,
+      "tokens_seen": 233701376
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007508445336008024,
+      "loss": 1.6472,
+      "theoretical_loss": 4.30059089478708,
+      "tokens_seen": 233766912
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.000750828485456369,
+      "loss": 1.7246,
+      "theoretical_loss": 4.300444662773192,
+      "tokens_seen": 233832448
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007508124373119358,
+      "loss": 1.7401,
+      "theoretical_loss": 4.30029848320981,
+      "tokens_seen": 233897984
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007507963891675024,
+      "loss": 1.7319,
+      "theoretical_loss": 4.300152356063435,
+      "tokens_seen": 233963520
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007507803410230692,
+      "loss": 1.7894,
+      "theoretical_loss": 4.300006281300598,
+      "tokens_seen": 234029056
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007507642928786359,
+      "loss": 1.5778,
+      "theoretical_loss": 4.299860258887856,
+      "tokens_seen": 234094592
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007507482447342026,
+      "loss": 1.705,
+      "theoretical_loss": 4.299714288791802,
+      "tokens_seen": 234160128
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007507321965897694,
+      "loss": 1.7241,
+      "theoretical_loss": 4.299568370979056,
+      "tokens_seen": 234225664
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.000750716148445336,
+      "loss": 1.6857,
+      "theoretical_loss": 4.299422505416272,
+      "tokens_seen": 234291200
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007507001003009028,
+      "loss": 1.7266,
+      "theoretical_loss": 4.2992766920701335,
+      "tokens_seen": 234356736
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007506840521564694,
+      "loss": 1.589,
+      "theoretical_loss": 4.299130930907352,
+      "tokens_seen": 234422272
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007506680040120362,
+      "loss": 1.7423,
+      "theoretical_loss": 4.2989852218946725,
+      "tokens_seen": 234487808
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007506519558676028,
+      "loss": 1.6108,
+      "theoretical_loss": 4.298839564998869,
+      "tokens_seen": 234553344
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007506359077231696,
+      "loss": 1.7754,
+      "theoretical_loss": 4.298693960186747,
+      "tokens_seen": 234618880
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007506198595787362,
+      "loss": 1.7338,
+      "theoretical_loss": 4.298548407425141,
+      "tokens_seen": 234684416
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.000750603811434303,
+      "loss": 1.8418,
+      "theoretical_loss": 4.298402906680918,
+      "tokens_seen": 234749952
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007505877632898696,
+      "loss": 1.7488,
+      "theoretical_loss": 4.298257457920973,
+      "tokens_seen": 234815488
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007505717151454364,
+      "loss": 1.7443,
+      "theoretical_loss": 4.298112061112232,
+      "tokens_seen": 234881024
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.000750555667001003,
+      "loss": 1.8159,
+      "theoretical_loss": 4.297966716221653,
+      "tokens_seen": 234946560
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007505396188565698,
+      "loss": 1.685,
+      "theoretical_loss": 4.297821423216222,
+      "tokens_seen": 235012096
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007505235707121364,
+      "loss": 1.7925,
+      "theoretical_loss": 4.297676182062955,
+      "tokens_seen": 235077632
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007505075225677032,
+      "loss": 1.7185,
+      "theoretical_loss": 4.297530992728901,
+      "tokens_seen": 235143168
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007504914744232698,
+      "loss": 1.67,
+      "theoretical_loss": 4.297385855181136,
+      "tokens_seen": 235208704
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007504754262788366,
+      "loss": 1.728,
+      "theoretical_loss": 4.297240769386768,
+      "tokens_seen": 235274240
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007504593781344032,
+      "loss": 1.6595,
+      "theoretical_loss": 4.297095735312934,
+      "tokens_seen": 235339776
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00075044332998997,
+      "loss": 1.609,
+      "theoretical_loss": 4.296950752926801,
+      "tokens_seen": 235405312
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007504272818455366,
+      "loss": 1.7526,
+      "theoretical_loss": 4.296805822195568,
+      "tokens_seen": 235470848
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007504112337011034,
+      "loss": 1.6949,
+      "theoretical_loss": 4.29666094308646,
+      "tokens_seen": 235536384
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00075039518555667,
+      "loss": 1.7873,
+      "theoretical_loss": 4.296516115566736,
+      "tokens_seen": 235601920
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007503791374122368,
+      "loss": 1.7357,
+      "theoretical_loss": 4.296371339603684,
+      "tokens_seen": 235667456
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007503630892678034,
+      "loss": 1.7718,
+      "theoretical_loss": 4.296226615164618,
+      "tokens_seen": 235732992
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007503470411233702,
+      "loss": 1.6562,
+      "theoretical_loss": 4.296081942216887,
+      "tokens_seen": 235798528
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007503309929789368,
+      "loss": 1.7045,
+      "theoretical_loss": 4.295937320727866,
+      "tokens_seen": 235864064
+    },
+    {
+      "epoch": 0.07,
+      "objective/train/docs_used": 94470,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.7614660263061523,
+      "objective/train/theoretical_loss": 4.295792750664964,
+      "objective/train/tokens_used": 256389600,
+      "theoretical_loss": 4.295792750664964,
+      "tokens_seen": 235929600
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007503149448345036,
+      "loss": 1.7615,
+      "theoretical_loss": 4.295792750664964,
+      "tokens_seen": 235929600
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007502988966900702,
+      "loss": 1.5895,
+      "theoretical_loss": 4.295648231995613,
+      "tokens_seen": 235995136
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.000750282848545637,
+      "loss": 1.6453,
+      "theoretical_loss": 4.295503764687281,
+      "tokens_seen": 236060672
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007502668004012036,
+      "loss": 1.7145,
+      "theoretical_loss": 4.295359348707462,
+      "tokens_seen": 236126208
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007502507522567704,
+      "loss": 1.8196,
+      "theoretical_loss": 4.2952149840236835,
+      "tokens_seen": 236191744
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.000750234704112337,
+      "loss": 1.6946,
+      "theoretical_loss": 4.295070670603496,
+      "tokens_seen": 236257280
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007502186559679038,
+      "loss": 1.7409,
+      "theoretical_loss": 4.294926408414486,
+      "tokens_seen": 236322816
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007502026078234704,
+      "loss": 1.7793,
+      "theoretical_loss": 4.294782197424265,
+      "tokens_seen": 236388352
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007501865596790372,
+      "loss": 1.6357,
+      "theoretical_loss": 4.294638037600478,
+      "tokens_seen": 236453888
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007501705115346038,
+      "loss": 1.7133,
+      "theoretical_loss": 4.2944939289107955,
+      "tokens_seen": 236519424
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007501544633901706,
+      "loss": 1.7445,
+      "theoretical_loss": 4.29434987132292,
+      "tokens_seen": 236584960
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007501384152457372,
+      "loss": 1.6315,
+      "theoretical_loss": 4.294205864804581,
+      "tokens_seen": 236650496
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.000750122367101304,
+      "loss": 1.7542,
+      "theoretical_loss": 4.294061909323539,
+      "tokens_seen": 236716032
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007501063189568706,
+      "loss": 1.7551,
+      "theoretical_loss": 4.293918004847585,
+      "tokens_seen": 236781568
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007500902708124374,
+      "loss": 1.7341,
+      "theoretical_loss": 4.293774151344537,
+      "tokens_seen": 236847104
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.000750074222668004,
+      "loss": 1.7155,
+      "theoretical_loss": 4.2936303487822425,
+      "tokens_seen": 236912640
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007500581745235708,
+      "loss": 1.7439,
+      "theoretical_loss": 4.293486597128578,
+      "tokens_seen": 236978176
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007500421263791374,
+      "loss": 1.7712,
+      "theoretical_loss": 4.293342896351452,
+      "tokens_seen": 237043712
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007500260782347042,
+      "loss": 1.6608,
+      "theoretical_loss": 4.293199246418796,
+      "tokens_seen": 237109248
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007500100300902708,
+      "loss": 1.7285,
+      "theoretical_loss": 4.293055647298578,
+      "tokens_seen": 237174784
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007499939819458376,
+      "loss": 1.6871,
+      "theoretical_loss": 4.292912098958791,
+      "tokens_seen": 237240320
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007499779338014042,
+      "loss": 1.7474,
+      "theoretical_loss": 4.292768601367455,
+      "tokens_seen": 237305856
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.000749961885656971,
+      "loss": 1.7256,
+      "theoretical_loss": 4.292625154492623,
+      "tokens_seen": 237371392
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007499458375125376,
+      "loss": 1.6761,
+      "theoretical_loss": 4.292481758302375,
+      "tokens_seen": 237436928
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007499297893681044,
+      "loss": 1.6787,
+      "theoretical_loss": 4.2923384127648205,
+      "tokens_seen": 237502464
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.000749913741223671,
+      "loss": 1.5678,
+      "theoretical_loss": 4.292195117848098,
+      "tokens_seen": 237568000
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007498976930792378,
+      "loss": 1.6483,
+      "theoretical_loss": 4.292051873520373,
+      "tokens_seen": 237633536
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007498816449348044,
+      "loss": 1.7666,
+      "theoretical_loss": 4.291908679749842,
+      "tokens_seen": 237699072
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007498655967903712,
+      "loss": 1.6182,
+      "theoretical_loss": 4.2917655365047285,
+      "tokens_seen": 237764608
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007498495486459378,
+      "loss": 1.6742,
+      "theoretical_loss": 4.291622443753287,
+      "tokens_seen": 237830144
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007498335005015046,
+      "loss": 1.6149,
+      "theoretical_loss": 4.291479401463799,
+      "tokens_seen": 237895680
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007498174523570712,
+      "loss": 1.7008,
+      "theoretical_loss": 4.291336409604574,
+      "tokens_seen": 237961216
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.000749801404212638,
+      "loss": 1.6942,
+      "theoretical_loss": 4.291193468143952,
+      "tokens_seen": 238026752
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007497853560682046,
+      "loss": 1.66,
+      "theoretical_loss": 4.2910505770503,
+      "tokens_seen": 238092288
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007497693079237714,
+      "loss": 1.7244,
+      "theoretical_loss": 4.290907736292016,
+      "tokens_seen": 238157824
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.000749753259779338,
+      "loss": 1.7439,
+      "theoretical_loss": 4.290764945837522,
+      "tokens_seen": 238223360
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007497372116349048,
+      "loss": 1.6318,
+      "theoretical_loss": 4.290622205655273,
+      "tokens_seen": 238288896
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007497211634904714,
+      "loss": 1.6553,
+      "theoretical_loss": 4.290479515713751,
+      "tokens_seen": 238354432
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007497051153460382,
+      "loss": 1.7517,
+      "theoretical_loss": 4.290336875981465,
+      "tokens_seen": 238419968
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007496890672016048,
+      "loss": 1.637,
+      "theoretical_loss": 4.290194286426955,
+      "tokens_seen": 238485504
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007496730190571716,
+      "loss": 1.6891,
+      "theoretical_loss": 4.290051747018785,
+      "tokens_seen": 238551040
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007496569709127382,
+      "loss": 1.6317,
+      "theoretical_loss": 4.289909257725553,
+      "tokens_seen": 238616576
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.000749640922768305,
+      "loss": 1.666,
+      "theoretical_loss": 4.289766818515882,
+      "tokens_seen": 238682112
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007496248746238716,
+      "loss": 1.6351,
+      "theoretical_loss": 4.2896244293584225,
+      "tokens_seen": 238747648
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007496088264794384,
+      "loss": 1.7994,
+      "theoretical_loss": 4.2894820902218544,
+      "tokens_seen": 238813184
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.000749592778335005,
+      "loss": 1.7,
+      "theoretical_loss": 4.289339801074888,
+      "tokens_seen": 238878720
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007495767301905718,
+      "loss": 1.7788,
+      "theoretical_loss": 4.289197561886258,
+      "tokens_seen": 238944256
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007495606820461385,
+      "loss": 1.7326,
+      "theoretical_loss": 4.289055372624729,
+      "tokens_seen": 239009792
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007495446339017051,
+      "loss": 1.6691,
+      "theoretical_loss": 4.288913233259092,
+      "tokens_seen": 239075328
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007495285857572719,
+      "loss": 1.6302,
+      "theoretical_loss": 4.28877114375817,
+      "tokens_seen": 239140864
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007495125376128385,
+      "loss": 1.8139,
+      "theoretical_loss": 4.288629104090811,
+      "tokens_seen": 239206400
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007494964894684053,
+      "loss": 1.7045,
+      "theoretical_loss": 4.28848711422589,
+      "tokens_seen": 239271936
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007494804413239719,
+      "loss": 1.7913,
+      "theoretical_loss": 4.288345174132313,
+      "tokens_seen": 239337472
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007494643931795387,
+      "loss": 1.7044,
+      "theoretical_loss": 4.288203283779012,
+      "tokens_seen": 239403008
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007494483450351053,
+      "loss": 1.6033,
+      "theoretical_loss": 4.288061443134947,
+      "tokens_seen": 239468544
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007494322968906721,
+      "loss": 1.7749,
+      "theoretical_loss": 4.287919652169107,
+      "tokens_seen": 239534080
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007494162487462387,
+      "loss": 1.7446,
+      "theoretical_loss": 4.287777910850507,
+      "tokens_seen": 239599616
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007494002006018055,
+      "loss": 1.8335,
+      "theoretical_loss": 4.287636219148191,
+      "tokens_seen": 239665152
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007493841524573721,
+      "loss": 1.6768,
+      "theoretical_loss": 4.287494577031231,
+      "tokens_seen": 239730688
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007493681043129389,
+      "loss": 1.6988,
+      "theoretical_loss": 4.287352984468727,
+      "tokens_seen": 239796224
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007493520561685055,
+      "loss": 1.8031,
+      "theoretical_loss": 4.287211441429805,
+      "tokens_seen": 239861760
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007493360080240723,
+      "loss": 1.7529,
+      "theoretical_loss": 4.2870699478836185,
+      "tokens_seen": 239927296
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007493199598796389,
+      "loss": 1.6927,
+      "theoretical_loss": 4.286928503799352,
+      "tokens_seen": 239992832
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007493039117352057,
+      "loss": 1.7997,
+      "theoretical_loss": 4.286787109146213,
+      "tokens_seen": 240058368
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007492878635907723,
+      "loss": 1.6076,
+      "theoretical_loss": 4.286645763893442,
+      "tokens_seen": 240123904
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007492718154463391,
+      "loss": 1.6384,
+      "theoretical_loss": 4.2865044680103015,
+      "tokens_seen": 240189440
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007492557673019057,
+      "loss": 1.5811,
+      "theoretical_loss": 4.286363221466084,
+      "tokens_seen": 240254976
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007492397191574725,
+      "loss": 1.7501,
+      "theoretical_loss": 4.286222024230111,
+      "tokens_seen": 240320512
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007492236710130391,
+      "loss": 1.5877,
+      "theoretical_loss": 4.286080876271729,
+      "tokens_seen": 240386048
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007492076228686059,
+      "loss": 1.718,
+      "theoretical_loss": 4.285939777560312,
+      "tokens_seen": 240451584
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007491915747241725,
+      "loss": 1.7557,
+      "theoretical_loss": 4.285798728065264,
+      "tokens_seen": 240517120
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007491755265797393,
+      "loss": 1.7876,
+      "theoretical_loss": 4.285657727756012,
+      "tokens_seen": 240582656
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007491594784353059,
+      "loss": 1.6534,
+      "theoretical_loss": 4.285516776602016,
+      "tokens_seen": 240648192
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007491434302908727,
+      "loss": 1.6575,
+      "theoretical_loss": 4.2853758745727575,
+      "tokens_seen": 240713728
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007491273821464393,
+      "loss": 1.7133,
+      "theoretical_loss": 4.285235021637748,
+      "tokens_seen": 240779264
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007491113340020061,
+      "loss": 1.7635,
+      "theoretical_loss": 4.285094217766527,
+      "tokens_seen": 240844800
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007490952858575727,
+      "loss": 1.7549,
+      "theoretical_loss": 4.2849534629286605,
+      "tokens_seen": 240910336
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007490792377131395,
+      "loss": 1.7171,
+      "theoretical_loss": 4.284812757093741,
+      "tokens_seen": 240975872
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007490631895687061,
+      "loss": 1.6313,
+      "theoretical_loss": 4.284672100231388,
+      "tokens_seen": 241041408
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007490471414242729,
+      "loss": 1.6704,
+      "theoretical_loss": 4.284531492311248,
+      "tokens_seen": 241106944
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007490310932798395,
+      "loss": 1.6644,
+      "theoretical_loss": 4.284390933302997,
+      "tokens_seen": 241172480
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007490150451354063,
+      "loss": 1.7358,
+      "theoretical_loss": 4.284250423176335,
+      "tokens_seen": 241238016
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007489989969909729,
+      "loss": 1.6679,
+      "theoretical_loss": 4.28410996190099,
+      "tokens_seen": 241303552
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007489829488465397,
+      "loss": 1.562,
+      "theoretical_loss": 4.283969549446717,
+      "tokens_seen": 241369088
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007489669007021063,
+      "loss": 1.6372,
+      "theoretical_loss": 4.2838291857833,
+      "tokens_seen": 241434624
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007489508525576731,
+      "loss": 1.7309,
+      "theoretical_loss": 4.283688870880545,
+      "tokens_seen": 241500160
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007489348044132397,
+      "loss": 1.7814,
+      "theoretical_loss": 4.28354860470829,
+      "tokens_seen": 241565696
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007489187562688065,
+      "loss": 1.6999,
+      "theoretical_loss": 4.283408387236397,
+      "tokens_seen": 241631232
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007489027081243731,
+      "loss": 1.6034,
+      "theoretical_loss": 4.283268218434755,
+      "tokens_seen": 241696768
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007488866599799399,
+      "loss": 1.8609,
+      "theoretical_loss": 4.283128098273281,
+      "tokens_seen": 241762304
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007488706118355065,
+      "loss": 1.8951,
+      "theoretical_loss": 4.282988026721917,
+      "tokens_seen": 241827840
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007488545636910733,
+      "loss": 1.6055,
+      "theoretical_loss": 4.282848003750634,
+      "tokens_seen": 241893376
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007488385155466399,
+      "loss": 1.5376,
+      "theoretical_loss": 4.282708029329427,
+      "tokens_seen": 241958912
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007488224674022067,
+      "loss": 1.7143,
+      "theoretical_loss": 4.2825681034283205,
+      "tokens_seen": 242024448
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007488064192577733,
+      "loss": 1.6248,
+      "theoretical_loss": 4.2824282260173625,
+      "tokens_seen": 242089984
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007487903711133401,
+      "loss": 1.6945,
+      "theoretical_loss": 4.282288397066631,
+      "tokens_seen": 242155520
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007487743229689067,
+      "loss": 1.6584,
+      "theoretical_loss": 4.282148616546227,
+      "tokens_seen": 242221056
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007487582748244735,
+      "loss": 1.6498,
+      "theoretical_loss": 4.282008884426281,
+      "tokens_seen": 242286592
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007487422266800401,
+      "loss": 1.7651,
+      "theoretical_loss": 4.281869200676948,
+      "tokens_seen": 242352128
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007487261785356069,
+      "loss": 1.7393,
+      "theoretical_loss": 4.281729565268412,
+      "tokens_seen": 242417664
+    },
+    {
+      "epoch": 0.07,
+      "objective/train/docs_used": 97293,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.70721435546875,
+      "objective/train/theoretical_loss": 4.28158997817088,
+      "objective/train/tokens_used": 262943200,
+      "theoretical_loss": 4.28158997817088,
+      "tokens_seen": 242483200
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007487101303911735,
+      "loss": 1.7072,
+      "theoretical_loss": 4.28158997817088,
+      "tokens_seen": 242483200
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007486940822467403,
+      "loss": 1.6776,
+      "theoretical_loss": 4.281450439354587,
+      "tokens_seen": 242548736
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007486780341023069,
+      "loss": 1.6901,
+      "theoretical_loss": 4.281310948789796,
+      "tokens_seen": 242614272
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007486619859578737,
+      "loss": 1.6518,
+      "theoretical_loss": 4.2811715064467935,
+      "tokens_seen": 242679808
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007486459378134403,
+      "loss": 1.6409,
+      "theoretical_loss": 4.281032112295894,
+      "tokens_seen": 242745344
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007486298896690071,
+      "loss": 1.6907,
+      "theoretical_loss": 4.2808927663074385,
+      "tokens_seen": 242810880
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007486138415245737,
+      "loss": 1.7182,
+      "theoretical_loss": 4.280753468451793,
+      "tokens_seen": 242876416
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007485977933801405,
+      "loss": 1.6676,
+      "theoretical_loss": 4.28061421869935,
+      "tokens_seen": 242941952
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007485817452357071,
+      "loss": 1.6912,
+      "theoretical_loss": 4.28047501702053,
+      "tokens_seen": 243007488
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007485656970912739,
+      "loss": 1.7135,
+      "theoretical_loss": 4.280335863385778,
+      "tokens_seen": 243073024
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007485496489468405,
+      "loss": 1.6867,
+      "theoretical_loss": 4.280196757765564,
+      "tokens_seen": 243138560
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007485336008024073,
+      "loss": 1.7557,
+      "theoretical_loss": 4.280057700130388,
+      "tokens_seen": 243204096
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007485175526579739,
+      "loss": 1.6174,
+      "theoretical_loss": 4.279918690450771,
+      "tokens_seen": 243269632
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007485015045135407,
+      "loss": 1.5945,
+      "theoretical_loss": 4.279779728697263,
+      "tokens_seen": 243335168
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007484854563691073,
+      "loss": 1.7017,
+      "theoretical_loss": 4.279640814840441,
+      "tokens_seen": 243400704
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007484694082246741,
+      "loss": 1.8184,
+      "theoretical_loss": 4.279501948850905,
+      "tokens_seen": 243466240
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007484533600802407,
+      "loss": 1.6976,
+      "theoretical_loss": 4.279363130699284,
+      "tokens_seen": 243531776
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007484373119358075,
+      "loss": 1.7968,
+      "theoretical_loss": 4.279224360356231,
+      "tokens_seen": 243597312
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007484212637913741,
+      "loss": 1.671,
+      "theoretical_loss": 4.279085637792425,
+      "tokens_seen": 243662848
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007484052156469409,
+      "loss": 1.7158,
+      "theoretical_loss": 4.278946962978573,
+      "tokens_seen": 243728384
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007483891675025075,
+      "loss": 1.6697,
+      "theoretical_loss": 4.278808335885403,
+      "tokens_seen": 243793920
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007483731193580743,
+      "loss": 1.8423,
+      "theoretical_loss": 4.278669756483674,
+      "tokens_seen": 243859456
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007483570712136409,
+      "loss": 1.7238,
+      "theoretical_loss": 4.278531224744167,
+      "tokens_seen": 243924992
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007483410230692077,
+      "loss": 1.6312,
+      "theoretical_loss": 4.278392740637694,
+      "tokens_seen": 243990528
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007483249749247745,
+      "loss": 1.6698,
+      "theoretical_loss": 4.278254304135085,
+      "tokens_seen": 244056064
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007483089267803411,
+      "loss": 1.6407,
+      "theoretical_loss": 4.2781159152072,
+      "tokens_seen": 244121600
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007482928786359077,
+      "loss": 1.6967,
+      "theoretical_loss": 4.277977573824928,
+      "tokens_seen": 244187136
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007482768304914745,
+      "loss": 1.6572,
+      "theoretical_loss": 4.277839279959177,
+      "tokens_seen": 244252672
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007482607823470411,
+      "loss": 1.581,
+      "theoretical_loss": 4.277701033580884,
+      "tokens_seen": 244318208
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007482447342026079,
+      "loss": 1.7374,
+      "theoretical_loss": 4.277562834661013,
+      "tokens_seen": 244383744
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007482286860581746,
+      "loss": 1.6872,
+      "theoretical_loss": 4.277424683170549,
+      "tokens_seen": 244449280
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007482126379137413,
+      "loss": 1.7246,
+      "theoretical_loss": 4.277286579080508,
+      "tokens_seen": 244514816
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.000748196589769308,
+      "loss": 1.6254,
+      "theoretical_loss": 4.277148522361927,
+      "tokens_seen": 244580352
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007481805416248747,
+      "loss": 1.6898,
+      "theoretical_loss": 4.277010512985871,
+      "tokens_seen": 244645888
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007481644934804414,
+      "loss": 1.6823,
+      "theoretical_loss": 4.2768725509234296,
+      "tokens_seen": 244711424
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.000748148445336008,
+      "loss": 1.6525,
+      "theoretical_loss": 4.276734636145717,
+      "tokens_seen": 244776960
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007481323971915748,
+      "loss": 1.6133,
+      "theoretical_loss": 4.276596768623875,
+      "tokens_seen": 244842496
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007481163490471414,
+      "loss": 1.6995,
+      "theoretical_loss": 4.276458948329068,
+      "tokens_seen": 244908032
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007481003009027082,
+      "loss": 1.6637,
+      "theoretical_loss": 4.276321175232487,
+      "tokens_seen": 244973568
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007480842527582748,
+      "loss": 1.7109,
+      "theoretical_loss": 4.276183449305348,
+      "tokens_seen": 245039104
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007480682046138416,
+      "loss": 1.6925,
+      "theoretical_loss": 4.276045770518895,
+      "tokens_seen": 245104640
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007480521564694082,
+      "loss": 1.632,
+      "theoretical_loss": 4.275908138844392,
+      "tokens_seen": 245170176
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.000748036108324975,
+      "loss": 1.696,
+      "theoretical_loss": 4.275770554253132,
+      "tokens_seen": 245235712
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007480200601805416,
+      "loss": 1.636,
+      "theoretical_loss": 4.275633016716432,
+      "tokens_seen": 245301248
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007480040120361084,
+      "loss": 1.6925,
+      "theoretical_loss": 4.275495526205635,
+      "tokens_seen": 245366784
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.000747987963891675,
+      "loss": 1.7086,
+      "theoretical_loss": 4.275358082692106,
+      "tokens_seen": 245432320
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007479719157472418,
+      "loss": 1.6197,
+      "theoretical_loss": 4.275220686147241,
+      "tokens_seen": 245497856
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007479558676028084,
+      "loss": 1.7857,
+      "theoretical_loss": 4.275083336542455,
+      "tokens_seen": 245563392
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007479398194583752,
+      "loss": 1.7038,
+      "theoretical_loss": 4.274946033849189,
+      "tokens_seen": 245628928
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007479237713139418,
+      "loss": 1.6159,
+      "theoretical_loss": 4.2748087780389135,
+      "tokens_seen": 245694464
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007479077231695086,
+      "loss": 1.7605,
+      "theoretical_loss": 4.27467156908312,
+      "tokens_seen": 245760000
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007478916750250752,
+      "loss": 1.6676,
+      "theoretical_loss": 4.274534406953325,
+      "tokens_seen": 245825536
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.000747875626880642,
+      "loss": 1.6935,
+      "theoretical_loss": 4.274397291621073,
+      "tokens_seen": 245891072
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007478595787362086,
+      "loss": 1.7408,
+      "theoretical_loss": 4.274260223057928,
+      "tokens_seen": 245956608
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007478435305917754,
+      "loss": 1.747,
+      "theoretical_loss": 4.274123201235483,
+      "tokens_seen": 246022144
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.000747827482447342,
+      "loss": 1.7192,
+      "theoretical_loss": 4.273986226125356,
+      "tokens_seen": 246087680
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007478114343029088,
+      "loss": 1.6998,
+      "theoretical_loss": 4.273849297699186,
+      "tokens_seen": 246153216
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007477953861584754,
+      "loss": 1.6497,
+      "theoretical_loss": 4.273712415928643,
+      "tokens_seen": 246218752
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007477793380140422,
+      "loss": 1.6862,
+      "theoretical_loss": 4.273575580785415,
+      "tokens_seen": 246284288
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007477632898696088,
+      "loss": 1.7198,
+      "theoretical_loss": 4.27343879224122,
+      "tokens_seen": 246349824
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007477472417251756,
+      "loss": 1.7479,
+      "theoretical_loss": 4.273302050267796,
+      "tokens_seen": 246415360
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007477311935807422,
+      "loss": 1.7629,
+      "theoretical_loss": 4.27316535483691,
+      "tokens_seen": 246480896
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.000747715145436309,
+      "loss": 1.6332,
+      "theoretical_loss": 4.273028705920351,
+      "tokens_seen": 246546432
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007476990972918756,
+      "loss": 1.7132,
+      "theoretical_loss": 4.272892103489934,
+      "tokens_seen": 246611968
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007476830491474424,
+      "loss": 1.8742,
+      "theoretical_loss": 4.272755547517495,
+      "tokens_seen": 246677504
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.000747667001003009,
+      "loss": 1.6886,
+      "theoretical_loss": 4.272619037974902,
+      "tokens_seen": 246743040
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007476509528585758,
+      "loss": 1.7306,
+      "theoretical_loss": 4.272482574834039,
+      "tokens_seen": 246808576
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007476349047141424,
+      "loss": 1.6748,
+      "theoretical_loss": 4.272346158066821,
+      "tokens_seen": 246874112
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007476188565697092,
+      "loss": 1.6958,
+      "theoretical_loss": 4.272209787645183,
+      "tokens_seen": 246939648
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007476028084252758,
+      "loss": 1.7104,
+      "theoretical_loss": 4.272073463541087,
+      "tokens_seen": 247005184
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007475867602808426,
+      "loss": 1.6468,
+      "theoretical_loss": 4.271937185726519,
+      "tokens_seen": 247070720
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007475707121364092,
+      "loss": 1.6516,
+      "theoretical_loss": 4.2718009541734885,
+      "tokens_seen": 247136256
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.000747554663991976,
+      "loss": 1.6801,
+      "theoretical_loss": 4.27166476885403,
+      "tokens_seen": 247201792
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007475386158475426,
+      "loss": 1.792,
+      "theoretical_loss": 4.271528629740201,
+      "tokens_seen": 247267328
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007475225677031094,
+      "loss": 1.7957,
+      "theoretical_loss": 4.271392536804088,
+      "tokens_seen": 247332864
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.000747506519558676,
+      "loss": 1.6654,
+      "theoretical_loss": 4.2712564900177945,
+      "tokens_seen": 247398400
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007474904714142428,
+      "loss": 1.826,
+      "theoretical_loss": 4.271120489353454,
+      "tokens_seen": 247463936
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007474744232698094,
+      "loss": 1.6521,
+      "theoretical_loss": 4.270984534783221,
+      "tokens_seen": 247529472
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007474583751253762,
+      "loss": 1.7131,
+      "theoretical_loss": 4.270848626279275,
+      "tokens_seen": 247595008
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007474423269809428,
+      "loss": 1.6221,
+      "theoretical_loss": 4.270712763813822,
+      "tokens_seen": 247660544
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007474262788365096,
+      "loss": 1.7582,
+      "theoretical_loss": 4.270576947359088,
+      "tokens_seen": 247726080
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007474102306920762,
+      "loss": 1.6828,
+      "theoretical_loss": 4.270441176887327,
+      "tokens_seen": 247791616
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.000747394182547643,
+      "loss": 1.695,
+      "theoretical_loss": 4.270305452370814,
+      "tokens_seen": 247857152
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007473781344032096,
+      "loss": 1.5398,
+      "theoretical_loss": 4.27016977378185,
+      "tokens_seen": 247922688
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007473620862587764,
+      "loss": 1.694,
+      "theoretical_loss": 4.270034141092759,
+      "tokens_seen": 247988224
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.000747346038114343,
+      "loss": 1.654,
+      "theoretical_loss": 4.269898554275889,
+      "tokens_seen": 248053760
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007473299899699098,
+      "loss": 1.703,
+      "theoretical_loss": 4.269763013303613,
+      "tokens_seen": 248119296
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007473139418254764,
+      "loss": 1.7145,
+      "theoretical_loss": 4.269627518148326,
+      "tokens_seen": 248184832
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007472978936810432,
+      "loss": 1.6864,
+      "theoretical_loss": 4.26949206878245,
+      "tokens_seen": 248250368
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007472818455366098,
+      "loss": 1.6432,
+      "theoretical_loss": 4.269356665178426,
+      "tokens_seen": 248315904
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007472657973921766,
+      "loss": 1.7075,
+      "theoretical_loss": 4.269221307308725,
+      "tokens_seen": 248381440
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007472497492477432,
+      "loss": 1.5608,
+      "theoretical_loss": 4.269085995145837,
+      "tokens_seen": 248446976
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00074723370110331,
+      "loss": 1.6955,
+      "theoretical_loss": 4.268950728662278,
+      "tokens_seen": 248512512
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007472176529588766,
+      "loss": 1.6484,
+      "theoretical_loss": 4.268815507830587,
+      "tokens_seen": 248578048
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007472016048144434,
+      "loss": 1.7139,
+      "theoretical_loss": 4.268680332623328,
+      "tokens_seen": 248643584
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00074718555667001,
+      "loss": 1.6881,
+      "theoretical_loss": 4.268545203013085,
+      "tokens_seen": 248709120
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007471695085255768,
+      "loss": 1.7474,
+      "theoretical_loss": 4.26841011897247,
+      "tokens_seen": 248774656
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007471534603811434,
+      "loss": 1.6635,
+      "theoretical_loss": 4.268275080474118,
+      "tokens_seen": 248840192
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007471374122367102,
+      "loss": 1.7351,
+      "theoretical_loss": 4.268140087490686,
+      "tokens_seen": 248905728
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.000747121364092277,
+      "loss": 1.7056,
+      "theoretical_loss": 4.2680051399948535,
+      "tokens_seen": 248971264
+    },
+    {
+      "epoch": 0.08,
+      "objective/train/docs_used": 99585,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.5767475366592407,
+      "objective/train/theoretical_loss": 4.267870237959327,
+      "objective/train/tokens_used": 269496800,
+      "theoretical_loss": 4.267870237959327,
+      "tokens_seen": 249036800
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007471053159478436,
+      "loss": 1.5767,
+      "theoretical_loss": 4.267870237959327,
+      "tokens_seen": 249036800
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007470892678034103,
+      "loss": 1.8116,
+      "theoretical_loss": 4.267735381356835,
+      "tokens_seen": 249102336
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.000747073219658977,
+      "loss": 1.8217,
+      "theoretical_loss": 4.26760057016013,
+      "tokens_seen": 249167872
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007470571715145438,
+      "loss": 1.7518,
+      "theoretical_loss": 4.2674658043419855,
+      "tokens_seen": 249233408
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007470411233701104,
+      "loss": 1.7357,
+      "theoretical_loss": 4.267331083875202,
+      "tokens_seen": 249298944
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007470250752256772,
+      "loss": 1.6793,
+      "theoretical_loss": 4.267196408732601,
+      "tokens_seen": 249364480
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007470090270812438,
+      "loss": 1.7845,
+      "theoretical_loss": 4.267061778887029,
+      "tokens_seen": 249430016
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007469929789368106,
+      "loss": 1.6544,
+      "theoretical_loss": 4.2669271943113545,
+      "tokens_seen": 249495552
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007469769307923772,
+      "loss": 1.6735,
+      "theoretical_loss": 4.26679265497847,
+      "tokens_seen": 249561088
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.000746960882647944,
+      "loss": 1.5407,
+      "theoretical_loss": 4.266658160861293,
+      "tokens_seen": 249626624
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007469448345035106,
+      "loss": 1.7296,
+      "theoretical_loss": 4.266523711932761,
+      "tokens_seen": 249692160
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007469287863590774,
+      "loss": 1.6292,
+      "theoretical_loss": 4.266389308165837,
+      "tokens_seen": 249757696
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.000746912738214644,
+      "loss": 1.6179,
+      "theoretical_loss": 4.266254949533507,
+      "tokens_seen": 249823232
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007468966900702106,
+      "loss": 1.6641,
+      "theoretical_loss": 4.26612063600878,
+      "tokens_seen": 249888768
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007468806419257774,
+      "loss": 1.6639,
+      "theoretical_loss": 4.265986367564688,
+      "tokens_seen": 249954304
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.000746864593781344,
+      "loss": 1.6032,
+      "theoretical_loss": 4.265852144174286,
+      "tokens_seen": 250019840
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007468485456369108,
+      "loss": 1.7491,
+      "theoretical_loss": 4.265717965810653,
+      "tokens_seen": 250085376
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007468324974924775,
+      "loss": 1.6103,
+      "theoretical_loss": 4.265583832446892,
+      "tokens_seen": 250150912
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007468164493480442,
+      "loss": 1.7388,
+      "theoretical_loss": 4.265449744056126,
+      "tokens_seen": 250216448
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007468004012036109,
+      "loss": 1.6815,
+      "theoretical_loss": 4.265315700611504,
+      "tokens_seen": 250281984
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007467843530591776,
+      "loss": 1.7446,
+      "theoretical_loss": 4.2651817020861955,
+      "tokens_seen": 250347520
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007467683049147443,
+      "loss": 1.6566,
+      "theoretical_loss": 4.265047748453396,
+      "tokens_seen": 250413056
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007467522567703109,
+      "loss": 1.6856,
+      "theoretical_loss": 4.264913839686323,
+      "tokens_seen": 250478592
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007467362086258777,
+      "loss": 1.7246,
+      "theoretical_loss": 4.264779975758215,
+      "tokens_seen": 250544128
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007467201604814443,
+      "loss": 1.7964,
+      "theoretical_loss": 4.2646461566423355,
+      "tokens_seen": 250609664
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007467041123370111,
+      "loss": 1.6955,
+      "theoretical_loss": 4.26451238231197,
+      "tokens_seen": 250675200
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007466880641925777,
+      "loss": 1.6534,
+      "theoretical_loss": 4.2643786527404295,
+      "tokens_seen": 250740736
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007466720160481445,
+      "loss": 1.6161,
+      "theoretical_loss": 4.264244967901043,
+      "tokens_seen": 250806272
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007466559679037111,
+      "loss": 1.679,
+      "theoretical_loss": 4.264111327767164,
+      "tokens_seen": 250871808
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007466399197592779,
+      "loss": 1.7451,
+      "theoretical_loss": 4.263977732312174,
+      "tokens_seen": 250937344
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007466238716148445,
+      "loss": 1.6456,
+      "theoretical_loss": 4.26384418150947,
+      "tokens_seen": 251002880
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007466078234704113,
+      "loss": 1.6419,
+      "theoretical_loss": 4.263710675332476,
+      "tokens_seen": 251068416
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007465917753259779,
+      "loss": 1.6311,
+      "theoretical_loss": 4.263577213754638,
+      "tokens_seen": 251133952
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007465757271815447,
+      "loss": 1.5964,
+      "theoretical_loss": 4.2634437967494225,
+      "tokens_seen": 251199488
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007465596790371113,
+      "loss": 1.7977,
+      "theoretical_loss": 4.2633104242903235,
+      "tokens_seen": 251265024
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007465436308926781,
+      "loss": 1.7178,
+      "theoretical_loss": 4.263177096350852,
+      "tokens_seen": 251330560
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007465275827482447,
+      "loss": 1.6314,
+      "theoretical_loss": 4.263043812904547,
+      "tokens_seen": 251396096
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007465115346038115,
+      "loss": 1.7299,
+      "theoretical_loss": 4.262910573924965,
+      "tokens_seen": 251461632
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007464954864593781,
+      "loss": 1.575,
+      "theoretical_loss": 4.26277737938569,
+      "tokens_seen": 251527168
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007464794383149449,
+      "loss": 1.6518,
+      "theoretical_loss": 4.262644229260324,
+      "tokens_seen": 251592704
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007464633901705115,
+      "loss": 1.6766,
+      "theoretical_loss": 4.262511123522495,
+      "tokens_seen": 251658240
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007464473420260783,
+      "loss": 1.6524,
+      "theoretical_loss": 4.262378062145853,
+      "tokens_seen": 251723776
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007464312938816449,
+      "loss": 1.6439,
+      "theoretical_loss": 4.262245045104068,
+      "tokens_seen": 251789312
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007464152457372117,
+      "loss": 1.7256,
+      "theoretical_loss": 4.262112072370837,
+      "tokens_seen": 251854848
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007463991975927783,
+      "loss": 1.7312,
+      "theoretical_loss": 4.261979143919873,
+      "tokens_seen": 251920384
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007463831494483451,
+      "loss": 1.6506,
+      "theoretical_loss": 4.261846259724917,
+      "tokens_seen": 251985920
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007463671013039117,
+      "loss": 1.7237,
+      "theoretical_loss": 4.261713419759731,
+      "tokens_seen": 252051456
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007463510531594785,
+      "loss": 1.7041,
+      "theoretical_loss": 4.2615806239981,
+      "tokens_seen": 252116992
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007463350050150451,
+      "loss": 1.7052,
+      "theoretical_loss": 4.261447872413827,
+      "tokens_seen": 252182528
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007463189568706119,
+      "loss": 1.699,
+      "theoretical_loss": 4.261315164980743,
+      "tokens_seen": 252248064
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007463029087261785,
+      "loss": 1.6671,
+      "theoretical_loss": 4.261182501672698,
+      "tokens_seen": 252313600
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007462868605817453,
+      "loss": 1.722,
+      "theoretical_loss": 4.261049882463565,
+      "tokens_seen": 252379136
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007462708124373119,
+      "loss": 1.7046,
+      "theoretical_loss": 4.26091730732724,
+      "tokens_seen": 252444672
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007462547642928787,
+      "loss": 1.7357,
+      "theoretical_loss": 4.2607847762376405,
+      "tokens_seen": 252510208
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007462387161484453,
+      "loss": 1.5987,
+      "theoretical_loss": 4.260652289168706,
+      "tokens_seen": 252575744
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007462226680040121,
+      "loss": 1.6433,
+      "theoretical_loss": 4.260519846094399,
+      "tokens_seen": 252641280
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007462066198595787,
+      "loss": 1.659,
+      "theoretical_loss": 4.2603874469887035,
+      "tokens_seen": 252706816
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007461905717151455,
+      "loss": 1.6698,
+      "theoretical_loss": 4.260255091825625,
+      "tokens_seen": 252772352
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007461745235707121,
+      "loss": 1.5611,
+      "theoretical_loss": 4.260122780579193,
+      "tokens_seen": 252837888
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007461584754262789,
+      "loss": 1.5983,
+      "theoretical_loss": 4.259990513223458,
+      "tokens_seen": 252903424
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007461424272818455,
+      "loss": 1.7221,
+      "theoretical_loss": 4.259858289732491,
+      "tokens_seen": 252968960
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007461263791374123,
+      "loss": 1.6704,
+      "theoretical_loss": 4.259726110080388,
+      "tokens_seen": 253034496
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007461103309929789,
+      "loss": 1.7931,
+      "theoretical_loss": 4.259593974241264,
+      "tokens_seen": 253100032
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007460942828485457,
+      "loss": 1.664,
+      "theoretical_loss": 4.25946188218926,
+      "tokens_seen": 253165568
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007460782347041123,
+      "loss": 1.7322,
+      "theoretical_loss": 4.259329833898534,
+      "tokens_seen": 253231104
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007460621865596791,
+      "loss": 1.825,
+      "theoretical_loss": 4.25919782934327,
+      "tokens_seen": 253296640
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007460461384152457,
+      "loss": 1.7371,
+      "theoretical_loss": 4.259065868497672,
+      "tokens_seen": 253362176
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007460300902708125,
+      "loss": 1.6705,
+      "theoretical_loss": 4.258933951335965,
+      "tokens_seen": 253427712
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007460140421263791,
+      "loss": 1.6526,
+      "theoretical_loss": 4.258802077832399,
+      "tokens_seen": 253493248
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007459979939819459,
+      "loss": 1.5827,
+      "theoretical_loss": 4.258670247961243,
+      "tokens_seen": 253558784
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007459819458375125,
+      "loss": 1.7689,
+      "theoretical_loss": 4.258538461696787,
+      "tokens_seen": 253624320
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007459658976930793,
+      "loss": 1.5772,
+      "theoretical_loss": 4.258406719013347,
+      "tokens_seen": 253689856
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007459498495486459,
+      "loss": 1.7136,
+      "theoretical_loss": 4.258275019885257,
+      "tokens_seen": 253755392
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007459338014042127,
+      "loss": 1.7662,
+      "theoretical_loss": 4.258143364286875,
+      "tokens_seen": 253820928
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007459177532597794,
+      "loss": 1.6651,
+      "theoretical_loss": 4.258011752192579,
+      "tokens_seen": 253886464
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007459017051153461,
+      "loss": 1.6266,
+      "theoretical_loss": 4.257880183576768,
+      "tokens_seen": 253952000
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007458856569709128,
+      "loss": 1.5498,
+      "theoretical_loss": 4.257748658413867,
+      "tokens_seen": 254017536
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007458696088264795,
+      "loss": 1.6362,
+      "theoretical_loss": 4.257617176678317,
+      "tokens_seen": 254083072
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007458535606820462,
+      "loss": 1.6602,
+      "theoretical_loss": 4.2574857383445845,
+      "tokens_seen": 254148608
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007458375125376129,
+      "loss": 1.7236,
+      "theoretical_loss": 4.257354343387156,
+      "tokens_seen": 254214144
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007458214643931796,
+      "loss": 1.6552,
+      "theoretical_loss": 4.257222991780541,
+      "tokens_seen": 254279680
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007458054162487463,
+      "loss": 1.6718,
+      "theoretical_loss": 4.257091683499268,
+      "tokens_seen": 254345216
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.000745789368104313,
+      "loss": 1.6494,
+      "theoretical_loss": 4.25696041851789,
+      "tokens_seen": 254410752
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007457733199598797,
+      "loss": 1.6609,
+      "theoretical_loss": 4.256829196810978,
+      "tokens_seen": 254476288
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007457572718154464,
+      "loss": 1.7628,
+      "theoretical_loss": 4.256698018353129,
+      "tokens_seen": 254541824
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007457412236710131,
+      "loss": 1.7087,
+      "theoretical_loss": 4.256566883118958,
+      "tokens_seen": 254607360
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007457251755265798,
+      "loss": 1.7376,
+      "theoretical_loss": 4.256435791083101,
+      "tokens_seen": 254672896
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007457091273821465,
+      "loss": 1.7467,
+      "theoretical_loss": 4.256304742220219,
+      "tokens_seen": 254738432
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007456930792377133,
+      "loss": 1.6381,
+      "theoretical_loss": 4.25617373650499,
+      "tokens_seen": 254803968
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007456770310932799,
+      "loss": 1.7607,
+      "theoretical_loss": 4.256042773912117,
+      "tokens_seen": 254869504
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007456609829488467,
+      "loss": 1.697,
+      "theoretical_loss": 4.255911854416323,
+      "tokens_seen": 254935040
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007456449348044133,
+      "loss": 1.5103,
+      "theoretical_loss": 4.255780977992352,
+      "tokens_seen": 255000576
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00074562888665998,
+      "loss": 1.6685,
+      "theoretical_loss": 4.255650144614968,
+      "tokens_seen": 255066112
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007456128385155467,
+      "loss": 1.623,
+      "theoretical_loss": 4.255519354258959,
+      "tokens_seen": 255131648
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007455967903711135,
+      "loss": 1.6435,
+      "theoretical_loss": 4.255388606899133,
+      "tokens_seen": 255197184
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007455807422266801,
+      "loss": 1.5703,
+      "theoretical_loss": 4.255257902510319,
+      "tokens_seen": 255262720
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007455646940822469,
+      "loss": 1.7121,
+      "theoretical_loss": 4.255127241067368,
+      "tokens_seen": 255328256
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007455486459378135,
+      "loss": 1.5956,
+      "theoretical_loss": 4.25499662254515,
+      "tokens_seen": 255393792
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007455325977933803,
+      "loss": 1.58,
+      "theoretical_loss": 4.254866046918559,
+      "tokens_seen": 255459328
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007455165496489469,
+      "loss": 1.7176,
+      "theoretical_loss": 4.254735514162508,
+      "tokens_seen": 255524864
+    },
+    {
+      "epoch": 0.08,
+      "objective/train/docs_used": 101923,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.7840094566345215,
+      "objective/train/theoretical_loss": 4.254605024251933,
+      "objective/train/tokens_used": 276050400,
+      "theoretical_loss": 4.254605024251933,
+      "tokens_seen": 255590400
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007455005015045136,
+      "loss": 1.784,
+      "theoretical_loss": 4.254605024251933,
+      "tokens_seen": 255590400
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007454844533600803,
+      "loss": 1.7116,
+      "theoretical_loss": 4.254474577161789,
+      "tokens_seen": 255655936
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.000745468405215647,
+      "loss": 1.6839,
+      "theoretical_loss": 4.254344172867053,
+      "tokens_seen": 255721472
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007454523570712137,
+      "loss": 1.6665,
+      "theoretical_loss": 4.254213811342723,
+      "tokens_seen": 255787008
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007454363089267804,
+      "loss": 1.7717,
+      "theoretical_loss": 4.254083492563819,
+      "tokens_seen": 255852544
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007454202607823471,
+      "loss": 1.6835,
+      "theoretical_loss": 4.253953216505381,
+      "tokens_seen": 255918080
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007454042126379138,
+      "loss": 1.6512,
+      "theoretical_loss": 4.253822983142468,
+      "tokens_seen": 255983616
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007453881644934805,
+      "loss": 1.695,
+      "theoretical_loss": 4.253692792450165,
+      "tokens_seen": 256049152
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007453721163490472,
+      "loss": 1.6956,
+      "theoretical_loss": 4.253562644403573,
+      "tokens_seen": 256114688
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007453560682046138,
+      "loss": 1.6822,
+      "theoretical_loss": 4.253432538977816,
+      "tokens_seen": 256180224
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007453400200601806,
+      "loss": 1.5904,
+      "theoretical_loss": 4.253302476148039,
+      "tokens_seen": 256245760
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007453239719157472,
+      "loss": 1.7748,
+      "theoretical_loss": 4.253172455889407,
+      "tokens_seen": 256311296
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.000745307923771314,
+      "loss": 1.654,
+      "theoretical_loss": 4.2530424781771075,
+      "tokens_seen": 256376832
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007452918756268806,
+      "loss": 1.7685,
+      "theoretical_loss": 4.252912542986345,
+      "tokens_seen": 256442368
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007452758274824474,
+      "loss": 1.5897,
+      "theoretical_loss": 4.2527826502923505,
+      "tokens_seen": 256507904
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.000745259779338014,
+      "loss": 1.7146,
+      "theoretical_loss": 4.25265280007037,
+      "tokens_seen": 256573440
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007452437311935808,
+      "loss": 1.6872,
+      "theoretical_loss": 4.252522992295676,
+      "tokens_seen": 256638976
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007452276830491474,
+      "loss": 1.7714,
+      "theoretical_loss": 4.252393226943555,
+      "tokens_seen": 256704512
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007452116349047142,
+      "loss": 1.6481,
+      "theoretical_loss": 4.252263503989321,
+      "tokens_seen": 256770048
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007451955867602808,
+      "loss": 1.8716,
+      "theoretical_loss": 4.252133823408303,
+      "tokens_seen": 256835584
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007451795386158476,
+      "loss": 1.7463,
+      "theoretical_loss": 4.2520041851758545,
+      "tokens_seen": 256901120
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007451634904714142,
+      "loss": 1.6302,
+      "theoretical_loss": 4.251874589267347,
+      "tokens_seen": 256966656
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.000745147442326981,
+      "loss": 1.788,
+      "theoretical_loss": 4.251745035658176,
+      "tokens_seen": 257032192
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007451313941825476,
+      "loss": 1.5937,
+      "theoretical_loss": 4.251615524323752,
+      "tokens_seen": 257097728
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007451153460381144,
+      "loss": 1.6688,
+      "theoretical_loss": 4.251486055239512,
+      "tokens_seen": 257163264
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.000745099297893681,
+      "loss": 1.7729,
+      "theoretical_loss": 4.251356628380911,
+      "tokens_seen": 257228800
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007450832497492478,
+      "loss": 1.7324,
+      "theoretical_loss": 4.251227243723424,
+      "tokens_seen": 257294336
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007450672016048144,
+      "loss": 1.7021,
+      "theoretical_loss": 4.251097901242546,
+      "tokens_seen": 257359872
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007450511534603812,
+      "loss": 1.6532,
+      "theoretical_loss": 4.250968600913795,
+      "tokens_seen": 257425408
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007450351053159478,
+      "loss": 1.689,
+      "theoretical_loss": 4.250839342712707,
+      "tokens_seen": 257490944
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007450190571715146,
+      "loss": 1.6999,
+      "theoretical_loss": 4.250710126614838,
+      "tokens_seen": 257556480
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007450030090270812,
+      "loss": 1.663,
+      "theoretical_loss": 4.250580952595769,
+      "tokens_seen": 257622016
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.000744986960882648,
+      "loss": 1.5993,
+      "theoretical_loss": 4.250451820631095,
+      "tokens_seen": 257687552
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007449709127382146,
+      "loss": 1.7228,
+      "theoretical_loss": 4.2503227306964355,
+      "tokens_seen": 257753088
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007449548645937814,
+      "loss": 1.7235,
+      "theoretical_loss": 4.25019368276743,
+      "tokens_seen": 257818624
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.000744938816449348,
+      "loss": 1.8186,
+      "theoretical_loss": 4.250064676819735,
+      "tokens_seen": 257884160
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007449227683049148,
+      "loss": 1.7635,
+      "theoretical_loss": 4.249935712829033,
+      "tokens_seen": 257949696
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007449067201604814,
+      "loss": 1.6662,
+      "theoretical_loss": 4.249806790771022,
+      "tokens_seen": 258015232
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007448906720160482,
+      "loss": 1.6486,
+      "theoretical_loss": 4.249677910621422,
+      "tokens_seen": 258080768
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007448746238716148,
+      "loss": 1.7737,
+      "theoretical_loss": 4.249549072355974,
+      "tokens_seen": 258146304
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007448585757271816,
+      "loss": 1.7493,
+      "theoretical_loss": 4.2494202759504365,
+      "tokens_seen": 258211840
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007448425275827482,
+      "loss": 1.6564,
+      "theoretical_loss": 4.249291521380591,
+      "tokens_seen": 258277376
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.000744826479438315,
+      "loss": 1.688,
+      "theoretical_loss": 4.249162808622237,
+      "tokens_seen": 258342912
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007448104312938816,
+      "loss": 1.7598,
+      "theoretical_loss": 4.249034137651197,
+      "tokens_seen": 258408448
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007447943831494484,
+      "loss": 1.6927,
+      "theoretical_loss": 4.248905508443311,
+      "tokens_seen": 258473984
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.000744778335005015,
+      "loss": 1.6818,
+      "theoretical_loss": 4.248776920974439,
+      "tokens_seen": 258539520
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007447622868605818,
+      "loss": 1.6458,
+      "theoretical_loss": 4.248648375220464,
+      "tokens_seen": 258605056
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007447462387161484,
+      "loss": 1.7227,
+      "theoretical_loss": 4.248519871157285,
+      "tokens_seen": 258670592
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007447301905717152,
+      "loss": 1.7057,
+      "theoretical_loss": 4.248391408760824,
+      "tokens_seen": 258736128
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007447141424272819,
+      "loss": 1.63,
+      "theoretical_loss": 4.248262988007022,
+      "tokens_seen": 258801664
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007446980942828486,
+      "loss": 1.7302,
+      "theoretical_loss": 4.248134608871841,
+      "tokens_seen": 258867200
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007446820461384153,
+      "loss": 1.6269,
+      "theoretical_loss": 4.24800627133126,
+      "tokens_seen": 258932736
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.000744665997993982,
+      "loss": 1.6952,
+      "theoretical_loss": 4.247877975361281,
+      "tokens_seen": 258998272
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007446499498495487,
+      "loss": 1.6942,
+      "theoretical_loss": 4.247749720937925,
+      "tokens_seen": 259063808
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007446339017051154,
+      "loss": 1.6828,
+      "theoretical_loss": 4.247621508037232,
+      "tokens_seen": 259129344
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007446178535606821,
+      "loss": 1.75,
+      "theoretical_loss": 4.247493336635264,
+      "tokens_seen": 259194880
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007446018054162488,
+      "loss": 1.6661,
+      "theoretical_loss": 4.2473652067081,
+      "tokens_seen": 259260416
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007445857572718155,
+      "loss": 1.6707,
+      "theoretical_loss": 4.247237118231841,
+      "tokens_seen": 259325952
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007445697091273822,
+      "loss": 1.6324,
+      "theoretical_loss": 4.247109071182607,
+      "tokens_seen": 259391488
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007445536609829489,
+      "loss": 1.6038,
+      "theoretical_loss": 4.246981065536538,
+      "tokens_seen": 259457024
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007445376128385156,
+      "loss": 1.5675,
+      "theoretical_loss": 4.246853101269793,
+      "tokens_seen": 259522560
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007445215646940823,
+      "loss": 1.7639,
+      "theoretical_loss": 4.2467251783585525,
+      "tokens_seen": 259588096
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.000744505516549649,
+      "loss": 1.7606,
+      "theoretical_loss": 4.246597296779014,
+      "tokens_seen": 259653632
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007444894684052157,
+      "loss": 1.7239,
+      "theoretical_loss": 4.246469456507398,
+      "tokens_seen": 259719168
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007444734202607824,
+      "loss": 1.725,
+      "theoretical_loss": 4.246341657519943,
+      "tokens_seen": 259784704
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007444573721163491,
+      "loss": 1.7266,
+      "theoretical_loss": 4.246213899792905,
+      "tokens_seen": 259850240
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007444413239719158,
+      "loss": 1.6228,
+      "theoretical_loss": 4.246086183302564,
+      "tokens_seen": 259915776
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007444252758274825,
+      "loss": 1.5962,
+      "theoretical_loss": 4.2459585080252165,
+      "tokens_seen": 259981312
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007444092276830492,
+      "loss": 1.6643,
+      "theoretical_loss": 4.245830873937179,
+      "tokens_seen": 260046848
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.000744393179538616,
+      "loss": 1.768,
+      "theoretical_loss": 4.24570328101479,
+      "tokens_seen": 260112384
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007443771313941826,
+      "loss": 1.73,
+      "theoretical_loss": 4.245575729234403,
+      "tokens_seen": 260177920
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007443610832497493,
+      "loss": 1.5881,
+      "theoretical_loss": 4.245448218572396,
+      "tokens_seen": 260243456
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.000744345035105316,
+      "loss": 1.7745,
+      "theoretical_loss": 4.245320749005163,
+      "tokens_seen": 260308992
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007443289869608828,
+      "loss": 1.6264,
+      "theoretical_loss": 4.245193320509118,
+      "tokens_seen": 260374528
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007443129388164494,
+      "loss": 1.6443,
+      "theoretical_loss": 4.245065933060697,
+      "tokens_seen": 260440064
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007442968906720162,
+      "loss": 1.6297,
+      "theoretical_loss": 4.244938586636351,
+      "tokens_seen": 260505600
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007442808425275828,
+      "loss": 1.6536,
+      "theoretical_loss": 4.244811281212556,
+      "tokens_seen": 260571136
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007442647943831496,
+      "loss": 1.7296,
+      "theoretical_loss": 4.244684016765802,
+      "tokens_seen": 260636672
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007442487462387162,
+      "loss": 1.647,
+      "theoretical_loss": 4.2445567932726025,
+      "tokens_seen": 260702208
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.000744232698094283,
+      "loss": 1.7275,
+      "theoretical_loss": 4.244429610709488,
+      "tokens_seen": 260767744
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007442166499498496,
+      "loss": 1.5733,
+      "theoretical_loss": 4.244302469053007,
+      "tokens_seen": 260833280
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007442006018054164,
+      "loss": 1.6508,
+      "theoretical_loss": 4.244175368279733,
+      "tokens_seen": 260898816
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.000744184553660983,
+      "loss": 1.6887,
+      "theoretical_loss": 4.244048308366253,
+      "tokens_seen": 260964352
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007441685055165498,
+      "loss": 1.5841,
+      "theoretical_loss": 4.243921289289175,
+      "tokens_seen": 261029888
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007441524573721164,
+      "loss": 1.6457,
+      "theoretical_loss": 4.243794311025128,
+      "tokens_seen": 261095424
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007441364092276832,
+      "loss": 1.7269,
+      "theoretical_loss": 4.243667373550757,
+      "tokens_seen": 261160960
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007441203610832498,
+      "loss": 1.6444,
+      "theoretical_loss": 4.24354047684273,
+      "tokens_seen": 261226496
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007441043129388166,
+      "loss": 1.6782,
+      "theoretical_loss": 4.243413620877732,
+      "tokens_seen": 261292032
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007440882647943832,
+      "loss": 1.5928,
+      "theoretical_loss": 4.243286805632467,
+      "tokens_seen": 261357568
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007440722166499499,
+      "loss": 1.6889,
+      "theoretical_loss": 4.243160031083658,
+      "tokens_seen": 261423104
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007440561685055166,
+      "loss": 1.5918,
+      "theoretical_loss": 4.24303329720805,
+      "tokens_seen": 261488640
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007440401203610833,
+      "loss": 1.6905,
+      "theoretical_loss": 4.242906603982402,
+      "tokens_seen": 261554176
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00074402407221665,
+      "loss": 1.7469,
+      "theoretical_loss": 4.242779951383497,
+      "tokens_seen": 261619712
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007440080240722167,
+      "loss": 1.6461,
+      "theoretical_loss": 4.242653339388135,
+      "tokens_seen": 261685248
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007439919759277834,
+      "loss": 1.7377,
+      "theoretical_loss": 4.242526767973134,
+      "tokens_seen": 261750784
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007439759277833501,
+      "loss": 1.6465,
+      "theoretical_loss": 4.242400237115333,
+      "tokens_seen": 261816320
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007439598796389168,
+      "loss": 1.5911,
+      "theoretical_loss": 4.242273746791589,
+      "tokens_seen": 261881856
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007439438314944835,
+      "loss": 1.765,
+      "theoretical_loss": 4.242147296978778,
+      "tokens_seen": 261947392
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007439277833500501,
+      "loss": 1.7125,
+      "theoretical_loss": 4.242020887653795,
+      "tokens_seen": 262012928
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007439117352056169,
+      "loss": 1.6799,
+      "theoretical_loss": 4.241894518793555,
+      "tokens_seen": 262078464
+    },
+    {
+      "epoch": 0.08,
+      "objective/train/docs_used": 104233,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.6709940433502197,
+      "objective/train/theoretical_loss": 4.24176819037499,
+      "objective/train/tokens_used": 282604000,
+      "theoretical_loss": 4.24176819037499,
+      "tokens_seen": 262144000
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007438956870611835,
+      "loss": 1.671,
+      "theoretical_loss": 4.24176819037499,
+      "tokens_seen": 262144000
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007438796389167503,
+      "loss": 1.7132,
+      "theoretical_loss": 4.2416419023750525,
+      "tokens_seen": 262209536
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007438635907723169,
+      "loss": 1.6513,
+      "theoretical_loss": 4.241515654770713,
+      "tokens_seen": 262275072
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007438475426278837,
+      "loss": 1.6345,
+      "theoretical_loss": 4.241389447538961,
+      "tokens_seen": 262340608
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007438314944834503,
+      "loss": 1.6632,
+      "theoretical_loss": 4.241263280656806,
+      "tokens_seen": 262406144
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007438154463390171,
+      "loss": 1.6645,
+      "theoretical_loss": 4.241137154101275,
+      "tokens_seen": 262471680
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007437993981945837,
+      "loss": 1.6946,
+      "theoretical_loss": 4.2410110678494135,
+      "tokens_seen": 262537216
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007437833500501505,
+      "loss": 1.6594,
+      "theoretical_loss": 4.240885021878286,
+      "tokens_seen": 262602752
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007437673019057171,
+      "loss": 1.6579,
+      "theoretical_loss": 4.240759016164978,
+      "tokens_seen": 262668288
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007437512537612839,
+      "loss": 1.682,
+      "theoretical_loss": 4.240633050686592,
+      "tokens_seen": 262733824
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007437352056168505,
+      "loss": 1.6489,
+      "theoretical_loss": 4.2405071254202475,
+      "tokens_seen": 262799360
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007437191574724173,
+      "loss": 1.7881,
+      "theoretical_loss": 4.240381240343085,
+      "tokens_seen": 262864896
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007437031093279839,
+      "loss": 1.7095,
+      "theoretical_loss": 4.240255395432264,
+      "tokens_seen": 262930432
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007436870611835507,
+      "loss": 1.6394,
+      "theoretical_loss": 4.240129590664961,
+      "tokens_seen": 262995968
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007436710130391173,
+      "loss": 1.6244,
+      "theoretical_loss": 4.240003826018373,
+      "tokens_seen": 263061504
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007436549648946841,
+      "loss": 1.7085,
+      "theoretical_loss": 4.239878101469713,
+      "tokens_seen": 263127040
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007436389167502507,
+      "loss": 1.722,
+      "theoretical_loss": 4.239752416996215,
+      "tokens_seen": 263192576
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007436228686058175,
+      "loss": 1.7182,
+      "theoretical_loss": 4.239626772575131,
+      "tokens_seen": 263258112
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007436068204613841,
+      "loss": 1.5903,
+      "theoretical_loss": 4.239501168183732,
+      "tokens_seen": 263323648
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007435907723169509,
+      "loss": 1.5815,
+      "theoretical_loss": 4.239375603799305,
+      "tokens_seen": 263389184
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007435747241725175,
+      "loss": 1.6607,
+      "theoretical_loss": 4.2392500793991585,
+      "tokens_seen": 263454720
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007435586760280843,
+      "loss": 1.5409,
+      "theoretical_loss": 4.239124594960619,
+      "tokens_seen": 263520256
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.000743542627883651,
+      "loss": 1.6256,
+      "theoretical_loss": 4.23899915046103,
+      "tokens_seen": 263585792
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007435265797392177,
+      "loss": 1.6867,
+      "theoretical_loss": 4.238873745877756,
+      "tokens_seen": 263651328
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007435105315947844,
+      "loss": 1.6541,
+      "theoretical_loss": 4.238748381188176,
+      "tokens_seen": 263716864
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007434944834503511,
+      "loss": 1.8033,
+      "theoretical_loss": 4.238623056369692,
+      "tokens_seen": 263782400
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007434784353059178,
+      "loss": 1.6045,
+      "theoretical_loss": 4.238497771399721,
+      "tokens_seen": 263847936
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007434623871614845,
+      "loss": 1.7191,
+      "theoretical_loss": 4.2383725262556995,
+      "tokens_seen": 263913472
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007434463390170512,
+      "loss": 1.6445,
+      "theoretical_loss": 4.238247320915083,
+      "tokens_seen": 263979008
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007434302908726179,
+      "loss": 1.5632,
+      "theoretical_loss": 4.238122155355345,
+      "tokens_seen": 264044544
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007434142427281846,
+      "loss": 1.702,
+      "theoretical_loss": 4.237997029553977,
+      "tokens_seen": 264110080
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007433981945837513,
+      "loss": 1.6446,
+      "theoretical_loss": 4.237871943488488,
+      "tokens_seen": 264175616
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.000743382146439318,
+      "loss": 1.7052,
+      "theoretical_loss": 4.237746897136408,
+      "tokens_seen": 264241152
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007433660982948847,
+      "loss": 1.6234,
+      "theoretical_loss": 4.237621890475284,
+      "tokens_seen": 264306688
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007433500501504514,
+      "loss": 1.6944,
+      "theoretical_loss": 4.237496923482678,
+      "tokens_seen": 264372224
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007433340020060181,
+      "loss": 1.6701,
+      "theoretical_loss": 4.237371996136175,
+      "tokens_seen": 264437760
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007433179538615848,
+      "loss": 1.6702,
+      "theoretical_loss": 4.237247108413376,
+      "tokens_seen": 264503296
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007433019057171515,
+      "loss": 1.5667,
+      "theoretical_loss": 4.237122260291901,
+      "tokens_seen": 264568832
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007432858575727182,
+      "loss": 1.6129,
+      "theoretical_loss": 4.236997451749387,
+      "tokens_seen": 264634368
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007432698094282849,
+      "loss": 1.7445,
+      "theoretical_loss": 4.236872682763489,
+      "tokens_seen": 264699904
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007432537612838516,
+      "loss": 1.6724,
+      "theoretical_loss": 4.236747953311884,
+      "tokens_seen": 264765440
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007432377131394183,
+      "loss": 1.6016,
+      "theoretical_loss": 4.236623263372261,
+      "tokens_seen": 264830976
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.000743221664994985,
+      "loss": 1.6213,
+      "theoretical_loss": 4.236498612922331,
+      "tokens_seen": 264896512
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007432056168505517,
+      "loss": 1.6128,
+      "theoretical_loss": 4.236374001939822,
+      "tokens_seen": 264962048
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007431895687061184,
+      "loss": 1.6469,
+      "theoretical_loss": 4.2362494304024825,
+      "tokens_seen": 265027584
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007431735205616851,
+      "loss": 1.7452,
+      "theoretical_loss": 4.236124898288074,
+      "tokens_seen": 265093120
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007431574724172518,
+      "loss": 1.6845,
+      "theoretical_loss": 4.236000405574382,
+      "tokens_seen": 265158656
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007431414242728185,
+      "loss": 1.7285,
+      "theoretical_loss": 4.235875952239204,
+      "tokens_seen": 265224192
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007431253761283852,
+      "loss": 1.6745,
+      "theoretical_loss": 4.235751538260359,
+      "tokens_seen": 265289728
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007431093279839519,
+      "loss": 1.6798,
+      "theoretical_loss": 4.2356271636156855,
+      "tokens_seen": 265355264
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007430932798395186,
+      "loss": 1.6917,
+      "theoretical_loss": 4.235502828283035,
+      "tokens_seen": 265420800
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007430772316950853,
+      "loss": 1.7893,
+      "theoretical_loss": 4.235378532240283,
+      "tokens_seen": 265486336
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.000743061183550652,
+      "loss": 1.7136,
+      "theoretical_loss": 4.235254275465317,
+      "tokens_seen": 265551872
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007430451354062187,
+      "loss": 1.6035,
+      "theoretical_loss": 4.235130057936045,
+      "tokens_seen": 265617408
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007430290872617854,
+      "loss": 1.5738,
+      "theoretical_loss": 4.235005879630395,
+      "tokens_seen": 265682944
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007430130391173521,
+      "loss": 1.6699,
+      "theoretical_loss": 4.23488174052631,
+      "tokens_seen": 265748480
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007429969909729188,
+      "loss": 1.6366,
+      "theoretical_loss": 4.234757640601751,
+      "tokens_seen": 265814016
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007429809428284855,
+      "loss": 1.5538,
+      "theoretical_loss": 4.234633579834698,
+      "tokens_seen": 265879552
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007429648946840523,
+      "loss": 1.6776,
+      "theoretical_loss": 4.234509558203149,
+      "tokens_seen": 265945088
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007429488465396189,
+      "loss": 1.6846,
+      "theoretical_loss": 4.234385575685116,
+      "tokens_seen": 266010624
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007429327983951857,
+      "loss": 1.7382,
+      "theoretical_loss": 4.234261632258636,
+      "tokens_seen": 266076160
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007429167502507523,
+      "loss": 1.7313,
+      "theoretical_loss": 4.234137727901757,
+      "tokens_seen": 266141696
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.000742900702106319,
+      "loss": 1.5754,
+      "theoretical_loss": 4.234013862592548,
+      "tokens_seen": 266207232
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007428846539618857,
+      "loss": 1.7733,
+      "theoretical_loss": 4.233890036309095,
+      "tokens_seen": 266272768
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007428686058174525,
+      "loss": 1.7302,
+      "theoretical_loss": 4.233766249029501,
+      "tokens_seen": 266338304
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007428525576730191,
+      "loss": 1.6632,
+      "theoretical_loss": 4.233642500731889,
+      "tokens_seen": 266403840
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007428365095285859,
+      "loss": 1.6486,
+      "theoretical_loss": 4.233518791394396,
+      "tokens_seen": 266469376
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007428204613841525,
+      "loss": 1.5989,
+      "theoretical_loss": 4.233395120995181,
+      "tokens_seen": 266534912
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007428044132397193,
+      "loss": 1.6482,
+      "theoretical_loss": 4.233271489512416,
+      "tokens_seen": 266600448
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007427883650952859,
+      "loss": 1.6986,
+      "theoretical_loss": 4.233147896924294,
+      "tokens_seen": 266665984
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007427723169508527,
+      "loss": 1.7408,
+      "theoretical_loss": 4.233024343209023,
+      "tokens_seen": 266731520
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007427562688064193,
+      "loss": 1.5948,
+      "theoretical_loss": 4.232900828344833,
+      "tokens_seen": 266797056
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007427402206619861,
+      "loss": 1.7545,
+      "theoretical_loss": 4.232777352309966,
+      "tokens_seen": 266862592
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007427241725175527,
+      "loss": 1.7316,
+      "theoretical_loss": 4.232653915082684,
+      "tokens_seen": 266928128
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007427081243731195,
+      "loss": 1.6597,
+      "theoretical_loss": 4.232530516641267,
+      "tokens_seen": 266993664
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007426920762286861,
+      "loss": 1.6585,
+      "theoretical_loss": 4.232407156964014,
+      "tokens_seen": 267059200
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007426760280842528,
+      "loss": 1.6008,
+      "theoretical_loss": 4.232283836029237,
+      "tokens_seen": 267124736
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007426599799398195,
+      "loss": 1.6104,
+      "theoretical_loss": 4.232160553815269,
+      "tokens_seen": 267190272
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007426439317953862,
+      "loss": 1.6741,
+      "theoretical_loss": 4.232037310300458,
+      "tokens_seen": 267255808
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007426278836509529,
+      "loss": 1.6344,
+      "theoretical_loss": 4.231914105463173,
+      "tokens_seen": 267321344
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007426118355065196,
+      "loss": 1.7326,
+      "theoretical_loss": 4.231790939281797,
+      "tokens_seen": 267386880
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007425957873620863,
+      "loss": 1.7199,
+      "theoretical_loss": 4.231667811734732,
+      "tokens_seen": 267452416
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.000742579739217653,
+      "loss": 1.602,
+      "theoretical_loss": 4.231544722800395,
+      "tokens_seen": 267517952
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007425636910732197,
+      "loss": 1.5649,
+      "theoretical_loss": 4.231421672457225,
+      "tokens_seen": 267583488
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007425476429287864,
+      "loss": 1.6479,
+      "theoretical_loss": 4.231298660683675,
+      "tokens_seen": 267649024
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.000742531594784353,
+      "loss": 1.772,
+      "theoretical_loss": 4.231175687458214,
+      "tokens_seen": 267714560
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007425155466399198,
+      "loss": 1.696,
+      "theoretical_loss": 4.231052752759332,
+      "tokens_seen": 267780096
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007424994984954864,
+      "loss": 1.6956,
+      "theoretical_loss": 4.230929856565535,
+      "tokens_seen": 267845632
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007424834503510532,
+      "loss": 1.7029,
+      "theoretical_loss": 4.230806998855343,
+      "tokens_seen": 267911168
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007424674022066198,
+      "loss": 1.7181,
+      "theoretical_loss": 4.230684179607297,
+      "tokens_seen": 267976704
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007424513540621866,
+      "loss": 1.7139,
+      "theoretical_loss": 4.2305613987999555,
+      "tokens_seen": 268042240
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007424353059177532,
+      "loss": 1.7342,
+      "theoretical_loss": 4.230438656411891,
+      "tokens_seen": 268107776
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00074241925777332,
+      "loss": 1.691,
+      "theoretical_loss": 4.230315952421697,
+      "tokens_seen": 268173312
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007424032096288866,
+      "loss": 1.6905,
+      "theoretical_loss": 4.230193286807981,
+      "tokens_seen": 268238848
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007423871614844534,
+      "loss": 1.6338,
+      "theoretical_loss": 4.230070659549368,
+      "tokens_seen": 268304384
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00074237111334002,
+      "loss": 1.719,
+      "theoretical_loss": 4.229948070624502,
+      "tokens_seen": 268369920
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007423550651955868,
+      "loss": 1.6452,
+      "theoretical_loss": 4.229825520012042,
+      "tokens_seen": 268435456
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007423390170511535,
+      "loss": 1.6339,
+      "theoretical_loss": 4.2297030076906665,
+      "tokens_seen": 268500992
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007423229689067202,
+      "loss": 1.6726,
+      "theoretical_loss": 4.229580533639068,
+      "tokens_seen": 268566528
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007423069207622869,
+      "loss": 1.6218,
+      "theoretical_loss": 4.229458097835959,
+      "tokens_seen": 268632064
+    },
+    {
+      "epoch": 0.08,
+      "objective/train/docs_used": 106547,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.7015962600708008,
+      "objective/train/theoretical_loss": 4.2293357002600676,
+      "objective/train/tokens_used": 289157600,
+      "theoretical_loss": 4.2293357002600676,
+      "tokens_seen": 268697600
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007422908726178536,
+      "loss": 1.7016,
+      "theoretical_loss": 4.2293357002600676,
+      "tokens_seen": 268697600
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007422748244734203,
+      "loss": 1.7635,
+      "theoretical_loss": 4.229213340890139,
+      "tokens_seen": 268763136
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.000742258776328987,
+      "loss": 1.669,
+      "theoretical_loss": 4.229091019704935,
+      "tokens_seen": 268828672
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007422427281845537,
+      "loss": 1.5246,
+      "theoretical_loss": 4.2289687366832345,
+      "tokens_seen": 268894208
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007422266800401204,
+      "loss": 1.6254,
+      "theoretical_loss": 4.228846491803835,
+      "tokens_seen": 268959744
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007422106318956871,
+      "loss": 1.6714,
+      "theoretical_loss": 4.228724285045549,
+      "tokens_seen": 269025280
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007421945837512538,
+      "loss": 1.5393,
+      "theoretical_loss": 4.228602116387206,
+      "tokens_seen": 269090816
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007421785356068205,
+      "loss": 1.6996,
+      "theoretical_loss": 4.228479985807653,
+      "tokens_seen": 269156352
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007421624874623872,
+      "loss": 1.8339,
+      "theoretical_loss": 4.228357893285755,
+      "tokens_seen": 269221888
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007421464393179539,
+      "loss": 1.7352,
+      "theoretical_loss": 4.2282358388003916,
+      "tokens_seen": 269287424
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007421303911735206,
+      "loss": 1.5974,
+      "theoretical_loss": 4.228113822330461,
+      "tokens_seen": 269352960
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007421143430290873,
+      "loss": 1.7363,
+      "theoretical_loss": 4.227991843854878,
+      "tokens_seen": 269418496
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.000742098294884654,
+      "loss": 1.7108,
+      "theoretical_loss": 4.227869903352572,
+      "tokens_seen": 269484032
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007420822467402207,
+      "loss": 1.6682,
+      "theoretical_loss": 4.227748000802493,
+      "tokens_seen": 269549568
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007420661985957874,
+      "loss": 1.6003,
+      "theoretical_loss": 4.227626136183606,
+      "tokens_seen": 269615104
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007420501504513541,
+      "loss": 1.5697,
+      "theoretical_loss": 4.227504309474892,
+      "tokens_seen": 269680640
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007420341023069208,
+      "loss": 1.7448,
+      "theoretical_loss": 4.227382520655348,
+      "tokens_seen": 269746176
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007420180541624875,
+      "loss": 1.7072,
+      "theoretical_loss": 4.227260769703991,
+      "tokens_seen": 269811712
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007420020060180542,
+      "loss": 1.6838,
+      "theoretical_loss": 4.227139056599851,
+      "tokens_seen": 269877248
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007419859578736209,
+      "loss": 1.6778,
+      "theoretical_loss": 4.2270173813219785,
+      "tokens_seen": 269942784
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007419699097291876,
+      "loss": 1.6301,
+      "theoretical_loss": 4.226895743849437,
+      "tokens_seen": 270008320
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007419538615847543,
+      "loss": 1.6792,
+      "theoretical_loss": 4.226774144161309,
+      "tokens_seen": 270073856
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.000741937813440321,
+      "loss": 1.6911,
+      "theoretical_loss": 4.226652582236694,
+      "tokens_seen": 270139392
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007419217652958877,
+      "loss": 1.5549,
+      "theoretical_loss": 4.226531058054706,
+      "tokens_seen": 270204928
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007419057171514544,
+      "loss": 1.5975,
+      "theoretical_loss": 4.226409571594477,
+      "tokens_seen": 270270464
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007418896690070211,
+      "loss": 1.6987,
+      "theoretical_loss": 4.226288122835156,
+      "tokens_seen": 270336000
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007418736208625878,
+      "loss": 1.5463,
+      "theoretical_loss": 4.226166711755906,
+      "tokens_seen": 270401536
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007418575727181545,
+      "loss": 1.7582,
+      "theoretical_loss": 4.2260453383359105,
+      "tokens_seen": 270467072
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007418415245737212,
+      "loss": 1.7113,
+      "theoretical_loss": 4.2259240025543665,
+      "tokens_seen": 270532608
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007418254764292879,
+      "loss": 1.6721,
+      "theoretical_loss": 4.225802704390488,
+      "tokens_seen": 270598144
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007418094282848546,
+      "loss": 1.6108,
+      "theoretical_loss": 4.2256814438235075,
+      "tokens_seen": 270663680
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007417933801404213,
+      "loss": 1.6607,
+      "theoretical_loss": 4.225560220832672,
+      "tokens_seen": 270729216
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.000741777331995988,
+      "loss": 1.681,
+      "theoretical_loss": 4.225439035397246,
+      "tokens_seen": 270794752
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007417612838515547,
+      "loss": 1.7115,
+      "theoretical_loss": 4.225317887496508,
+      "tokens_seen": 270860288
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007417452357071214,
+      "loss": 1.6303,
+      "theoretical_loss": 4.2251967771097565,
+      "tokens_seen": 270925824
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007417291875626881,
+      "loss": 1.6108,
+      "theoretical_loss": 4.225075704216305,
+      "tokens_seen": 270991360
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007417131394182548,
+      "loss": 1.746,
+      "theoretical_loss": 4.224954668795483,
+      "tokens_seen": 271056896
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007416970912738215,
+      "loss": 1.6751,
+      "theoretical_loss": 4.224833670826636,
+      "tokens_seen": 271122432
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007416810431293882,
+      "loss": 1.5968,
+      "theoretical_loss": 4.224712710289127,
+      "tokens_seen": 271187968
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.000741664994984955,
+      "loss": 1.6471,
+      "theoretical_loss": 4.2245917871623355,
+      "tokens_seen": 271253504
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007416489468405216,
+      "loss": 1.4978,
+      "theoretical_loss": 4.224470901425655,
+      "tokens_seen": 271319040
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007416328986960883,
+      "loss": 1.674,
+      "theoretical_loss": 4.224350053058499,
+      "tokens_seen": 271384576
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.000741616850551655,
+      "loss": 1.6004,
+      "theoretical_loss": 4.224229242040295,
+      "tokens_seen": 271450112
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007416008024072218,
+      "loss": 1.6213,
+      "theoretical_loss": 4.224108468350485,
+      "tokens_seen": 271515648
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007415847542627884,
+      "loss": 1.7061,
+      "theoretical_loss": 4.223987731968532,
+      "tokens_seen": 271581184
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007415687061183552,
+      "loss": 1.6753,
+      "theoretical_loss": 4.223867032873912,
+      "tokens_seen": 271646720
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007415526579739218,
+      "loss": 1.8097,
+      "theoretical_loss": 4.223746371046116,
+      "tokens_seen": 271712256
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007415366098294886,
+      "loss": 1.5684,
+      "theoretical_loss": 4.223625746464656,
+      "tokens_seen": 271777792
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007415205616850552,
+      "loss": 1.6576,
+      "theoretical_loss": 4.223505159109054,
+      "tokens_seen": 271843328
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.000741504513540622,
+      "loss": 1.7029,
+      "theoretical_loss": 4.223384608958855,
+      "tokens_seen": 271908864
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007414884653961886,
+      "loss": 1.7804,
+      "theoretical_loss": 4.223264095993615,
+      "tokens_seen": 271974400
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007414724172517554,
+      "loss": 1.6577,
+      "theoretical_loss": 4.223143620192907,
+      "tokens_seen": 272039936
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.000741456369107322,
+      "loss": 1.5331,
+      "theoretical_loss": 4.223023181536323,
+      "tokens_seen": 272105472
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007414403209628888,
+      "loss": 1.6128,
+      "theoretical_loss": 4.2229027800034675,
+      "tokens_seen": 272171008
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007414242728184554,
+      "loss": 1.5997,
+      "theoretical_loss": 4.222782415573962,
+      "tokens_seen": 272236544
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007414082246740222,
+      "loss": 1.703,
+      "theoretical_loss": 4.222662088227447,
+      "tokens_seen": 272302080
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007413921765295888,
+      "loss": 1.5279,
+      "theoretical_loss": 4.222541797943576,
+      "tokens_seen": 272367616
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007413761283851556,
+      "loss": 1.7537,
+      "theoretical_loss": 4.222421544702018,
+      "tokens_seen": 272433152
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007413600802407222,
+      "loss": 1.7776,
+      "theoretical_loss": 4.2223013284824615,
+      "tokens_seen": 272498688
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.000741344032096289,
+      "loss": 1.7529,
+      "theoretical_loss": 4.222181149264608,
+      "tokens_seen": 272564224
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007413279839518556,
+      "loss": 1.6538,
+      "theoretical_loss": 4.222061007028175,
+      "tokens_seen": 272629760
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007413119358074224,
+      "loss": 1.647,
+      "theoretical_loss": 4.221940901752899,
+      "tokens_seen": 272695296
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.000741295887662989,
+      "loss": 1.6851,
+      "theoretical_loss": 4.221820833418529,
+      "tokens_seen": 272760832
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007412798395185557,
+      "loss": 1.6389,
+      "theoretical_loss": 4.221700802004832,
+      "tokens_seen": 272826368
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007412637913741224,
+      "loss": 1.7206,
+      "theoretical_loss": 4.2215808074915895,
+      "tokens_seen": 272891904
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007412477432296891,
+      "loss": 1.5648,
+      "theoretical_loss": 4.221460849858601,
+      "tokens_seen": 272957440
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007412316950852558,
+      "loss": 1.6165,
+      "theoretical_loss": 4.221340929085679,
+      "tokens_seen": 273022976
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007412156469408225,
+      "loss": 1.582,
+      "theoretical_loss": 4.221221045152655,
+      "tokens_seen": 273088512
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007411995987963892,
+      "loss": 1.7083,
+      "theoretical_loss": 4.221101198039375,
+      "tokens_seen": 273154048
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007411835506519559,
+      "loss": 1.6301,
+      "theoretical_loss": 4.2209813877257,
+      "tokens_seen": 273219584
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007411675025075226,
+      "loss": 1.6637,
+      "theoretical_loss": 4.220861614191508,
+      "tokens_seen": 273285120
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007411514543630893,
+      "loss": 1.674,
+      "theoretical_loss": 4.220741877416691,
+      "tokens_seen": 273350656
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.000741135406218656,
+      "loss": 1.7409,
+      "theoretical_loss": 4.22062217738116,
+      "tokens_seen": 273416192
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007411193580742227,
+      "loss": 1.7326,
+      "theoretical_loss": 4.22050251406484,
+      "tokens_seen": 273481728
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007411033099297894,
+      "loss": 1.6014,
+      "theoretical_loss": 4.220382887447671,
+      "tokens_seen": 273547264
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007410872617853561,
+      "loss": 1.676,
+      "theoretical_loss": 4.22026329750961,
+      "tokens_seen": 273612800
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007410712136409228,
+      "loss": 1.6044,
+      "theoretical_loss": 4.220143744230629,
+      "tokens_seen": 273678336
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007410551654964895,
+      "loss": 1.7051,
+      "theoretical_loss": 4.220024227590717,
+      "tokens_seen": 273743872
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007410391173520562,
+      "loss": 1.628,
+      "theoretical_loss": 4.219904747569877,
+      "tokens_seen": 273809408
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007410230692076229,
+      "loss": 1.72,
+      "theoretical_loss": 4.219785304148129,
+      "tokens_seen": 273874944
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007410070210631896,
+      "loss": 1.7064,
+      "theoretical_loss": 4.219665897305506,
+      "tokens_seen": 273940480
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007409909729187563,
+      "loss": 1.6476,
+      "theoretical_loss": 4.219546527022062,
+      "tokens_seen": 274006016
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.000740974924774323,
+      "loss": 1.6121,
+      "theoretical_loss": 4.219427193277861,
+      "tokens_seen": 274071552
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007409588766298897,
+      "loss": 1.8465,
+      "theoretical_loss": 4.219307896052987,
+      "tokens_seen": 274137088
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007409428284854564,
+      "loss": 1.7803,
+      "theoretical_loss": 4.219188635327535,
+      "tokens_seen": 274202624
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007409267803410231,
+      "loss": 1.6073,
+      "theoretical_loss": 4.219069411081622,
+      "tokens_seen": 274268160
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007409107321965898,
+      "loss": 1.7041,
+      "theoretical_loss": 4.218950223295374,
+      "tokens_seen": 274333696
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007408946840521565,
+      "loss": 1.6319,
+      "theoretical_loss": 4.218831071948936,
+      "tokens_seen": 274399232
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007408786359077232,
+      "loss": 1.6451,
+      "theoretical_loss": 4.218711957022469,
+      "tokens_seen": 274464768
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007408625877632899,
+      "loss": 1.7653,
+      "theoretical_loss": 4.218592878496148,
+      "tokens_seen": 274530304
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007408465396188566,
+      "loss": 1.6506,
+      "theoretical_loss": 4.218473836350164,
+      "tokens_seen": 274595840
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007408304914744233,
+      "loss": 1.6903,
+      "theoretical_loss": 4.2183548305647225,
+      "tokens_seen": 274661376
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00074081444332999,
+      "loss": 1.7129,
+      "theoretical_loss": 4.218235861120047,
+      "tokens_seen": 274726912
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007407983951855567,
+      "loss": 1.6008,
+      "theoretical_loss": 4.218116927996375,
+      "tokens_seen": 274792448
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007407823470411234,
+      "loss": 1.6658,
+      "theoretical_loss": 4.217998031173959,
+      "tokens_seen": 274857984
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007407662988966901,
+      "loss": 1.5342,
+      "theoretical_loss": 4.217879170633067,
+      "tokens_seen": 274923520
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007407502507522568,
+      "loss": 1.6918,
+      "theoretical_loss": 4.217760346353984,
+      "tokens_seen": 274989056
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007407342026078235,
+      "loss": 1.6561,
+      "theoretical_loss": 4.217641558317008,
+      "tokens_seen": 275054592
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007407181544633902,
+      "loss": 1.6212,
+      "theoretical_loss": 4.217522806502455,
+      "tokens_seen": 275120128
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007407021063189569,
+      "loss": 1.5607,
+      "theoretical_loss": 4.217404090890653,
+      "tokens_seen": 275185664
+    },
+    {
+      "epoch": 0.08,
+      "objective/train/docs_used": 108722,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.6769760847091675,
+      "objective/train/theoretical_loss": 4.217285411461949,
+      "objective/train/tokens_used": 295711200,
+      "theoretical_loss": 4.217285411461949,
+      "tokens_seen": 275251200
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007406860581745236,
+      "loss": 1.677,
+      "theoretical_loss": 4.217285411461949,
+      "tokens_seen": 275251200
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007406700100300903,
+      "loss": 1.6009,
+      "theoretical_loss": 4.217166768196704,
+      "tokens_seen": 275316736
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.000740653961885657,
+      "loss": 1.7233,
+      "theoretical_loss": 4.217048161075294,
+      "tokens_seen": 275382272
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007406379137412237,
+      "loss": 1.6772,
+      "theoretical_loss": 4.2169295900781085,
+      "tokens_seen": 275447808
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007406218655967904,
+      "loss": 1.6218,
+      "theoretical_loss": 4.216811055185556,
+      "tokens_seen": 275513344
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007406058174523571,
+      "loss": 1.6164,
+      "theoretical_loss": 4.216692556378058,
+      "tokens_seen": 275578880
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007405897693079238,
+      "loss": 1.5954,
+      "theoretical_loss": 4.216574093636053,
+      "tokens_seen": 275644416
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007405737211634905,
+      "loss": 1.5856,
+      "theoretical_loss": 4.216455666939991,
+      "tokens_seen": 275709952
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007405576730190572,
+      "loss": 1.7238,
+      "theoretical_loss": 4.216337276270341,
+      "tokens_seen": 275775488
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007405416248746239,
+      "loss": 1.7341,
+      "theoretical_loss": 4.216218921607586,
+      "tokens_seen": 275841024
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007405255767301906,
+      "loss": 1.6413,
+      "theoretical_loss": 4.216100602932224,
+      "tokens_seen": 275906560
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007405095285857573,
+      "loss": 1.6513,
+      "theoretical_loss": 4.21598232022477,
+      "tokens_seen": 275972096
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.000740493480441324,
+      "loss": 1.6008,
+      "theoretical_loss": 4.215864073465749,
+      "tokens_seen": 276037632
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007404774322968907,
+      "loss": 1.6939,
+      "theoretical_loss": 4.215745862635708,
+      "tokens_seen": 276103168
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007404613841524574,
+      "loss": 1.639,
+      "theoretical_loss": 4.2156276877152035,
+      "tokens_seen": 276168704
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007404453360080241,
+      "loss": 1.6366,
+      "theoretical_loss": 4.21550954868481,
+      "tokens_seen": 276234240
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007404292878635908,
+      "loss": 1.5483,
+      "theoretical_loss": 4.2153914455251185,
+      "tokens_seen": 276299776
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007404132397191575,
+      "loss": 1.648,
+      "theoretical_loss": 4.215273378216731,
+      "tokens_seen": 276365312
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007403971915747242,
+      "loss": 1.686,
+      "theoretical_loss": 4.215155346740268,
+      "tokens_seen": 276430848
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007403811434302909,
+      "loss": 1.616,
+      "theoretical_loss": 4.215037351076363,
+      "tokens_seen": 276496384
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007403650952858576,
+      "loss": 1.6878,
+      "theoretical_loss": 4.214919391205665,
+      "tokens_seen": 276561920
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007403490471414243,
+      "loss": 1.6904,
+      "theoretical_loss": 4.21480146710884,
+      "tokens_seen": 276627456
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.000740332998996991,
+      "loss": 1.6566,
+      "theoretical_loss": 4.214683578766566,
+      "tokens_seen": 276692992
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007403169508525577,
+      "loss": 1.5896,
+      "theoretical_loss": 4.214565726159539,
+      "tokens_seen": 276758528
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007403009027081244,
+      "loss": 1.6907,
+      "theoretical_loss": 4.214447909268467,
+      "tokens_seen": 276824064
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007402848545636911,
+      "loss": 1.6596,
+      "theoretical_loss": 4.214330128074074,
+      "tokens_seen": 276889600
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007402688064192578,
+      "loss": 1.7149,
+      "theoretical_loss": 4.214212382557101,
+      "tokens_seen": 276955136
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007402527582748245,
+      "loss": 1.6435,
+      "theoretical_loss": 4.214094672698302,
+      "tokens_seen": 277020672
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007402367101303912,
+      "loss": 1.5875,
+      "theoretical_loss": 4.213976998478445,
+      "tokens_seen": 277086208
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007402206619859579,
+      "loss": 1.6132,
+      "theoretical_loss": 4.213859359878316,
+      "tokens_seen": 277151744
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007402046138415247,
+      "loss": 1.6561,
+      "theoretical_loss": 4.213741756878713,
+      "tokens_seen": 277217280
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007401885656970913,
+      "loss": 1.5125,
+      "theoretical_loss": 4.213624189460449,
+      "tokens_seen": 277282816
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.000740172517552658,
+      "loss": 1.742,
+      "theoretical_loss": 4.213506657604356,
+      "tokens_seen": 277348352
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007401564694082247,
+      "loss": 1.6136,
+      "theoretical_loss": 4.213389161291273,
+      "tokens_seen": 277413888
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007401404212637915,
+      "loss": 1.6431,
+      "theoretical_loss": 4.213271700502062,
+      "tokens_seen": 277479424
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007401243731193581,
+      "loss": 1.6536,
+      "theoretical_loss": 4.213154275217597,
+      "tokens_seen": 277544960
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007401083249749249,
+      "loss": 1.5936,
+      "theoretical_loss": 4.213036885418763,
+      "tokens_seen": 277610496
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007400922768304915,
+      "loss": 1.6004,
+      "theoretical_loss": 4.212919531086466,
+      "tokens_seen": 277676032
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007400762286860583,
+      "loss": 1.7002,
+      "theoretical_loss": 4.2128022122016215,
+      "tokens_seen": 277741568
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007400601805416249,
+      "loss": 1.7299,
+      "theoretical_loss": 4.212684928745163,
+      "tokens_seen": 277807104
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007400441323971917,
+      "loss": 1.7295,
+      "theoretical_loss": 4.212567680698038,
+      "tokens_seen": 277872640
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007400280842527583,
+      "loss": 1.5592,
+      "theoretical_loss": 4.212450468041208,
+      "tokens_seen": 277938176
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007400120361083251,
+      "loss": 1.609,
+      "theoretical_loss": 4.21233329075565,
+      "tokens_seen": 278003712
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007399959879638917,
+      "loss": 1.6057,
+      "theoretical_loss": 4.212216148822355,
+      "tokens_seen": 278069248
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007399799398194585,
+      "loss": 1.5876,
+      "theoretical_loss": 4.212099042222331,
+      "tokens_seen": 278134784
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007399638916750251,
+      "loss": 1.5245,
+      "theoretical_loss": 4.211981970936595,
+      "tokens_seen": 278200320
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007399478435305919,
+      "loss": 1.5892,
+      "theoretical_loss": 4.2118649349461865,
+      "tokens_seen": 278265856
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007399317953861585,
+      "loss": 1.5567,
+      "theoretical_loss": 4.211747934232153,
+      "tokens_seen": 278331392
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007399157472417253,
+      "loss": 1.584,
+      "theoretical_loss": 4.211630968775561,
+      "tokens_seen": 278396928
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007398996990972919,
+      "loss": 1.6937,
+      "theoretical_loss": 4.211514038557488,
+      "tokens_seen": 278462464
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007398836509528586,
+      "loss": 1.6164,
+      "theoretical_loss": 4.211397143559029,
+      "tokens_seen": 278528000
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007398676028084253,
+      "loss": 1.5245,
+      "theoretical_loss": 4.211280283761294,
+      "tokens_seen": 278593536
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.000739851554663992,
+      "loss": 1.66,
+      "theoretical_loss": 4.211163459145403,
+      "tokens_seen": 278659072
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007398355065195587,
+      "loss": 1.6867,
+      "theoretical_loss": 4.211046669692496,
+      "tokens_seen": 278724608
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007398194583751254,
+      "loss": 1.5391,
+      "theoretical_loss": 4.2109299153837245,
+      "tokens_seen": 278790144
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007398034102306921,
+      "loss": 1.6044,
+      "theoretical_loss": 4.210813196200255,
+      "tokens_seen": 278855680
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007397873620862588,
+      "loss": 1.7526,
+      "theoretical_loss": 4.21069651212327,
+      "tokens_seen": 278921216
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007397713139418255,
+      "loss": 1.6291,
+      "theoretical_loss": 4.2105798631339635,
+      "tokens_seen": 278986752
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007397552657973922,
+      "loss": 1.6543,
+      "theoretical_loss": 4.2104632492135465,
+      "tokens_seen": 279052288
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007397392176529589,
+      "loss": 1.6405,
+      "theoretical_loss": 4.210346670343244,
+      "tokens_seen": 279117824
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007397231695085256,
+      "loss": 1.7012,
+      "theoretical_loss": 4.210230126504296,
+      "tokens_seen": 279183360
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007397071213640923,
+      "loss": 1.6207,
+      "theoretical_loss": 4.210113617677955,
+      "tokens_seen": 279248896
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.000739691073219659,
+      "loss": 1.6666,
+      "theoretical_loss": 4.20999714384549,
+      "tokens_seen": 279314432
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007396750250752257,
+      "loss": 1.6338,
+      "theoretical_loss": 4.209880704988181,
+      "tokens_seen": 279379968
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007396589769307924,
+      "loss": 1.6907,
+      "theoretical_loss": 4.209764301087327,
+      "tokens_seen": 279445504
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007396429287863591,
+      "loss": 1.7038,
+      "theoretical_loss": 4.20964793212424,
+      "tokens_seen": 279511040
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007396268806419258,
+      "loss": 1.6824,
+      "theoretical_loss": 4.209531598080245,
+      "tokens_seen": 279576576
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007396108324974925,
+      "loss": 1.6282,
+      "theoretical_loss": 4.20941529893668,
+      "tokens_seen": 279642112
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007395947843530592,
+      "loss": 1.545,
+      "theoretical_loss": 4.209299034674902,
+      "tokens_seen": 279707648
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007395787362086259,
+      "loss": 1.586,
+      "theoretical_loss": 4.209182805276278,
+      "tokens_seen": 279773184
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007395626880641926,
+      "loss": 1.7454,
+      "theoretical_loss": 4.209066610722192,
+      "tokens_seen": 279838720
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007395466399197593,
+      "loss": 1.5542,
+      "theoretical_loss": 4.208950450994041,
+      "tokens_seen": 279904256
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.000739530591775326,
+      "loss": 1.6669,
+      "theoretical_loss": 4.2088343260732355,
+      "tokens_seen": 279969792
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007395145436308927,
+      "loss": 1.605,
+      "theoretical_loss": 4.208718235941204,
+      "tokens_seen": 280035328
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007394984954864594,
+      "loss": 1.7347,
+      "theoretical_loss": 4.2086021805793825,
+      "tokens_seen": 280100864
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007394824473420261,
+      "loss": 1.719,
+      "theoretical_loss": 4.208486159969229,
+      "tokens_seen": 280166400
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007394663991975928,
+      "loss": 1.701,
+      "theoretical_loss": 4.20837017409221,
+      "tokens_seen": 280231936
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007394503510531595,
+      "loss": 1.6611,
+      "theoretical_loss": 4.20825422292981,
+      "tokens_seen": 280297472
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007394343029087262,
+      "loss": 1.5767,
+      "theoretical_loss": 4.208138306463524,
+      "tokens_seen": 280363008
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007394182547642929,
+      "loss": 1.68,
+      "theoretical_loss": 4.208022424674864,
+      "tokens_seen": 280428544
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007394022066198596,
+      "loss": 1.6801,
+      "theoretical_loss": 4.207906577545356,
+      "tokens_seen": 280494080
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007393861584754263,
+      "loss": 1.5856,
+      "theoretical_loss": 4.207790765056538,
+      "tokens_seen": 280559616
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.000739370110330993,
+      "loss": 1.6837,
+      "theoretical_loss": 4.207674987189965,
+      "tokens_seen": 280625152
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007393540621865597,
+      "loss": 1.8647,
+      "theoretical_loss": 4.2075592439272045,
+      "tokens_seen": 280690688
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007393380140421264,
+      "loss": 1.6042,
+      "theoretical_loss": 4.207443535249838,
+      "tokens_seen": 280756224
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007393219658976931,
+      "loss": 1.5646,
+      "theoretical_loss": 4.207327861139461,
+      "tokens_seen": 280821760
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007393059177532598,
+      "loss": 1.7132,
+      "theoretical_loss": 4.207212221577684,
+      "tokens_seen": 280887296
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007392898696088265,
+      "loss": 1.731,
+      "theoretical_loss": 4.207096616546132,
+      "tokens_seen": 280952832
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007392738214643932,
+      "loss": 1.6299,
+      "theoretical_loss": 4.206981046026442,
+      "tokens_seen": 281018368
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007392577733199599,
+      "loss": 1.5683,
+      "theoretical_loss": 4.2068655100002665,
+      "tokens_seen": 281083904
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007392417251755266,
+      "loss": 1.5759,
+      "theoretical_loss": 4.206750008449272,
+      "tokens_seen": 281149440
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007392256770310933,
+      "loss": 1.6443,
+      "theoretical_loss": 4.206634541355138,
+      "tokens_seen": 281214976
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.00073920962888666,
+      "loss": 1.7292,
+      "theoretical_loss": 4.20651910869956,
+      "tokens_seen": 281280512
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007391935807422267,
+      "loss": 1.6339,
+      "theoretical_loss": 4.2064037104642455,
+      "tokens_seen": 281346048
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007391775325977934,
+      "loss": 1.5848,
+      "theoretical_loss": 4.206288346630917,
+      "tokens_seen": 281411584
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007391614844533601,
+      "loss": 1.6348,
+      "theoretical_loss": 4.206173017181309,
+      "tokens_seen": 281477120
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007391454363089268,
+      "loss": 1.7069,
+      "theoretical_loss": 4.206057722097174,
+      "tokens_seen": 281542656
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007391293881644935,
+      "loss": 1.5232,
+      "theoretical_loss": 4.205942461360276,
+      "tokens_seen": 281608192
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007391133400200602,
+      "loss": 1.5416,
+      "theoretical_loss": 4.205827234952391,
+      "tokens_seen": 281673728
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007390972918756269,
+      "loss": 1.5954,
+      "theoretical_loss": 4.205712042855312,
+      "tokens_seen": 281739264
+    },
+    {
+      "epoch": 0.09,
+      "objective/train/docs_used": 111427,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.5511796474456787,
+      "objective/train/theoretical_loss": 4.205596885050846,
+      "objective/train/tokens_used": 302264800,
+      "theoretical_loss": 4.205596885050846,
+      "tokens_seen": 281804800
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007390812437311936,
+      "loss": 1.5512,
+      "theoretical_loss": 4.205596885050846,
+      "tokens_seen": 281804800
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007390651955867603,
+      "loss": 1.6241,
+      "theoretical_loss": 4.205481761520811,
+      "tokens_seen": 281870336
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.000739049147442327,
+      "loss": 1.6993,
+      "theoretical_loss": 4.20536667224704,
+      "tokens_seen": 281935872
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007390330992978937,
+      "loss": 1.6109,
+      "theoretical_loss": 4.205251617211382,
+      "tokens_seen": 282001408
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007390170511534604,
+      "loss": 1.693,
+      "theoretical_loss": 4.205136596395698,
+      "tokens_seen": 282066944
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007390010030090271,
+      "loss": 1.6325,
+      "theoretical_loss": 4.205021609781861,
+      "tokens_seen": 282132480
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007389849548645938,
+      "loss": 1.5679,
+      "theoretical_loss": 4.204906657351763,
+      "tokens_seen": 282198016
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007389689067201605,
+      "loss": 1.5551,
+      "theoretical_loss": 4.2047917390873035,
+      "tokens_seen": 282263552
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007389528585757272,
+      "loss": 1.5724,
+      "theoretical_loss": 4.204676854970401,
+      "tokens_seen": 282329088
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.000738936810431294,
+      "loss": 1.7219,
+      "theoretical_loss": 4.204562004982986,
+      "tokens_seen": 282394624
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007389207622868606,
+      "loss": 1.6348,
+      "theoretical_loss": 4.204447189107,
+      "tokens_seen": 282460160
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007389047141424273,
+      "loss": 1.6085,
+      "theoretical_loss": 4.204332407324403,
+      "tokens_seen": 282525696
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.000738888665997994,
+      "loss": 1.6378,
+      "theoretical_loss": 4.204217659617164,
+      "tokens_seen": 282591232
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007388726178535607,
+      "loss": 1.5899,
+      "theoretical_loss": 4.2041029459672705,
+      "tokens_seen": 282656768
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007388565697091274,
+      "loss": 1.5708,
+      "theoretical_loss": 4.2039882663567205,
+      "tokens_seen": 282722304
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007388405215646942,
+      "loss": 1.6657,
+      "theoretical_loss": 4.203873620767526,
+      "tokens_seen": 282787840
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007388244734202608,
+      "loss": 1.6038,
+      "theoretical_loss": 4.203759009181714,
+      "tokens_seen": 282853376
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007388084252758276,
+      "loss": 1.813,
+      "theoretical_loss": 4.203644431581323,
+      "tokens_seen": 282918912
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007387923771313942,
+      "loss": 1.6019,
+      "theoretical_loss": 4.203529887948408,
+      "tokens_seen": 282984448
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.000738776328986961,
+      "loss": 1.6598,
+      "theoretical_loss": 4.203415378265034,
+      "tokens_seen": 283049984
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007387602808425276,
+      "loss": 1.5217,
+      "theoretical_loss": 4.203300902513284,
+      "tokens_seen": 283115520
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007387442326980944,
+      "loss": 1.6503,
+      "theoretical_loss": 4.203186460675251,
+      "tokens_seen": 283181056
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007387281845536611,
+      "loss": 1.5696,
+      "theoretical_loss": 4.203072052733043,
+      "tokens_seen": 283246592
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007387121364092278,
+      "loss": 1.683,
+      "theoretical_loss": 4.202957678668782,
+      "tokens_seen": 283312128
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007386960882647945,
+      "loss": 1.6325,
+      "theoretical_loss": 4.202843338464602,
+      "tokens_seen": 283377664
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007386800401203612,
+      "loss": 1.7068,
+      "theoretical_loss": 4.202729032102653,
+      "tokens_seen": 283443200
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007386639919759279,
+      "loss": 1.6819,
+      "theoretical_loss": 4.202614759565096,
+      "tokens_seen": 283508736
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007386479438314946,
+      "loss": 1.6655,
+      "theoretical_loss": 4.202500520834106,
+      "tokens_seen": 283574272
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007386318956870613,
+      "loss": 1.6321,
+      "theoretical_loss": 4.202386315891874,
+      "tokens_seen": 283639808
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.000738615847542628,
+      "loss": 1.6632,
+      "theoretical_loss": 4.202272144720602,
+      "tokens_seen": 283705344
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007385997993981946,
+      "loss": 1.6903,
+      "theoretical_loss": 4.202158007302506,
+      "tokens_seen": 283770880
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007385837512537614,
+      "loss": 1.6503,
+      "theoretical_loss": 4.202043903619814,
+      "tokens_seen": 283836416
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.000738567703109328,
+      "loss": 1.6348,
+      "theoretical_loss": 4.201929833654772,
+      "tokens_seen": 283901952
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007385516549648948,
+      "loss": 1.6455,
+      "theoretical_loss": 4.201815797389634,
+      "tokens_seen": 283967488
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007385356068204614,
+      "loss": 1.6176,
+      "theoretical_loss": 4.201701794806672,
+      "tokens_seen": 284033024
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007385195586760282,
+      "loss": 1.656,
+      "theoretical_loss": 4.201587825888167,
+      "tokens_seen": 284098560
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007385035105315948,
+      "loss": 1.6294,
+      "theoretical_loss": 4.201473890616418,
+      "tokens_seen": 284164096
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007384874623871615,
+      "loss": 1.5932,
+      "theoretical_loss": 4.201359988973733,
+      "tokens_seen": 284229632
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007384714142427282,
+      "loss": 1.6699,
+      "theoretical_loss": 4.201246120942438,
+      "tokens_seen": 284295168
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007384553660982949,
+      "loss": 1.6361,
+      "theoretical_loss": 4.201132286504867,
+      "tokens_seen": 284360704
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007384393179538616,
+      "loss": 1.6764,
+      "theoretical_loss": 4.201018485643373,
+      "tokens_seen": 284426240
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007384232698094283,
+      "loss": 1.7661,
+      "theoretical_loss": 4.200904718340317,
+      "tokens_seen": 284491776
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.000738407221664995,
+      "loss": 1.6275,
+      "theoretical_loss": 4.200790984578078,
+      "tokens_seen": 284557312
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007383911735205617,
+      "loss": 1.6574,
+      "theoretical_loss": 4.200677284339045,
+      "tokens_seen": 284622848
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007383751253761284,
+      "loss": 1.6439,
+      "theoretical_loss": 4.2005636176056225,
+      "tokens_seen": 284688384
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007383590772316951,
+      "loss": 1.6343,
+      "theoretical_loss": 4.200449984360226,
+      "tokens_seen": 284753920
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007383430290872618,
+      "loss": 1.6418,
+      "theoretical_loss": 4.200336384585286,
+      "tokens_seen": 284819456
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007383269809428285,
+      "loss": 1.6593,
+      "theoretical_loss": 4.200222818263246,
+      "tokens_seen": 284884992
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007383109327983952,
+      "loss": 1.6162,
+      "theoretical_loss": 4.200109285376563,
+      "tokens_seen": 284950528
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007382948846539619,
+      "loss": 1.5544,
+      "theoretical_loss": 4.1999957859077055,
+      "tokens_seen": 285016064
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007382788365095286,
+      "loss": 1.6838,
+      "theoretical_loss": 4.1998823198391575,
+      "tokens_seen": 285081600
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007382627883650953,
+      "loss": 1.5549,
+      "theoretical_loss": 4.199768887153415,
+      "tokens_seen": 285147136
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.000738246740220662,
+      "loss": 1.7648,
+      "theoretical_loss": 4.199655487832987,
+      "tokens_seen": 285212672
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007382306920762287,
+      "loss": 1.6146,
+      "theoretical_loss": 4.199542121860397,
+      "tokens_seen": 285278208
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007382146439317954,
+      "loss": 1.5645,
+      "theoretical_loss": 4.199428789218181,
+      "tokens_seen": 285343744
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007381985957873621,
+      "loss": 1.6011,
+      "theoretical_loss": 4.199315489888885,
+      "tokens_seen": 285409280
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007381825476429288,
+      "loss": 1.6522,
+      "theoretical_loss": 4.199202223855075,
+      "tokens_seen": 285474816
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007381664994984955,
+      "loss": 1.6987,
+      "theoretical_loss": 4.199088991099323,
+      "tokens_seen": 285540352
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007381504513540622,
+      "loss": 1.7089,
+      "theoretical_loss": 4.19897579160422,
+      "tokens_seen": 285605888
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007381344032096289,
+      "loss": 1.5844,
+      "theoretical_loss": 4.1988626253523655,
+      "tokens_seen": 285671424
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007381183550651956,
+      "loss": 1.5387,
+      "theoretical_loss": 4.198749492326374,
+      "tokens_seen": 285736960
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007381023069207623,
+      "loss": 1.553,
+      "theoretical_loss": 4.198636392508874,
+      "tokens_seen": 285802496
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.000738086258776329,
+      "loss": 1.6633,
+      "theoretical_loss": 4.198523325882505,
+      "tokens_seen": 285868032
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007380702106318957,
+      "loss": 1.6429,
+      "theoretical_loss": 4.198410292429922,
+      "tokens_seen": 285933568
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007380541624874624,
+      "loss": 1.6591,
+      "theoretical_loss": 4.19829729213379,
+      "tokens_seen": 285999104
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007380381143430291,
+      "loss": 1.6118,
+      "theoretical_loss": 4.198184324976791,
+      "tokens_seen": 286064640
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007380220661985958,
+      "loss": 1.6294,
+      "theoretical_loss": 4.198071390941616,
+      "tokens_seen": 286130176
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007380060180541625,
+      "loss": 1.6333,
+      "theoretical_loss": 4.197958490010971,
+      "tokens_seen": 286195712
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007379899699097292,
+      "loss": 1.6301,
+      "theoretical_loss": 4.197845622167575,
+      "tokens_seen": 286261248
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007379739217652959,
+      "loss": 1.6499,
+      "theoretical_loss": 4.197732787394161,
+      "tokens_seen": 286326784
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007379578736208626,
+      "loss": 1.69,
+      "theoretical_loss": 4.197619985673471,
+      "tokens_seen": 286392320
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007379418254764293,
+      "loss": 1.6694,
+      "theoretical_loss": 4.197507216988265,
+      "tokens_seen": 286457856
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.000737925777331996,
+      "loss": 1.621,
+      "theoretical_loss": 4.197394481321312,
+      "tokens_seen": 286523392
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007379097291875627,
+      "loss": 1.6053,
+      "theoretical_loss": 4.1972817786553955,
+      "tokens_seen": 286588928
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007378936810431294,
+      "loss": 1.7107,
+      "theoretical_loss": 4.197169108973313,
+      "tokens_seen": 286654464
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007378776328986961,
+      "loss": 1.6213,
+      "theoretical_loss": 4.197056472257874,
+      "tokens_seen": 286720000
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007378615847542628,
+      "loss": 1.6317,
+      "theoretical_loss": 4.196943868491899,
+      "tokens_seen": 286785536
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007378455366098295,
+      "loss": 1.5911,
+      "theoretical_loss": 4.196831297658225,
+      "tokens_seen": 286851072
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007378294884653962,
+      "loss": 1.6587,
+      "theoretical_loss": 4.1967187597396975,
+      "tokens_seen": 286916608
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007378134403209629,
+      "loss": 1.5819,
+      "theoretical_loss": 4.196606254719179,
+      "tokens_seen": 286982144
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007377973921765296,
+      "loss": 1.6383,
+      "theoretical_loss": 4.196493782579544,
+      "tokens_seen": 287047680
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007377813440320963,
+      "loss": 1.5129,
+      "theoretical_loss": 4.196381343303676,
+      "tokens_seen": 287113216
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.000737765295887663,
+      "loss": 1.6588,
+      "theoretical_loss": 4.1962689368744766,
+      "tokens_seen": 287178752
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007377492477432297,
+      "loss": 1.73,
+      "theoretical_loss": 4.196156563274858,
+      "tokens_seen": 287244288
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007377331995987964,
+      "loss": 1.7149,
+      "theoretical_loss": 4.1960442224877434,
+      "tokens_seen": 287309824
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007377171514543631,
+      "loss": 1.6242,
+      "theoretical_loss": 4.195931914496072,
+      "tokens_seen": 287375360
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007377011033099298,
+      "loss": 1.6117,
+      "theoretical_loss": 4.195819639282792,
+      "tokens_seen": 287440896
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007376850551654965,
+      "loss": 1.6406,
+      "theoretical_loss": 4.195707396830869,
+      "tokens_seen": 287506432
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007376690070210632,
+      "loss": 1.6764,
+      "theoretical_loss": 4.195595187123277,
+      "tokens_seen": 287571968
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007376529588766299,
+      "loss": 1.5057,
+      "theoretical_loss": 4.195483010143006,
+      "tokens_seen": 287637504
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007376369107321966,
+      "loss": 1.5263,
+      "theoretical_loss": 4.1953708658730555,
+      "tokens_seen": 287703040
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007376208625877633,
+      "loss": 1.5509,
+      "theoretical_loss": 4.195258754296441,
+      "tokens_seen": 287768576
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.00073760481444333,
+      "loss": 1.5322,
+      "theoretical_loss": 4.195146675396189,
+      "tokens_seen": 287834112
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007375887662988967,
+      "loss": 1.5192,
+      "theoretical_loss": 4.195034629155338,
+      "tokens_seen": 287899648
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007375727181544634,
+      "loss": 1.6154,
+      "theoretical_loss": 4.194922615556941,
+      "tokens_seen": 287965184
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007375566700100301,
+      "loss": 1.6231,
+      "theoretical_loss": 4.194810634584061,
+      "tokens_seen": 288030720
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007375406218655968,
+      "loss": 1.6379,
+      "theoretical_loss": 4.194698686219778,
+      "tokens_seen": 288096256
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007375245737211636,
+      "loss": 1.6091,
+      "theoretical_loss": 4.194586770447179,
+      "tokens_seen": 288161792
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007375085255767302,
+      "loss": 1.5601,
+      "theoretical_loss": 4.194474887249369,
+      "tokens_seen": 288227328
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.000737492477432297,
+      "loss": 1.5731,
+      "theoretical_loss": 4.19436303660946,
+      "tokens_seen": 288292864
+    },
+    {
+      "epoch": 0.09,
+      "objective/train/docs_used": 113697,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.635483741760254,
+      "objective/train/theoretical_loss": 4.194251218510583,
+      "objective/train/tokens_used": 308818400,
+      "theoretical_loss": 4.194251218510583,
+      "tokens_seen": 288358400
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007374764292878637,
+      "loss": 1.6355,
+      "theoretical_loss": 4.194251218510583,
+      "tokens_seen": 288358400
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007374603811434304,
+      "loss": 1.6591,
+      "theoretical_loss": 4.194139432935877,
+      "tokens_seen": 288423936
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.000737444332998997,
+      "loss": 1.6107,
+      "theoretical_loss": 4.194027679868495,
+      "tokens_seen": 288489472
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007374282848545638,
+      "loss": 1.6476,
+      "theoretical_loss": 4.193915959291601,
+      "tokens_seen": 288555008
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007374122367101305,
+      "loss": 1.6513,
+      "theoretical_loss": 4.193804271188376,
+      "tokens_seen": 288620544
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007373961885656972,
+      "loss": 1.5988,
+      "theoretical_loss": 4.193692615542007,
+      "tokens_seen": 288686080
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007373801404212639,
+      "loss": 1.6891,
+      "theoretical_loss": 4.193580992335699,
+      "tokens_seen": 288751616
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007373640922768306,
+      "loss": 1.6256,
+      "theoretical_loss": 4.193469401552667,
+      "tokens_seen": 288817152
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007373480441323973,
+      "loss": 1.5912,
+      "theoretical_loss": 4.193357843176139,
+      "tokens_seen": 288882688
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.000737331995987964,
+      "loss": 1.6378,
+      "theoretical_loss": 4.193246317189356,
+      "tokens_seen": 288948224
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007373159478435307,
+      "loss": 1.5431,
+      "theoretical_loss": 4.19313482357557,
+      "tokens_seen": 289013760
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007372998996990974,
+      "loss": 1.5822,
+      "theoretical_loss": 4.193023362318047,
+      "tokens_seen": 289079296
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007372838515546641,
+      "loss": 1.7027,
+      "theoretical_loss": 4.1929119334000635,
+      "tokens_seen": 289144832
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007372678034102308,
+      "loss": 1.635,
+      "theoretical_loss": 4.192800536804912,
+      "tokens_seen": 289210368
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007372517552657975,
+      "loss": 1.6755,
+      "theoretical_loss": 4.192689172515893,
+      "tokens_seen": 289275904
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007372357071213642,
+      "loss": 1.6572,
+      "theoretical_loss": 4.192577840516323,
+      "tokens_seen": 289341440
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007372196589769309,
+      "loss": 1.6666,
+      "theoretical_loss": 4.192466540789528,
+      "tokens_seen": 289406976
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007372036108324975,
+      "loss": 1.6041,
+      "theoretical_loss": 4.19235527331885,
+      "tokens_seen": 289472512
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007371875626880643,
+      "loss": 1.5248,
+      "theoretical_loss": 4.19224403808764,
+      "tokens_seen": 289538048
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007371715145436309,
+      "loss": 1.7546,
+      "theoretical_loss": 4.192132835079262,
+      "tokens_seen": 289603584
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007371554663991977,
+      "loss": 1.5898,
+      "theoretical_loss": 4.192021664277094,
+      "tokens_seen": 289669120
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007371394182547643,
+      "loss": 1.6041,
+      "theoretical_loss": 4.191910525664524,
+      "tokens_seen": 289734656
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007371233701103311,
+      "loss": 1.7532,
+      "theoretical_loss": 4.191799419224955,
+      "tokens_seen": 289800192
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007371073219658977,
+      "loss": 1.5373,
+      "theoretical_loss": 4.1916883449418,
+      "tokens_seen": 289865728
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007370912738214645,
+      "loss": 1.5238,
+      "theoretical_loss": 4.1915773027984855,
+      "tokens_seen": 289931264
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007370752256770311,
+      "loss": 1.6475,
+      "theoretical_loss": 4.19146629277845,
+      "tokens_seen": 289996800
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007370591775325978,
+      "loss": 1.6677,
+      "theoretical_loss": 4.1913553148651435,
+      "tokens_seen": 290062336
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007370431293881645,
+      "loss": 1.6085,
+      "theoretical_loss": 4.191244369042029,
+      "tokens_seen": 290127872
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007370270812437312,
+      "loss": 1.616,
+      "theoretical_loss": 4.191133455292583,
+      "tokens_seen": 290193408
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007370110330992979,
+      "loss": 1.652,
+      "theoretical_loss": 4.191022573600293,
+      "tokens_seen": 290258944
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007369949849548646,
+      "loss": 1.5688,
+      "theoretical_loss": 4.190911723948657,
+      "tokens_seen": 290324480
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007369789368104313,
+      "loss": 1.5422,
+      "theoretical_loss": 4.190800906321188,
+      "tokens_seen": 290390016
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.000736962888665998,
+      "loss": 1.7319,
+      "theoretical_loss": 4.190690120701412,
+      "tokens_seen": 290455552
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007369468405215647,
+      "loss": 1.5619,
+      "theoretical_loss": 4.190579367072861,
+      "tokens_seen": 290521088
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007369307923771314,
+      "loss": 1.674,
+      "theoretical_loss": 4.190468645419086,
+      "tokens_seen": 290586624
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007369147442326981,
+      "loss": 1.6522,
+      "theoretical_loss": 4.190357955723649,
+      "tokens_seen": 290652160
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007368986960882648,
+      "loss": 1.6744,
+      "theoretical_loss": 4.190247297970121,
+      "tokens_seen": 290717696
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007368826479438315,
+      "loss": 1.6186,
+      "theoretical_loss": 4.190136672142088,
+      "tokens_seen": 290783232
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007368665997993982,
+      "loss": 1.6333,
+      "theoretical_loss": 4.190026078223145,
+      "tokens_seen": 290848768
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007368505516549649,
+      "loss": 1.6789,
+      "theoretical_loss": 4.189915516196904,
+      "tokens_seen": 290914304
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007368345035105316,
+      "loss": 1.7059,
+      "theoretical_loss": 4.189804986046984,
+      "tokens_seen": 290979840
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007368184553660983,
+      "loss": 1.6899,
+      "theoretical_loss": 4.18969448775702,
+      "tokens_seen": 291045376
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.000736802407221665,
+      "loss": 1.7178,
+      "theoretical_loss": 4.189584021310657,
+      "tokens_seen": 291110912
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007367863590772317,
+      "loss": 1.5942,
+      "theoretical_loss": 4.189473586691554,
+      "tokens_seen": 291176448
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007367703109327984,
+      "loss": 1.6982,
+      "theoretical_loss": 4.189363183883377,
+      "tokens_seen": 291241984
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007367542627883651,
+      "loss": 1.6072,
+      "theoretical_loss": 4.189252812869811,
+      "tokens_seen": 291307520
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007367382146439318,
+      "loss": 1.6422,
+      "theoretical_loss": 4.18914247363455,
+      "tokens_seen": 291373056
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007367221664994985,
+      "loss": 1.5706,
+      "theoretical_loss": 4.189032166161297,
+      "tokens_seen": 291438592
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007367061183550652,
+      "loss": 1.6603,
+      "theoretical_loss": 4.188921890433772,
+      "tokens_seen": 291504128
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007366900702106319,
+      "loss": 1.6234,
+      "theoretical_loss": 4.1888116464357035,
+      "tokens_seen": 291569664
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007366740220661986,
+      "loss": 1.654,
+      "theoretical_loss": 4.188701434150834,
+      "tokens_seen": 291635200
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007366579739217653,
+      "loss": 1.6638,
+      "theoretical_loss": 4.188591253562918,
+      "tokens_seen": 291700736
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.000736641925777332,
+      "loss": 1.6495,
+      "theoretical_loss": 4.18848110465572,
+      "tokens_seen": 291766272
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007366258776328987,
+      "loss": 1.5695,
+      "theoretical_loss": 4.188370987413018,
+      "tokens_seen": 291831808
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007366098294884654,
+      "loss": 1.5701,
+      "theoretical_loss": 4.188260901818602,
+      "tokens_seen": 291897344
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007365937813440321,
+      "loss": 1.5787,
+      "theoretical_loss": 4.1881508478562735,
+      "tokens_seen": 291962880
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007365777331995988,
+      "loss": 1.5897,
+      "theoretical_loss": 4.188040825509845,
+      "tokens_seen": 292028416
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007365616850551655,
+      "loss": 1.6281,
+      "theoretical_loss": 4.1879308347631445,
+      "tokens_seen": 292093952
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007365456369107322,
+      "loss": 1.5942,
+      "theoretical_loss": 4.187820875600006,
+      "tokens_seen": 292159488
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007365295887662989,
+      "loss": 1.6395,
+      "theoretical_loss": 4.187710948004281,
+      "tokens_seen": 292225024
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007365135406218656,
+      "loss": 1.706,
+      "theoretical_loss": 4.187601051959831,
+      "tokens_seen": 292290560
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007364974924774323,
+      "loss": 1.7282,
+      "theoretical_loss": 4.187491187450527,
+      "tokens_seen": 292356096
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.000736481444332999,
+      "loss": 1.6488,
+      "theoretical_loss": 4.187381354460255,
+      "tokens_seen": 292421632
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007364653961885657,
+      "loss": 1.7034,
+      "theoretical_loss": 4.187271552972911,
+      "tokens_seen": 292487168
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007364493480441324,
+      "loss": 1.6495,
+      "theoretical_loss": 4.187161782972405,
+      "tokens_seen": 292552704
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007364332998996991,
+      "loss": 1.7075,
+      "theoretical_loss": 4.187052044442655,
+      "tokens_seen": 292618240
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007364172517552658,
+      "loss": 1.6143,
+      "theoretical_loss": 4.1869423373675945,
+      "tokens_seen": 292683776
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007364012036108325,
+      "loss": 1.6848,
+      "theoretical_loss": 4.1868326617311675,
+      "tokens_seen": 292749312
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007363851554663992,
+      "loss": 1.6273,
+      "theoretical_loss": 4.18672301751733,
+      "tokens_seen": 292814848
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007363691073219659,
+      "loss": 1.6245,
+      "theoretical_loss": 4.186613404710048,
+      "tokens_seen": 292880384
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007363530591775326,
+      "loss": 1.6515,
+      "theoretical_loss": 4.186503823293303,
+      "tokens_seen": 292945920
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007363370110330993,
+      "loss": 1.6875,
+      "theoretical_loss": 4.186394273251084,
+      "tokens_seen": 293011456
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007363209628886661,
+      "loss": 1.7158,
+      "theoretical_loss": 4.186284754567395,
+      "tokens_seen": 293076992
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007363049147442327,
+      "loss": 1.6544,
+      "theoretical_loss": 4.1861752672262496,
+      "tokens_seen": 293142528
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007362888665997995,
+      "loss": 1.6383,
+      "theoretical_loss": 4.186065811211675,
+      "tokens_seen": 293208064
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007362728184553661,
+      "loss": 1.6027,
+      "theoretical_loss": 4.185956386507708,
+      "tokens_seen": 293273600
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007362567703109329,
+      "loss": 1.5945,
+      "theoretical_loss": 4.185846993098398,
+      "tokens_seen": 293339136
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007362407221664995,
+      "loss": 1.6038,
+      "theoretical_loss": 4.185737630967808,
+      "tokens_seen": 293404672
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007362246740220663,
+      "loss": 1.6761,
+      "theoretical_loss": 4.18562830010001,
+      "tokens_seen": 293470208
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.000736208625877633,
+      "loss": 1.707,
+      "theoretical_loss": 4.185519000479089,
+      "tokens_seen": 293535744
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007361925777331997,
+      "loss": 1.5523,
+      "theoretical_loss": 4.18540973208914,
+      "tokens_seen": 293601280
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007361765295887663,
+      "loss": 1.6931,
+      "theoretical_loss": 4.185300494914273,
+      "tokens_seen": 293666816
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007361604814443331,
+      "loss": 1.555,
+      "theoretical_loss": 4.185191288938606,
+      "tokens_seen": 293732352
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007361444332998997,
+      "loss": 1.6381,
+      "theoretical_loss": 4.185082114146271,
+      "tokens_seen": 293797888
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007361283851554665,
+      "loss": 1.7155,
+      "theoretical_loss": 4.184972970521409,
+      "tokens_seen": 293863424
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007361123370110332,
+      "loss": 1.5649,
+      "theoretical_loss": 4.184863858048177,
+      "tokens_seen": 293928960
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007360962888665999,
+      "loss": 1.6585,
+      "theoretical_loss": 4.18475477671074,
+      "tokens_seen": 293994496
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007360802407221666,
+      "loss": 1.6703,
+      "theoretical_loss": 4.184645726493275,
+      "tokens_seen": 294060032
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007360641925777333,
+      "loss": 1.5013,
+      "theoretical_loss": 4.184536707379972,
+      "tokens_seen": 294125568
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007360481444333,
+      "loss": 1.6214,
+      "theoretical_loss": 4.184427719355031,
+      "tokens_seen": 294191104
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007360320962888667,
+      "loss": 1.6508,
+      "theoretical_loss": 4.184318762402664,
+      "tokens_seen": 294256640
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007360160481444334,
+      "loss": 1.576,
+      "theoretical_loss": 4.184209836507096,
+      "tokens_seen": 294322176
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007360000000000001,
+      "loss": 1.6761,
+      "theoretical_loss": 4.18410094165256,
+      "tokens_seen": 294387712
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007359839518555668,
+      "loss": 1.5281,
+      "theoretical_loss": 4.183992077823305,
+      "tokens_seen": 294453248
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007359679037111335,
+      "loss": 1.5962,
+      "theoretical_loss": 4.183883245003588,
+      "tokens_seen": 294518784
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007359518555667002,
+      "loss": 1.584,
+      "theoretical_loss": 4.183774443177679,
+      "tokens_seen": 294584320
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007359358074222669,
+      "loss": 1.6362,
+      "theoretical_loss": 4.183665672329861,
+      "tokens_seen": 294649856
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007359197592778336,
+      "loss": 1.5156,
+      "theoretical_loss": 4.183556932444424,
+      "tokens_seen": 294715392
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007359037111334003,
+      "loss": 1.5991,
+      "theoretical_loss": 4.183448223505673,
+      "tokens_seen": 294780928
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.000735887662988967,
+      "loss": 1.7028,
+      "theoretical_loss": 4.183339545497924,
+      "tokens_seen": 294846464
+    },
+    {
+      "epoch": 0.09,
+      "objective/train/docs_used": 116257,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.6783287525177002,
+      "objective/train/theoretical_loss": 4.1832308984055055,
+      "objective/train/tokens_used": 315372000,
+      "theoretical_loss": 4.1832308984055055,
+      "tokens_seen": 294912000
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007358716148445337,
+      "loss": 1.6783,
+      "theoretical_loss": 4.1832308984055055,
+      "tokens_seen": 294912000
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007358555667001004,
+      "loss": 1.5823,
+      "theoretical_loss": 4.183122282212754,
+      "tokens_seen": 294977536
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007358395185556671,
+      "loss": 1.695,
+      "theoretical_loss": 4.183013696904018,
+      "tokens_seen": 295043072
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007358234704112338,
+      "loss": 1.5844,
+      "theoretical_loss": 4.182905142463662,
+      "tokens_seen": 295108608
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007358074222668004,
+      "loss": 1.4783,
+      "theoretical_loss": 4.182796618876057,
+      "tokens_seen": 295174144
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007357913741223672,
+      "loss": 1.7181,
+      "theoretical_loss": 4.182688126125587,
+      "tokens_seen": 295239680
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007357753259779338,
+      "loss": 1.652,
+      "theoretical_loss": 4.182579664196648,
+      "tokens_seen": 295305216
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007357592778335006,
+      "loss": 1.6404,
+      "theoretical_loss": 4.182471233073647,
+      "tokens_seen": 295370752
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007357432296890672,
+      "loss": 1.5395,
+      "theoretical_loss": 4.182362832741001,
+      "tokens_seen": 295436288
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.000735727181544634,
+      "loss": 1.6276,
+      "theoretical_loss": 4.182254463183139,
+      "tokens_seen": 295501824
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007357111334002006,
+      "loss": 1.6724,
+      "theoretical_loss": 4.182146124384504,
+      "tokens_seen": 295567360
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007356950852557674,
+      "loss": 1.6746,
+      "theoretical_loss": 4.182037816329547,
+      "tokens_seen": 295632896
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.000735679037111334,
+      "loss": 1.6133,
+      "theoretical_loss": 4.181929539002731,
+      "tokens_seen": 295698432
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007356629889669007,
+      "loss": 1.7129,
+      "theoretical_loss": 4.1818212923885305,
+      "tokens_seen": 295763968
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007356469408224674,
+      "loss": 1.6663,
+      "theoretical_loss": 4.181713076471432,
+      "tokens_seen": 295829504
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007356308926780341,
+      "loss": 1.5642,
+      "theoretical_loss": 4.181604891235933,
+      "tokens_seen": 295895040
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007356148445336008,
+      "loss": 1.5794,
+      "theoretical_loss": 4.181496736666542,
+      "tokens_seen": 295960576
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007355987963891675,
+      "loss": 1.6991,
+      "theoretical_loss": 4.181388612747778,
+      "tokens_seen": 296026112
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007355827482447342,
+      "loss": 1.6103,
+      "theoretical_loss": 4.181280519464172,
+      "tokens_seen": 296091648
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007355667001003009,
+      "loss": 1.5653,
+      "theoretical_loss": 4.181172456800267,
+      "tokens_seen": 296157184
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007355506519558676,
+      "loss": 1.6646,
+      "theoretical_loss": 4.181064424740617,
+      "tokens_seen": 296222720
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007355346038114343,
+      "loss": 1.5364,
+      "theoretical_loss": 4.180956423269784,
+      "tokens_seen": 296288256
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.000735518555667001,
+      "loss": 1.565,
+      "theoretical_loss": 4.180848452372347,
+      "tokens_seen": 296353792
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007355025075225677,
+      "loss": 1.6551,
+      "theoretical_loss": 4.180740512032891,
+      "tokens_seen": 296419328
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007354864593781344,
+      "loss": 1.6264,
+      "theoretical_loss": 4.180632602236014,
+      "tokens_seen": 296484864
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007354704112337011,
+      "loss": 1.5132,
+      "theoretical_loss": 4.180524722966327,
+      "tokens_seen": 296550400
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007354543630892678,
+      "loss": 1.6651,
+      "theoretical_loss": 4.180416874208449,
+      "tokens_seen": 296615936
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007354383149448345,
+      "loss": 1.6225,
+      "theoretical_loss": 4.180309055947012,
+      "tokens_seen": 296681472
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007354222668004012,
+      "loss": 1.6646,
+      "theoretical_loss": 4.18020126816666,
+      "tokens_seen": 296747008
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007354062186559679,
+      "loss": 1.6745,
+      "theoretical_loss": 4.180093510852045,
+      "tokens_seen": 296812544
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007353901705115346,
+      "loss": 1.7138,
+      "theoretical_loss": 4.179985783987834,
+      "tokens_seen": 296878080
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007353741223671013,
+      "loss": 1.6246,
+      "theoretical_loss": 4.179878087558702,
+      "tokens_seen": 296943616
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.000735358074222668,
+      "loss": 1.5611,
+      "theoretical_loss": 4.179770421549336,
+      "tokens_seen": 297009152
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007353420260782347,
+      "loss": 1.6443,
+      "theoretical_loss": 4.179662785944435,
+      "tokens_seen": 297074688
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007353259779338014,
+      "loss": 1.4962,
+      "theoretical_loss": 4.179555180728708,
+      "tokens_seen": 297140224
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007353099297893681,
+      "loss": 1.6492,
+      "theoretical_loss": 4.179447605886876,
+      "tokens_seen": 297205760
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007352938816449348,
+      "loss": 1.6408,
+      "theoretical_loss": 4.1793400614036695,
+      "tokens_seen": 297271296
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007352778335005015,
+      "loss": 1.6694,
+      "theoretical_loss": 4.179232547263831,
+      "tokens_seen": 297336832
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007352617853560682,
+      "loss": 1.6532,
+      "theoretical_loss": 4.1791250634521155,
+      "tokens_seen": 297402368
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007352457372116349,
+      "loss": 1.7528,
+      "theoretical_loss": 4.179017609953287,
+      "tokens_seen": 297467904
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007352296890672016,
+      "loss": 1.6727,
+      "theoretical_loss": 4.17891018675212,
+      "tokens_seen": 297533440
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007352136409227683,
+      "loss": 1.7006,
+      "theoretical_loss": 4.178802793833402,
+      "tokens_seen": 297598976
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.000735197592778335,
+      "loss": 1.4933,
+      "theoretical_loss": 4.17869543118193,
+      "tokens_seen": 297664512
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007351815446339017,
+      "loss": 1.6455,
+      "theoretical_loss": 4.178588098782513,
+      "tokens_seen": 297730048
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007351654964894684,
+      "loss": 1.5437,
+      "theoretical_loss": 4.178480796619971,
+      "tokens_seen": 297795584
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007351494483450351,
+      "loss": 1.7077,
+      "theoretical_loss": 4.178373524679134,
+      "tokens_seen": 297861120
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007351334002006018,
+      "loss": 1.6845,
+      "theoretical_loss": 4.1782662829448425,
+      "tokens_seen": 297926656
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007351173520561686,
+      "loss": 1.6499,
+      "theoretical_loss": 4.17815907140195,
+      "tokens_seen": 297992192
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007351013039117352,
+      "loss": 1.5718,
+      "theoretical_loss": 4.178051890035319,
+      "tokens_seen": 298057728
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.000735085255767302,
+      "loss": 1.6246,
+      "theoretical_loss": 4.177944738829825,
+      "tokens_seen": 298123264
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007350692076228686,
+      "loss": 1.5704,
+      "theoretical_loss": 4.177837617770352,
+      "tokens_seen": 298188800
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007350531594784354,
+      "loss": 1.7485,
+      "theoretical_loss": 4.1777305268417955,
+      "tokens_seen": 298254336
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.000735037111334002,
+      "loss": 1.5856,
+      "theoretical_loss": 4.177623466029064,
+      "tokens_seen": 298319872
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007350210631895688,
+      "loss": 1.5798,
+      "theoretical_loss": 4.177516435317074,
+      "tokens_seen": 298385408
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007350050150451354,
+      "loss": 1.6708,
+      "theoretical_loss": 4.177409434690754,
+      "tokens_seen": 298450944
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007349889669007022,
+      "loss": 1.6167,
+      "theoretical_loss": 4.177302464135044,
+      "tokens_seen": 298516480
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007349729187562688,
+      "loss": 1.5887,
+      "theoretical_loss": 4.177195523634895,
+      "tokens_seen": 298582016
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007349568706118356,
+      "loss": 1.6746,
+      "theoretical_loss": 4.1770886131752665,
+      "tokens_seen": 298647552
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007349408224674022,
+      "loss": 1.6054,
+      "theoretical_loss": 4.176981732741131,
+      "tokens_seen": 298713088
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.000734924774322969,
+      "loss": 1.6931,
+      "theoretical_loss": 4.176874882317472,
+      "tokens_seen": 298778624
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007349087261785356,
+      "loss": 1.5524,
+      "theoretical_loss": 4.176768061889283,
+      "tokens_seen": 298844160
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007348926780341024,
+      "loss": 1.6189,
+      "theoretical_loss": 4.176661271441568,
+      "tokens_seen": 298909696
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.000734876629889669,
+      "loss": 1.635,
+      "theoretical_loss": 4.176554510959342,
+      "tokens_seen": 298975232
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007348605817452358,
+      "loss": 1.7508,
+      "theoretical_loss": 4.1764477804276305,
+      "tokens_seen": 299040768
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007348445336008024,
+      "loss": 1.5193,
+      "theoretical_loss": 4.176341079831471,
+      "tokens_seen": 299106304
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007348284854563692,
+      "loss": 1.7028,
+      "theoretical_loss": 4.176234409155912,
+      "tokens_seen": 299171840
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007348124373119358,
+      "loss": 1.6708,
+      "theoretical_loss": 4.176127768386008,
+      "tokens_seen": 299237376
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007347963891675026,
+      "loss": 1.5564,
+      "theoretical_loss": 4.176021157506832,
+      "tokens_seen": 299302912
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007347803410230692,
+      "loss": 1.6287,
+      "theoretical_loss": 4.1759145765034615,
+      "tokens_seen": 299368448
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.000734764292878636,
+      "loss": 1.6559,
+      "theoretical_loss": 4.175808025360988,
+      "tokens_seen": 299433984
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007347482447342027,
+      "loss": 1.6383,
+      "theoretical_loss": 4.175701504064511,
+      "tokens_seen": 299499520
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007347321965897694,
+      "loss": 1.595,
+      "theoretical_loss": 4.175595012599143,
+      "tokens_seen": 299565056
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.000734716148445336,
+      "loss": 1.6399,
+      "theoretical_loss": 4.175488550950007,
+      "tokens_seen": 299630592
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007347001003009028,
+      "loss": 1.628,
+      "theoretical_loss": 4.175382119102235,
+      "tokens_seen": 299696128
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007346840521564695,
+      "loss": 1.731,
+      "theoretical_loss": 4.175275717040972,
+      "tokens_seen": 299761664
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007346680040120362,
+      "loss": 1.5828,
+      "theoretical_loss": 4.175169344751371,
+      "tokens_seen": 299827200
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007346519558676029,
+      "loss": 1.6928,
+      "theoretical_loss": 4.175063002218598,
+      "tokens_seen": 299892736
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007346359077231696,
+      "loss": 1.6022,
+      "theoretical_loss": 4.174956689427829,
+      "tokens_seen": 299958272
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007346198595787363,
+      "loss": 1.6119,
+      "theoretical_loss": 4.17485040636425,
+      "tokens_seen": 300023808
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.000734603811434303,
+      "loss": 1.6451,
+      "theoretical_loss": 4.174744153013057,
+      "tokens_seen": 300089344
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007345877632898697,
+      "loss": 1.5967,
+      "theoretical_loss": 4.174637929359458,
+      "tokens_seen": 300154880
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007345717151454364,
+      "loss": 1.6597,
+      "theoretical_loss": 4.174531735388673,
+      "tokens_seen": 300220416
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007345556670010031,
+      "loss": 1.5078,
+      "theoretical_loss": 4.174425571085927,
+      "tokens_seen": 300285952
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007345396188565698,
+      "loss": 1.6415,
+      "theoretical_loss": 4.174319436436462,
+      "tokens_seen": 300351488
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007345235707121365,
+      "loss": 1.5989,
+      "theoretical_loss": 4.174213331425527,
+      "tokens_seen": 300417024
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007345075225677032,
+      "loss": 1.5678,
+      "theoretical_loss": 4.174107256038383,
+      "tokens_seen": 300482560
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007344914744232699,
+      "loss": 1.5294,
+      "theoretical_loss": 4.1740012102603,
+      "tokens_seen": 300548096
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007344754262788366,
+      "loss": 1.5301,
+      "theoretical_loss": 4.17389519407656,
+      "tokens_seen": 300613632
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007344593781344033,
+      "loss": 1.6422,
+      "theoretical_loss": 4.173789207472455,
+      "tokens_seen": 300679168
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.00073444332998997,
+      "loss": 1.7038,
+      "theoretical_loss": 4.173683250433287,
+      "tokens_seen": 300744704
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007344272818455367,
+      "loss": 1.7171,
+      "theoretical_loss": 4.17357732294437,
+      "tokens_seen": 300810240
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007344112337011033,
+      "loss": 1.6093,
+      "theoretical_loss": 4.1734714249910265,
+      "tokens_seen": 300875776
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007343951855566701,
+      "loss": 1.4467,
+      "theoretical_loss": 4.173365556558591,
+      "tokens_seen": 300941312
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007343791374122367,
+      "loss": 1.6797,
+      "theoretical_loss": 4.1732597176324076,
+      "tokens_seen": 301006848
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007343630892678035,
+      "loss": 1.662,
+      "theoretical_loss": 4.173153908197831,
+      "tokens_seen": 301072384
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007343470411233701,
+      "loss": 1.5466,
+      "theoretical_loss": 4.173048128240227,
+      "tokens_seen": 301137920
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007343309929789369,
+      "loss": 1.6066,
+      "theoretical_loss": 4.172942377744972,
+      "tokens_seen": 301203456
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007343149448345035,
+      "loss": 1.5853,
+      "theoretical_loss": 4.17283665669745,
+      "tokens_seen": 301268992
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007342988966900703,
+      "loss": 1.6151,
+      "theoretical_loss": 4.172730965083061,
+      "tokens_seen": 301334528
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007342828485456369,
+      "loss": 1.7252,
+      "theoretical_loss": 4.1726253028872105,
+      "tokens_seen": 301400064
+    },
+    {
+      "epoch": 0.09,
+      "objective/train/docs_used": 118641,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.509371042251587,
+      "objective/train/theoretical_loss": 4.172519670095315,
+      "objective/train/tokens_used": 321925600,
+      "theoretical_loss": 4.172519670095315,
+      "tokens_seen": 301465600
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007342668004012036,
+      "loss": 1.5094,
+      "theoretical_loss": 4.172519670095315,
+      "tokens_seen": 301465600
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007342507522567703,
+      "loss": 1.7312,
+      "theoretical_loss": 4.172414066692804,
+      "tokens_seen": 301531136
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.000734234704112337,
+      "loss": 1.6671,
+      "theoretical_loss": 4.172308492665114,
+      "tokens_seen": 301596672
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007342186559679037,
+      "loss": 1.6761,
+      "theoretical_loss": 4.172202947997695,
+      "tokens_seen": 301662208
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007342026078234704,
+      "loss": 1.5495,
+      "theoretical_loss": 4.172097432676006,
+      "tokens_seen": 301727744
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007341865596790371,
+      "loss": 1.5774,
+      "theoretical_loss": 4.171991946685516,
+      "tokens_seen": 301793280
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007341705115346038,
+      "loss": 1.6569,
+      "theoretical_loss": 4.171886490011705,
+      "tokens_seen": 301858816
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007341544633901705,
+      "loss": 1.6768,
+      "theoretical_loss": 4.171781062640063,
+      "tokens_seen": 301924352
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007341384152457372,
+      "loss": 1.747,
+      "theoretical_loss": 4.17167566455609,
+      "tokens_seen": 301989888
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007341223671013039,
+      "loss": 1.6123,
+      "theoretical_loss": 4.171570295745297,
+      "tokens_seen": 302055424
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007341063189568706,
+      "loss": 1.6138,
+      "theoretical_loss": 4.171464956193205,
+      "tokens_seen": 302120960
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007340902708124373,
+      "loss": 1.7125,
+      "theoretical_loss": 4.171359645885346,
+      "tokens_seen": 302186496
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.000734074222668004,
+      "loss": 1.6021,
+      "theoretical_loss": 4.17125436480726,
+      "tokens_seen": 302252032
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007340581745235707,
+      "loss": 1.5513,
+      "theoretical_loss": 4.171149112944501,
+      "tokens_seen": 302317568
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007340421263791374,
+      "loss": 1.6594,
+      "theoretical_loss": 4.171043890282629,
+      "tokens_seen": 302383104
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007340260782347041,
+      "loss": 1.6129,
+      "theoretical_loss": 4.170938696807218,
+      "tokens_seen": 302448640
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007340100300902708,
+      "loss": 1.534,
+      "theoretical_loss": 4.17083353250385,
+      "tokens_seen": 302514176
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007339939819458375,
+      "loss": 1.6151,
+      "theoretical_loss": 4.170728397358118,
+      "tokens_seen": 302579712
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007339779338014042,
+      "loss": 1.6966,
+      "theoretical_loss": 4.170623291355626,
+      "tokens_seen": 302645248
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007339618856569709,
+      "loss": 1.5945,
+      "theoretical_loss": 4.1705182144819855,
+      "tokens_seen": 302710784
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007339458375125376,
+      "loss": 1.5896,
+      "theoretical_loss": 4.170413166722822,
+      "tokens_seen": 302776320
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007339297893681043,
+      "loss": 1.717,
+      "theoretical_loss": 4.170308148063769,
+      "tokens_seen": 302841856
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007339137412236711,
+      "loss": 1.6862,
+      "theoretical_loss": 4.1702031584904695,
+      "tokens_seen": 302907392
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007338976930792377,
+      "loss": 1.5662,
+      "theoretical_loss": 4.170098197988578,
+      "tokens_seen": 302972928
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007338816449348045,
+      "loss": 1.6809,
+      "theoretical_loss": 4.169993266543759,
+      "tokens_seen": 303038464
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007338655967903711,
+      "loss": 1.6761,
+      "theoretical_loss": 4.169888364141689,
+      "tokens_seen": 303104000
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007338495486459379,
+      "loss": 1.6053,
+      "theoretical_loss": 4.169783490768049,
+      "tokens_seen": 303169536
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007338335005015045,
+      "loss": 1.7181,
+      "theoretical_loss": 4.169678646408536,
+      "tokens_seen": 303235072
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007338174523570713,
+      "loss": 1.5548,
+      "theoretical_loss": 4.169573831048854,
+      "tokens_seen": 303300608
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007338014042126379,
+      "loss": 1.597,
+      "theoretical_loss": 4.16946904467472,
+      "tokens_seen": 303366144
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007337853560682047,
+      "loss": 1.6502,
+      "theoretical_loss": 4.169364287271858,
+      "tokens_seen": 303431680
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007337693079237713,
+      "loss": 1.6468,
+      "theoretical_loss": 4.169259558826003,
+      "tokens_seen": 303497216
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007337532597793381,
+      "loss": 1.756,
+      "theoretical_loss": 4.169154859322901,
+      "tokens_seen": 303562752
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007337372116349047,
+      "loss": 1.6479,
+      "theoretical_loss": 4.169050188748307,
+      "tokens_seen": 303628288
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007337211634904715,
+      "loss": 1.6078,
+      "theoretical_loss": 4.168945547087987,
+      "tokens_seen": 303693824
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007337051153460381,
+      "loss": 1.6399,
+      "theoretical_loss": 4.168840934327717,
+      "tokens_seen": 303759360
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007336890672016049,
+      "loss": 1.7228,
+      "theoretical_loss": 4.1687363504532815,
+      "tokens_seen": 303824896
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007336730190571715,
+      "loss": 1.5608,
+      "theoretical_loss": 4.168631795450478,
+      "tokens_seen": 303890432
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007336569709127383,
+      "loss": 1.7236,
+      "theoretical_loss": 4.168527269305112,
+      "tokens_seen": 303955968
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007336409227683049,
+      "loss": 1.6453,
+      "theoretical_loss": 4.168422772002998,
+      "tokens_seen": 304021504
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007336248746238717,
+      "loss": 1.6206,
+      "theoretical_loss": 4.168318303529963,
+      "tokens_seen": 304087040
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007336088264794383,
+      "loss": 1.5473,
+      "theoretical_loss": 4.168213863871843,
+      "tokens_seen": 304152576
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007335927783350051,
+      "loss": 1.6138,
+      "theoretical_loss": 4.168109453014484,
+      "tokens_seen": 304218112
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007335767301905717,
+      "loss": 1.6361,
+      "theoretical_loss": 4.1680050709437415,
+      "tokens_seen": 304283648
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007335606820461385,
+      "loss": 1.6445,
+      "theoretical_loss": 4.167900717645482,
+      "tokens_seen": 304349184
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007335446339017051,
+      "loss": 1.5995,
+      "theoretical_loss": 4.16779639310558,
+      "tokens_seen": 304414720
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007335285857572719,
+      "loss": 1.6864,
+      "theoretical_loss": 4.167692097309923,
+      "tokens_seen": 304480256
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007335125376128385,
+      "loss": 1.5532,
+      "theoretical_loss": 4.167587830244408,
+      "tokens_seen": 304545792
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007334964894684053,
+      "loss": 1.6382,
+      "theoretical_loss": 4.167483591894937,
+      "tokens_seen": 304611328
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.000733480441323972,
+      "loss": 1.6111,
+      "theoretical_loss": 4.167379382247429,
+      "tokens_seen": 304676864
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007334643931795387,
+      "loss": 1.6875,
+      "theoretical_loss": 4.167275201287808,
+      "tokens_seen": 304742400
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007334483450351053,
+      "loss": 1.5534,
+      "theoretical_loss": 4.16717104900201,
+      "tokens_seen": 304807936
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007334322968906721,
+      "loss": 1.5838,
+      "theoretical_loss": 4.167066925375981,
+      "tokens_seen": 304873472
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007334162487462387,
+      "loss": 1.574,
+      "theoretical_loss": 4.1669628303956765,
+      "tokens_seen": 304939008
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007334002006018055,
+      "loss": 1.5887,
+      "theoretical_loss": 4.166858764047062,
+      "tokens_seen": 305004544
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007333841524573721,
+      "loss": 1.5975,
+      "theoretical_loss": 4.166754726316111,
+      "tokens_seen": 305070080
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007333681043129389,
+      "loss": 1.5917,
+      "theoretical_loss": 4.16665071718881,
+      "tokens_seen": 305135616
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007333520561685056,
+      "loss": 1.5893,
+      "theoretical_loss": 4.1665467366511555,
+      "tokens_seen": 305201152
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007333360080240723,
+      "loss": 1.6601,
+      "theoretical_loss": 4.166442784689149,
+      "tokens_seen": 305266688
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.000733319959879639,
+      "loss": 1.668,
+      "theoretical_loss": 4.166338861288808,
+      "tokens_seen": 305332224
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007333039117352057,
+      "loss": 1.5953,
+      "theoretical_loss": 4.166234966436155,
+      "tokens_seen": 305397760
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007332878635907724,
+      "loss": 1.6166,
+      "theoretical_loss": 4.166131100117226,
+      "tokens_seen": 305463296
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007332718154463391,
+      "loss": 1.6878,
+      "theoretical_loss": 4.166027262318064,
+      "tokens_seen": 305528832
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007332557673019058,
+      "loss": 1.5142,
+      "theoretical_loss": 4.165923453024724,
+      "tokens_seen": 305594368
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007332397191574725,
+      "loss": 1.6742,
+      "theoretical_loss": 4.165819672223269,
+      "tokens_seen": 305659904
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007332236710130392,
+      "loss": 1.6469,
+      "theoretical_loss": 4.165715919899773,
+      "tokens_seen": 305725440
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007332076228686059,
+      "loss": 1.5844,
+      "theoretical_loss": 4.165612196040319,
+      "tokens_seen": 305790976
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007331915747241726,
+      "loss": 1.77,
+      "theoretical_loss": 4.1655085006310015,
+      "tokens_seen": 305856512
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007331755265797393,
+      "loss": 1.5801,
+      "theoretical_loss": 4.165404833657923,
+      "tokens_seen": 305922048
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.000733159478435306,
+      "loss": 1.6918,
+      "theoretical_loss": 4.165301195107195,
+      "tokens_seen": 305987584
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007331434302908727,
+      "loss": 1.5664,
+      "theoretical_loss": 4.1651975849649405,
+      "tokens_seen": 306053120
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007331273821464394,
+      "loss": 1.6055,
+      "theoretical_loss": 4.165094003217293,
+      "tokens_seen": 306118656
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007331113340020061,
+      "loss": 1.704,
+      "theoretical_loss": 4.164990449850393,
+      "tokens_seen": 306184192
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007330952858575728,
+      "loss": 1.7277,
+      "theoretical_loss": 4.164886924850393,
+      "tokens_seen": 306249728
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007330792377131395,
+      "loss": 1.5589,
+      "theoretical_loss": 4.164783428203453,
+      "tokens_seen": 306315264
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007330631895687062,
+      "loss": 1.6528,
+      "theoretical_loss": 4.1646799598957465,
+      "tokens_seen": 306380800
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007330471414242729,
+      "loss": 1.6215,
+      "theoretical_loss": 4.164576519913453,
+      "tokens_seen": 306446336
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007330310932798396,
+      "loss": 1.6438,
+      "theoretical_loss": 4.164473108242763,
+      "tokens_seen": 306511872
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007330150451354062,
+      "loss": 1.6968,
+      "theoretical_loss": 4.164369724869878,
+      "tokens_seen": 306577408
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.000732998996990973,
+      "loss": 1.6686,
+      "theoretical_loss": 4.164266369781005,
+      "tokens_seen": 306642944
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007329829488465396,
+      "loss": 1.5945,
+      "theoretical_loss": 4.164163042962366,
+      "tokens_seen": 306708480
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007329669007021064,
+      "loss": 1.6439,
+      "theoretical_loss": 4.1640597444001894,
+      "tokens_seen": 306774016
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.000732950852557673,
+      "loss": 1.6308,
+      "theoretical_loss": 4.163956474080715,
+      "tokens_seen": 306839552
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007329348044132398,
+      "loss": 1.6596,
+      "theoretical_loss": 4.16385323199019,
+      "tokens_seen": 306905088
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007329187562688064,
+      "loss": 1.627,
+      "theoretical_loss": 4.1637500181148726,
+      "tokens_seen": 306970624
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007329027081243732,
+      "loss": 1.6526,
+      "theoretical_loss": 4.163646832441031,
+      "tokens_seen": 307036160
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007328866599799398,
+      "loss": 1.6554,
+      "theoretical_loss": 4.163543674954942,
+      "tokens_seen": 307101696
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007328706118355065,
+      "loss": 1.6339,
+      "theoretical_loss": 4.163440545642894,
+      "tokens_seen": 307167232
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007328545636910732,
+      "loss": 1.6537,
+      "theoretical_loss": 4.163337444491183,
+      "tokens_seen": 307232768
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007328385155466399,
+      "loss": 1.5867,
+      "theoretical_loss": 4.163234371486114,
+      "tokens_seen": 307298304
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007328224674022066,
+      "loss": 1.5403,
+      "theoretical_loss": 4.163131326614003,
+      "tokens_seen": 307363840
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007328064192577733,
+      "loss": 1.6822,
+      "theoretical_loss": 4.163028309861176,
+      "tokens_seen": 307429376
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.00073279037111334,
+      "loss": 1.6459,
+      "theoretical_loss": 4.162925321213968,
+      "tokens_seen": 307494912
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007327743229689067,
+      "loss": 1.585,
+      "theoretical_loss": 4.162822360658723,
+      "tokens_seen": 307560448
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007327582748244734,
+      "loss": 1.6779,
+      "theoretical_loss": 4.162719428181794,
+      "tokens_seen": 307625984
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007327422266800401,
+      "loss": 1.6442,
+      "theoretical_loss": 4.162616523769547,
+      "tokens_seen": 307691520
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007327261785356068,
+      "loss": 1.5705,
+      "theoretical_loss": 4.1625136474083515,
+      "tokens_seen": 307757056
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007327101303911736,
+      "loss": 1.6629,
+      "theoretical_loss": 4.1624107990845935,
+      "tokens_seen": 307822592
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007326940822467402,
+      "loss": 1.607,
+      "theoretical_loss": 4.162307978784662,
+      "tokens_seen": 307888128
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.000732678034102307,
+      "loss": 1.6092,
+      "theoretical_loss": 4.162205186494962,
+      "tokens_seen": 307953664
+    },
+    {
+      "epoch": 0.09,
+      "objective/train/docs_used": 120992,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.5098984241485596,
+      "objective/train/theoretical_loss": 4.162102422201901,
+      "objective/train/tokens_used": 328479200,
+      "theoretical_loss": 4.162102422201901,
+      "tokens_seen": 308019200
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007326619859578736,
+      "loss": 1.5099,
+      "theoretical_loss": 4.162102422201901,
+      "tokens_seen": 308019200
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007326459378134404,
+      "loss": 1.737,
+      "theoretical_loss": 4.161999685891901,
+      "tokens_seen": 308084736
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.000732629889669007,
+      "loss": 1.5714,
+      "theoretical_loss": 4.161896977551393,
+      "tokens_seen": 308150272
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007326138415245738,
+      "loss": 1.6119,
+      "theoretical_loss": 4.161794297166815,
+      "tokens_seen": 308215808
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007325977933801404,
+      "loss": 1.6664,
+      "theoretical_loss": 4.161691644724616,
+      "tokens_seen": 308281344
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007325817452357072,
+      "loss": 1.4754,
+      "theoretical_loss": 4.161589020211256,
+      "tokens_seen": 308346880
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007325656970912738,
+      "loss": 1.6394,
+      "theoretical_loss": 4.1614864236132,
+      "tokens_seen": 308412416
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007325496489468406,
+      "loss": 1.6824,
+      "theoretical_loss": 4.161383854916926,
+      "tokens_seen": 308477952
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007325336008024072,
+      "loss": 1.6069,
+      "theoretical_loss": 4.161281314108923,
+      "tokens_seen": 308543488
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.000732517552657974,
+      "loss": 1.56,
+      "theoretical_loss": 4.1611788011756845,
+      "tokens_seen": 308609024
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007325015045135406,
+      "loss": 1.5524,
+      "theoretical_loss": 4.161076316103717,
+      "tokens_seen": 308674560
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007324854563691074,
+      "loss": 1.6594,
+      "theoretical_loss": 4.160973858879535,
+      "tokens_seen": 308740096
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.000732469408224674,
+      "loss": 1.5346,
+      "theoretical_loss": 4.160871429489663,
+      "tokens_seen": 308805632
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007324533600802408,
+      "loss": 1.5763,
+      "theoretical_loss": 4.160769027920635,
+      "tokens_seen": 308871168
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007324373119358074,
+      "loss": 1.7231,
+      "theoretical_loss": 4.160666654158993,
+      "tokens_seen": 308936704
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007324212637913742,
+      "loss": 1.6798,
+      "theoretical_loss": 4.160564308191289,
+      "tokens_seen": 309002240
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007324052156469408,
+      "loss": 1.5925,
+      "theoretical_loss": 4.1604619900040865,
+      "tokens_seen": 309067776
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007323891675025076,
+      "loss": 1.6104,
+      "theoretical_loss": 4.160359699583955,
+      "tokens_seen": 309133312
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007323731193580742,
+      "loss": 1.5745,
+      "theoretical_loss": 4.160257436917476,
+      "tokens_seen": 309198848
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.000732357071213641,
+      "loss": 1.5826,
+      "theoretical_loss": 4.160155201991238,
+      "tokens_seen": 309264384
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007323410230692076,
+      "loss": 1.56,
+      "theoretical_loss": 4.160052994791841,
+      "tokens_seen": 309329920
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007323249749247744,
+      "loss": 1.6606,
+      "theoretical_loss": 4.159950815305893,
+      "tokens_seen": 309395456
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.000732308926780341,
+      "loss": 1.708,
+      "theoretical_loss": 4.1598486635200125,
+      "tokens_seen": 309460992
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007322928786359078,
+      "loss": 1.5914,
+      "theoretical_loss": 4.159746539420825,
+      "tokens_seen": 309526528
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007322768304914744,
+      "loss": 1.5279,
+      "theoretical_loss": 4.159644442994967,
+      "tokens_seen": 309592064
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007322607823470412,
+      "loss": 1.7254,
+      "theoretical_loss": 4.159542374229084,
+      "tokens_seen": 309657600
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007322447342026078,
+      "loss": 1.5446,
+      "theoretical_loss": 4.159440333109832,
+      "tokens_seen": 309723136
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007322286860581746,
+      "loss": 1.6927,
+      "theoretical_loss": 4.159338319623874,
+      "tokens_seen": 309788672
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007322126379137412,
+      "loss": 1.6384,
+      "theoretical_loss": 4.1592363337578835,
+      "tokens_seen": 309854208
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.000732196589769308,
+      "loss": 1.586,
+      "theoretical_loss": 4.159134375498542,
+      "tokens_seen": 309919744
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007321805416248746,
+      "loss": 1.4855,
+      "theoretical_loss": 4.159032444832543,
+      "tokens_seen": 309985280
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007321644934804414,
+      "loss": 1.616,
+      "theoretical_loss": 4.158930541746587,
+      "tokens_seen": 310050816
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.000732148445336008,
+      "loss": 1.6506,
+      "theoretical_loss": 4.158828666227383,
+      "tokens_seen": 310116352
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007321323971915748,
+      "loss": 1.6238,
+      "theoretical_loss": 4.158726818261652,
+      "tokens_seen": 310181888
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007321163490471414,
+      "loss": 1.64,
+      "theoretical_loss": 4.158624997836123,
+      "tokens_seen": 310247424
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007321003009027082,
+      "loss": 1.7264,
+      "theoretical_loss": 4.158523204937531,
+      "tokens_seen": 310312960
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007320842527582748,
+      "loss": 1.6462,
+      "theoretical_loss": 4.158421439552626,
+      "tokens_seen": 310378496
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007320682046138416,
+      "loss": 1.6258,
+      "theoretical_loss": 4.158319701668162,
+      "tokens_seen": 310444032
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007320521564694082,
+      "loss": 1.6203,
+      "theoretical_loss": 4.158217991270906,
+      "tokens_seen": 310509568
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.000732036108324975,
+      "loss": 1.7848,
+      "theoretical_loss": 4.158116308347632,
+      "tokens_seen": 310575104
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007320200601805416,
+      "loss": 1.7354,
+      "theoretical_loss": 4.1580146528851225,
+      "tokens_seen": 310640640
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007320040120361084,
+      "loss": 1.5643,
+      "theoretical_loss": 4.157913024870172,
+      "tokens_seen": 310706176
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.000731987963891675,
+      "loss": 1.6105,
+      "theoretical_loss": 4.157811424289582,
+      "tokens_seen": 310771712
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007319719157472418,
+      "loss": 1.6735,
+      "theoretical_loss": 4.1577098511301624,
+      "tokens_seen": 310837248
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007319558676028085,
+      "loss": 1.6173,
+      "theoretical_loss": 4.157608305378735,
+      "tokens_seen": 310902784
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007319398194583752,
+      "loss": 1.7382,
+      "theoretical_loss": 4.157506787022128,
+      "tokens_seen": 310968320
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007319237713139419,
+      "loss": 1.6054,
+      "theoretical_loss": 4.15740529604718,
+      "tokens_seen": 311033856
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007319077231695086,
+      "loss": 1.5183,
+      "theoretical_loss": 4.15730383244074,
+      "tokens_seen": 311099392
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007318916750250753,
+      "loss": 1.5937,
+      "theoretical_loss": 4.157202396189662,
+      "tokens_seen": 311164928
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.000731875626880642,
+      "loss": 1.5996,
+      "theoretical_loss": 4.157100987280814,
+      "tokens_seen": 311230464
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007318595787362087,
+      "loss": 1.6759,
+      "theoretical_loss": 4.15699960570107,
+      "tokens_seen": 311296000
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007318435305917754,
+      "loss": 1.7264,
+      "theoretical_loss": 4.156898251437312,
+      "tokens_seen": 311361536
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007318274824473421,
+      "loss": 1.6315,
+      "theoretical_loss": 4.156796924476435,
+      "tokens_seen": 311427072
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007318114343029088,
+      "loss": 1.7101,
+      "theoretical_loss": 4.156695624805341,
+      "tokens_seen": 311492608
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007317953861584755,
+      "loss": 1.6406,
+      "theoretical_loss": 4.15659435241094,
+      "tokens_seen": 311558144
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007317793380140422,
+      "loss": 1.5463,
+      "theoretical_loss": 4.15649310728015,
+      "tokens_seen": 311623680
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007317632898696089,
+      "loss": 1.7026,
+      "theoretical_loss": 4.156391889399904,
+      "tokens_seen": 311689216
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007317472417251756,
+      "loss": 1.5756,
+      "theoretical_loss": 4.156290698757139,
+      "tokens_seen": 311754752
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007317311935807423,
+      "loss": 1.5774,
+      "theoretical_loss": 4.156189535338799,
+      "tokens_seen": 311820288
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.000731715145436309,
+      "loss": 1.5761,
+      "theoretical_loss": 4.156088399131844,
+      "tokens_seen": 311885824
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007316990972918757,
+      "loss": 1.4766,
+      "theoretical_loss": 4.155987290123236,
+      "tokens_seen": 311951360
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007316830491474424,
+      "loss": 1.5592,
+      "theoretical_loss": 4.15588620829995,
+      "tokens_seen": 312016896
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007316670010030091,
+      "loss": 1.6467,
+      "theoretical_loss": 4.15578515364897,
+      "tokens_seen": 312082432
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007316509528585758,
+      "loss": 1.6672,
+      "theoretical_loss": 4.155684126157286,
+      "tokens_seen": 312147968
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007316349047141425,
+      "loss": 1.6334,
+      "theoretical_loss": 4.1555831258119005,
+      "tokens_seen": 312213504
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007316188565697092,
+      "loss": 1.6194,
+      "theoretical_loss": 4.155482152599823,
+      "tokens_seen": 312279040
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007316028084252759,
+      "loss": 1.6357,
+      "theoretical_loss": 4.155381206508071,
+      "tokens_seen": 312344576
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007315867602808425,
+      "loss": 1.7414,
+      "theoretical_loss": 4.155280287523674,
+      "tokens_seen": 312410112
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007315707121364093,
+      "loss": 1.6323,
+      "theoretical_loss": 4.155179395633667,
+      "tokens_seen": 312475648
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007315546639919759,
+      "loss": 1.6686,
+      "theoretical_loss": 4.1550785308250955,
+      "tokens_seen": 312541184
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007315386158475427,
+      "loss": 1.5955,
+      "theoretical_loss": 4.154977693085016,
+      "tokens_seen": 312606720
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007315225677031093,
+      "loss": 1.633,
+      "theoretical_loss": 4.154876882400489,
+      "tokens_seen": 312672256
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007315065195586761,
+      "loss": 1.5965,
+      "theoretical_loss": 4.154776098758589,
+      "tokens_seen": 312737792
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007314904714142427,
+      "loss": 1.5974,
+      "theoretical_loss": 4.154675342146397,
+      "tokens_seen": 312803328
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007314744232698095,
+      "loss": 1.7647,
+      "theoretical_loss": 4.154574612551001,
+      "tokens_seen": 312868864
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007314583751253761,
+      "loss": 1.5976,
+      "theoretical_loss": 4.154473909959503,
+      "tokens_seen": 312934400
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007314423269809429,
+      "loss": 1.7073,
+      "theoretical_loss": 4.154373234359007,
+      "tokens_seen": 312999936
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007314262788365095,
+      "loss": 1.7557,
+      "theoretical_loss": 4.154272585736632,
+      "tokens_seen": 313065472
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007314102306920763,
+      "loss": 1.6123,
+      "theoretical_loss": 4.154171964079504,
+      "tokens_seen": 313131008
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007313941825476429,
+      "loss": 1.6845,
+      "theoretical_loss": 4.154071369374755,
+      "tokens_seen": 313196544
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007313781344032097,
+      "loss": 1.5383,
+      "theoretical_loss": 4.15397080160953,
+      "tokens_seen": 313262080
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007313620862587763,
+      "loss": 1.5845,
+      "theoretical_loss": 4.153870260770979,
+      "tokens_seen": 313327616
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007313460381143431,
+      "loss": 1.6715,
+      "theoretical_loss": 4.153769746846265,
+      "tokens_seen": 313393152
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007313299899699097,
+      "loss": 1.8037,
+      "theoretical_loss": 4.153669259822556,
+      "tokens_seen": 313458688
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0007313139418254765,
+      "loss": 1.716,
+      "theoretical_loss": 4.153568799687031,
+      "tokens_seen": 313524224
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0007312978936810431,
+      "loss": 1.6919,
+      "theoretical_loss": 4.153468366426877,
+      "tokens_seen": 313589760
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0007312818455366099,
+      "loss": 1.587,
+      "theoretical_loss": 4.153367960029288,
+      "tokens_seen": 313655296
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0007312657973921765,
+      "loss": 1.5868,
+      "theoretical_loss": 4.153267580481472,
+      "tokens_seen": 313720832
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0007312497492477433,
+      "loss": 1.57,
+      "theoretical_loss": 4.15316722777064,
+      "tokens_seen": 313786368
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0007312337011033099,
+      "loss": 1.6001,
+      "theoretical_loss": 4.153066901884016,
+      "tokens_seen": 313851904
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0007312176529588767,
+      "loss": 1.7275,
+      "theoretical_loss": 4.15296660280883,
+      "tokens_seen": 313917440
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0007312016048144433,
+      "loss": 1.6171,
+      "theoretical_loss": 4.152866330532321,
+      "tokens_seen": 313982976
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0007311855566700101,
+      "loss": 1.6996,
+      "theoretical_loss": 4.152766085041737,
+      "tokens_seen": 314048512
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0007311695085255767,
+      "loss": 1.6482,
+      "theoretical_loss": 4.152665866324337,
+      "tokens_seen": 314114048
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0007311534603811435,
+      "loss": 1.5343,
+      "theoretical_loss": 4.152565674367387,
+      "tokens_seen": 314179584
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0007311374122367101,
+      "loss": 1.6345,
+      "theoretical_loss": 4.15246550915816,
+      "tokens_seen": 314245120
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0007311213640922769,
+      "loss": 1.635,
+      "theoretical_loss": 4.15236537068394,
+      "tokens_seen": 314310656
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0007311053159478435,
+      "loss": 1.6151,
+      "theoretical_loss": 4.152265258932019,
+      "tokens_seen": 314376192
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0007310892678034103,
+      "loss": 1.5423,
+      "theoretical_loss": 4.152165173889697,
+      "tokens_seen": 314441728
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0007310732196589769,
+      "loss": 1.5504,
+      "theoretical_loss": 4.152065115544284,
+      "tokens_seen": 314507264
+    },
+    {
+      "epoch": 0.1,
+      "objective/train/docs_used": 123650,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.5619208812713623,
+      "objective/train/theoretical_loss": 4.151965083883098,
+      "objective/train/tokens_used": 335032800,
+      "theoretical_loss": 4.151965083883098,
+      "tokens_seen": 314572800
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0007310571715145437,
+      "loss": 1.5619,
+      "theoretical_loss": 4.151965083883098,
+      "tokens_seen": 314572800
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0007310411233701103,
+      "loss": 1.5635,
+      "theoretical_loss": 4.151865078893466,
+      "tokens_seen": 314638336
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0007310250752256771,
+      "loss": 1.6122,
+      "theoretical_loss": 4.151765100562723,
+      "tokens_seen": 314703872
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0007310090270812437,
+      "loss": 1.6883,
+      "theoretical_loss": 4.151665148878212,
+      "tokens_seen": 314769408
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0007309929789368105,
+      "loss": 1.6062,
+      "theoretical_loss": 4.151565223827288,
+      "tokens_seen": 314834944
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0007309769307923771,
+      "loss": 1.5542,
+      "theoretical_loss": 4.15146532539731,
+      "tokens_seen": 314900480
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0007309608826479439,
+      "loss": 1.6221,
+      "theoretical_loss": 4.151365453575648,
+      "tokens_seen": 314966016
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0007309448345035105,
+      "loss": 1.5982,
+      "theoretical_loss": 4.151265608349682,
+      "tokens_seen": 315031552
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0007309287863590773,
+      "loss": 1.5673,
+      "theoretical_loss": 4.151165789706798,
+      "tokens_seen": 315097088
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0007309127382146439,
+      "loss": 1.6323,
+      "theoretical_loss": 4.151065997634393,
+      "tokens_seen": 315162624
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0007308966900702107,
+      "loss": 1.5159,
+      "theoretical_loss": 4.1509662321198695,
+      "tokens_seen": 315228160
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0007308806419257773,
+      "loss": 1.5806,
+      "theoretical_loss": 4.150866493150642,
+      "tokens_seen": 315293696
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0007308645937813441,
+      "loss": 1.5716,
+      "theoretical_loss": 4.150766780714131,
+      "tokens_seen": 315359232
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0007308485456369107,
+      "loss": 1.7149,
+      "theoretical_loss": 4.150667094797766,
+      "tokens_seen": 315424768
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0007308324974924775,
+      "loss": 1.6208,
+      "theoretical_loss": 4.150567435388988,
+      "tokens_seen": 315490304
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0007308164493480441,
+      "loss": 1.6424,
+      "theoretical_loss": 4.150467802475241,
+      "tokens_seen": 315555840
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0007308004012036109,
+      "loss": 1.5372,
+      "theoretical_loss": 4.150368196043983,
+      "tokens_seen": 315621376
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0007307843530591775,
+      "loss": 1.5855,
+      "theoretical_loss": 4.150268616082678,
+      "tokens_seen": 315686912
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0007307683049147443,
+      "loss": 1.6299,
+      "theoretical_loss": 4.150169062578798,
+      "tokens_seen": 315752448
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0007307522567703109,
+      "loss": 1.6293,
+      "theoretical_loss": 4.150069535519825,
+      "tokens_seen": 315817984
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0007307362086258777,
+      "loss": 1.6443,
+      "theoretical_loss": 4.149970034893249,
+      "tokens_seen": 315883520
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0007307201604814443,
+      "loss": 1.5875,
+      "theoretical_loss": 4.149870560686567,
+      "tokens_seen": 315949056
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0007307041123370111,
+      "loss": 1.6472,
+      "theoretical_loss": 4.149771112887288,
+      "tokens_seen": 316014592
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0007306880641925777,
+      "loss": 1.5727,
+      "theoretical_loss": 4.149671691482926,
+      "tokens_seen": 316080128
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0007306720160481445,
+      "loss": 1.7287,
+      "theoretical_loss": 4.149572296461005,
+      "tokens_seen": 316145664
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0007306559679037111,
+      "loss": 1.5935,
+      "theoretical_loss": 4.149472927809057,
+      "tokens_seen": 316211200
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0007306399197592779,
+      "loss": 1.6763,
+      "theoretical_loss": 4.149373585514625,
+      "tokens_seen": 316276736
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0007306238716148446,
+      "loss": 1.6598,
+      "theoretical_loss": 4.149274269565256,
+      "tokens_seen": 316342272
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0007306078234704113,
+      "loss": 1.676,
+      "theoretical_loss": 4.149174979948507,
+      "tokens_seen": 316407808
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.000730591775325978,
+      "loss": 1.5757,
+      "theoretical_loss": 4.1490757166519465,
+      "tokens_seen": 316473344
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0007305757271815447,
+      "loss": 1.5618,
+      "theoretical_loss": 4.148976479663148,
+      "tokens_seen": 316538880
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0007305596790371114,
+      "loss": 1.5078,
+      "theoretical_loss": 4.148877268969694,
+      "tokens_seen": 316604416
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0007305436308926781,
+      "loss": 1.6376,
+      "theoretical_loss": 4.1487780845591775,
+      "tokens_seen": 316669952
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0007305275827482448,
+      "loss": 1.6326,
+      "theoretical_loss": 4.148678926419197,
+      "tokens_seen": 316735488
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0007305115346038115,
+      "loss": 1.6067,
+      "theoretical_loss": 4.148579794537362,
+      "tokens_seen": 316801024
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0007304954864593782,
+      "loss": 1.578,
+      "theoretical_loss": 4.148480688901288,
+      "tokens_seen": 316866560
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0007304794383149449,
+      "loss": 1.694,
+      "theoretical_loss": 4.148381609498602,
+      "tokens_seen": 316932096
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0007304633901705116,
+      "loss": 1.5567,
+      "theoretical_loss": 4.148282556316936,
+      "tokens_seen": 316997632
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0007304473420260783,
+      "loss": 1.6117,
+      "theoretical_loss": 4.148183529343932,
+      "tokens_seen": 317063168
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.000730431293881645,
+      "loss": 1.5606,
+      "theoretical_loss": 4.148084528567241,
+      "tokens_seen": 317128704
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0007304152457372117,
+      "loss": 1.6511,
+      "theoretical_loss": 4.147985553974522,
+      "tokens_seen": 317194240
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0007303991975927784,
+      "loss": 1.5511,
+      "theoretical_loss": 4.14788660555344,
+      "tokens_seen": 317259776
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0007303831494483451,
+      "loss": 1.5398,
+      "theoretical_loss": 4.147787683291672,
+      "tokens_seen": 317325312
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0007303671013039118,
+      "loss": 1.7024,
+      "theoretical_loss": 4.147688787176902,
+      "tokens_seen": 317390848
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0007303510531594785,
+      "loss": 1.5716,
+      "theoretical_loss": 4.147589917196821,
+      "tokens_seen": 317456384
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0007303350050150452,
+      "loss": 1.5588,
+      "theoretical_loss": 4.14749107333913,
+      "tokens_seen": 317521920
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0007303189568706119,
+      "loss": 1.6169,
+      "theoretical_loss": 4.147392255591538,
+      "tokens_seen": 317587456
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0007303029087261786,
+      "loss": 1.7814,
+      "theoretical_loss": 4.147293463941762,
+      "tokens_seen": 317652992
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0007302868605817453,
+      "loss": 1.4771,
+      "theoretical_loss": 4.147194698377526,
+      "tokens_seen": 317718528
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.000730270812437312,
+      "loss": 1.6387,
+      "theoretical_loss": 4.147095958886566,
+      "tokens_seen": 317784064
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0007302547642928787,
+      "loss": 1.6541,
+      "theoretical_loss": 4.146997245456623,
+      "tokens_seen": 317849600
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0007302387161484454,
+      "loss": 1.6418,
+      "theoretical_loss": 4.146898558075446,
+      "tokens_seen": 317915136
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0007302226680040121,
+      "loss": 1.5862,
+      "theoretical_loss": 4.146799896730794,
+      "tokens_seen": 317980672
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0007302066198595788,
+      "loss": 1.5395,
+      "theoretical_loss": 4.146701261410435,
+      "tokens_seen": 318046208
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0007301905717151454,
+      "loss": 1.5877,
+      "theoretical_loss": 4.1466026521021435,
+      "tokens_seen": 318111744
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0007301745235707122,
+      "loss": 1.7137,
+      "theoretical_loss": 4.146504068793703,
+      "tokens_seen": 318177280
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0007301584754262788,
+      "loss": 1.6126,
+      "theoretical_loss": 4.146405511472905,
+      "tokens_seen": 318242816
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0007301424272818456,
+      "loss": 1.6433,
+      "theoretical_loss": 4.146306980127548,
+      "tokens_seen": 318308352
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0007301263791374122,
+      "loss": 1.6642,
+      "theoretical_loss": 4.146208474745442,
+      "tokens_seen": 318373888
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.000730110330992979,
+      "loss": 1.6511,
+      "theoretical_loss": 4.1461099953144025,
+      "tokens_seen": 318439424
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0007300942828485456,
+      "loss": 1.5287,
+      "theoretical_loss": 4.146011541822253,
+      "tokens_seen": 318504960
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0007300782347041124,
+      "loss": 1.538,
+      "theoretical_loss": 4.145913114256829,
+      "tokens_seen": 318570496
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.000730062186559679,
+      "loss": 1.5778,
+      "theoretical_loss": 4.145814712605969,
+      "tokens_seen": 318636032
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0007300461384152458,
+      "loss": 1.5975,
+      "theoretical_loss": 4.1457163368575225,
+      "tokens_seen": 318701568
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0007300300902708124,
+      "loss": 1.552,
+      "theoretical_loss": 4.145617986999348,
+      "tokens_seen": 318767104
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0007300140421263792,
+      "loss": 1.6324,
+      "theoretical_loss": 4.145519663019309,
+      "tokens_seen": 318832640
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0007299979939819458,
+      "loss": 1.5314,
+      "theoretical_loss": 4.145421364905282,
+      "tokens_seen": 318898176
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0007299819458375126,
+      "loss": 1.5986,
+      "theoretical_loss": 4.145323092645146,
+      "tokens_seen": 318963712
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0007299658976930792,
+      "loss": 1.6539,
+      "theoretical_loss": 4.145224846226793,
+      "tokens_seen": 319029248
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.000729949849548646,
+      "loss": 1.6846,
+      "theoretical_loss": 4.14512662563812,
+      "tokens_seen": 319094784
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0007299338014042126,
+      "loss": 1.5786,
+      "theoretical_loss": 4.145028430867035,
+      "tokens_seen": 319160320
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0007299177532597794,
+      "loss": 1.6125,
+      "theoretical_loss": 4.144930261901451,
+      "tokens_seen": 319225856
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.000729901705115346,
+      "loss": 1.5453,
+      "theoretical_loss": 4.144832118729291,
+      "tokens_seen": 319291392
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0007298856569709128,
+      "loss": 1.6036,
+      "theoretical_loss": 4.144734001338486,
+      "tokens_seen": 319356928
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0007298696088264794,
+      "loss": 1.7081,
+      "theoretical_loss": 4.144635909716976,
+      "tokens_seen": 319422464
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0007298535606820462,
+      "loss": 1.6158,
+      "theoretical_loss": 4.144537843852705,
+      "tokens_seen": 319488000
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0007298375125376128,
+      "loss": 1.4767,
+      "theoretical_loss": 4.144439803733631,
+      "tokens_seen": 319553536
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0007298214643931796,
+      "loss": 1.5602,
+      "theoretical_loss": 4.144341789347715,
+      "tokens_seen": 319619072
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0007298054162487462,
+      "loss": 1.6508,
+      "theoretical_loss": 4.144243800682931,
+      "tokens_seen": 319684608
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.000729789368104313,
+      "loss": 1.5997,
+      "theoretical_loss": 4.1441458377272555,
+      "tokens_seen": 319750144
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0007297733199598796,
+      "loss": 1.6316,
+      "theoretical_loss": 4.144047900468678,
+      "tokens_seen": 319815680
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0007297572718154464,
+      "loss": 1.6584,
+      "theoretical_loss": 4.143949988895193,
+      "tokens_seen": 319881216
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.000729741223671013,
+      "loss": 1.6522,
+      "theoretical_loss": 4.1438521029948046,
+      "tokens_seen": 319946752
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0007297251755265798,
+      "loss": 1.5663,
+      "theoretical_loss": 4.1437542427555245,
+      "tokens_seen": 320012288
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0007297091273821464,
+      "loss": 1.5859,
+      "theoretical_loss": 4.143656408165373,
+      "tokens_seen": 320077824
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0007296930792377132,
+      "loss": 1.6449,
+      "theoretical_loss": 4.143558599212376,
+      "tokens_seen": 320143360
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0007296770310932798,
+      "loss": 1.6123,
+      "theoretical_loss": 4.143460815884571,
+      "tokens_seen": 320208896
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0007296609829488466,
+      "loss": 1.6351,
+      "theoretical_loss": 4.143363058170001,
+      "tokens_seen": 320274432
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0007296449348044132,
+      "loss": 1.6385,
+      "theoretical_loss": 4.143265326056719,
+      "tokens_seen": 320339968
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.00072962888665998,
+      "loss": 1.6856,
+      "theoretical_loss": 4.1431676195327825,
+      "tokens_seen": 320405504
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0007296128385155466,
+      "loss": 1.7328,
+      "theoretical_loss": 4.143069938586263,
+      "tokens_seen": 320471040
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0007295967903711134,
+      "loss": 1.7697,
+      "theoretical_loss": 4.142972283205233,
+      "tokens_seen": 320536576
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.00072958074222668,
+      "loss": 1.5611,
+      "theoretical_loss": 4.142874653377778,
+      "tokens_seen": 320602112
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0007295646940822468,
+      "loss": 1.6218,
+      "theoretical_loss": 4.14277704909199,
+      "tokens_seen": 320667648
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0007295486459378134,
+      "loss": 1.5247,
+      "theoretical_loss": 4.142679470335969,
+      "tokens_seen": 320733184
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0007295325977933802,
+      "loss": 1.5723,
+      "theoretical_loss": 4.142581917097821,
+      "tokens_seen": 320798720
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0007295165496489468,
+      "loss": 1.6569,
+      "theoretical_loss": 4.142484389365664,
+      "tokens_seen": 320864256
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0007295005015045136,
+      "loss": 1.5901,
+      "theoretical_loss": 4.14238688712762,
+      "tokens_seen": 320929792
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0007294844533600802,
+      "loss": 1.6504,
+      "theoretical_loss": 4.142289410371823,
+      "tokens_seen": 320995328
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.000729468405215647,
+      "loss": 1.6673,
+      "theoretical_loss": 4.14219195908641,
+      "tokens_seen": 321060864
+    },
+    {
+      "epoch": 0.1,
+      "objective/train/docs_used": 125851,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.5615171194076538,
+      "objective/train/theoretical_loss": 4.14209453325953,
+      "objective/train/tokens_used": 341586400,
+      "theoretical_loss": 4.14209453325953,
+      "tokens_seen": 321126400
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0007294523570712136,
+      "loss": 1.5615,
+      "theoretical_loss": 4.14209453325953,
+      "tokens_seen": 321126400
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0007294363089267804,
+      "loss": 1.6042,
+      "theoretical_loss": 4.1419971328793395,
+      "tokens_seen": 321191936
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.000729420260782347,
+      "loss": 1.6517,
+      "theoretical_loss": 4.141899757934,
+      "tokens_seen": 321257472
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0007294042126379138,
+      "loss": 1.5173,
+      "theoretical_loss": 4.141802408411683,
+      "tokens_seen": 321323008
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0007293881644934804,
+      "loss": 1.595,
+      "theoretical_loss": 4.14170508430057,
+      "tokens_seen": 321388544
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0007293721163490472,
+      "loss": 1.6864,
+      "theoretical_loss": 4.141607785588846,
+      "tokens_seen": 321454080
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0007293560682046138,
+      "loss": 1.6457,
+      "theoretical_loss": 4.141510512264707,
+      "tokens_seen": 321519616
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0007293400200601806,
+      "loss": 1.6577,
+      "theoretical_loss": 4.141413264316356,
+      "tokens_seen": 321585152
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0007293239719157472,
+      "loss": 1.7241,
+      "theoretical_loss": 4.141316041732004,
+      "tokens_seen": 321650688
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.000729307923771314,
+      "loss": 1.657,
+      "theoretical_loss": 4.141218844499869,
+      "tokens_seen": 321716224
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0007292918756268806,
+      "loss": 1.6004,
+      "theoretical_loss": 4.141121672608178,
+      "tokens_seen": 321781760
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0007292758274824474,
+      "loss": 1.6061,
+      "theoretical_loss": 4.141024526045166,
+      "tokens_seen": 321847296
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.000729259779338014,
+      "loss": 1.725,
+      "theoretical_loss": 4.140927404799075,
+      "tokens_seen": 321912832
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0007292437311935808,
+      "loss": 1.6464,
+      "theoretical_loss": 4.140830308858154,
+      "tokens_seen": 321978368
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0007292276830491475,
+      "loss": 1.5866,
+      "theoretical_loss": 4.140733238210664,
+      "tokens_seen": 322043904
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0007292116349047142,
+      "loss": 1.5196,
+      "theoretical_loss": 4.140636192844867,
+      "tokens_seen": 322109440
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0007291955867602809,
+      "loss": 1.6769,
+      "theoretical_loss": 4.1405391727490395,
+      "tokens_seen": 322174976
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0007291795386158476,
+      "loss": 1.6522,
+      "theoretical_loss": 4.140442177911462,
+      "tokens_seen": 322240512
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0007291634904714143,
+      "loss": 1.5604,
+      "theoretical_loss": 4.140345208320425,
+      "tokens_seen": 322306048
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.000729147442326981,
+      "loss": 1.6293,
+      "theoretical_loss": 4.1402482639642235,
+      "tokens_seen": 322371584
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0007291313941825477,
+      "loss": 1.5509,
+      "theoretical_loss": 4.140151344831164,
+      "tokens_seen": 322437120
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0007291153460381144,
+      "loss": 1.6253,
+      "theoretical_loss": 4.14005445090956,
+      "tokens_seen": 322502656
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0007290992978936812,
+      "loss": 1.619,
+      "theoretical_loss": 4.13995758218773,
+      "tokens_seen": 322568192
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0007290832497492478,
+      "loss": 1.6907,
+      "theoretical_loss": 4.1398607386540025,
+      "tokens_seen": 322633728
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0007290672016048146,
+      "loss": 1.6889,
+      "theoretical_loss": 4.139763920296716,
+      "tokens_seen": 322699264
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0007290511534603812,
+      "loss": 1.6077,
+      "theoretical_loss": 4.139667127104211,
+      "tokens_seen": 322764800
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.000729035105315948,
+      "loss": 1.5675,
+      "theoretical_loss": 4.139570359064842,
+      "tokens_seen": 322830336
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0007290190571715146,
+      "loss": 1.5447,
+      "theoretical_loss": 4.139473616166967,
+      "tokens_seen": 322895872
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0007290030090270814,
+      "loss": 1.4649,
+      "theoretical_loss": 4.139376898398953,
+      "tokens_seen": 322961408
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.000728986960882648,
+      "loss": 1.6747,
+      "theoretical_loss": 4.139280205749175,
+      "tokens_seen": 323026944
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0007289709127382148,
+      "loss": 1.4629,
+      "theoretical_loss": 4.139183538206017,
+      "tokens_seen": 323092480
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0007289548645937814,
+      "loss": 1.7784,
+      "theoretical_loss": 4.139086895757868,
+      "tokens_seen": 323158016
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0007289388164493481,
+      "loss": 1.6418,
+      "theoretical_loss": 4.138990278393125,
+      "tokens_seen": 323223552
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0007289227683049148,
+      "loss": 1.5323,
+      "theoretical_loss": 4.138893686100196,
+      "tokens_seen": 323289088
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0007289067201604815,
+      "loss": 1.5261,
+      "theoretical_loss": 4.138797118867493,
+      "tokens_seen": 323354624
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0007288906720160482,
+      "loss": 1.6058,
+      "theoretical_loss": 4.138700576683437,
+      "tokens_seen": 323420160
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0007288746238716149,
+      "loss": 1.5333,
+      "theoretical_loss": 4.138604059536458,
+      "tokens_seen": 323485696
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0007288585757271816,
+      "loss": 1.5597,
+      "theoretical_loss": 4.138507567414993,
+      "tokens_seen": 323551232
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0007288425275827483,
+      "loss": 1.5813,
+      "theoretical_loss": 4.138411100307485,
+      "tokens_seen": 323616768
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.000728826479438315,
+      "loss": 1.6585,
+      "theoretical_loss": 4.138314658202385,
+      "tokens_seen": 323682304
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0007288104312938817,
+      "loss": 1.6357,
+      "theoretical_loss": 4.138218241088156,
+      "tokens_seen": 323747840
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0007287943831494483,
+      "loss": 1.5418,
+      "theoretical_loss": 4.138121848953261,
+      "tokens_seen": 323813376
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0007287783350050151,
+      "loss": 1.6357,
+      "theoretical_loss": 4.138025481786178,
+      "tokens_seen": 323878912
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0007287622868605817,
+      "loss": 1.599,
+      "theoretical_loss": 4.137929139575387,
+      "tokens_seen": 323944448
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0007287462387161485,
+      "loss": 1.6741,
+      "theoretical_loss": 4.137832822309382,
+      "tokens_seen": 324009984
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0007287301905717151,
+      "loss": 1.6565,
+      "theoretical_loss": 4.1377365299766575,
+      "tokens_seen": 324075520
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0007287141424272819,
+      "loss": 1.5809,
+      "theoretical_loss": 4.13764026256572,
+      "tokens_seen": 324141056
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0007286980942828485,
+      "loss": 1.6851,
+      "theoretical_loss": 4.137544020065082,
+      "tokens_seen": 324206592
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0007286820461384153,
+      "loss": 1.5922,
+      "theoretical_loss": 4.1374478024632655,
+      "tokens_seen": 324272128
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0007286659979939819,
+      "loss": 1.7437,
+      "theoretical_loss": 4.137351609748798,
+      "tokens_seen": 324337664
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0007286499498495487,
+      "loss": 1.6601,
+      "theoretical_loss": 4.137255441910215,
+      "tokens_seen": 324403200
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0007286339017051153,
+      "loss": 1.6656,
+      "theoretical_loss": 4.13715929893606,
+      "tokens_seen": 324468736
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0007286178535606821,
+      "loss": 1.6646,
+      "theoretical_loss": 4.137063180814886,
+      "tokens_seen": 324534272
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0007286018054162487,
+      "loss": 1.5173,
+      "theoretical_loss": 4.136967087535249,
+      "tokens_seen": 324599808
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0007285857572718155,
+      "loss": 1.4389,
+      "theoretical_loss": 4.136871019085717,
+      "tokens_seen": 324665344
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0007285697091273821,
+      "loss": 1.6198,
+      "theoretical_loss": 4.136774975454863,
+      "tokens_seen": 324730880
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0007285536609829489,
+      "loss": 1.5379,
+      "theoretical_loss": 4.136678956631269,
+      "tokens_seen": 324796416
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0007285376128385155,
+      "loss": 1.5541,
+      "theoretical_loss": 4.136582962603523,
+      "tokens_seen": 324861952
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0007285215646940823,
+      "loss": 1.6251,
+      "theoretical_loss": 4.136486993360222,
+      "tokens_seen": 324927488
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0007285055165496489,
+      "loss": 1.564,
+      "theoretical_loss": 4.136391048889971,
+      "tokens_seen": 324993024
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0007284894684052157,
+      "loss": 1.5952,
+      "theoretical_loss": 4.136295129181381,
+      "tokens_seen": 325058560
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0007284734202607823,
+      "loss": 1.6159,
+      "theoretical_loss": 4.136199234223069,
+      "tokens_seen": 325124096
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0007284573721163491,
+      "loss": 1.5685,
+      "theoretical_loss": 4.136103364003665,
+      "tokens_seen": 325189632
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0007284413239719157,
+      "loss": 1.6032,
+      "theoretical_loss": 4.1360075185118,
+      "tokens_seen": 325255168
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0007284252758274825,
+      "loss": 1.7056,
+      "theoretical_loss": 4.135911697736118,
+      "tokens_seen": 325320704
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0007284092276830491,
+      "loss": 1.5234,
+      "theoretical_loss": 4.135815901665267,
+      "tokens_seen": 325386240
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0007283931795386159,
+      "loss": 1.65,
+      "theoretical_loss": 4.135720130287904,
+      "tokens_seen": 325451776
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0007283771313941825,
+      "loss": 1.6589,
+      "theoretical_loss": 4.135624383592694,
+      "tokens_seen": 325517312
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0007283610832497493,
+      "loss": 1.572,
+      "theoretical_loss": 4.135528661568307,
+      "tokens_seen": 325582848
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0007283450351053159,
+      "loss": 1.5901,
+      "theoretical_loss": 4.135432964203422,
+      "tokens_seen": 325648384
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0007283289869608827,
+      "loss": 1.7418,
+      "theoretical_loss": 4.135337291486727,
+      "tokens_seen": 325713920
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0007283129388164493,
+      "loss": 1.6472,
+      "theoretical_loss": 4.135241643406915,
+      "tokens_seen": 325779456
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0007282968906720161,
+      "loss": 1.629,
+      "theoretical_loss": 4.135146019952688,
+      "tokens_seen": 325844992
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0007282808425275827,
+      "loss": 1.5419,
+      "theoretical_loss": 4.135050421112755,
+      "tokens_seen": 325910528
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0007282647943831495,
+      "loss": 1.646,
+      "theoretical_loss": 4.134954846875832,
+      "tokens_seen": 325976064
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0007282487462387161,
+      "loss": 1.6814,
+      "theoretical_loss": 4.134859297230644,
+      "tokens_seen": 326041600
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0007282326980942829,
+      "loss": 1.6225,
+      "theoretical_loss": 4.13476377216592,
+      "tokens_seen": 326107136
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0007282166499498495,
+      "loss": 1.651,
+      "theoretical_loss": 4.134668271670401,
+      "tokens_seen": 326172672
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0007282006018054163,
+      "loss": 1.6105,
+      "theoretical_loss": 4.134572795732832,
+      "tokens_seen": 326238208
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0007281845536609829,
+      "loss": 1.6282,
+      "theoretical_loss": 4.134477344341967,
+      "tokens_seen": 326303744
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0007281685055165497,
+      "loss": 1.6764,
+      "theoretical_loss": 4.134381917486565,
+      "tokens_seen": 326369280
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0007281524573721163,
+      "loss": 1.5892,
+      "theoretical_loss": 4.134286515155398,
+      "tokens_seen": 326434816
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0007281364092276831,
+      "loss": 1.5875,
+      "theoretical_loss": 4.134191137337238,
+      "tokens_seen": 326500352
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0007281203610832497,
+      "loss": 1.6182,
+      "theoretical_loss": 4.134095784020871,
+      "tokens_seen": 326565888
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0007281043129388165,
+      "loss": 1.5375,
+      "theoretical_loss": 4.1340004551950855,
+      "tokens_seen": 326631424
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0007280882647943831,
+      "loss": 1.5434,
+      "theoretical_loss": 4.133905150848681,
+      "tokens_seen": 326696960
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0007280722166499499,
+      "loss": 1.5871,
+      "theoretical_loss": 4.133809870970461,
+      "tokens_seen": 326762496
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0007280561685055165,
+      "loss": 1.6211,
+      "theoretical_loss": 4.1337146155492395,
+      "tokens_seen": 326828032
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0007280401203610833,
+      "loss": 1.4744,
+      "theoretical_loss": 4.133619384573836,
+      "tokens_seen": 326893568
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0007280240722166499,
+      "loss": 1.5541,
+      "theoretical_loss": 4.1335241780330785,
+      "tokens_seen": 326959104
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0007280080240722167,
+      "loss": 1.5424,
+      "theoretical_loss": 4.1334289959158,
+      "tokens_seen": 327024640
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0007279919759277833,
+      "loss": 1.5648,
+      "theoretical_loss": 4.133333838210844,
+      "tokens_seen": 327090176
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0007279759277833501,
+      "loss": 1.7727,
+      "theoretical_loss": 4.133238704907059,
+      "tokens_seen": 327155712
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0007279598796389167,
+      "loss": 1.6525,
+      "theoretical_loss": 4.133143595993303,
+      "tokens_seen": 327221248
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0007279438314944835,
+      "loss": 1.6713,
+      "theoretical_loss": 4.133048511458438,
+      "tokens_seen": 327286784
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0007279277833500503,
+      "loss": 1.705,
+      "theoretical_loss": 4.132953451291337,
+      "tokens_seen": 327352320
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0007279117352056169,
+      "loss": 1.5638,
+      "theoretical_loss": 4.132858415480877,
+      "tokens_seen": 327417856
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0007278956870611837,
+      "loss": 1.6074,
+      "theoretical_loss": 4.132763404015946,
+      "tokens_seen": 327483392
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0007278796389167503,
+      "loss": 1.5912,
+      "theoretical_loss": 4.132668416885435,
+      "tokens_seen": 327548928
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0007278635907723171,
+      "loss": 1.5823,
+      "theoretical_loss": 4.132573454078246,
+      "tokens_seen": 327614464
+    },
+    {
+      "epoch": 0.1,
+      "objective/train/docs_used": 128204,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.5737180709838867,
+      "objective/train/theoretical_loss": 4.132478515583286,
+      "objective/train/tokens_used": 348140000,
+      "theoretical_loss": 4.132478515583286,
+      "tokens_seen": 327680000
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0007278475426278837,
+      "loss": 1.5737,
+      "theoretical_loss": 4.132478515583286,
+      "tokens_seen": 327680000
+    }
+  ],
+  "max_steps": 50354,
+  "num_train_epochs": 9223372036854775807,
+  "total_flos": 1.6722690048e+17,
+  "trial_name": null,
+  "trial_params": null
+}