diff --git "a/checkpoint-5000/trainer_state.json" "b/checkpoint-5000/trainer_state.json"
new file mode 100644--- /dev/null
+++ "b/checkpoint-5000/trainer_state.json"
@@ -0,0 +1,36123 @@
+{
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 0.09929697740000794,
+  "global_step": 5000,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.0,
+      "learning_rate": 1.5873015873015873e-06,
+      "loss": 10.5638,
+      "theoretical_loss": 20.84128112621979,
+      "tokens_seen": 65536
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 3.1746031746031746e-06,
+      "loss": 10.5611,
+      "theoretical_loss": 17.594664429001284,
+      "tokens_seen": 131072
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 4.7619047619047615e-06,
+      "loss": 10.4527,
+      "theoretical_loss": 15.967940417509208,
+      "tokens_seen": 196608
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 6.349206349206349e-06,
+      "loss": 10.2555,
+      "theoretical_loss": 14.92078008066121,
+      "tokens_seen": 262144
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 7.936507936507936e-06,
+      "loss": 9.989,
+      "theoretical_loss": 14.164680262435617,
+      "tokens_seen": 327680
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 9.523809523809523e-06,
+      "loss": 9.6996,
+      "theoretical_loss": 13.581024797222863,
+      "tokens_seen": 393216
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 1.1111111111111112e-05,
+      "loss": 9.4982,
+      "theoretical_loss": 13.11027232607383,
+      "tokens_seen": 458752
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 1.2698412698412699e-05,
+      "loss": 9.3018,
+      "theoretical_loss": 12.718592950155966,
+      "tokens_seen": 524288
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 1.4285714285714285e-05,
+      "loss": 9.1658,
+      "theoretical_loss": 12.385055788546264,
+      "tokens_seen": 589824
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 1.5873015873015872e-05,
+      "loss": 9.026,
+      "theoretical_loss": 12.09587593170772,
+      "tokens_seen": 655360
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 1.746031746031746e-05,
+      "loss": 8.8595,
+      "theoretical_loss": 11.841529555774319,
+      "tokens_seen": 720896
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 1.9047619047619046e-05,
+      "loss": 8.9419,
+      "theoretical_loss": 11.615182533379372,
+      "tokens_seen": 786432
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 2.0634920634920636e-05,
+      "loss": 8.6758,
+      "theoretical_loss": 11.41177746485943,
+      "tokens_seen": 851968
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 2.2222222222222223e-05,
+      "loss": 8.6859,
+      "theoretical_loss": 11.227475026784514,
+      "tokens_seen": 917504
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 2.380952380952381e-05,
+      "loss": 8.6441,
+      "theoretical_loss": 11.059297242650555,
+      "tokens_seen": 983040
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 2.5396825396825397e-05,
+      "loss": 8.5614,
+      "theoretical_loss": 10.90489141112959,
+      "tokens_seen": 1048576
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 2.6984126984126984e-05,
+      "loss": 8.4541,
+      "theoretical_loss": 10.762369064735367,
+      "tokens_seen": 1114112
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 2.857142857142857e-05,
+      "loss": 8.3028,
+      "theoretical_loss": 10.63019320090292,
+      "tokens_seen": 1179648
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 3.0158730158730158e-05,
+      "loss": 8.2986,
+      "theoretical_loss": 10.507097497029282,
+      "tokens_seen": 1245184
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 3.1746031746031745e-05,
+      "loss": 8.2865,
+      "theoretical_loss": 10.392027268435971,
+      "tokens_seen": 1310720
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 3.3333333333333335e-05,
+      "loss": 8.213,
+      "theoretical_loss": 10.284095545569386,
+      "tokens_seen": 1376256
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 3.492063492063492e-05,
+      "loss": 8.1072,
+      "theoretical_loss": 10.182549877942659,
+      "tokens_seen": 1441792
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 3.650793650793651e-05,
+      "loss": 8.0311,
+      "theoretical_loss": 10.08674688640524,
+      "tokens_seen": 1507328
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 3.809523809523809e-05,
+      "loss": 7.9282,
+      "theoretical_loss": 9.99613250351292,
+      "tokens_seen": 1572864
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 3.968253968253968e-05,
+      "loss": 7.9728,
+      "theoretical_loss": 9.91022645106575,
+      "tokens_seen": 1638400
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 4.126984126984127e-05,
+      "loss": 7.7709,
+      "theoretical_loss": 9.828609916213201,
+      "tokens_seen": 1703936
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 4.2857142857142856e-05,
+      "loss": 7.8065,
+      "theoretical_loss": 9.750915671580087,
+      "tokens_seen": 1769472
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 4.4444444444444447e-05,
+      "loss": 7.6517,
+      "theoretical_loss": 9.676820083754187,
+      "tokens_seen": 1835008
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 4.603174603174603e-05,
+      "loss": 7.6361,
+      "theoretical_loss": 9.606036595818281,
+      "tokens_seen": 1900544
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 4.761904761904762e-05,
+      "loss": 7.53,
+      "theoretical_loss": 9.538310371437493,
+      "tokens_seen": 1966080
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 4.9206349206349204e-05,
+      "loss": 7.5888,
+      "theoretical_loss": 9.473413862304575,
+      "tokens_seen": 2031616
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 5.0793650793650794e-05,
+      "loss": 7.4044,
+      "theoretical_loss": 9.411143115583098,
+      "tokens_seen": 2097152
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 5.2380952380952384e-05,
+      "loss": 7.4525,
+      "theoretical_loss": 9.351314678906622,
+      "tokens_seen": 2162688
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 5.396825396825397e-05,
+      "loss": 7.3801,
+      "theoretical_loss": 9.293762991332915,
+      "tokens_seen": 2228224
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 5.555555555555556e-05,
+      "loss": 7.3191,
+      "theoretical_loss": 9.238338172117398,
+      "tokens_seen": 2293760
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 5.714285714285714e-05,
+      "loss": 7.2429,
+      "theoretical_loss": 9.184904137180933,
+      "tokens_seen": 2359296
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 5.873015873015873e-05,
+      "loss": 7.2826,
+      "theoretical_loss": 9.133336987085992,
+      "tokens_seen": 2424832
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 6.0317460317460316e-05,
+      "loss": 7.1569,
+      "theoretical_loss": 9.083523621206375,
+      "tokens_seen": 2490368
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 6.19047619047619e-05,
+      "loss": 7.1954,
+      "theoretical_loss": 9.03536054131647,
+      "tokens_seen": 2555904
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 6.349206349206349e-05,
+      "loss": 7.0611,
+      "theoretical_loss": 8.988752814581996,
+      "tokens_seen": 2621440
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 6.507936507936509e-05,
+      "loss": 7.0413,
+      "theoretical_loss": 8.943613171313253,
+      "tokens_seen": 2686976
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 6.666666666666667e-05,
+      "loss": 6.9267,
+      "theoretical_loss": 8.899861217150864,
+      "tokens_seen": 2752512
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 6.825396825396825e-05,
+      "loss": 6.9239,
+      "theoretical_loss": 8.85742274282614,
+      "tokens_seen": 2818048
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 6.984126984126984e-05,
+      "loss": 6.9214,
+      "theoretical_loss": 8.816229117451055,
+      "tokens_seen": 2883584
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 7.142857142857143e-05,
+      "loss": 6.8661,
+      "theoretical_loss": 8.776216753584098,
+      "tokens_seen": 2949120
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 7.301587301587302e-05,
+      "loss": 6.6479,
+      "theoretical_loss": 8.737326634193474,
+      "tokens_seen": 3014656
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 7.460317460317461e-05,
+      "loss": 6.8119,
+      "theoretical_loss": 8.699503893181488,
+      "tokens_seen": 3080192
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 7.619047619047618e-05,
+      "loss": 6.6601,
+      "theoretical_loss": 8.662697442408115,
+      "tokens_seen": 3145728
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 7.777777777777778e-05,
+      "loss": 6.5707,
+      "theoretical_loss": 8.626859639208966,
+      "tokens_seen": 3211264
+    },
+    {
+      "epoch": 0.0,
+      "objective/train/docs_used": 7637,
+      "objective/train/instantaneous_batch_size": 32,
+      "objective/train/instantaneous_microbatch_size": 32768,
+      "objective/train/original_loss": 6.526330947875977,
+      "objective/train/theoretical_loss": 8.591945989283708,
+      "objective/train/tokens_used": 23736800,
+      "theoretical_loss": 8.591945989283708,
+      "tokens_seen": 3276800
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 7.936507936507937e-05,
+      "loss": 6.522,
+      "theoretical_loss": 8.591945989283708,
+      "tokens_seen": 3276800
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 8.095238095238096e-05,
+      "loss": 6.6008,
+      "theoretical_loss": 8.557914880567758,
+      "tokens_seen": 3342336
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 8.253968253968255e-05,
+      "loss": 6.3612,
+      "theoretical_loss": 8.524727344318643,
+      "tokens_seen": 3407872
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 8.412698412698413e-05,
+      "loss": 6.3783,
+      "theoretical_loss": 8.492346840169828,
+      "tokens_seen": 3473408
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 8.571428571428571e-05,
+      "loss": 6.3441,
+      "theoretical_loss": 8.460739062345422,
+      "tokens_seen": 3538944
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 8.730158730158731e-05,
+      "loss": 6.3382,
+      "theoretical_loss": 8.429871764603359,
+      "tokens_seen": 3604480
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 8.888888888888889e-05,
+      "loss": 6.2292,
+      "theoretical_loss": 8.399714601792851,
+      "tokens_seen": 3670016
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 9.047619047619048e-05,
+      "loss": 6.1924,
+      "theoretical_loss": 8.370238986183718,
+      "tokens_seen": 3735552
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 9.206349206349206e-05,
+      "loss": 6.1078,
+      "theoretical_loss": 8.34141795695797,
+      "tokens_seen": 3801088
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 9.365079365079366e-05,
+      "loss": 5.9921,
+      "theoretical_loss": 8.313226061453822,
+      "tokens_seen": 3866624
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 9.523809523809524e-05,
+      "loss": 6.0694,
+      "theoretical_loss": 8.285639246924504,
+      "tokens_seen": 3932160
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 9.682539682539682e-05,
+      "loss": 5.9962,
+      "theoretical_loss": 8.258634761722755,
+      "tokens_seen": 3997696
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 9.841269841269841e-05,
+      "loss": 5.8399,
+      "theoretical_loss": 8.232191064950612,
+      "tokens_seen": 4063232
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.0001,
+      "loss": 5.9487,
+      "theoretical_loss": 8.206287743725687,
+      "tokens_seen": 4128768
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00010158730158730159,
+      "loss": 5.9181,
+      "theoretical_loss": 8.180905437312258,
+      "tokens_seen": 4194304
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00010317460317460319,
+      "loss": 5.8804,
+      "theoretical_loss": 8.15602576745014,
+      "tokens_seen": 4259840
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00010476190476190477,
+      "loss": 5.9051,
+      "theoretical_loss": 8.13163127428835,
+      "tokens_seen": 4325376
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00010634920634920635,
+      "loss": 5.8469,
+      "theoretical_loss": 8.107705357395282,
+      "tokens_seen": 4390912
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00010793650793650794,
+      "loss": 5.6132,
+      "theoretical_loss": 8.084232221374057,
+      "tokens_seen": 4456448
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00010952380952380953,
+      "loss": 5.6537,
+      "theoretical_loss": 8.06119682566165,
+      "tokens_seen": 4521984
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00011111111111111112,
+      "loss": 5.7699,
+      "theoretical_loss": 8.038584838134476,
+      "tokens_seen": 4587520
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.0001126984126984127,
+      "loss": 5.6984,
+      "theoretical_loss": 8.016382592182019,
+      "tokens_seen": 4653056
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00011428571428571428,
+      "loss": 5.511,
+      "theoretical_loss": 7.994577046944442,
+      "tokens_seen": 4718592
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.0001158730158730159,
+      "loss": 5.5143,
+      "theoretical_loss": 7.973155750440629,
+      "tokens_seen": 4784128
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00011746031746031746,
+      "loss": 5.5075,
+      "theoretical_loss": 7.95210680534016,
+      "tokens_seen": 4849664
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00011904761904761905,
+      "loss": 5.6961,
+      "theoretical_loss": 7.931418837156708,
+      "tokens_seen": 4915200
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00012063492063492063,
+      "loss": 5.49,
+      "theoretical_loss": 7.911080964661844,
+      "tokens_seen": 4980736
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00012222222222222224,
+      "loss": 5.5264,
+      "theoretical_loss": 7.891082772337269,
+      "tokens_seen": 5046272
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.0001238095238095238,
+      "loss": 5.4852,
+      "theoretical_loss": 7.871414284700578,
+      "tokens_seen": 5111808
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.0001253968253968254,
+      "loss": 5.3432,
+      "theoretical_loss": 7.852065942354954,
+      "tokens_seen": 5177344
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00012698412698412698,
+      "loss": 5.4524,
+      "theoretical_loss": 7.833028579626806,
+      "tokens_seen": 5242880
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00012857142857142858,
+      "loss": 5.2613,
+      "theoretical_loss": 7.814293403667714,
+      "tokens_seen": 5308416
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00013015873015873017,
+      "loss": 5.4356,
+      "theoretical_loss": 7.795851974908005,
+      "tokens_seen": 5373952
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00013174603174603174,
+      "loss": 5.3659,
+      "theoretical_loss": 7.77769618875926,
+      "tokens_seen": 5439488
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00013333333333333334,
+      "loss": 5.5456,
+      "theoretical_loss": 7.7598182584719595,
+      "tokens_seen": 5505024
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00013492063492063494,
+      "loss": 5.3546,
+      "theoretical_loss": 7.742210699062561,
+      "tokens_seen": 5570560
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.0001365079365079365,
+      "loss": 5.3844,
+      "theoretical_loss": 7.724866312231575,
+      "tokens_seen": 5636096
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.0001380952380952381,
+      "loss": 5.3161,
+      "theoretical_loss": 7.707778172200822,
+      "tokens_seen": 5701632
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00013968253968253967,
+      "loss": 5.2856,
+      "theoretical_loss": 7.6909396124039695,
+      "tokens_seen": 5767168
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.0001412698412698413,
+      "loss": 5.2451,
+      "theoretical_loss": 7.67434421296992,
+      "tokens_seen": 5832704
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00014285714285714287,
+      "loss": 5.2539,
+      "theoretical_loss": 7.657985788943502,
+      "tokens_seen": 5898240
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00014444444444444444,
+      "loss": 5.2143,
+      "theoretical_loss": 7.641858379192337,
+      "tokens_seen": 5963776
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00014603174603174603,
+      "loss": 5.2365,
+      "theoretical_loss": 7.625956235952921,
+      "tokens_seen": 6029312
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00014761904761904763,
+      "loss": 5.2243,
+      "theoretical_loss": 7.610273814972529,
+      "tokens_seen": 6094848
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00014920634920634923,
+      "loss": 5.3252,
+      "theoretical_loss": 7.594805766207012,
+      "tokens_seen": 6160384
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.0001507936507936508,
+      "loss": 5.1686,
+      "theoretical_loss": 7.57954692503759,
+      "tokens_seen": 6225920
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00015238095238095237,
+      "loss": 5.1521,
+      "theoretical_loss": 7.5644923039725604,
+      "tokens_seen": 6291456
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.000153968253968254,
+      "loss": 5.1351,
+      "theoretical_loss": 7.549637084802403,
+      "tokens_seen": 6356992
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00015555555555555556,
+      "loss": 5.0053,
+      "theoretical_loss": 7.534976611179138,
+      "tokens_seen": 6422528
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00015714285714285716,
+      "loss": 5.0737,
+      "theoretical_loss": 7.52050638159289,
+      "tokens_seen": 6488064
+    },
+    {
+      "epoch": 0.0,
+      "objective/train/docs_used": 8815,
+      "objective/train/instantaneous_batch_size": 32,
+      "objective/train/instantaneous_microbatch_size": 32768,
+      "objective/train/original_loss": 4.870777130126953,
+      "objective/train/theoretical_loss": 7.506222042720672,
+      "objective/train/tokens_used": 27013600,
+      "theoretical_loss": 7.506222042720672,
+      "tokens_seen": 6553600
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00015873015873015873,
+      "loss": 5.001,
+      "theoretical_loss": 7.506222042720672,
+      "tokens_seen": 6553600
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00016031746031746033,
+      "loss": 5.0853,
+      "theoretical_loss": 7.492119383124103,
+      "tokens_seen": 6619136
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00016190476190476192,
+      "loss": 5.1408,
+      "theoretical_loss": 7.478194327274557,
+      "tokens_seen": 6684672
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.0001634920634920635,
+      "loss": 5.0878,
+      "theoretical_loss": 7.46444292988566,
+      "tokens_seen": 6750208
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.0001650793650793651,
+      "loss": 4.9552,
+      "theoretical_loss": 7.450861370534546,
+      "tokens_seen": 6815744
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.0001666666666666667,
+      "loss": 5.0462,
+      "theoretical_loss": 7.437445948554516,
+      "tokens_seen": 6881280
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00016825396825396826,
+      "loss": 5.0117,
+      "theoretical_loss": 7.424193078182986,
+      "tokens_seen": 6946816
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00016984126984126986,
+      "loss": 5.0001,
+      "theoretical_loss": 7.411099283949674,
+      "tokens_seen": 7012352
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00017142857142857143,
+      "loss": 4.9158,
+      "theoretical_loss": 7.3981611962910145,
+      "tokens_seen": 7077888
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00017301587301587302,
+      "loss": 4.949,
+      "theoretical_loss": 7.38537554737773,
+      "tokens_seen": 7143424
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00017460317460317462,
+      "loss": 5.0563,
+      "theoretical_loss": 7.372739167143329,
+      "tokens_seen": 7208960
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.0001761904761904762,
+      "loss": 4.9183,
+      "theoretical_loss": 7.36024897950216,
+      "tokens_seen": 7274496
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00017777777777777779,
+      "loss": 4.9242,
+      "theoretical_loss": 7.347901998746318,
+      "tokens_seen": 7340032
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00017936507936507938,
+      "loss": 4.869,
+      "theoretical_loss": 7.335695326111465,
+      "tokens_seen": 7405568
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00018095238095238095,
+      "loss": 5.025,
+      "theoretical_loss": 7.323626146502205,
+      "tokens_seen": 7471104
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00018253968253968255,
+      "loss": 4.8879,
+      "theoretical_loss": 7.311691725368293,
+      "tokens_seen": 7536640
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00018412698412698412,
+      "loss": 4.9441,
+      "theoretical_loss": 7.299889405723473,
+      "tokens_seen": 7602176
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00018571428571428572,
+      "loss": 4.9486,
+      "theoretical_loss": 7.288216605299264,
+      "tokens_seen": 7667712
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00018730158730158731,
+      "loss": 4.9052,
+      "theoretical_loss": 7.276670813826511,
+      "tokens_seen": 7733248
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00018888888888888888,
+      "loss": 4.8581,
+      "theoretical_loss": 7.265249590437903,
+      "tokens_seen": 7798784
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00019047619047619048,
+      "loss": 4.9746,
+      "theoretical_loss": 7.253950561185138,
+      "tokens_seen": 7864320
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00019206349206349208,
+      "loss": 4.8516,
+      "theoretical_loss": 7.242771416664761,
+      "tokens_seen": 7929856
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00019365079365079365,
+      "loss": 4.9576,
+      "theoretical_loss": 7.231709909747042,
+      "tokens_seen": 7995392
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00019523809523809525,
+      "loss": 4.8637,
+      "theoretical_loss": 7.220763853402656,
+      "tokens_seen": 8060928
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00019682539682539682,
+      "loss": 4.8775,
+      "theoretical_loss": 7.209931118622157,
+      "tokens_seen": 8126464
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00019841269841269844,
+      "loss": 4.866,
+      "theoretical_loss": 7.199209632423606,
+      "tokens_seen": 8192000
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.0002,
+      "loss": 4.8242,
+      "theoretical_loss": 7.188597375943913,
+      "tokens_seen": 8257536
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00020158730158730158,
+      "loss": 4.7343,
+      "theoretical_loss": 7.178092382609779,
+      "tokens_seen": 8323072
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00020317460317460318,
+      "loss": 4.766,
+      "theoretical_loss": 7.1676927363842795,
+      "tokens_seen": 8388608
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00020476190476190475,
+      "loss": 4.8334,
+      "theoretical_loss": 7.157396570085435,
+      "tokens_seen": 8454144
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00020634920634920637,
+      "loss": 4.776,
+      "theoretical_loss": 7.147202063773257,
+      "tokens_seen": 8519680
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00020793650793650797,
+      "loss": 4.7742,
+      "theoretical_loss": 7.137107443201986,
+      "tokens_seen": 8585216
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00020952380952380954,
+      "loss": 4.7856,
+      "theoretical_loss": 7.127110978334411,
+      "tokens_seen": 8650752
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00021111111111111113,
+      "loss": 4.8232,
+      "theoretical_loss": 7.117210981915334,
+      "tokens_seen": 8716288
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.0002126984126984127,
+      "loss": 4.774,
+      "theoretical_loss": 7.10740580810139,
+      "tokens_seen": 8781824
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.0002142857142857143,
+      "loss": 4.7729,
+      "theoretical_loss": 7.097693851144607,
+      "tokens_seen": 8847360
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00021587301587301587,
+      "loss": 4.7239,
+      "theoretical_loss": 7.088073544127209,
+      "tokens_seen": 8912896
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00021746031746031744,
+      "loss": 4.8107,
+      "theoretical_loss": 7.07854335774531,
+      "tokens_seen": 8978432
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00021904761904761907,
+      "loss": 4.7937,
+      "theoretical_loss": 7.069101799139266,
+      "tokens_seen": 9043968
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00022063492063492066,
+      "loss": 4.685,
+      "theoretical_loss": 7.0597474107685745,
+      "tokens_seen": 9109504
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00022222222222222223,
+      "loss": 4.7767,
+      "theoretical_loss": 7.050478769329319,
+      "tokens_seen": 9175040
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00022380952380952383,
+      "loss": 4.7015,
+      "theoretical_loss": 7.0412944847122505,
+      "tokens_seen": 9240576
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.0002253968253968254,
+      "loss": 4.7519,
+      "theoretical_loss": 7.032193198999712,
+      "tokens_seen": 9306112
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.000226984126984127,
+      "loss": 4.7746,
+      "theoretical_loss": 7.023173585499698,
+      "tokens_seen": 9371648
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00022857142857142857,
+      "loss": 4.7763,
+      "theoretical_loss": 7.014234347815421,
+      "tokens_seen": 9437184
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00023015873015873016,
+      "loss": 4.6207,
+      "theoretical_loss": 7.00537421894883,
+      "tokens_seen": 9502720
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.0002317460317460318,
+      "loss": 4.5372,
+      "theoretical_loss": 6.996591960436656,
+      "tokens_seen": 9568256
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00023333333333333336,
+      "loss": 4.6846,
+      "theoretical_loss": 6.9878863615175355,
+      "tokens_seen": 9633792
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00023492063492063493,
+      "loss": 4.7459,
+      "theoretical_loss": 6.97925623832895,
+      "tokens_seen": 9699328
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00023650793650793653,
+      "loss": 4.6728,
+      "theoretical_loss": 6.97070043313267,
+      "tokens_seen": 9764864
+    },
+    {
+      "epoch": 0.0,
+      "objective/train/docs_used": 9875,
+      "objective/train/instantaneous_batch_size": 32,
+      "objective/train/instantaneous_microbatch_size": 32768,
+      "objective/train/original_loss": 4.381937503814697,
+      "objective/train/theoretical_loss": 6.962217813567541,
+      "objective/train/tokens_used": 30290400,
+      "theoretical_loss": 6.962217813567541,
+      "tokens_seen": 9830400
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.0002380952380952381,
+      "loss": 4.5442,
+      "theoretical_loss": 6.962217813567541,
+      "tokens_seen": 9830400
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.0002396825396825397,
+      "loss": 4.4541,
+      "theoretical_loss": 6.953807271928458,
+      "tokens_seen": 9895936
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00024126984126984126,
+      "loss": 4.7125,
+      "theoretical_loss": 6.945467724470439,
+      "tokens_seen": 9961472
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00024285714285714286,
+      "loss": 4.6732,
+      "theoretical_loss": 6.937198110736788,
+      "tokens_seen": 10027008
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.0002444444444444445,
+      "loss": 4.6423,
+      "theoretical_loss": 6.928997392910329,
+      "tokens_seen": 10092544
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00024603174603174605,
+      "loss": 4.5394,
+      "theoretical_loss": 6.920864555186801,
+      "tokens_seen": 10158080
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.0002476190476190476,
+      "loss": 4.5712,
+      "theoretical_loss": 6.912798603169512,
+      "tokens_seen": 10223616
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00024920634920634925,
+      "loss": 4.5111,
+      "theoretical_loss": 6.904798563284391,
+      "tokens_seen": 10289152
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.0002507936507936508,
+      "loss": 4.6929,
+      "theoretical_loss": 6.896863482214638,
+      "tokens_seen": 10354688
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.0002523809523809524,
+      "loss": 4.5159,
+      "theoretical_loss": 6.888992426354192,
+      "tokens_seen": 10420224
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00025396825396825396,
+      "loss": 4.6576,
+      "theoretical_loss": 6.881184481279271,
+      "tokens_seen": 10485760
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00025555555555555553,
+      "loss": 4.7163,
+      "theoretical_loss": 6.873438751237283,
+      "tokens_seen": 10551296
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00025714285714285715,
+      "loss": 4.6156,
+      "theoretical_loss": 6.865754358652436,
+      "tokens_seen": 10616832
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.0002587301587301588,
+      "loss": 4.4927,
+      "theoretical_loss": 6.85813044364739,
+      "tokens_seen": 10682368
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00026031746031746035,
+      "loss": 4.6024,
+      "theoretical_loss": 6.85056616358036,
+      "tokens_seen": 10747904
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.0002619047619047619,
+      "loss": 4.614,
+      "theoretical_loss": 6.843060692597036,
+      "tokens_seen": 10813440
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.0002634920634920635,
+      "loss": 4.6141,
+      "theoretical_loss": 6.835613221196822,
+      "tokens_seen": 10878976
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.0002650793650793651,
+      "loss": 4.6311,
+      "theoretical_loss": 6.8282229558127865,
+      "tokens_seen": 10944512
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.0002666666666666667,
+      "loss": 4.5575,
+      "theoretical_loss": 6.820889118404866,
+      "tokens_seen": 11010048
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00026825396825396825,
+      "loss": 4.727,
+      "theoretical_loss": 6.813610946065789,
+      "tokens_seen": 11075584
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.0002698412698412699,
+      "loss": 4.6214,
+      "theoretical_loss": 6.806387690639279,
+      "tokens_seen": 11141120
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00027142857142857144,
+      "loss": 4.6214,
+      "theoretical_loss": 6.799218618350064,
+      "tokens_seen": 11206656
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.000273015873015873,
+      "loss": 4.3948,
+      "theoretical_loss": 6.792103009445266,
+      "tokens_seen": 11272192
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00027460317460317464,
+      "loss": 4.5582,
+      "theoretical_loss": 6.7850401578467645,
+      "tokens_seen": 11337728
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.0002761904761904762,
+      "loss": 4.4994,
+      "theoretical_loss": 6.778029370814127,
+      "tokens_seen": 11403264
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.0002777777777777778,
+      "loss": 4.6215,
+      "theoretical_loss": 6.771069968617732,
+      "tokens_seen": 11468800
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00027936507936507935,
+      "loss": 4.5369,
+      "theoretical_loss": 6.764161284221716,
+      "tokens_seen": 11534336
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00028095238095238097,
+      "loss": 4.6742,
+      "theoretical_loss": 6.757302662976402,
+      "tokens_seen": 11599872
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.0002825396825396826,
+      "loss": 4.7673,
+      "theoretical_loss": 6.750493462319867,
+      "tokens_seen": 11665408
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00028412698412698417,
+      "loss": 4.5183,
+      "theoretical_loss": 6.74373305148833,
+      "tokens_seen": 11730944
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00028571428571428574,
+      "loss": 4.422,
+      "theoretical_loss": 6.737020811235054,
+      "tokens_seen": 11796480
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.0002873015873015873,
+      "loss": 4.6342,
+      "theoretical_loss": 6.730356133557465,
+      "tokens_seen": 11862016
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.0002888888888888889,
+      "loss": 4.4275,
+      "theoretical_loss": 6.7237384214322,
+      "tokens_seen": 11927552
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.0002904761904761905,
+      "loss": 4.5501,
+      "theoretical_loss": 6.717167088557821,
+      "tokens_seen": 11993088
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00029206349206349207,
+      "loss": 4.5781,
+      "theoretical_loss": 6.710641559104915,
+      "tokens_seen": 12058624
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00029365079365079364,
+      "loss": 4.6419,
+      "theoretical_loss": 6.704161267473356,
+      "tokens_seen": 12124160
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00029523809523809526,
+      "loss": 4.5732,
+      "theoretical_loss": 6.697725658056456,
+      "tokens_seen": 12189696
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00029682539682539683,
+      "loss": 4.5047,
+      "theoretical_loss": 6.691334185011799,
+      "tokens_seen": 12255232
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00029841269841269846,
+      "loss": 4.5736,
+      "theoretical_loss": 6.68498631203852,
+      "tokens_seen": 12320768
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00030000000000000003,
+      "loss": 4.3565,
+      "theoretical_loss": 6.678681512160812,
+      "tokens_seen": 12386304
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.0003015873015873016,
+      "loss": 4.5654,
+      "theoretical_loss": 6.672419267517472,
+      "tokens_seen": 12451840
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00030317460317460317,
+      "loss": 4.4851,
+      "theoretical_loss": 6.666199069157267,
+      "tokens_seen": 12517376
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00030476190476190474,
+      "loss": 4.4389,
+      "theoretical_loss": 6.660020416839945,
+      "tokens_seen": 12582912
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00030634920634920636,
+      "loss": 4.566,
+      "theoretical_loss": 6.65388281884269,
+      "tokens_seen": 12648448
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.000307936507936508,
+      "loss": 4.4483,
+      "theoretical_loss": 6.647785791771864,
+      "tokens_seen": 12713984
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00030952380952380956,
+      "loss": 4.4135,
+      "theoretical_loss": 6.641728860379852,
+      "tokens_seen": 12779520
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.0003111111111111111,
+      "loss": 4.3407,
+      "theoretical_loss": 6.635711557386854,
+      "tokens_seen": 12845056
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.0003126984126984127,
+      "loss": 4.5253,
+      "theoretical_loss": 6.629733423307451,
+      "tokens_seen": 12910592
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.0003142857142857143,
+      "loss": 4.3576,
+      "theoretical_loss": 6.6237940062818215,
+      "tokens_seen": 12976128
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.0003158730158730159,
+      "loss": 4.3529,
+      "theoretical_loss": 6.617892861911426,
+      "tokens_seen": 13041664
+    },
+    {
+      "epoch": 0.0,
+      "objective/train/docs_used": 11229,
+      "objective/train/instantaneous_batch_size": 32,
+      "objective/train/instantaneous_microbatch_size": 32768,
+      "objective/train/original_loss": 4.286226272583008,
+      "objective/train/theoretical_loss": 6.612029553099055,
+      "objective/train/tokens_used": 33567200,
+      "theoretical_loss": 6.612029553099055,
+      "tokens_seen": 13107200
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00031746031746031746,
+      "loss": 4.3359,
+      "theoretical_loss": 6.612029553099055,
+      "tokens_seen": 13107200
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.0003190476190476191,
+      "loss": 4.3879,
+      "theoretical_loss": 6.606203649893072,
+      "tokens_seen": 13172736
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00032063492063492065,
+      "loss": 4.3955,
+      "theoretical_loss": 6.600414729335739,
+      "tokens_seen": 13238272
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.0003222222222222222,
+      "loss": 4.4644,
+      "theoretical_loss": 6.5946623753155045,
+      "tokens_seen": 13303808
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00032380952380952385,
+      "loss": 4.4955,
+      "theoretical_loss": 6.588946178423104,
+      "tokens_seen": 13369344
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.0003253968253968254,
+      "loss": 4.3927,
+      "theoretical_loss": 6.583265735811386,
+      "tokens_seen": 13434880
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.000326984126984127,
+      "loss": 4.4351,
+      "theoretical_loss": 6.577620651058732,
+      "tokens_seen": 13500416
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00032857142857142856,
+      "loss": 4.4853,
+      "theoretical_loss": 6.572010534035971,
+      "tokens_seen": 13565952
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.0003301587301587302,
+      "loss": 4.4636,
+      "theoretical_loss": 6.566435000776667,
+      "tokens_seen": 13631488
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.0003317460317460318,
+      "loss": 4.5298,
+      "theoretical_loss": 6.5608936733507095,
+      "tokens_seen": 13697024
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.0003333333333333334,
+      "loss": 4.4583,
+      "theoretical_loss": 6.555386179741061,
+      "tokens_seen": 13762560
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00033492063492063495,
+      "loss": 4.3963,
+      "theoretical_loss": 6.549912153723619,
+      "tokens_seen": 13828096
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.0003365079365079365,
+      "loss": 4.2565,
+      "theoretical_loss": 6.544471234750057,
+      "tokens_seen": 13893632
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.0003380952380952381,
+      "loss": 4.5186,
+      "theoretical_loss": 6.539063067833591,
+      "tokens_seen": 13959168
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.0003396825396825397,
+      "loss": 4.3913,
+      "theoretical_loss": 6.533687303437551,
+      "tokens_seen": 14024704
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.0003412698412698413,
+      "loss": 4.4186,
+      "theoretical_loss": 6.528343597366709,
+      "tokens_seen": 14090240
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00034285714285714285,
+      "loss": 4.4453,
+      "theoretical_loss": 6.523031610661259,
+      "tokens_seen": 14155776
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.0003444444444444445,
+      "loss": 4.3743,
+      "theoretical_loss": 6.517751009493386,
+      "tokens_seen": 14221312
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00034603174603174604,
+      "loss": 4.386,
+      "theoretical_loss": 6.51250146506634,
+      "tokens_seen": 14286848
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00034761904761904767,
+      "loss": 4.531,
+      "theoretical_loss": 6.507282653515952,
+      "tokens_seen": 14352384
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00034920634920634924,
+      "loss": 4.4029,
+      "theoretical_loss": 6.5020942558145105,
+      "tokens_seen": 14417920
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.0003507936507936508,
+      "loss": 4.4528,
+      "theoretical_loss": 6.496935957676949,
+      "tokens_seen": 14483456
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.0003523809523809524,
+      "loss": 4.3805,
+      "theoretical_loss": 6.491807449469257,
+      "tokens_seen": 14548992
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00035396825396825395,
+      "loss": 4.3879,
+      "theoretical_loss": 6.486708426119071,
+      "tokens_seen": 14614528
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00035555555555555557,
+      "loss": 4.2345,
+      "theoretical_loss": 6.481638587028371,
+      "tokens_seen": 14680064
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.0003571428571428572,
+      "loss": 4.3617,
+      "theoretical_loss": 6.4765976359882345,
+      "tokens_seen": 14745600
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00035873015873015877,
+      "loss": 4.2069,
+      "theoretical_loss": 6.47158528109558,
+      "tokens_seen": 14811136
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00036031746031746034,
+      "loss": 4.3384,
+      "theoretical_loss": 6.466601234671855,
+      "tokens_seen": 14876672
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.0003619047619047619,
+      "loss": 4.5406,
+      "theoretical_loss": 6.461645213183605,
+      "tokens_seen": 14942208
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00036349206349206353,
+      "loss": 4.3038,
+      "theoretical_loss": 6.45671693716488,
+      "tokens_seen": 15007744
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.0003650793650793651,
+      "loss": 4.3228,
+      "theoretical_loss": 6.451816131141427,
+      "tokens_seen": 15073280
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00036666666666666667,
+      "loss": 4.2239,
+      "theoretical_loss": 6.4469425235566105,
+      "tokens_seen": 15138816
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00036825396825396824,
+      "loss": 4.2529,
+      "theoretical_loss": 6.442095846699032,
+      "tokens_seen": 15204352
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00036984126984126986,
+      "loss": 4.4215,
+      "theoretical_loss": 6.437275836631781,
+      "tokens_seen": 15269888
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00037142857142857143,
+      "loss": 4.2747,
+      "theoretical_loss": 6.432482233123297,
+      "tokens_seen": 15335424
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00037301587301587306,
+      "loss": 4.2938,
+      "theoretical_loss": 6.427714779579773,
+      "tokens_seen": 15400960
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00037460317460317463,
+      "loss": 4.2046,
+      "theoretical_loss": 6.422973222979093,
+      "tokens_seen": 15466496
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.0003761904761904762,
+      "loss": 4.2806,
+      "theoretical_loss": 6.418257313806215,
+      "tokens_seen": 15532032
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00037777777777777777,
+      "loss": 4.2457,
+      "theoretical_loss": 6.413566805990028,
+      "tokens_seen": 15597568
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.0003793650793650794,
+      "loss": 4.2291,
+      "theoretical_loss": 6.408901456841564,
+      "tokens_seen": 15663104
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00038095238095238096,
+      "loss": 4.3285,
+      "theoretical_loss": 6.404261026993607,
+      "tokens_seen": 15728640
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.0003825396825396826,
+      "loss": 4.3042,
+      "theoretical_loss": 6.399645280341607,
+      "tokens_seen": 15794176
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00038412698412698416,
+      "loss": 4.2348,
+      "theoretical_loss": 6.395053983985889,
+      "tokens_seen": 15859712
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.0003857142857142857,
+      "loss": 4.2749,
+      "theoretical_loss": 6.390486908175121,
+      "tokens_seen": 15925248
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.0003873015873015873,
+      "loss": 4.2945,
+      "theoretical_loss": 6.385943826251012,
+      "tokens_seen": 15990784
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.0003888888888888889,
+      "loss": 4.3808,
+      "theoretical_loss": 6.381424514594194,
+      "tokens_seen": 16056320
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.0003904761904761905,
+      "loss": 4.261,
+      "theoretical_loss": 6.376928752571271,
+      "tokens_seen": 16121856
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00039206349206349206,
+      "loss": 4.3231,
+      "theoretical_loss": 6.372456322483007,
+      "tokens_seen": 16187392
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00039365079365079363,
+      "loss": 4.2677,
+      "theoretical_loss": 6.368007009513611,
+      "tokens_seen": 16252928
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00039523809523809526,
+      "loss": 4.3075,
+      "theoretical_loss": 6.363580601681101,
+      "tokens_seen": 16318464
+    },
+    {
+      "epoch": 0.0,
+      "objective/train/docs_used": 12091,
+      "objective/train/instantaneous_batch_size": 32,
+      "objective/train/instantaneous_microbatch_size": 32768,
+      "objective/train/original_loss": 4.485805034637451,
+      "objective/train/theoretical_loss": 6.359176889788724,
+      "objective/train/tokens_used": 36844000,
+      "theoretical_loss": 6.359176889788724,
+      "tokens_seen": 16384000
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.0003968253968253969,
+      "loss": 4.4192,
+      "theoretical_loss": 6.359176889788724,
+      "tokens_seen": 16384000
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00039841269841269845,
+      "loss": 4.2839,
+      "theoretical_loss": 6.354795667377392,
+      "tokens_seen": 16449536
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004,
+      "loss": 4.2816,
+      "theoretical_loss": 6.350436730679111,
+      "tokens_seen": 16515072
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004015873015873016,
+      "loss": 4.1834,
+      "theoretical_loss": 6.346099878571405,
+      "tokens_seen": 16580608
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00040317460317460316,
+      "loss": 4.0906,
+      "theoretical_loss": 6.341784912532662,
+      "tokens_seen": 16646144
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004047619047619048,
+      "loss": 4.2961,
+      "theoretical_loss": 6.337491636598426,
+      "tokens_seen": 16711680
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00040634920634920635,
+      "loss": 4.2197,
+      "theoretical_loss": 6.33321985731858,
+      "tokens_seen": 16777216
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004079365079365079,
+      "loss": 4.275,
+      "theoretical_loss": 6.328969383715416,
+      "tokens_seen": 16842752
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004095238095238095,
+      "loss": 4.1974,
+      "theoretical_loss": 6.324740027242559,
+      "tokens_seen": 16908288
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004111111111111111,
+      "loss": 4.1405,
+      "theoretical_loss": 6.32053160174474,
+      "tokens_seen": 16973824
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00041269841269841274,
+      "loss": 4.1761,
+      "theoretical_loss": 6.316343923418371,
+      "tokens_seen": 17039360
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00041428571428571437,
+      "loss": 4.2186,
+      "theoretical_loss": 6.312176810772938,
+      "tokens_seen": 17104896
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00041587301587301594,
+      "loss": 4.1414,
+      "theoretical_loss": 6.308030084593148,
+      "tokens_seen": 17170432
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004174603174603175,
+      "loss": 4.1507,
+      "theoretical_loss": 6.30390356790187,
+      "tokens_seen": 17235968
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004190476190476191,
+      "loss": 4.2286,
+      "theoretical_loss": 6.2997970859237835,
+      "tokens_seen": 17301504
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00042063492063492065,
+      "loss": 4.3572,
+      "theoretical_loss": 6.295710466049778,
+      "tokens_seen": 17367040
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00042222222222222227,
+      "loss": 4.2201,
+      "theoretical_loss": 6.29164353780205,
+      "tokens_seen": 17432576
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00042380952380952384,
+      "loss": 4.2053,
+      "theoretical_loss": 6.287596132799887,
+      "tokens_seen": 17498112
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004253968253968254,
+      "loss": 4.0628,
+      "theoretical_loss": 6.283568084726138,
+      "tokens_seen": 17563648
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.000426984126984127,
+      "loss": 4.2117,
+      "theoretical_loss": 6.27955922929433,
+      "tokens_seen": 17629184
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004285714285714286,
+      "loss": 4.1475,
+      "theoretical_loss": 6.2755694042164425,
+      "tokens_seen": 17694720
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004301587301587302,
+      "loss": 4.1126,
+      "theoretical_loss": 6.271598449171293,
+      "tokens_seen": 17760256
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00043174603174603174,
+      "loss": 4.186,
+      "theoretical_loss": 6.267646205773557,
+      "tokens_seen": 17825792
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004333333333333333,
+      "loss": 4.1588,
+      "theoretical_loss": 6.263712517543373,
+      "tokens_seen": 17891328
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004349206349206349,
+      "loss": 4.1403,
+      "theoretical_loss": 6.25979722987654,
+      "tokens_seen": 17956864
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004365079365079365,
+      "loss": 4.2176,
+      "theoretical_loss": 6.255900190015284,
+      "tokens_seen": 18022400
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00043809523809523813,
+      "loss": 4.167,
+      "theoretical_loss": 6.252021247019596,
+      "tokens_seen": 18087936
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00043968253968253976,
+      "loss": 4.1248,
+      "theoretical_loss": 6.2481602517390975,
+      "tokens_seen": 18153472
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004412698412698413,
+      "loss": 4.1501,
+      "theoretical_loss": 6.244317056785462,
+      "tokens_seen": 18219008
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004428571428571429,
+      "loss": 4.1834,
+      "theoretical_loss": 6.240491516505343,
+      "tokens_seen": 18284544
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00044444444444444447,
+      "loss": 3.9826,
+      "theoretical_loss": 6.236683486953819,
+      "tokens_seen": 18350080
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004460317460317461,
+      "loss": 4.0851,
+      "theoretical_loss": 6.232892825868342,
+      "tokens_seen": 18415616
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00044761904761904766,
+      "loss": 4.2053,
+      "theoretical_loss": 6.229119392643171,
+      "tokens_seen": 18481152
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00044920634920634923,
+      "loss": 4.2157,
+      "theoretical_loss": 6.225363048304281,
+      "tokens_seen": 18546688
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004507936507936508,
+      "loss": 4.0984,
+      "theoretical_loss": 6.221623655484739,
+      "tokens_seen": 18612224
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00045238095238095237,
+      "loss": 4.1741,
+      "theoretical_loss": 6.217901078400542,
+      "tokens_seen": 18677760
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.000453968253968254,
+      "loss": 4.1512,
+      "theoretical_loss": 6.214195182826902,
+      "tokens_seen": 18743296
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00045555555555555556,
+      "loss": 4.2334,
+      "theoretical_loss": 6.2105058360749545,
+      "tokens_seen": 18808832
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00045714285714285713,
+      "loss": 4.0211,
+      "theoretical_loss": 6.206832906968911,
+      "tokens_seen": 18874368
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004587301587301587,
+      "loss": 3.9994,
+      "theoretical_loss": 6.203176265823624,
+      "tokens_seen": 18939904
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00046031746031746033,
+      "loss": 4.1663,
+      "theoretical_loss": 6.1995357844225545,
+      "tokens_seen": 19005440
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004619047619047619,
+      "loss": 4.0203,
+      "theoretical_loss": 6.195911335996151,
+      "tokens_seen": 19070976
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004634920634920636,
+      "loss": 4.0235,
+      "theoretical_loss": 6.192302795200606,
+      "tokens_seen": 19136512
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00046507936507936515,
+      "loss": 4.2248,
+      "theoretical_loss": 6.188710038097013,
+      "tokens_seen": 19202048
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004666666666666667,
+      "loss": 4.0779,
+      "theoretical_loss": 6.185132942130884,
+      "tokens_seen": 19267584
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004682539682539683,
+      "loss": 4.1567,
+      "theoretical_loss": 6.181571386112038,
+      "tokens_seen": 19333120
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00046984126984126986,
+      "loss": 4.1237,
+      "theoretical_loss": 6.178025250194852,
+      "tokens_seen": 19398656
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004714285714285715,
+      "loss": 4.117,
+      "theoretical_loss": 6.174494415858863,
+      "tokens_seen": 19464192
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00047301587301587305,
+      "loss": 4.0048,
+      "theoretical_loss": 6.170978765889705,
+      "tokens_seen": 19529728
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004746031746031746,
+      "loss": 4.1286,
+      "theoretical_loss": 6.167478184360402,
+      "tokens_seen": 19595264
+    },
+    {
+      "epoch": 0.01,
+      "objective/train/docs_used": 13329,
+      "objective/train/instantaneous_batch_size": 32,
+      "objective/train/instantaneous_microbatch_size": 32768,
+      "objective/train/original_loss": 4.195092678070068,
+      "objective/train/theoretical_loss": 6.163992556612967,
+      "objective/train/tokens_used": 40120800,
+      "theoretical_loss": 6.163992556612967,
+      "tokens_seen": 19660800
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004761904761904762,
+      "loss": 4.0441,
+      "theoretical_loss": 6.163992556612967,
+      "tokens_seen": 19660800
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004777777777777778,
+      "loss": 3.9248,
+      "theoretical_loss": 6.160521769240346,
+      "tokens_seen": 19726336
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004793650793650794,
+      "loss": 4.0968,
+      "theoretical_loss": 6.157065710068663,
+      "tokens_seen": 19791872
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00048095238095238095,
+      "loss": 4.0649,
+      "theoretical_loss": 6.1536242681397795,
+      "tokens_seen": 19857408
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004825396825396825,
+      "loss": 3.9885,
+      "theoretical_loss": 6.150197333694162,
+      "tokens_seen": 19922944
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004841269841269841,
+      "loss": 3.9086,
+      "theoretical_loss": 6.146784798154039,
+      "tokens_seen": 19988480
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004857142857142857,
+      "loss": 3.9802,
+      "theoretical_loss": 6.143386554106854,
+      "tokens_seen": 20054016
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004873015873015873,
+      "loss": 3.9978,
+      "theoretical_loss": 6.140002495289,
+      "tokens_seen": 20119552
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.000488888888888889,
+      "loss": 4.0074,
+      "theoretical_loss": 6.136632516569836,
+      "tokens_seen": 20185088
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004904761904761905,
+      "loss": 4.0614,
+      "theoretical_loss": 6.133276513935977,
+      "tokens_seen": 20250624
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004920634920634921,
+      "loss": 3.9051,
+      "theoretical_loss": 6.129934384475845,
+      "tokens_seen": 20316160
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004936507936507937,
+      "loss": 3.7378,
+      "theoretical_loss": 6.126606026364497,
+      "tokens_seen": 20381696
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004952380952380952,
+      "loss": 4.0041,
+      "theoretical_loss": 6.123291338848694,
+      "tokens_seen": 20447232
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004968253968253969,
+      "loss": 4.0944,
+      "theoretical_loss": 6.119990222232234,
+      "tokens_seen": 20512768
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004984126984126985,
+      "loss": 4.0604,
+      "theoretical_loss": 6.116702577861526,
+      "tokens_seen": 20578304
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0005,
+      "loss": 4.2215,
+      "theoretical_loss": 6.113428308111404,
+      "tokens_seen": 20643840
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0005015873015873016,
+      "loss": 4.1514,
+      "theoretical_loss": 6.110167316371188,
+      "tokens_seen": 20709376
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0005031746031746032,
+      "loss": 3.9323,
+      "theoretical_loss": 6.106919507030966,
+      "tokens_seen": 20774912
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0005047619047619048,
+      "loss": 3.9757,
+      "theoretical_loss": 6.103684785468113,
+      "tokens_seen": 20840448
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0005063492063492064,
+      "loss": 3.9713,
+      "theoretical_loss": 6.100463058034027,
+      "tokens_seen": 20905984
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0005079365079365079,
+      "loss": 4.0141,
+      "theoretical_loss": 6.097254232041089,
+      "tokens_seen": 20971520
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0005095238095238095,
+      "loss": 3.999,
+      "theoretical_loss": 6.094058215749833,
+      "tokens_seen": 21037056
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0005111111111111111,
+      "loss": 4.0366,
+      "theoretical_loss": 6.090874918356328,
+      "tokens_seen": 21102592
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0005126984126984128,
+      "loss": 4.0464,
+      "theoretical_loss": 6.087704249979775,
+      "tokens_seen": 21168128
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0005142857142857143,
+      "loss": 3.8926,
+      "theoretical_loss": 6.084546121650289,
+      "tokens_seen": 21233664
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0005158730158730159,
+      "loss": 4.118,
+      "theoretical_loss": 6.081400445296899,
+      "tokens_seen": 21299200
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0005174603174603176,
+      "loss": 3.9569,
+      "theoretical_loss": 6.078267133735724,
+      "tokens_seen": 21364736
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0005190476190476191,
+      "loss": 4.0569,
+      "theoretical_loss": 6.075146100658356,
+      "tokens_seen": 21430272
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0005206349206349207,
+      "loss": 3.8271,
+      "theoretical_loss": 6.072037260620421,
+      "tokens_seen": 21495808
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0005222222222222222,
+      "loss": 3.9131,
+      "theoretical_loss": 6.0689405290303275,
+      "tokens_seen": 21561344
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0005238095238095238,
+      "loss": 3.9558,
+      "theoretical_loss": 6.065855822138193,
+      "tokens_seen": 21626880
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0005253968253968255,
+      "loss": 3.9374,
+      "theoretical_loss": 6.06278305702495,
+      "tokens_seen": 21692416
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.000526984126984127,
+      "loss": 4.0054,
+      "theoretical_loss": 6.05972215159162,
+      "tokens_seen": 21757952
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0005285714285714286,
+      "loss": 3.8966,
+      "theoretical_loss": 6.056673024548766,
+      "tokens_seen": 21823488
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0005301587301587302,
+      "loss": 4.0723,
+      "theoretical_loss": 6.053635595406105,
+      "tokens_seen": 21889024
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0005317460317460317,
+      "loss": 4.0181,
+      "theoretical_loss": 6.050609784462289,
+      "tokens_seen": 21954560
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0005333333333333334,
+      "loss": 4.029,
+      "theoretical_loss": 6.047595512794841,
+      "tokens_seen": 22020096
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0005349206349206349,
+      "loss": 3.8708,
+      "theoretical_loss": 6.044592702250255,
+      "tokens_seen": 22085632
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0005365079365079365,
+      "loss": 3.9511,
+      "theoretical_loss": 6.041601275434251,
+      "tokens_seen": 22151168
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0005380952380952381,
+      "loss": 3.9624,
+      "theoretical_loss": 6.038621155702174,
+      "tokens_seen": 22216704
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0005396825396825397,
+      "loss": 3.9308,
+      "theoretical_loss": 6.0356522671495485,
+      "tokens_seen": 22282240
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0005412698412698414,
+      "loss": 4.0214,
+      "theoretical_loss": 6.032694534602784,
+      "tokens_seen": 22347776
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0005428571428571429,
+      "loss": 3.8022,
+      "theoretical_loss": 6.029747883610009,
+      "tokens_seen": 22413312
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0005444444444444445,
+      "loss": 3.9222,
+      "theoretical_loss": 6.026812240432067,
+      "tokens_seen": 22478848
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.000546031746031746,
+      "loss": 3.7736,
+      "theoretical_loss": 6.023887532033628,
+      "tokens_seen": 22544384
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0005476190476190477,
+      "loss": 3.9178,
+      "theoretical_loss": 6.020973686074459,
+      "tokens_seen": 22609920
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0005492063492063493,
+      "loss": 3.815,
+      "theoretical_loss": 6.018070630900809,
+      "tokens_seen": 22675456
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0005507936507936508,
+      "loss": 3.9869,
+      "theoretical_loss": 6.015178295536933,
+      "tokens_seen": 22740992
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0005523809523809524,
+      "loss": 3.9219,
+      "theoretical_loss": 6.012296609676753,
+      "tokens_seen": 22806528
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0005539682539682539,
+      "loss": 3.9291,
+      "theoretical_loss": 6.009425503675624,
+      "tokens_seen": 22872064
+    },
+    {
+      "epoch": 0.01,
+      "objective/train/docs_used": 14650,
+      "objective/train/instantaneous_batch_size": 32,
+      "objective/train/instantaneous_microbatch_size": 32768,
+      "objective/train/original_loss": 3.9398677349090576,
+      "objective/train/theoretical_loss": 6.006564908542249,
+      "objective/train/tokens_used": 43397600,
+      "theoretical_loss": 6.006564908542249,
+      "tokens_seen": 22937600
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0005555555555555556,
+      "loss": 3.9023,
+      "theoretical_loss": 6.006564908542249,
+      "tokens_seen": 22937600
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0005571428571428572,
+      "loss": 3.9074,
+      "theoretical_loss": 6.003714755930698,
+      "tokens_seen": 23003136
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0005587301587301587,
+      "loss": 3.7779,
+      "theoretical_loss": 6.000874978132554,
+      "tokens_seen": 23068672
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0005603174603174603,
+      "loss": 3.8925,
+      "theoretical_loss": 5.998045508069175,
+      "tokens_seen": 23134208
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0005619047619047619,
+      "loss": 3.7435,
+      "theoretical_loss": 5.995226279284073,
+      "tokens_seen": 23199744
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0005634920634920636,
+      "loss": 3.8925,
+      "theoretical_loss": 5.992417225935405,
+      "tokens_seen": 23265280
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0005650793650793652,
+      "loss": 3.6862,
+      "theoretical_loss": 5.989618282788578,
+      "tokens_seen": 23330816
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0005666666666666667,
+      "loss": 3.8544,
+      "theoretical_loss": 5.9868293852089565,
+      "tokens_seen": 23396352
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0005682539682539683,
+      "loss": 3.924,
+      "theoretical_loss": 5.984050469154685,
+      "tokens_seen": 23461888
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0005698412698412698,
+      "loss": 3.6818,
+      "theoretical_loss": 5.981281471169619,
+      "tokens_seen": 23527424
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0005714285714285715,
+      "loss": 3.8683,
+      "theoretical_loss": 5.978522328376346,
+      "tokens_seen": 23592960
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0005730158730158731,
+      "loss": 3.8562,
+      "theoretical_loss": 5.975772978469321,
+      "tokens_seen": 23658496
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0005746031746031746,
+      "loss": 3.7012,
+      "theoretical_loss": 5.973033359708099,
+      "tokens_seen": 23724032
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0005761904761904762,
+      "loss": 3.8193,
+      "theoretical_loss": 5.9703034109106685,
+      "tokens_seen": 23789568
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0005777777777777778,
+      "loss": 3.8737,
+      "theoretical_loss": 5.967583071446869,
+      "tokens_seen": 23855104
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0005793650793650794,
+      "loss": 3.8975,
+      "theoretical_loss": 5.9648722812319255,
+      "tokens_seen": 23920640
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.000580952380952381,
+      "loss": 3.7711,
+      "theoretical_loss": 5.962170980720055,
+      "tokens_seen": 23986176
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0005825396825396825,
+      "loss": 3.7881,
+      "theoretical_loss": 5.959479110898181,
+      "tokens_seen": 24051712
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0005841269841269841,
+      "loss": 3.8026,
+      "theoretical_loss": 5.956796613279727,
+      "tokens_seen": 24117248
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0005857142857142857,
+      "loss": 3.844,
+      "theoretical_loss": 5.954123429898508,
+      "tokens_seen": 24182784
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0005873015873015873,
+      "loss": 3.7136,
+      "theoretical_loss": 5.951459503302701,
+      "tokens_seen": 24248320
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0005888888888888889,
+      "loss": 3.7391,
+      "theoretical_loss": 5.948804776548908,
+      "tokens_seen": 24313856
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0005904761904761905,
+      "loss": 3.8898,
+      "theoretical_loss": 5.9461591931963,
+      "tokens_seen": 24379392
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0005920634920634922,
+      "loss": 3.8351,
+      "theoretical_loss": 5.943522697300843,
+      "tokens_seen": 24444928
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0005936507936507937,
+      "loss": 3.7491,
+      "theoretical_loss": 5.940895233409613,
+      "tokens_seen": 24510464
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0005952380952380953,
+      "loss": 3.6488,
+      "theoretical_loss": 5.938276746555175,
+      "tokens_seen": 24576000
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0005968253968253969,
+      "loss": 3.7852,
+      "theoretical_loss": 5.9356671822500635,
+      "tokens_seen": 24641536
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0005984126984126984,
+      "loss": 3.728,
+      "theoretical_loss": 5.933066486481318,
+      "tokens_seen": 24707072
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0006000000000000001,
+      "loss": 3.8557,
+      "theoretical_loss": 5.930474605705113,
+      "tokens_seen": 24772608
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0006015873015873016,
+      "loss": 3.7144,
+      "theoretical_loss": 5.927891486841449,
+      "tokens_seen": 24838144
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0006031746031746032,
+      "loss": 3.7249,
+      "theoretical_loss": 5.925317077268927,
+      "tokens_seen": 24903680
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0006047619047619048,
+      "loss": 3.7556,
+      "theoretical_loss": 5.92275132481959,
+      "tokens_seen": 24969216
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0006063492063492063,
+      "loss": 3.7397,
+      "theoretical_loss": 5.9201941777738405,
+      "tokens_seen": 25034752
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.000607936507936508,
+      "loss": 3.6145,
+      "theoretical_loss": 5.917645584855421,
+      "tokens_seen": 25100288
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0006095238095238095,
+      "loss": 3.671,
+      "theoretical_loss": 5.915105495226474,
+      "tokens_seen": 25165824
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0006111111111111111,
+      "loss": 3.7142,
+      "theoretical_loss": 5.912573858482657,
+      "tokens_seen": 25231360
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0006126984126984127,
+      "loss": 3.8682,
+      "theoretical_loss": 5.910050624648335,
+      "tokens_seen": 25296896
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0006142857142857143,
+      "loss": 3.7272,
+      "theoretical_loss": 5.907535744171835,
+      "tokens_seen": 25362432
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.000615873015873016,
+      "loss": 3.7217,
+      "theoretical_loss": 5.905029167920765,
+      "tokens_seen": 25427968
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0006174603174603175,
+      "loss": 3.7212,
+      "theoretical_loss": 5.902530847177394,
+      "tokens_seen": 25493504
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0006190476190476191,
+      "loss": 3.77,
+      "theoretical_loss": 5.900040733634098,
+      "tokens_seen": 25559040
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0006206349206349206,
+      "loss": 3.7162,
+      "theoretical_loss": 5.897558779388872,
+      "tokens_seen": 25624576
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0006222222222222223,
+      "loss": 3.6215,
+      "theoretical_loss": 5.895084936940887,
+      "tokens_seen": 25690112
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0006238095238095239,
+      "loss": 3.5772,
+      "theoretical_loss": 5.892619159186127,
+      "tokens_seen": 25755648
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0006253968253968254,
+      "loss": 3.671,
+      "theoretical_loss": 5.890161399413071,
+      "tokens_seen": 25821184
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.000626984126984127,
+      "loss": 3.6441,
+      "theoretical_loss": 5.887711611298428,
+      "tokens_seen": 25886720
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0006285714285714286,
+      "loss": 3.7587,
+      "theoretical_loss": 5.885269748902955,
+      "tokens_seen": 25952256
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0006301587301587302,
+      "loss": 3.6738,
+      "theoretical_loss": 5.882835766667295,
+      "tokens_seen": 26017792
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0006317460317460318,
+      "loss": 3.7029,
+      "theoretical_loss": 5.880409619407899,
+      "tokens_seen": 26083328
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0006333333333333333,
+      "loss": 3.634,
+      "theoretical_loss": 5.877991262312986,
+      "tokens_seen": 26148864
+    },
+    {
+      "epoch": 0.01,
+      "objective/train/docs_used": 15510,
+      "objective/train/instantaneous_batch_size": 32,
+      "objective/train/instantaneous_microbatch_size": 32768,
+      "objective/train/original_loss": 3.562167167663574,
+      "objective/train/theoretical_loss": 5.875580650938565,
+      "objective/train/tokens_used": 46674400,
+      "theoretical_loss": 5.875580650938565,
+      "tokens_seen": 26214400
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0006349206349206349,
+      "loss": 3.6464,
+      "theoretical_loss": 5.875580650938565,
+      "tokens_seen": 26214400
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0006365079365079365,
+      "loss": 3.7448,
+      "theoretical_loss": 5.873177741204501,
+      "tokens_seen": 26279936
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0006380952380952382,
+      "loss": 3.6473,
+      "theoretical_loss": 5.8707824893906455,
+      "tokens_seen": 26345472
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0006396825396825398,
+      "loss": 3.5706,
+      "theoretical_loss": 5.868394852133001,
+      "tokens_seen": 26411008
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0006412698412698413,
+      "loss": 3.8032,
+      "theoretical_loss": 5.866014786419951,
+      "tokens_seen": 26476544
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0006428571428571429,
+      "loss": 3.7774,
+      "theoretical_loss": 5.86364224958853,
+      "tokens_seen": 26542080
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0006444444444444444,
+      "loss": 3.7486,
+      "theoretical_loss": 5.861277199320743,
+      "tokens_seen": 26607616
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0006460317460317461,
+      "loss": 3.6064,
+      "theoretical_loss": 5.858919593639941,
+      "tokens_seen": 26673152
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0006476190476190477,
+      "loss": 3.8279,
+      "theoretical_loss": 5.856569390907229,
+      "tokens_seen": 26738688
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0006492063492063492,
+      "loss": 3.6837,
+      "theoretical_loss": 5.854226549817936,
+      "tokens_seen": 26804224
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0006507936507936508,
+      "loss": 3.6649,
+      "theoretical_loss": 5.851891029398114,
+      "tokens_seen": 26869760
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0006523809523809524,
+      "loss": 3.6972,
+      "theoretical_loss": 5.8495627890011015,
+      "tokens_seen": 26935296
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.000653968253968254,
+      "loss": 3.5475,
+      "theoretical_loss": 5.847241788304114,
+      "tokens_seen": 27000832
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0006555555555555556,
+      "loss": 3.6434,
+      "theoretical_loss": 5.844927987304884,
+      "tokens_seen": 27066368
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0006571428571428571,
+      "loss": 3.6581,
+      "theoretical_loss": 5.842621346318348,
+      "tokens_seen": 27131904
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0006587301587301587,
+      "loss": 3.6019,
+      "theoretical_loss": 5.84032182597337,
+      "tokens_seen": 27197440
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0006603174603174604,
+      "loss": 3.5183,
+      "theoretical_loss": 5.8380293872095095,
+      "tokens_seen": 27262976
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0006619047619047619,
+      "loss": 3.6166,
+      "theoretical_loss": 5.835743991273832,
+      "tokens_seen": 27328512
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0006634920634920636,
+      "loss": 3.6293,
+      "theoretical_loss": 5.833465599717752,
+      "tokens_seen": 27394048
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0006650793650793651,
+      "loss": 3.7443,
+      "theoretical_loss": 5.831194174393929,
+      "tokens_seen": 27459584
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0006666666666666668,
+      "loss": 3.6967,
+      "theoretical_loss": 5.828929677453187,
+      "tokens_seen": 27525120
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0006682539682539683,
+      "loss": 3.6127,
+      "theoretical_loss": 5.826672071341488,
+      "tokens_seen": 27590656
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0006698412698412699,
+      "loss": 3.5727,
+      "theoretical_loss": 5.824421318796932,
+      "tokens_seen": 27656192
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0006714285714285715,
+      "loss": 3.7115,
+      "theoretical_loss": 5.822177382846799,
+      "tokens_seen": 27721728
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.000673015873015873,
+      "loss": 3.7053,
+      "theoretical_loss": 5.8199402268046265,
+      "tokens_seen": 27787264
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0006746031746031747,
+      "loss": 3.5489,
+      "theoretical_loss": 5.817709814267329,
+      "tokens_seen": 27852800
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0006761904761904762,
+      "loss": 3.6114,
+      "theoretical_loss": 5.815486109112342,
+      "tokens_seen": 27918336
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0006777777777777778,
+      "loss": 3.6637,
+      "theoretical_loss": 5.813269075494812,
+      "tokens_seen": 27983872
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0006793650793650794,
+      "loss": 3.5809,
+      "theoretical_loss": 5.811058677844817,
+      "tokens_seen": 28049408
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0006809523809523809,
+      "loss": 3.6056,
+      "theoretical_loss": 5.808854880864618,
+      "tokens_seen": 28114944
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0006825396825396826,
+      "loss": 3.5561,
+      "theoretical_loss": 5.8066576495259525,
+      "tokens_seen": 28180480
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0006841269841269841,
+      "loss": 3.588,
+      "theoretical_loss": 5.804466949067352,
+      "tokens_seen": 28246016
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0006857142857142857,
+      "loss": 3.554,
+      "theoretical_loss": 5.8022827449914995,
+      "tokens_seen": 28311552
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0006873015873015873,
+      "loss": 3.4486,
+      "theoretical_loss": 5.800105003062612,
+      "tokens_seen": 28377088
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.000688888888888889,
+      "loss": 3.5973,
+      "theoretical_loss": 5.797933689303866,
+      "tokens_seen": 28442624
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0006904761904761906,
+      "loss": 3.5119,
+      "theoretical_loss": 5.795768769994842,
+      "tokens_seen": 28508160
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0006920634920634921,
+      "loss": 3.5986,
+      "theoretical_loss": 5.793610211669004,
+      "tokens_seen": 28573696
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0006936507936507937,
+      "loss": 3.4873,
+      "theoretical_loss": 5.79145798111122,
+      "tokens_seen": 28639232
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0006952380952380953,
+      "loss": 3.6591,
+      "theoretical_loss": 5.789312045355292,
+      "tokens_seen": 28704768
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0006968253968253969,
+      "loss": 3.4647,
+      "theoretical_loss": 5.787172371681535,
+      "tokens_seen": 28770304
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0006984126984126985,
+      "loss": 3.4834,
+      "theoretical_loss": 5.785038927614373,
+      "tokens_seen": 28835840
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007,
+      "loss": 3.666,
+      "theoretical_loss": 5.782911680919971,
+      "tokens_seen": 28901376
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007015873015873016,
+      "loss": 3.6369,
+      "theoretical_loss": 5.78079059960389,
+      "tokens_seen": 28966912
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007031746031746032,
+      "loss": 3.5484,
+      "theoretical_loss": 5.77867565190877,
+      "tokens_seen": 29032448
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007047619047619048,
+      "loss": 3.5085,
+      "theoretical_loss": 5.776566806312052,
+      "tokens_seen": 29097984
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007063492063492064,
+      "loss": 3.6073,
+      "theoretical_loss": 5.774464031523705,
+      "tokens_seen": 29163520
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007079365079365079,
+      "loss": 3.6575,
+      "theoretical_loss": 5.772367296484001,
+      "tokens_seen": 29229056
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007095238095238095,
+      "loss": 3.6789,
+      "theoretical_loss": 5.770276570361309,
+      "tokens_seen": 29294592
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007111111111111111,
+      "loss": 3.6252,
+      "theoretical_loss": 5.768191822549908,
+      "tokens_seen": 29360128
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007126984126984128,
+      "loss": 3.483,
+      "theoretical_loss": 5.766113022667838,
+      "tokens_seen": 29425664
+    },
+    {
+      "epoch": 0.01,
+      "objective/train/docs_used": 16736,
+      "objective/train/instantaneous_batch_size": 32,
+      "objective/train/instantaneous_microbatch_size": 32768,
+      "objective/train/original_loss": 3.615713596343994,
+      "objective/train/theoretical_loss": 5.764040140554766,
+      "objective/train/tokens_used": 49951200,
+      "theoretical_loss": 5.764040140554766,
+      "tokens_seen": 29491200
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007142857142857144,
+      "loss": 3.5998,
+      "theoretical_loss": 5.764040140554766,
+      "tokens_seen": 29491200
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007158730158730159,
+      "loss": 3.5203,
+      "theoretical_loss": 5.761973146269881,
+      "tokens_seen": 29556736
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007174603174603175,
+      "loss": 3.3129,
+      "theoretical_loss": 5.759912010089819,
+      "tokens_seen": 29622272
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.000719047619047619,
+      "loss": 3.5524,
+      "theoretical_loss": 5.757856702506597,
+      "tokens_seen": 29687808
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007206349206349207,
+      "loss": 3.6247,
+      "theoretical_loss": 5.755807194225595,
+      "tokens_seen": 29753344
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007222222222222223,
+      "loss": 3.5035,
+      "theoretical_loss": 5.753763456163529,
+      "tokens_seen": 29818880
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007238095238095238,
+      "loss": 3.587,
+      "theoretical_loss": 5.751725459446487,
+      "tokens_seen": 29884416
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007253968253968254,
+      "loss": 3.6204,
+      "theoretical_loss": 5.7496931754079466,
+      "tokens_seen": 29949952
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007269841269841271,
+      "loss": 3.5093,
+      "theoretical_loss": 5.747666575586849,
+      "tokens_seen": 30015488
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007285714285714286,
+      "loss": 3.6138,
+      "theoretical_loss": 5.7456456317256785,
+      "tokens_seen": 30081024
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007301587301587302,
+      "loss": 3.5145,
+      "theoretical_loss": 5.743630315768563,
+      "tokens_seen": 30146560
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007317460317460317,
+      "loss": 3.5539,
+      "theoretical_loss": 5.741620599859402,
+      "tokens_seen": 30212096
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007333333333333333,
+      "loss": 3.441,
+      "theoretical_loss": 5.739616456340021,
+      "tokens_seen": 30277632
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.000734920634920635,
+      "loss": 3.478,
+      "theoretical_loss": 5.737617857748329,
+      "tokens_seen": 30343168
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007365079365079365,
+      "loss": 3.6403,
+      "theoretical_loss": 5.73562477681652,
+      "tokens_seen": 30408704
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007380952380952382,
+      "loss": 3.5638,
+      "theoretical_loss": 5.733637186469274,
+      "tokens_seen": 30474240
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007396825396825397,
+      "loss": 3.5231,
+      "theoretical_loss": 5.731655059821993,
+      "tokens_seen": 30539776
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007412698412698414,
+      "loss": 3.4033,
+      "theoretical_loss": 5.729678370179052,
+      "tokens_seen": 30605312
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007428571428571429,
+      "loss": 3.5958,
+      "theoretical_loss": 5.727707091032062,
+      "tokens_seen": 30670848
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007444444444444445,
+      "loss": 3.6203,
+      "theoretical_loss": 5.725741196058175,
+      "tokens_seen": 30736384
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007460317460317461,
+      "loss": 3.4194,
+      "theoretical_loss": 5.723780659118377,
+      "tokens_seen": 30801920
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007476190476190476,
+      "loss": 3.4464,
+      "theoretical_loss": 5.721825454255825,
+      "tokens_seen": 30867456
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007492063492063493,
+      "loss": 3.484,
+      "theoretical_loss": 5.71987555569419,
+      "tokens_seen": 30932992
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007507936507936508,
+      "loss": 3.429,
+      "theoretical_loss": 5.717930937836029,
+      "tokens_seen": 30998528
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007523809523809524,
+      "loss": 3.6263,
+      "theoretical_loss": 5.715991575261159,
+      "tokens_seen": 31064064
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.000753968253968254,
+      "loss": 3.4072,
+      "theoretical_loss": 5.714057442725068,
+      "tokens_seen": 31129600
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007555555555555555,
+      "loss": 3.5214,
+      "theoretical_loss": 5.712128515157324,
+      "tokens_seen": 31195136
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007571428571428572,
+      "loss": 3.3546,
+      "theoretical_loss": 5.710204767660022,
+      "tokens_seen": 31260672
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007587301587301588,
+      "loss": 3.4292,
+      "theoretical_loss": 5.708286175506232,
+      "tokens_seen": 31326208
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007603174603174603,
+      "loss": 3.515,
+      "theoretical_loss": 5.706372714138472,
+      "tokens_seen": 31391744
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007619047619047619,
+      "loss": 3.5369,
+      "theoretical_loss": 5.704464359167195,
+      "tokens_seen": 31457280
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007634920634920635,
+      "loss": 3.2781,
+      "theoretical_loss": 5.702561086369294,
+      "tokens_seen": 31522816
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007650793650793652,
+      "loss": 3.4947,
+      "theoretical_loss": 5.7006628716866246,
+      "tokens_seen": 31588352
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007666666666666667,
+      "loss": 3.5344,
+      "theoretical_loss": 5.698769691224539,
+      "tokens_seen": 31653888
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007682539682539683,
+      "loss": 3.5101,
+      "theoretical_loss": 5.696881521250441,
+      "tokens_seen": 31719424
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007698412698412699,
+      "loss": 3.4994,
+      "theoretical_loss": 5.694998338192356,
+      "tokens_seen": 31784960
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007714285714285715,
+      "loss": 3.4336,
+      "theoretical_loss": 5.693120118637513,
+      "tokens_seen": 31850496
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007730158730158731,
+      "loss": 3.5182,
+      "theoretical_loss": 5.691246839330946,
+      "tokens_seen": 31916032
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007746031746031746,
+      "loss": 3.5079,
+      "theoretical_loss": 5.689378477174106,
+      "tokens_seen": 31981568
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007761904761904762,
+      "loss": 3.3034,
+      "theoretical_loss": 5.687515009223498,
+      "tokens_seen": 32047104
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007777777777777778,
+      "loss": 3.412,
+      "theoretical_loss": 5.685656412689318,
+      "tokens_seen": 32112640
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007793650793650794,
+      "loss": 3.5224,
+      "theoretical_loss": 5.683802664934115,
+      "tokens_seen": 32178176
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.000780952380952381,
+      "loss": 3.338,
+      "theoretical_loss": 5.681953743471466,
+      "tokens_seen": 32243712
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007825396825396825,
+      "loss": 3.552,
+      "theoretical_loss": 5.680109625964663,
+      "tokens_seen": 32309248
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007841269841269841,
+      "loss": 3.3683,
+      "theoretical_loss": 5.678270290225415,
+      "tokens_seen": 32374784
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007857142857142857,
+      "loss": 3.4463,
+      "theoretical_loss": 5.676435714212562,
+      "tokens_seen": 32440320
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007873015873015873,
+      "loss": 3.3597,
+      "theoretical_loss": 5.674605876030808,
+      "tokens_seen": 32505856
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.000788888888888889,
+      "loss": 3.4494,
+      "theoretical_loss": 5.672780753929461,
+      "tokens_seen": 32571392
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007904761904761905,
+      "loss": 3.4102,
+      "theoretical_loss": 5.67096032630119,
+      "tokens_seen": 32636928
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007920634920634921,
+      "loss": 3.4447,
+      "theoretical_loss": 5.669144571680797,
+      "tokens_seen": 32702464
+    },
+    {
+      "epoch": 0.01,
+      "objective/train/docs_used": 17598,
+      "objective/train/instantaneous_batch_size": 32,
+      "objective/train/instantaneous_microbatch_size": 32768,
+      "objective/train/original_loss": 3.3977389335632324,
+      "objective/train/theoretical_loss": 5.667333468743994,
+      "objective/train/tokens_used": 53228000,
+      "theoretical_loss": 5.667333468743994,
+      "tokens_seen": 32768000
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007936507936507938,
+      "loss": 3.416,
+      "theoretical_loss": 5.667333468743994,
+      "tokens_seen": 32768000
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007952380952380953,
+      "loss": 3.3995,
+      "theoretical_loss": 5.665526996306205,
+      "tokens_seen": 32833536
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007968253968253969,
+      "loss": 3.4332,
+      "theoretical_loss": 5.663725133321369,
+      "tokens_seen": 32899072
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007984126984126984,
+      "loss": 3.4056,
+      "theoretical_loss": 5.661927858880761,
+      "tokens_seen": 32964608
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0008,
+      "loss": 3.4997,
+      "theoretical_loss": 5.6601351522118275,
+      "tokens_seen": 33030144
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007999839518555668,
+      "loss": 3.5224,
+      "theoretical_loss": 5.65834699267703,
+      "tokens_seen": 33095680
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007999679037111334,
+      "loss": 3.5053,
+      "theoretical_loss": 5.6565633597727025,
+      "tokens_seen": 33161216
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007999518555667002,
+      "loss": 3.5716,
+      "theoretical_loss": 5.65478423312792,
+      "tokens_seen": 33226752
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007999358074222668,
+      "loss": 3.496,
+      "theoretical_loss": 5.65300959250338,
+      "tokens_seen": 33292288
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007999197592778336,
+      "loss": 3.3879,
+      "theoretical_loss": 5.651239417790293,
+      "tokens_seen": 33357824
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007999037111334002,
+      "loss": 3.3997,
+      "theoretical_loss": 5.649473689009292,
+      "tokens_seen": 33423360
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.000799887662988967,
+      "loss": 3.6344,
+      "theoretical_loss": 5.6477123863093395,
+      "tokens_seen": 33488896
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007998716148445336,
+      "loss": 3.4542,
+      "theoretical_loss": 5.645955489966661,
+      "tokens_seen": 33554432
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007998555667001004,
+      "loss": 3.3641,
+      "theoretical_loss": 5.644202980383678,
+      "tokens_seen": 33619968
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.000799839518555667,
+      "loss": 3.3726,
+      "theoretical_loss": 5.6424548380879616,
+      "tokens_seen": 33685504
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007998234704112338,
+      "loss": 3.4834,
+      "theoretical_loss": 5.640711043731184,
+      "tokens_seen": 33751040
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007998074222668004,
+      "loss": 3.2946,
+      "theoretical_loss": 5.6389715780880945,
+      "tokens_seen": 33816576
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007997913741223672,
+      "loss": 3.3852,
+      "theoretical_loss": 5.6372364220555,
+      "tokens_seen": 33882112
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007997753259779338,
+      "loss": 3.409,
+      "theoretical_loss": 5.635505556651251,
+      "tokens_seen": 33947648
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007997592778335006,
+      "loss": 3.4924,
+      "theoretical_loss": 5.633778963013247,
+      "tokens_seen": 34013184
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007997432296890672,
+      "loss": 3.3312,
+      "theoretical_loss": 5.632056622398448,
+      "tokens_seen": 34078720
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.000799727181544634,
+      "loss": 3.2852,
+      "theoretical_loss": 5.63033851618189,
+      "tokens_seen": 34144256
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007997111334002006,
+      "loss": 3.2534,
+      "theoretical_loss": 5.628624625855725,
+      "tokens_seen": 34209792
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007996950852557674,
+      "loss": 3.4196,
+      "theoretical_loss": 5.626914933028255,
+      "tokens_seen": 34275328
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.000799679037111334,
+      "loss": 3.4043,
+      "theoretical_loss": 5.625209419422983,
+      "tokens_seen": 34340864
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007996629889669008,
+      "loss": 3.4019,
+      "theoretical_loss": 5.623508066877678,
+      "tokens_seen": 34406400
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007996469408224675,
+      "loss": 3.3266,
+      "theoretical_loss": 5.621810857343441,
+      "tokens_seen": 34471936
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007996308926780342,
+      "loss": 3.4346,
+      "theoretical_loss": 5.620117772883784,
+      "tokens_seen": 34537472
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007996148445336009,
+      "loss": 3.181,
+      "theoretical_loss": 5.618428795673718,
+      "tokens_seen": 34603008
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007995987963891676,
+      "loss": 3.2602,
+      "theoretical_loss": 5.616743907998853,
+      "tokens_seen": 34668544
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007995827482447343,
+      "loss": 3.2791,
+      "theoretical_loss": 5.6150630922545,
+      "tokens_seen": 34734080
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.000799566700100301,
+      "loss": 3.3478,
+      "theoretical_loss": 5.613386330944792,
+      "tokens_seen": 34799616
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007995506519558677,
+      "loss": 3.3831,
+      "theoretical_loss": 5.6117136066818,
+      "tokens_seen": 34865152
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007995346038114344,
+      "loss": 3.3701,
+      "theoretical_loss": 5.610044902184672,
+      "tokens_seen": 34930688
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007995185556670011,
+      "loss": 3.4321,
+      "theoretical_loss": 5.608380200278775,
+      "tokens_seen": 34996224
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007995025075225678,
+      "loss": 3.3108,
+      "theoretical_loss": 5.606719483894837,
+      "tokens_seen": 35061760
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007994864593781345,
+      "loss": 3.1902,
+      "theoretical_loss": 5.605062736068113,
+      "tokens_seen": 35127296
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007994704112337012,
+      "loss": 3.3093,
+      "theoretical_loss": 5.603409939937547,
+      "tokens_seen": 35192832
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007994543630892679,
+      "loss": 3.2981,
+      "theoretical_loss": 5.601761078744952,
+      "tokens_seen": 35258368
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007994383149448346,
+      "loss": 3.2821,
+      "theoretical_loss": 5.600116135834183,
+      "tokens_seen": 35323904
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007994222668004013,
+      "loss": 3.237,
+      "theoretical_loss": 5.598475094650334,
+      "tokens_seen": 35389440
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.000799406218655968,
+      "loss": 3.4141,
+      "theoretical_loss": 5.59683793873894,
+      "tokens_seen": 35454976
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007993901705115347,
+      "loss": 3.3493,
+      "theoretical_loss": 5.595204651745176,
+      "tokens_seen": 35520512
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007993741223671014,
+      "loss": 3.2313,
+      "theoretical_loss": 5.593575217413074,
+      "tokens_seen": 35586048
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007993580742226681,
+      "loss": 3.3789,
+      "theoretical_loss": 5.591949619584746,
+      "tokens_seen": 35651584
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007993420260782347,
+      "loss": 3.1288,
+      "theoretical_loss": 5.590327842199611,
+      "tokens_seen": 35717120
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007993259779338015,
+      "loss": 3.3649,
+      "theoretical_loss": 5.588709869293634,
+      "tokens_seen": 35782656
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007993099297893681,
+      "loss": 3.3458,
+      "theoretical_loss": 5.5870956849985705,
+      "tokens_seen": 35848192
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007992938816449349,
+      "loss": 3.285,
+      "theoretical_loss": 5.585485273541212,
+      "tokens_seen": 35913728
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007992778335005015,
+      "loss": 3.1898,
+      "theoretical_loss": 5.583878619242652,
+      "tokens_seen": 35979264
+    },
+    {
+      "epoch": 0.01,
+      "objective/train/docs_used": 18750,
+      "objective/train/instantaneous_batch_size": 32,
+      "objective/train/instantaneous_microbatch_size": 32768,
+      "objective/train/original_loss": 3.3048973083496094,
+      "objective/train/theoretical_loss": 5.582275706517549,
+      "objective/train/tokens_used": 56504800,
+      "theoretical_loss": 5.582275706517549,
+      "tokens_seen": 36044800
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007992617853560683,
+      "loss": 3.3286,
+      "theoretical_loss": 5.582275706517549,
+      "tokens_seen": 36044800
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007992457372116349,
+      "loss": 3.2911,
+      "theoretical_loss": 5.580676519873396,
+      "tokens_seen": 36110336
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007992296890672017,
+      "loss": 3.2725,
+      "theoretical_loss": 5.579081043909808,
+      "tokens_seen": 36175872
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007992136409227683,
+      "loss": 3.4246,
+      "theoretical_loss": 5.577489263317796,
+      "tokens_seen": 36241408
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.000799197592778335,
+      "loss": 3.306,
+      "theoretical_loss": 5.5759011628790764,
+      "tokens_seen": 36306944
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007991815446339017,
+      "loss": 3.2385,
+      "theoretical_loss": 5.574316727465358,
+      "tokens_seen": 36372480
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007991654964894684,
+      "loss": 3.1253,
+      "theoretical_loss": 5.572735942037655,
+      "tokens_seen": 36438016
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007991494483450351,
+      "loss": 3.392,
+      "theoretical_loss": 5.571158791645597,
+      "tokens_seen": 36503552
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007991334002006018,
+      "loss": 3.3194,
+      "theoretical_loss": 5.569585261426752,
+      "tokens_seen": 36569088
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007991173520561685,
+      "loss": 3.3435,
+      "theoretical_loss": 5.568015336605953,
+      "tokens_seen": 36634624
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007991013039117352,
+      "loss": 3.2141,
+      "theoretical_loss": 5.566449002494627,
+      "tokens_seen": 36700160
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007990852557673019,
+      "loss": 3.3243,
+      "theoretical_loss": 5.564886244490136,
+      "tokens_seen": 36765696
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007990692076228686,
+      "loss": 3.2836,
+      "theoretical_loss": 5.563327048075124,
+      "tokens_seen": 36831232
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007990531594784353,
+      "loss": 3.2402,
+      "theoretical_loss": 5.5617713988168616,
+      "tokens_seen": 36896768
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.000799037111334002,
+      "loss": 3.1849,
+      "theoretical_loss": 5.560219282366614,
+      "tokens_seen": 36962304
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007990210631895687,
+      "loss": 3.2411,
+      "theoretical_loss": 5.558670684458994,
+      "tokens_seen": 37027840
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007990050150451354,
+      "loss": 3.2418,
+      "theoretical_loss": 5.557125590911339,
+      "tokens_seen": 37093376
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007989889669007021,
+      "loss": 3.3203,
+      "theoretical_loss": 5.5555839876230815,
+      "tokens_seen": 37158912
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007989729187562688,
+      "loss": 3.2477,
+      "theoretical_loss": 5.55404586057513,
+      "tokens_seen": 37224448
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007989568706118355,
+      "loss": 3.2417,
+      "theoretical_loss": 5.5525111958292594,
+      "tokens_seen": 37289984
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007989408224674022,
+      "loss": 3.2639,
+      "theoretical_loss": 5.550979979527501,
+      "tokens_seen": 37355520
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007989247743229689,
+      "loss": 3.3022,
+      "theoretical_loss": 5.549452197891538,
+      "tokens_seen": 37421056
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007989087261785356,
+      "loss": 3.2796,
+      "theoretical_loss": 5.547927837222119,
+      "tokens_seen": 37486592
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007988926780341023,
+      "loss": 3.3177,
+      "theoretical_loss": 5.546406883898451,
+      "tokens_seen": 37552128
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.000798876629889669,
+      "loss": 3.3537,
+      "theoretical_loss": 5.544889324377628,
+      "tokens_seen": 37617664
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007988605817452357,
+      "loss": 3.365,
+      "theoretical_loss": 5.543375145194048,
+      "tokens_seen": 37683200
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007988445336008024,
+      "loss": 3.2576,
+      "theoretical_loss": 5.541864332958831,
+      "tokens_seen": 37748736
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007988284854563691,
+      "loss": 3.1739,
+      "theoretical_loss": 5.540356874359257,
+      "tokens_seen": 37814272
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007988124373119358,
+      "loss": 3.1861,
+      "theoretical_loss": 5.538852756158201,
+      "tokens_seen": 37879808
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007987963891675025,
+      "loss": 3.177,
+      "theoretical_loss": 5.537351965193569,
+      "tokens_seen": 37945344
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007987803410230693,
+      "loss": 3.1705,
+      "theoretical_loss": 5.535854488377751,
+      "tokens_seen": 38010880
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007987642928786359,
+      "loss": 3.1531,
+      "theoretical_loss": 5.534360312697064,
+      "tokens_seen": 38076416
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007987482447342027,
+      "loss": 3.1667,
+      "theoretical_loss": 5.532869425211214,
+      "tokens_seen": 38141952
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007987321965897693,
+      "loss": 3.3777,
+      "theoretical_loss": 5.531381813052757,
+      "tokens_seen": 38207488
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007987161484453361,
+      "loss": 3.1323,
+      "theoretical_loss": 5.529897463426559,
+      "tokens_seen": 38273024
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007987001003009027,
+      "loss": 3.2019,
+      "theoretical_loss": 5.528416363609276,
+      "tokens_seen": 38338560
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007986840521564695,
+      "loss": 3.3026,
+      "theoretical_loss": 5.526938500948817,
+      "tokens_seen": 38404096
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007986680040120361,
+      "loss": 3.1529,
+      "theoretical_loss": 5.525463862863836,
+      "tokens_seen": 38469632
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007986519558676029,
+      "loss": 3.24,
+      "theoretical_loss": 5.523992436843209,
+      "tokens_seen": 38535168
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007986359077231695,
+      "loss": 3.2625,
+      "theoretical_loss": 5.522524210445528,
+      "tokens_seen": 38600704
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007986198595787363,
+      "loss": 3.0977,
+      "theoretical_loss": 5.521059171298592,
+      "tokens_seen": 38666240
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007986038114343029,
+      "loss": 3.2564,
+      "theoretical_loss": 5.519597307098907,
+      "tokens_seen": 38731776
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007985877632898697,
+      "loss": 3.2323,
+      "theoretical_loss": 5.518138605611188,
+      "tokens_seen": 38797312
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007985717151454363,
+      "loss": 3.1504,
+      "theoretical_loss": 5.516683054667868,
+      "tokens_seen": 38862848
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007985556670010031,
+      "loss": 3.2903,
+      "theoretical_loss": 5.515230642168607,
+      "tokens_seen": 38928384
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007985396188565697,
+      "loss": 3.1036,
+      "theoretical_loss": 5.513781356079811,
+      "tokens_seen": 38993920
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007985235707121365,
+      "loss": 3.1206,
+      "theoretical_loss": 5.512335184434152,
+      "tokens_seen": 39059456
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007985075225677031,
+      "loss": 3.217,
+      "theoretical_loss": 5.5108921153300905,
+      "tokens_seen": 39124992
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007984914744232699,
+      "loss": 3.1188,
+      "theoretical_loss": 5.5094521369314045,
+      "tokens_seen": 39190528
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007984754262788365,
+      "loss": 3.1188,
+      "theoretical_loss": 5.5080152374667275,
+      "tokens_seen": 39256064
+    },
+    {
+      "epoch": 0.01,
+      "objective/train/docs_used": 19586,
+      "objective/train/instantaneous_batch_size": 32,
+      "objective/train/instantaneous_microbatch_size": 32768,
+      "objective/train/original_loss": 2.9256014823913574,
+      "objective/train/theoretical_loss": 5.506581405229079,
+      "objective/train/tokens_used": 59781600,
+      "theoretical_loss": 5.506581405229079,
+      "tokens_seen": 39321600
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007984593781344033,
+      "loss": 3.0216,
+      "theoretical_loss": 5.506581405229079,
+      "tokens_seen": 39321600
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007984433299899699,
+      "loss": 3.1897,
+      "theoretical_loss": 5.505150628575409,
+      "tokens_seen": 39387136
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007984272818455367,
+      "loss": 3.2565,
+      "theoretical_loss": 5.503722895926143,
+      "tokens_seen": 39452672
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007984112337011033,
+      "loss": 3.1173,
+      "theoretical_loss": 5.50229819576473,
+      "tokens_seen": 39518208
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007983951855566701,
+      "loss": 3.1802,
+      "theoretical_loss": 5.500876516637199,
+      "tokens_seen": 39583744
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007983791374122367,
+      "loss": 3.1397,
+      "theoretical_loss": 5.49945784715171,
+      "tokens_seen": 39649280
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007983630892678035,
+      "loss": 3.0917,
+      "theoretical_loss": 5.498042175978123,
+      "tokens_seen": 39714816
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007983470411233701,
+      "loss": 3.12,
+      "theoretical_loss": 5.496629491847553,
+      "tokens_seen": 39780352
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007983309929789369,
+      "loss": 3.1142,
+      "theoretical_loss": 5.495219783551947,
+      "tokens_seen": 39845888
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007983149448345036,
+      "loss": 3.1628,
+      "theoretical_loss": 5.493813039943654,
+      "tokens_seen": 39911424
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007982988966900703,
+      "loss": 3.2016,
+      "theoretical_loss": 5.492409249934996,
+      "tokens_seen": 39976960
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.000798282848545637,
+      "loss": 2.9627,
+      "theoretical_loss": 5.491008402497855,
+      "tokens_seen": 40042496
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007982668004012037,
+      "loss": 3.1029,
+      "theoretical_loss": 5.489610486663252,
+      "tokens_seen": 40108032
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007982507522567704,
+      "loss": 3.0547,
+      "theoretical_loss": 5.488215491520933,
+      "tokens_seen": 40173568
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007982347041123371,
+      "loss": 3.2048,
+      "theoretical_loss": 5.486823406218962,
+      "tokens_seen": 40239104
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007982186559679038,
+      "loss": 3.2584,
+      "theoretical_loss": 5.485434219963315,
+      "tokens_seen": 40304640
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007982026078234705,
+      "loss": 3.2753,
+      "theoretical_loss": 5.484047922017476,
+      "tokens_seen": 40370176
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007981865596790372,
+      "loss": 2.9574,
+      "theoretical_loss": 5.482664501702036,
+      "tokens_seen": 40435712
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007981705115346039,
+      "loss": 2.9897,
+      "theoretical_loss": 5.481283948394303,
+      "tokens_seen": 40501248
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007981544633901706,
+      "loss": 3.0212,
+      "theoretical_loss": 5.479906251527901,
+      "tokens_seen": 40566784
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007981384152457373,
+      "loss": 2.8446,
+      "theoretical_loss": 5.478531400592393,
+      "tokens_seen": 40632320
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.000798122367101304,
+      "loss": 3.111,
+      "theoretical_loss": 5.4771593851328815,
+      "tokens_seen": 40697856
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007981063189568707,
+      "loss": 3.0204,
+      "theoretical_loss": 5.475790194749637,
+      "tokens_seen": 40763392
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007980902708124374,
+      "loss": 2.9798,
+      "theoretical_loss": 5.474423819097713,
+      "tokens_seen": 40828928
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007980742226680041,
+      "loss": 3.0839,
+      "theoretical_loss": 5.473060247886572,
+      "tokens_seen": 40894464
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007980581745235708,
+      "loss": 3.006,
+      "theoretical_loss": 5.471699470879715,
+      "tokens_seen": 40960000
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007980421263791375,
+      "loss": 3.0602,
+      "theoretical_loss": 5.470341477894303,
+      "tokens_seen": 41025536
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007980260782347042,
+      "loss": 3.097,
+      "theoretical_loss": 5.468986258800805,
+      "tokens_seen": 41091072
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007980100300902709,
+      "loss": 3.1927,
+      "theoretical_loss": 5.4676338035226175,
+      "tokens_seen": 41156608
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007979939819458376,
+      "loss": 3.0428,
+      "theoretical_loss": 5.466284102035719,
+      "tokens_seen": 41222144
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007979779338014043,
+      "loss": 3.0008,
+      "theoretical_loss": 5.464937144368307,
+      "tokens_seen": 41287680
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.000797961885656971,
+      "loss": 3.1651,
+      "theoretical_loss": 5.463592920600435,
+      "tokens_seen": 41353216
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007979458375125376,
+      "loss": 3.1196,
+      "theoretical_loss": 5.46225142086368,
+      "tokens_seen": 41418752
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007979297893681044,
+      "loss": 3.1898,
+      "theoretical_loss": 5.460912635340775,
+      "tokens_seen": 41484288
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.000797913741223671,
+      "loss": 2.9941,
+      "theoretical_loss": 5.459576554265276,
+      "tokens_seen": 41549824
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007978976930792378,
+      "loss": 3.0188,
+      "theoretical_loss": 5.458243167921212,
+      "tokens_seen": 41615360
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007978816449348044,
+      "loss": 3.0855,
+      "theoretical_loss": 5.456912466642748,
+      "tokens_seen": 41680896
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007978655967903712,
+      "loss": 3.0476,
+      "theoretical_loss": 5.455584440813853,
+      "tokens_seen": 41746432
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007978495486459378,
+      "loss": 3.0335,
+      "theoretical_loss": 5.454259080867951,
+      "tokens_seen": 41811968
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007978335005015046,
+      "loss": 3.0883,
+      "theoretical_loss": 5.4529363772876085,
+      "tokens_seen": 41877504
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007978174523570712,
+      "loss": 3.0041,
+      "theoretical_loss": 5.451616320604193,
+      "tokens_seen": 41943040
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007978014042126379,
+      "loss": 3.1093,
+      "theoretical_loss": 5.450298901397547,
+      "tokens_seen": 42008576
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007977853560682046,
+      "loss": 3.1478,
+      "theoretical_loss": 5.448984110295672,
+      "tokens_seen": 42074112
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007977693079237713,
+      "loss": 3.1955,
+      "theoretical_loss": 5.447671937974404,
+      "tokens_seen": 42139648
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.000797753259779338,
+      "loss": 3.042,
+      "theoretical_loss": 5.446362375157092,
+      "tokens_seen": 42205184
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007977372116349047,
+      "loss": 2.9404,
+      "theoretical_loss": 5.445055412614287,
+      "tokens_seen": 42270720
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007977211634904714,
+      "loss": 3.1725,
+      "theoretical_loss": 5.443751041163428,
+      "tokens_seen": 42336256
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007977051153460381,
+      "loss": 2.9953,
+      "theoretical_loss": 5.442449251668533,
+      "tokens_seen": 42401792
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007976890672016048,
+      "loss": 2.9237,
+      "theoretical_loss": 5.441150035039886,
+      "tokens_seen": 42467328
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007976730190571715,
+      "loss": 3.0381,
+      "theoretical_loss": 5.439853382233738,
+      "tokens_seen": 42532864
+    },
+    {
+      "epoch": 0.01,
+      "objective/train/docs_used": 20818,
+      "objective/train/instantaneous_batch_size": 32,
+      "objective/train/instantaneous_microbatch_size": 32768,
+      "objective/train/original_loss": 2.9582602977752686,
+      "objective/train/theoretical_loss": 5.438559284252003,
+      "objective/train/tokens_used": 63058400,
+      "theoretical_loss": 5.438559284252003,
+      "tokens_seen": 42598400
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007976569709127382,
+      "loss": 2.898,
+      "theoretical_loss": 5.438559284252003,
+      "tokens_seen": 42598400
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007976409227683049,
+      "loss": 2.9189,
+      "theoretical_loss": 5.437267732141949,
+      "tokens_seen": 42663936
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007976248746238716,
+      "loss": 3.0087,
+      "theoretical_loss": 5.435978716995917,
+      "tokens_seen": 42729472
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007976088264794383,
+      "loss": 2.9993,
+      "theoretical_loss": 5.434692229951009,
+      "tokens_seen": 42795008
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.000797592778335005,
+      "loss": 2.992,
+      "theoretical_loss": 5.433408262188802,
+      "tokens_seen": 42860544
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007975767301905718,
+      "loss": 3.0014,
+      "theoretical_loss": 5.432126804935059,
+      "tokens_seen": 42926080
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007975606820461384,
+      "loss": 2.8421,
+      "theoretical_loss": 5.430847849459438,
+      "tokens_seen": 42991616
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007975446339017052,
+      "loss": 3.1039,
+      "theoretical_loss": 5.429571387075203,
+      "tokens_seen": 43057152
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007975285857572718,
+      "loss": 2.868,
+      "theoretical_loss": 5.428297409138947,
+      "tokens_seen": 43122688
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007975125376128386,
+      "loss": 3.0765,
+      "theoretical_loss": 5.427025907050307,
+      "tokens_seen": 43188224
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007974964894684052,
+      "loss": 3.0613,
+      "theoretical_loss": 5.425756872251682,
+      "tokens_seen": 43253760
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.000797480441323972,
+      "loss": 3.1449,
+      "theoretical_loss": 5.42449029622796,
+      "tokens_seen": 43319296
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007974643931795386,
+      "loss": 2.8422,
+      "theoretical_loss": 5.423226170506243,
+      "tokens_seen": 43384832
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007974483450351054,
+      "loss": 3.0436,
+      "theoretical_loss": 5.421964486655567,
+      "tokens_seen": 43450368
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.000797432296890672,
+      "loss": 2.98,
+      "theoretical_loss": 5.420705236286646,
+      "tokens_seen": 43515904
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007974162487462388,
+      "loss": 3.0859,
+      "theoretical_loss": 5.41944841105159,
+      "tokens_seen": 43581440
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007974002006018054,
+      "loss": 2.9513,
+      "theoretical_loss": 5.418194002643643,
+      "tokens_seen": 43646976
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007973841524573722,
+      "loss": 2.8869,
+      "theoretical_loss": 5.416942002796926,
+      "tokens_seen": 43712512
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007973681043129388,
+      "loss": 3.0067,
+      "theoretical_loss": 5.415692403286162,
+      "tokens_seen": 43778048
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007973520561685056,
+      "loss": 2.8628,
+      "theoretical_loss": 5.414445195926426,
+      "tokens_seen": 43843584
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007973360080240722,
+      "loss": 3.0188,
+      "theoretical_loss": 5.413200372572884,
+      "tokens_seen": 43909120
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.000797319959879639,
+      "loss": 3.0607,
+      "theoretical_loss": 5.411957925120538,
+      "tokens_seen": 43974656
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007973039117352056,
+      "loss": 3.07,
+      "theoretical_loss": 5.410717845503969,
+      "tokens_seen": 44040192
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007972878635907724,
+      "loss": 2.8619,
+      "theoretical_loss": 5.40948012569709,
+      "tokens_seen": 44105728
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.000797271815446339,
+      "loss": 2.9487,
+      "theoretical_loss": 5.408244757712893,
+      "tokens_seen": 44171264
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007972557673019058,
+      "loss": 2.9891,
+      "theoretical_loss": 5.407011733603199,
+      "tokens_seen": 44236800
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007972397191574724,
+      "loss": 2.9547,
+      "theoretical_loss": 5.4057810454584185,
+      "tokens_seen": 44302336
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007972236710130392,
+      "loss": 2.808,
+      "theoretical_loss": 5.404552685407301,
+      "tokens_seen": 44367872
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007972076228686058,
+      "loss": 2.945,
+      "theoretical_loss": 5.403326645616698,
+      "tokens_seen": 44433408
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007971915747241726,
+      "loss": 3.0102,
+      "theoretical_loss": 5.402102918291316,
+      "tokens_seen": 44498944
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007971755265797392,
+      "loss": 2.9457,
+      "theoretical_loss": 5.400881495673488,
+      "tokens_seen": 44564480
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.000797159478435306,
+      "loss": 2.9013,
+      "theoretical_loss": 5.3996623700429245,
+      "tokens_seen": 44630016
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007971434302908726,
+      "loss": 2.9583,
+      "theoretical_loss": 5.398445533716492,
+      "tokens_seen": 44695552
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007971273821464394,
+      "loss": 2.9018,
+      "theoretical_loss": 5.3972309790479684,
+      "tokens_seen": 44761088
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.000797111334002006,
+      "loss": 2.9923,
+      "theoretical_loss": 5.39601869842782,
+      "tokens_seen": 44826624
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007970952858575728,
+      "loss": 2.8066,
+      "theoretical_loss": 5.394808684282968,
+      "tokens_seen": 44892160
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007970792377131394,
+      "loss": 2.8783,
+      "theoretical_loss": 5.393600929076564,
+      "tokens_seen": 44957696
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007970631895687062,
+      "loss": 3.0059,
+      "theoretical_loss": 5.392395425307758,
+      "tokens_seen": 45023232
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007970471414242728,
+      "loss": 3.0282,
+      "theoretical_loss": 5.391192165511484,
+      "tokens_seen": 45088768
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007970310932798396,
+      "loss": 2.9522,
+      "theoretical_loss": 5.389991142258227,
+      "tokens_seen": 45154304
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007970150451354062,
+      "loss": 2.9479,
+      "theoretical_loss": 5.38879234815381,
+      "tokens_seen": 45219840
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.000796998996990973,
+      "loss": 2.8979,
+      "theoretical_loss": 5.387595775839175,
+      "tokens_seen": 45285376
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007969829488465396,
+      "loss": 2.9946,
+      "theoretical_loss": 5.386401417990159,
+      "tokens_seen": 45350912
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007969669007021064,
+      "loss": 2.9795,
+      "theoretical_loss": 5.385209267317286,
+      "tokens_seen": 45416448
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.000796950852557673,
+      "loss": 3.0915,
+      "theoretical_loss": 5.3840193165655466,
+      "tokens_seen": 45481984
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007969348044132398,
+      "loss": 2.9921,
+      "theoretical_loss": 5.382831558514191,
+      "tokens_seen": 45547520
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007969187562688065,
+      "loss": 2.9256,
+      "theoretical_loss": 5.3816459859765144,
+      "tokens_seen": 45613056
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007969027081243732,
+      "loss": 2.9078,
+      "theoretical_loss": 5.380462591799651,
+      "tokens_seen": 45678592
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007968866599799399,
+      "loss": 2.8356,
+      "theoretical_loss": 5.379281368864362,
+      "tokens_seen": 45744128
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007968706118355066,
+      "loss": 2.9709,
+      "theoretical_loss": 5.378102310084834,
+      "tokens_seen": 45809664
+    },
+    {
+      "epoch": 0.01,
+      "objective/train/docs_used": 21811,
+      "objective/train/instantaneous_batch_size": 32,
+      "objective/train/instantaneous_microbatch_size": 32768,
+      "objective/train/original_loss": 2.8180923461914062,
+      "objective/train/theoretical_loss": 5.376925408408475,
+      "objective/train/tokens_used": 66335200,
+      "theoretical_loss": 5.376925408408475,
+      "tokens_seen": 45875200
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007968545636910733,
+      "loss": 2.8521,
+      "theoretical_loss": 5.376925408408475,
+      "tokens_seen": 45875200
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00079683851554664,
+      "loss": 2.865,
+      "theoretical_loss": 5.375750656815705,
+      "tokens_seen": 45940736
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007968224674022067,
+      "loss": 2.844,
+      "theoretical_loss": 5.37457804831976,
+      "tokens_seen": 46006272
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007968064192577734,
+      "loss": 2.8022,
+      "theoretical_loss": 5.373407575966493,
+      "tokens_seen": 46071808
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007967903711133401,
+      "loss": 2.8001,
+      "theoretical_loss": 5.372239232834173,
+      "tokens_seen": 46137344
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007967743229689068,
+      "loss": 2.8544,
+      "theoretical_loss": 5.3710730120332855,
+      "tokens_seen": 46202880
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007967582748244735,
+      "loss": 2.7719,
+      "theoretical_loss": 5.369908906706346,
+      "tokens_seen": 46268416
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007967422266800402,
+      "loss": 2.7981,
+      "theoretical_loss": 5.368746910027696,
+      "tokens_seen": 46333952
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007967261785356069,
+      "loss": 2.825,
+      "theoretical_loss": 5.3675870152033145,
+      "tokens_seen": 46399488
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007967101303911736,
+      "loss": 2.8073,
+      "theoretical_loss": 5.36642921547063,
+      "tokens_seen": 46465024
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007966940822467403,
+      "loss": 2.8989,
+      "theoretical_loss": 5.365273504098326,
+      "tokens_seen": 46530560
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.000796678034102307,
+      "loss": 2.7738,
+      "theoretical_loss": 5.364119874386157,
+      "tokens_seen": 46596096
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007966619859578737,
+      "loss": 2.7075,
+      "theoretical_loss": 5.362968319664756,
+      "tokens_seen": 46661632
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007966459378134404,
+      "loss": 2.8861,
+      "theoretical_loss": 5.361818833295455,
+      "tokens_seen": 46727168
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007966298896690071,
+      "loss": 2.8059,
+      "theoretical_loss": 5.3606714086701,
+      "tokens_seen": 46792704
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007966138415245738,
+      "loss": 2.9195,
+      "theoretical_loss": 5.359526039210861,
+      "tokens_seen": 46858240
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007965977933801405,
+      "loss": 2.9497,
+      "theoretical_loss": 5.358382718370063,
+      "tokens_seen": 46923776
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007965817452357072,
+      "loss": 2.7121,
+      "theoretical_loss": 5.357241439629992,
+      "tokens_seen": 46989312
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007965656970912739,
+      "loss": 2.7512,
+      "theoretical_loss": 5.3561021965027305,
+      "tokens_seen": 47054848
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007965496489468405,
+      "loss": 2.9349,
+      "theoretical_loss": 5.354964982529967,
+      "tokens_seen": 47120384
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007965336008024073,
+      "loss": 2.7165,
+      "theoretical_loss": 5.353829791282831,
+      "tokens_seen": 47185920
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007965175526579739,
+      "loss": 2.7052,
+      "theoretical_loss": 5.35269661636171,
+      "tokens_seen": 47251456
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007965015045135407,
+      "loss": 2.8222,
+      "theoretical_loss": 5.35156545139608,
+      "tokens_seen": 47316992
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007964854563691073,
+      "loss": 2.7788,
+      "theoretical_loss": 5.350436290044332,
+      "tokens_seen": 47382528
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007964694082246741,
+      "loss": 2.7784,
+      "theoretical_loss": 5.3493091259936,
+      "tokens_seen": 47448064
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007964533600802407,
+      "loss": 2.8736,
+      "theoretical_loss": 5.348183952959596,
+      "tokens_seen": 47513600
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007964373119358075,
+      "loss": 2.7399,
+      "theoretical_loss": 5.347060764686436,
+      "tokens_seen": 47579136
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007964212637913741,
+      "loss": 2.8886,
+      "theoretical_loss": 5.345939554946472,
+      "tokens_seen": 47644672
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007964052156469408,
+      "loss": 2.7161,
+      "theoretical_loss": 5.344820317540132,
+      "tokens_seen": 47710208
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007963891675025075,
+      "loss": 2.6917,
+      "theoretical_loss": 5.343703046295751,
+      "tokens_seen": 47775744
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007963731193580743,
+      "loss": 2.8907,
+      "theoretical_loss": 5.342587735069408,
+      "tokens_seen": 47841280
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007963570712136409,
+      "loss": 2.8617,
+      "theoretical_loss": 5.3414743777447615,
+      "tokens_seen": 47906816
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007963410230692077,
+      "loss": 2.7052,
+      "theoretical_loss": 5.340362968232892,
+      "tokens_seen": 47972352
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007963249749247743,
+      "loss": 2.88,
+      "theoretical_loss": 5.339253500472138,
+      "tokens_seen": 48037888
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007963089267803411,
+      "loss": 2.8261,
+      "theoretical_loss": 5.338145968427941,
+      "tokens_seen": 48103424
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007962928786359077,
+      "loss": 2.7997,
+      "theoretical_loss": 5.337040366092687,
+      "tokens_seen": 48168960
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007962768304914745,
+      "loss": 2.8677,
+      "theoretical_loss": 5.335936687485542,
+      "tokens_seen": 48234496
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007962607823470411,
+      "loss": 2.8327,
+      "theoretical_loss": 5.334834926652308,
+      "tokens_seen": 48300032
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007962447342026079,
+      "loss": 2.82,
+      "theoretical_loss": 5.33373507766526,
+      "tokens_seen": 48365568
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007962286860581745,
+      "loss": 2.7853,
+      "theoretical_loss": 5.332637134622999,
+      "tokens_seen": 48431104
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007962126379137413,
+      "loss": 2.8346,
+      "theoretical_loss": 5.331541091650294,
+      "tokens_seen": 48496640
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007961965897693079,
+      "loss": 2.7562,
+      "theoretical_loss": 5.330446942897931,
+      "tokens_seen": 48562176
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007961805416248747,
+      "loss": 2.6726,
+      "theoretical_loss": 5.32935468254257,
+      "tokens_seen": 48627712
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007961644934804413,
+      "loss": 2.7526,
+      "theoretical_loss": 5.328264304786586,
+      "tokens_seen": 48693248
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007961484453360081,
+      "loss": 2.7667,
+      "theoretical_loss": 5.327175803857929,
+      "tokens_seen": 48758784
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007961323971915747,
+      "loss": 2.7549,
+      "theoretical_loss": 5.326089174009972,
+      "tokens_seen": 48824320
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007961163490471415,
+      "loss": 2.6909,
+      "theoretical_loss": 5.325004409521368,
+      "tokens_seen": 48889856
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007961003009027081,
+      "loss": 2.6456,
+      "theoretical_loss": 5.323921504695903,
+      "tokens_seen": 48955392
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007960842527582749,
+      "loss": 2.7021,
+      "theoretical_loss": 5.322840453862356,
+      "tokens_seen": 49020928
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007960682046138415,
+      "loss": 2.6976,
+      "theoretical_loss": 5.3217612513743475,
+      "tokens_seen": 49086464
+    },
+    {
+      "epoch": 0.01,
+      "objective/train/docs_used": 22996,
+      "objective/train/instantaneous_batch_size": 32,
+      "objective/train/instantaneous_microbatch_size": 32768,
+      "objective/train/original_loss": 2.697645902633667,
+      "objective/train/theoretical_loss": 5.320683891610208,
+      "objective/train/tokens_used": 69612000,
+      "theoretical_loss": 5.320683891610208,
+      "tokens_seen": 49152000
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007960521564694083,
+      "loss": 2.7652,
+      "theoretical_loss": 5.320683891610208,
+      "tokens_seen": 49152000
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007960361083249749,
+      "loss": 2.7328,
+      "theoretical_loss": 5.3196083689728315,
+      "tokens_seen": 49217536
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007960200601805417,
+      "loss": 2.7463,
+      "theoretical_loss": 5.318534677889536,
+      "tokens_seen": 49283072
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007960040120361083,
+      "loss": 2.7242,
+      "theoretical_loss": 5.317462812811925,
+      "tokens_seen": 49348608
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007959879638916751,
+      "loss": 2.7408,
+      "theoretical_loss": 5.316392768215751,
+      "tokens_seen": 49414144
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0007959719157472417,
+      "loss": 2.708,
+      "theoretical_loss": 5.315324538600778,
+      "tokens_seen": 49479680
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007959558676028085,
+      "loss": 2.7238,
+      "theoretical_loss": 5.31425811849064,
+      "tokens_seen": 49545216
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007959398194583751,
+      "loss": 2.7962,
+      "theoretical_loss": 5.313193502432718,
+      "tokens_seen": 49610752
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007959237713139419,
+      "loss": 2.728,
+      "theoretical_loss": 5.312130684997992,
+      "tokens_seen": 49676288
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007959077231695085,
+      "loss": 2.6604,
+      "theoretical_loss": 5.31106966078092,
+      "tokens_seen": 49741824
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007958916750250753,
+      "loss": 2.6847,
+      "theoretical_loss": 5.310010424399295,
+      "tokens_seen": 49807360
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007958756268806419,
+      "loss": 2.6893,
+      "theoretical_loss": 5.308952970494124,
+      "tokens_seen": 49872896
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007958595787362087,
+      "loss": 2.8063,
+      "theoretical_loss": 5.307897293729489,
+      "tokens_seen": 49938432
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007958435305917753,
+      "loss": 2.7133,
+      "theoretical_loss": 5.306843388792423,
+      "tokens_seen": 50003968
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007958274824473421,
+      "loss": 2.7601,
+      "theoretical_loss": 5.305791250392778,
+      "tokens_seen": 50069504
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007958114343029087,
+      "loss": 2.5768,
+      "theoretical_loss": 5.304740873263098,
+      "tokens_seen": 50135040
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007957953861584755,
+      "loss": 2.5786,
+      "theoretical_loss": 5.303692252158496,
+      "tokens_seen": 50200576
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007957793380140421,
+      "loss": 2.5909,
+      "theoretical_loss": 5.302645381856522,
+      "tokens_seen": 50266112
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007957632898696089,
+      "loss": 2.7036,
+      "theoretical_loss": 5.30160025715704,
+      "tokens_seen": 50331648
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007957472417251755,
+      "loss": 2.6504,
+      "theoretical_loss": 5.300556872882108,
+      "tokens_seen": 50397184
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007957311935807423,
+      "loss": 2.7496,
+      "theoretical_loss": 5.299515223875848,
+      "tokens_seen": 50462720
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007957151454363089,
+      "loss": 2.5482,
+      "theoretical_loss": 5.2984753050043265,
+      "tokens_seen": 50528256
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007956990972918757,
+      "loss": 2.7896,
+      "theoretical_loss": 5.2974371111554355,
+      "tokens_seen": 50593792
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007956830491474423,
+      "loss": 2.6064,
+      "theoretical_loss": 5.296400637238764,
+      "tokens_seen": 50659328
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007956670010030091,
+      "loss": 2.5401,
+      "theoretical_loss": 5.295365878185488,
+      "tokens_seen": 50724864
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007956509528585757,
+      "loss": 2.6705,
+      "theoretical_loss": 5.294332828948243,
+      "tokens_seen": 50790400
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007956349047141425,
+      "loss": 2.7539,
+      "theoretical_loss": 5.2933014845010105,
+      "tokens_seen": 50855936
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007956188565697091,
+      "loss": 2.7565,
+      "theoretical_loss": 5.292271839838996,
+      "tokens_seen": 50921472
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007956028084252759,
+      "loss": 2.5354,
+      "theoretical_loss": 5.291243889978516,
+      "tokens_seen": 50987008
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007955867602808426,
+      "loss": 2.6721,
+      "theoretical_loss": 5.290217629956882,
+      "tokens_seen": 51052544
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007955707121364093,
+      "loss": 2.7336,
+      "theoretical_loss": 5.289193054832282,
+      "tokens_seen": 51118080
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.000795554663991976,
+      "loss": 2.6627,
+      "theoretical_loss": 5.288170159683668,
+      "tokens_seen": 51183616
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007955386158475427,
+      "loss": 2.6321,
+      "theoretical_loss": 5.287148939610645,
+      "tokens_seen": 51249152
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007955225677031094,
+      "loss": 2.6728,
+      "theoretical_loss": 5.28612938973335,
+      "tokens_seen": 51314688
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007955065195586761,
+      "loss": 2.5826,
+      "theoretical_loss": 5.285111505192349,
+      "tokens_seen": 51380224
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007954904714142428,
+      "loss": 2.5891,
+      "theoretical_loss": 5.284095281148522,
+      "tokens_seen": 51445760
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007954744232698095,
+      "loss": 2.6312,
+      "theoretical_loss": 5.283080712782951,
+      "tokens_seen": 51511296
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007954583751253762,
+      "loss": 2.576,
+      "theoretical_loss": 5.282067795296811,
+      "tokens_seen": 51576832
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007954423269809429,
+      "loss": 2.6877,
+      "theoretical_loss": 5.28105652391126,
+      "tokens_seen": 51642368
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007954262788365096,
+      "loss": 2.6855,
+      "theoretical_loss": 5.280046893867333,
+      "tokens_seen": 51707904
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007954102306920763,
+      "loss": 2.7701,
+      "theoretical_loss": 5.279038900425832,
+      "tokens_seen": 51773440
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.000795394182547643,
+      "loss": 2.7453,
+      "theoretical_loss": 5.278032538867221,
+      "tokens_seen": 51838976
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007953781344032097,
+      "loss": 2.581,
+      "theoretical_loss": 5.277027804491517,
+      "tokens_seen": 51904512
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007953620862587764,
+      "loss": 2.6354,
+      "theoretical_loss": 5.2760246926181855,
+      "tokens_seen": 51970048
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007953460381143431,
+      "loss": 2.6474,
+      "theoretical_loss": 5.275023198586037,
+      "tokens_seen": 52035584
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007953299899699098,
+      "loss": 2.5734,
+      "theoretical_loss": 5.274023317753125,
+      "tokens_seen": 52101120
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007953139418254765,
+      "loss": 2.4815,
+      "theoretical_loss": 5.273025045496631,
+      "tokens_seen": 52166656
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007952978936810432,
+      "loss": 2.5058,
+      "theoretical_loss": 5.272028377212777,
+      "tokens_seen": 52232192
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007952818455366099,
+      "loss": 2.5368,
+      "theoretical_loss": 5.271033308316715,
+      "tokens_seen": 52297728
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007952657973921766,
+      "loss": 2.8568,
+      "theoretical_loss": 5.270039834242425,
+      "tokens_seen": 52363264
+    },
+    {
+      "epoch": 0.02,
+      "objective/train/docs_used": 24258,
+      "objective/train/instantaneous_batch_size": 32,
+      "objective/train/instantaneous_microbatch_size": 32768,
+      "objective/train/original_loss": 2.8478150367736816,
+      "objective/train/theoretical_loss": 5.269047950442619,
+      "objective/train/tokens_used": 72888800,
+      "theoretical_loss": 5.269047950442619,
+      "tokens_seen": 52428800
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007952497492477433,
+      "loss": 2.7702,
+      "theoretical_loss": 5.269047950442619,
+      "tokens_seen": 52428800
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.00079523370110331,
+      "loss": 2.6321,
+      "theoretical_loss": 5.2680576523886335,
+      "tokens_seen": 52494336
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007952176529588767,
+      "loss": 2.6129,
+      "theoretical_loss": 5.2670689355703395,
+      "tokens_seen": 52559872
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007952016048144434,
+      "loss": 2.5308,
+      "theoretical_loss": 5.266081795496035,
+      "tokens_seen": 52625408
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007951855566700101,
+      "loss": 2.7037,
+      "theoretical_loss": 5.265096227692354,
+      "tokens_seen": 52690944
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007951695085255768,
+      "loss": 2.5846,
+      "theoretical_loss": 5.264112227704162,
+      "tokens_seen": 52756480
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007951534603811434,
+      "loss": 2.5396,
+      "theoretical_loss": 5.263129791094466,
+      "tokens_seen": 52822016
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007951374122367102,
+      "loss": 2.4622,
+      "theoretical_loss": 5.26214891344431,
+      "tokens_seen": 52887552
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007951213640922768,
+      "loss": 2.5156,
+      "theoretical_loss": 5.26116959035269,
+      "tokens_seen": 52953088
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007951053159478436,
+      "loss": 2.6235,
+      "theoretical_loss": 5.260191817436452,
+      "tokens_seen": 53018624
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007950892678034102,
+      "loss": 2.5897,
+      "theoretical_loss": 5.259215590330195,
+      "tokens_seen": 53084160
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.000795073219658977,
+      "loss": 2.6087,
+      "theoretical_loss": 5.258240904686186,
+      "tokens_seen": 53149696
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007950571715145436,
+      "loss": 2.4095,
+      "theoretical_loss": 5.25726775617426,
+      "tokens_seen": 53215232
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007950411233701104,
+      "loss": 2.6925,
+      "theoretical_loss": 5.256296140481728,
+      "tokens_seen": 53280768
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.000795025075225677,
+      "loss": 2.5187,
+      "theoretical_loss": 5.255326053313292,
+      "tokens_seen": 53346304
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007950090270812438,
+      "loss": 2.5804,
+      "theoretical_loss": 5.254357490390941,
+      "tokens_seen": 53411840
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007949929789368104,
+      "loss": 2.5054,
+      "theoretical_loss": 5.253390447453873,
+      "tokens_seen": 53477376
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007949769307923772,
+      "loss": 2.4199,
+      "theoretical_loss": 5.252424920258396,
+      "tokens_seen": 53542912
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007949608826479438,
+      "loss": 2.6297,
+      "theoretical_loss": 5.251460904577845,
+      "tokens_seen": 53608448
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007949448345035106,
+      "loss": 2.4558,
+      "theoretical_loss": 5.250498396202486,
+      "tokens_seen": 53673984
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007949287863590772,
+      "loss": 2.5582,
+      "theoretical_loss": 5.249537390939436,
+      "tokens_seen": 53739520
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.000794912738214644,
+      "loss": 2.5099,
+      "theoretical_loss": 5.248577884612564,
+      "tokens_seen": 53805056
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007948966900702106,
+      "loss": 2.468,
+      "theoretical_loss": 5.247619873062416,
+      "tokens_seen": 53870592
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007948806419257774,
+      "loss": 2.3527,
+      "theoretical_loss": 5.246663352146122,
+      "tokens_seen": 53936128
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.000794864593781344,
+      "loss": 2.4527,
+      "theoretical_loss": 5.245708317737305,
+      "tokens_seen": 54001664
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007948485456369108,
+      "loss": 2.3665,
+      "theoretical_loss": 5.244754765726007,
+      "tokens_seen": 54067200
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007948324974924774,
+      "loss": 2.4971,
+      "theoretical_loss": 5.243802692018596,
+      "tokens_seen": 54132736
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007948164493480442,
+      "loss": 2.6112,
+      "theoretical_loss": 5.242852092537677,
+      "tokens_seen": 54198272
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007948004012036108,
+      "loss": 2.4544,
+      "theoretical_loss": 5.241902963222023,
+      "tokens_seen": 54263808
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007947843530591776,
+      "loss": 2.61,
+      "theoretical_loss": 5.240955300026476,
+      "tokens_seen": 54329344
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007947683049147442,
+      "loss": 2.5205,
+      "theoretical_loss": 5.240009098921874,
+      "tokens_seen": 54394880
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.000794752256770311,
+      "loss": 2.518,
+      "theoretical_loss": 5.239064355894965,
+      "tokens_seen": 54460416
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007947362086258776,
+      "loss": 2.5668,
+      "theoretical_loss": 5.238121066948324,
+      "tokens_seen": 54525952
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007947201604814444,
+      "loss": 2.3089,
+      "theoretical_loss": 5.237179228100272,
+      "tokens_seen": 54591488
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.000794704112337011,
+      "loss": 2.4987,
+      "theoretical_loss": 5.2362388353848,
+      "tokens_seen": 54657024
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007946880641925778,
+      "loss": 2.6012,
+      "theoretical_loss": 5.23529988485148,
+      "tokens_seen": 54722560
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007946720160481444,
+      "loss": 2.3194,
+      "theoretical_loss": 5.2343623725653945,
+      "tokens_seen": 54788096
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007946559679037112,
+      "loss": 2.5002,
+      "theoretical_loss": 5.233426294607049,
+      "tokens_seen": 54853632
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007946399197592778,
+      "loss": 2.3917,
+      "theoretical_loss": 5.232491647072299,
+      "tokens_seen": 54919168
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007946238716148446,
+      "loss": 2.4287,
+      "theoretical_loss": 5.231558426072269,
+      "tokens_seen": 54984704
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007946078234704112,
+      "loss": 2.375,
+      "theoretical_loss": 5.230626627733274,
+      "tokens_seen": 55050240
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.000794591775325978,
+      "loss": 2.4455,
+      "theoretical_loss": 5.2296962481967455,
+      "tokens_seen": 55115776
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007945757271815446,
+      "loss": 2.5253,
+      "theoretical_loss": 5.228767283619151,
+      "tokens_seen": 55181312
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007945596790371114,
+      "loss": 2.5339,
+      "theoretical_loss": 5.227839730171918,
+      "tokens_seen": 55246848
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.000794543630892678,
+      "loss": 2.4102,
+      "theoretical_loss": 5.2269135840413625,
+      "tokens_seen": 55312384
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007945275827482448,
+      "loss": 2.5213,
+      "theoretical_loss": 5.225988841428606,
+      "tokens_seen": 55377920
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007945115346038114,
+      "loss": 2.4657,
+      "theoretical_loss": 5.225065498549508,
+      "tokens_seen": 55443456
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007944954864593782,
+      "loss": 2.413,
+      "theoretical_loss": 5.224143551634588,
+      "tokens_seen": 55508992
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007944794383149448,
+      "loss": 2.4655,
+      "theoretical_loss": 5.22322299692895,
+      "tokens_seen": 55574528
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007944633901705116,
+      "loss": 2.4596,
+      "theoretical_loss": 5.222303830692211,
+      "tokens_seen": 55640064
+    },
+    {
+      "epoch": 0.02,
+      "objective/train/docs_used": 25190,
+      "objective/train/instantaneous_batch_size": 32,
+      "objective/train/instantaneous_microbatch_size": 32768,
+      "objective/train/original_loss": 2.582289218902588,
+      "objective/train/theoretical_loss": 5.22138604919843,
+      "objective/train/tokens_used": 76165600,
+      "theoretical_loss": 5.22138604919843,
+      "tokens_seen": 55705600
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007944473420260782,
+      "loss": 2.4947,
+      "theoretical_loss": 5.22138604919843,
+      "tokens_seen": 55705600
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.000794431293881645,
+      "loss": 2.4601,
+      "theoretical_loss": 5.220469648736031,
+      "tokens_seen": 55771136
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007944152457372116,
+      "loss": 2.3962,
+      "theoretical_loss": 5.219554625607731,
+      "tokens_seen": 55836672
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007943991975927784,
+      "loss": 2.4124,
+      "theoretical_loss": 5.218640976130474,
+      "tokens_seen": 55902208
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.000794383149448345,
+      "loss": 2.462,
+      "theoretical_loss": 5.217728696635353,
+      "tokens_seen": 55967744
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007943671013039118,
+      "loss": 2.3987,
+      "theoretical_loss": 5.216817783467543,
+      "tokens_seen": 56033280
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007943510531594784,
+      "loss": 2.3923,
+      "theoretical_loss": 5.215908232986227,
+      "tokens_seen": 56098816
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007943350050150452,
+      "loss": 2.4097,
+      "theoretical_loss": 5.215000041564533,
+      "tokens_seen": 56164352
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007943189568706118,
+      "loss": 2.2344,
+      "theoretical_loss": 5.214093205589455,
+      "tokens_seen": 56229888
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007943029087261786,
+      "loss": 2.4473,
+      "theoretical_loss": 5.213187721461791,
+      "tokens_seen": 56295424
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007942868605817452,
+      "loss": 2.5236,
+      "theoretical_loss": 5.212283585596071,
+      "tokens_seen": 56360960
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.000794270812437312,
+      "loss": 2.4306,
+      "theoretical_loss": 5.211380794420492,
+      "tokens_seen": 56426496
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007942547642928786,
+      "loss": 2.2982,
+      "theoretical_loss": 5.210479344376845,
+      "tokens_seen": 56492032
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007942387161484454,
+      "loss": 2.3473,
+      "theoretical_loss": 5.209579231920451,
+      "tokens_seen": 56557568
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.000794222668004012,
+      "loss": 2.4044,
+      "theoretical_loss": 5.208680453520092,
+      "tokens_seen": 56623104
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007942066198595788,
+      "loss": 2.5501,
+      "theoretical_loss": 5.2077830056579515,
+      "tokens_seen": 56688640
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007941905717151455,
+      "loss": 2.3856,
+      "theoretical_loss": 5.206886884829535,
+      "tokens_seen": 56754176
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007941745235707122,
+      "loss": 2.3537,
+      "theoretical_loss": 5.205992087543614,
+      "tokens_seen": 56819712
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007941584754262789,
+      "loss": 2.3636,
+      "theoretical_loss": 5.205098610322162,
+      "tokens_seen": 56885248
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007941424272818456,
+      "loss": 2.4013,
+      "theoretical_loss": 5.204206449700279,
+      "tokens_seen": 56950784
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007941263791374123,
+      "loss": 2.4155,
+      "theoretical_loss": 5.203315602226139,
+      "tokens_seen": 57016320
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.000794110330992979,
+      "loss": 2.3567,
+      "theoretical_loss": 5.2024260644609175,
+      "tokens_seen": 57081856
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007940942828485457,
+      "loss": 2.3825,
+      "theoretical_loss": 5.201537832978731,
+      "tokens_seen": 57147392
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007940782347041124,
+      "loss": 2.3748,
+      "theoretical_loss": 5.200650904366574,
+      "tokens_seen": 57212928
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007940621865596791,
+      "loss": 2.3989,
+      "theoretical_loss": 5.199765275224252,
+      "tokens_seen": 57278464
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007940461384152458,
+      "loss": 2.3222,
+      "theoretical_loss": 5.198880942164323,
+      "tokens_seen": 57344000
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007940300902708125,
+      "loss": 2.472,
+      "theoretical_loss": 5.197997901812036,
+      "tokens_seen": 57409536
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007940140421263792,
+      "loss": 2.5758,
+      "theoretical_loss": 5.197116150805264,
+      "tokens_seen": 57475072
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007939979939819459,
+      "loss": 2.4398,
+      "theoretical_loss": 5.196235685794446,
+      "tokens_seen": 57540608
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007939819458375126,
+      "loss": 2.37,
+      "theoretical_loss": 5.195356503442525,
+      "tokens_seen": 57606144
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007939658976930794,
+      "loss": 2.3932,
+      "theoretical_loss": 5.194478600424889,
+      "tokens_seen": 57671680
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.000793949849548646,
+      "loss": 2.432,
+      "theoretical_loss": 5.193601973429306,
+      "tokens_seen": 57737216
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007939338014042128,
+      "loss": 2.3384,
+      "theoretical_loss": 5.192726619155867,
+      "tokens_seen": 57802752
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007939177532597794,
+      "loss": 2.466,
+      "theoretical_loss": 5.191852534316928,
+      "tokens_seen": 57868288
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007939017051153462,
+      "loss": 2.3083,
+      "theoretical_loss": 5.190979715637049,
+      "tokens_seen": 57933824
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007938856569709128,
+      "loss": 2.475,
+      "theoretical_loss": 5.190108159852929,
+      "tokens_seen": 57999360
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007938696088264795,
+      "loss": 2.3077,
+      "theoretical_loss": 5.189237863713357,
+      "tokens_seen": 58064896
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007938535606820462,
+      "loss": 2.3447,
+      "theoretical_loss": 5.188368823979152,
+      "tokens_seen": 58130432
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007938375125376129,
+      "loss": 2.4134,
+      "theoretical_loss": 5.187501037423096,
+      "tokens_seen": 58195968
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007938214643931796,
+      "loss": 2.3195,
+      "theoretical_loss": 5.186634500829886,
+      "tokens_seen": 58261504
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007938054162487463,
+      "loss": 2.4844,
+      "theoretical_loss": 5.185769210996076,
+      "tokens_seen": 58327040
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.000793789368104313,
+      "loss": 2.2643,
+      "theoretical_loss": 5.184905164730012,
+      "tokens_seen": 58392576
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007937733199598797,
+      "loss": 2.4695,
+      "theoretical_loss": 5.184042358851785,
+      "tokens_seen": 58458112
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007937572718154463,
+      "loss": 2.3881,
+      "theoretical_loss": 5.1831807901931715,
+      "tokens_seen": 58523648
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007937412236710131,
+      "loss": 2.2189,
+      "theoretical_loss": 5.18232045559757,
+      "tokens_seen": 58589184
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007937251755265797,
+      "loss": 2.1536,
+      "theoretical_loss": 5.181461351919959,
+      "tokens_seen": 58654720
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007937091273821465,
+      "loss": 2.4759,
+      "theoretical_loss": 5.180603476026832,
+      "tokens_seen": 58720256
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007936930792377131,
+      "loss": 2.2956,
+      "theoretical_loss": 5.179746824796146,
+      "tokens_seen": 58785792
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007936770310932799,
+      "loss": 2.4399,
+      "theoretical_loss": 5.178891395117262,
+      "tokens_seen": 58851328
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007936609829488465,
+      "loss": 2.3966,
+      "theoretical_loss": 5.178037183890902,
+      "tokens_seen": 58916864
+    },
+    {
+      "epoch": 0.02,
+      "objective/train/docs_used": 26426,
+      "objective/train/instantaneous_batch_size": 32,
+      "objective/train/instantaneous_microbatch_size": 32768,
+      "objective/train/original_loss": 2.25994873046875,
+      "objective/train/theoretical_loss": 5.177184188029082,
+      "objective/train/tokens_used": 79442400,
+      "theoretical_loss": 5.177184188029082,
+      "tokens_seen": 58982400
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007936449348044133,
+      "loss": 2.2624,
+      "theoretical_loss": 5.177184188029082,
+      "tokens_seen": 58982400
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007936288866599799,
+      "loss": 2.271,
+      "theoretical_loss": 5.1763324044550645,
+      "tokens_seen": 59047936
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007936128385155467,
+      "loss": 2.3542,
+      "theoretical_loss": 5.175481830103307,
+      "tokens_seen": 59113472
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007935967903711133,
+      "loss": 2.3463,
+      "theoretical_loss": 5.174632461919406,
+      "tokens_seen": 59179008
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007935807422266801,
+      "loss": 2.3343,
+      "theoretical_loss": 5.173784296860043,
+      "tokens_seen": 59244544
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007935646940822467,
+      "loss": 2.285,
+      "theoretical_loss": 5.172937331892934,
+      "tokens_seen": 59310080
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007935486459378135,
+      "loss": 2.2493,
+      "theoretical_loss": 5.17209156399678,
+      "tokens_seen": 59375616
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007935325977933801,
+      "loss": 2.3448,
+      "theoretical_loss": 5.171246990161208,
+      "tokens_seen": 59441152
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007935165496489469,
+      "loss": 2.2384,
+      "theoretical_loss": 5.17040360738673,
+      "tokens_seen": 59506688
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007935005015045135,
+      "loss": 2.2225,
+      "theoretical_loss": 5.169561412684677,
+      "tokens_seen": 59572224
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007934844533600803,
+      "loss": 2.3488,
+      "theoretical_loss": 5.168720403077165,
+      "tokens_seen": 59637760
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007934684052156469,
+      "loss": 2.4011,
+      "theoretical_loss": 5.167880575597031,
+      "tokens_seen": 59703296
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007934523570712137,
+      "loss": 2.2519,
+      "theoretical_loss": 5.167041927287787,
+      "tokens_seen": 59768832
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007934363089267803,
+      "loss": 2.4036,
+      "theoretical_loss": 5.166204455203575,
+      "tokens_seen": 59834368
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007934202607823471,
+      "loss": 2.4419,
+      "theoretical_loss": 5.16536815640911,
+      "tokens_seen": 59899904
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007934042126379137,
+      "loss": 2.5012,
+      "theoretical_loss": 5.1645330279796315,
+      "tokens_seen": 59965440
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007933881644934805,
+      "loss": 2.4089,
+      "theoretical_loss": 5.1636990670008585,
+      "tokens_seen": 60030976
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007933721163490471,
+      "loss": 2.4171,
+      "theoretical_loss": 5.162866270568938,
+      "tokens_seen": 60096512
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007933560682046139,
+      "loss": 2.3423,
+      "theoretical_loss": 5.162034635790397,
+      "tokens_seen": 60162048
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007933400200601805,
+      "loss": 2.2825,
+      "theoretical_loss": 5.1612041597820895,
+      "tokens_seen": 60227584
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007933239719157473,
+      "loss": 2.378,
+      "theoretical_loss": 5.160374839671159,
+      "tokens_seen": 60293120
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007933079237713139,
+      "loss": 2.3264,
+      "theoretical_loss": 5.1595466725949795,
+      "tokens_seen": 60358656
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007932918756268807,
+      "loss": 2.2196,
+      "theoretical_loss": 5.158719655701115,
+      "tokens_seen": 60424192
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007932758274824473,
+      "loss": 2.2858,
+      "theoretical_loss": 5.157893786147269,
+      "tokens_seen": 60489728
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007932597793380141,
+      "loss": 2.269,
+      "theoretical_loss": 5.157069061101238,
+      "tokens_seen": 60555264
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007932437311935807,
+      "loss": 2.3393,
+      "theoretical_loss": 5.156245477740866,
+      "tokens_seen": 60620800
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007932276830491475,
+      "loss": 2.3976,
+      "theoretical_loss": 5.155423033253997,
+      "tokens_seen": 60686336
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007932116349047141,
+      "loss": 2.3583,
+      "theoretical_loss": 5.154601724838431,
+      "tokens_seen": 60751872
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007931955867602809,
+      "loss": 2.2708,
+      "theoretical_loss": 5.153781549701872,
+      "tokens_seen": 60817408
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007931795386158475,
+      "loss": 2.4146,
+      "theoretical_loss": 5.15296250506189,
+      "tokens_seen": 60882944
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007931634904714143,
+      "loss": 2.3493,
+      "theoretical_loss": 5.152144588145871,
+      "tokens_seen": 60948480
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007931474423269809,
+      "loss": 2.3327,
+      "theoretical_loss": 5.151327796190975,
+      "tokens_seen": 61014016
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007931313941825477,
+      "loss": 2.2854,
+      "theoretical_loss": 5.1505121264440845,
+      "tokens_seen": 61079552
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007931153460381143,
+      "loss": 2.3792,
+      "theoretical_loss": 5.149697576161769,
+      "tokens_seen": 61145088
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007930992978936811,
+      "loss": 2.3118,
+      "theoretical_loss": 5.148884142610232,
+      "tokens_seen": 61210624
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007930832497492477,
+      "loss": 2.4018,
+      "theoretical_loss": 5.148071823065274,
+      "tokens_seen": 61276160
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007930672016048145,
+      "loss": 2.2722,
+      "theoretical_loss": 5.147260614812246,
+      "tokens_seen": 61341696
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007930511534603811,
+      "loss": 2.3685,
+      "theoretical_loss": 5.146450515146002,
+      "tokens_seen": 61407232
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007930351053159479,
+      "loss": 2.2692,
+      "theoretical_loss": 5.145641521370861,
+      "tokens_seen": 61472768
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007930190571715145,
+      "loss": 2.3931,
+      "theoretical_loss": 5.144833630800562,
+      "tokens_seen": 61538304
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007930030090270813,
+      "loss": 2.2115,
+      "theoretical_loss": 5.1440268407582215,
+      "tokens_seen": 61603840
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007929869608826479,
+      "loss": 2.3746,
+      "theoretical_loss": 5.143221148576288,
+      "tokens_seen": 61669376
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007929709127382147,
+      "loss": 2.2639,
+      "theoretical_loss": 5.142416551596505,
+      "tokens_seen": 61734912
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007929548645937813,
+      "loss": 2.3293,
+      "theoretical_loss": 5.141613047169862,
+      "tokens_seen": 61800448
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007929388164493481,
+      "loss": 2.4672,
+      "theoretical_loss": 5.14081063265656,
+      "tokens_seen": 61865984
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007929227683049147,
+      "loss": 2.3117,
+      "theoretical_loss": 5.140009305425963,
+      "tokens_seen": 61931520
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007929067201604815,
+      "loss": 2.4395,
+      "theoretical_loss": 5.13920906285656,
+      "tokens_seen": 61997056
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007928906720160481,
+      "loss": 2.3835,
+      "theoretical_loss": 5.138409902335925,
+      "tokens_seen": 62062592
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007928746238716149,
+      "loss": 2.2959,
+      "theoretical_loss": 5.137611821260673,
+      "tokens_seen": 62128128
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007928585757271815,
+      "loss": 2.3309,
+      "theoretical_loss": 5.13681481703642,
+      "tokens_seen": 62193664
+    },
+    {
+      "epoch": 0.02,
+      "objective/train/docs_used": 27339,
+      "objective/train/instantaneous_batch_size": 32,
+      "objective/train/instantaneous_microbatch_size": 32768,
+      "objective/train/original_loss": 2.3899831771850586,
+      "objective/train/theoretical_loss": 5.136018887077743,
+      "objective/train/tokens_used": 82719200,
+      "theoretical_loss": 5.136018887077743,
+      "tokens_seen": 62259200
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007928425275827483,
+      "loss": 2.315,
+      "theoretical_loss": 5.136018887077743,
+      "tokens_seen": 62259200
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.000792826479438315,
+      "loss": 2.1882,
+      "theoretical_loss": 5.13522402880814,
+      "tokens_seen": 62324736
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007928104312938817,
+      "loss": 2.2584,
+      "theoretical_loss": 5.13443023965999,
+      "tokens_seen": 62390272
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007927943831494484,
+      "loss": 2.2592,
+      "theoretical_loss": 5.13363751707451,
+      "tokens_seen": 62455808
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007927783350050151,
+      "loss": 2.3028,
+      "theoretical_loss": 5.132845858501721,
+      "tokens_seen": 62521344
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007927622868605819,
+      "loss": 2.1181,
+      "theoretical_loss": 5.132055261400403,
+      "tokens_seen": 62586880
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007927462387161485,
+      "loss": 2.3272,
+      "theoretical_loss": 5.131265723238059,
+      "tokens_seen": 62652416
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007927301905717153,
+      "loss": 2.3194,
+      "theoretical_loss": 5.130477241490876,
+      "tokens_seen": 62717952
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007927141424272819,
+      "loss": 2.1843,
+      "theoretical_loss": 5.129689813643683,
+      "tokens_seen": 62783488
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007926980942828487,
+      "loss": 2.257,
+      "theoretical_loss": 5.128903437189917,
+      "tokens_seen": 62849024
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007926820461384153,
+      "loss": 2.1401,
+      "theoretical_loss": 5.128118109631581,
+      "tokens_seen": 62914560
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007926659979939821,
+      "loss": 2.174,
+      "theoretical_loss": 5.127333828479208,
+      "tokens_seen": 62980096
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007926499498495487,
+      "loss": 2.2301,
+      "theoretical_loss": 5.126550591251821,
+      "tokens_seen": 63045632
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007926339017051155,
+      "loss": 2.219,
+      "theoretical_loss": 5.125768395476896,
+      "tokens_seen": 63111168
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007926178535606821,
+      "loss": 2.2643,
+      "theoretical_loss": 5.124987238690329,
+      "tokens_seen": 63176704
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007926018054162489,
+      "loss": 2.2679,
+      "theoretical_loss": 5.124207118436388,
+      "tokens_seen": 63242240
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007925857572718155,
+      "loss": 2.1915,
+      "theoretical_loss": 5.123428032267688,
+      "tokens_seen": 63307776
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007925697091273823,
+      "loss": 2.2756,
+      "theoretical_loss": 5.122649977745146,
+      "tokens_seen": 63373312
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007925536609829489,
+      "loss": 2.2722,
+      "theoretical_loss": 5.121872952437947,
+      "tokens_seen": 63438848
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007925376128385157,
+      "loss": 2.32,
+      "theoretical_loss": 5.121096953923507,
+      "tokens_seen": 63504384
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007925215646940823,
+      "loss": 2.3559,
+      "theoretical_loss": 5.1203219797874375,
+      "tokens_seen": 63569920
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007925055165496491,
+      "loss": 2.2186,
+      "theoretical_loss": 5.119548027623509,
+      "tokens_seen": 63635456
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007924894684052157,
+      "loss": 2.3859,
+      "theoretical_loss": 5.118775095033612,
+      "tokens_seen": 63700992
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007924734202607824,
+      "loss": 2.3028,
+      "theoretical_loss": 5.118003179627729,
+      "tokens_seen": 63766528
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007924573721163491,
+      "loss": 2.2712,
+      "theoretical_loss": 5.11723227902389,
+      "tokens_seen": 63832064
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007924413239719158,
+      "loss": 2.2979,
+      "theoretical_loss": 5.1164623908481435,
+      "tokens_seen": 63897600
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007924252758274825,
+      "loss": 2.3022,
+      "theoretical_loss": 5.115693512734515,
+      "tokens_seen": 63963136
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007924092276830492,
+      "loss": 2.2539,
+      "theoretical_loss": 5.114925642324982,
+      "tokens_seen": 64028672
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007923931795386159,
+      "loss": 2.1597,
+      "theoretical_loss": 5.114158777269429,
+      "tokens_seen": 64094208
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007923771313941826,
+      "loss": 2.1543,
+      "theoretical_loss": 5.113392915225617,
+      "tokens_seen": 64159744
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007923610832497493,
+      "loss": 2.1923,
+      "theoretical_loss": 5.112628053859153,
+      "tokens_seen": 64225280
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.000792345035105316,
+      "loss": 2.2827,
+      "theoretical_loss": 5.1118641908434475,
+      "tokens_seen": 64290816
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007923289869608826,
+      "loss": 2.3798,
+      "theoretical_loss": 5.111101323859687,
+      "tokens_seen": 64356352
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007923129388164494,
+      "loss": 2.2281,
+      "theoretical_loss": 5.1103394505968005,
+      "tokens_seen": 64421888
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.000792296890672016,
+      "loss": 2.2168,
+      "theoretical_loss": 5.109578568751417,
+      "tokens_seen": 64487424
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007922808425275828,
+      "loss": 2.3083,
+      "theoretical_loss": 5.108818676027845,
+      "tokens_seen": 64552960
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007922647943831494,
+      "loss": 2.128,
+      "theoretical_loss": 5.108059770138027,
+      "tokens_seen": 64618496
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007922487462387162,
+      "loss": 2.3606,
+      "theoretical_loss": 5.107301848801519,
+      "tokens_seen": 64684032
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007922326980942828,
+      "loss": 2.2326,
+      "theoretical_loss": 5.106544909745443,
+      "tokens_seen": 64749568
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007922166499498496,
+      "loss": 2.2915,
+      "theoretical_loss": 5.105788950704467,
+      "tokens_seen": 64815104
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007922006018054162,
+      "loss": 2.1764,
+      "theoretical_loss": 5.105033969420764,
+      "tokens_seen": 64880640
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.000792184553660983,
+      "loss": 2.1684,
+      "theoretical_loss": 5.104279963643982,
+      "tokens_seen": 64946176
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007921685055165496,
+      "loss": 2.0692,
+      "theoretical_loss": 5.103526931131217,
+      "tokens_seen": 65011712
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007921524573721164,
+      "loss": 2.1995,
+      "theoretical_loss": 5.102774869646972,
+      "tokens_seen": 65077248
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.000792136409227683,
+      "loss": 2.3402,
+      "theoretical_loss": 5.102023776963131,
+      "tokens_seen": 65142784
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007921203610832498,
+      "loss": 2.2009,
+      "theoretical_loss": 5.101273650858926,
+      "tokens_seen": 65208320
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007921043129388164,
+      "loss": 2.3025,
+      "theoretical_loss": 5.100524489120902,
+      "tokens_seen": 65273856
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007920882647943832,
+      "loss": 2.2621,
+      "theoretical_loss": 5.099776289542893,
+      "tokens_seen": 65339392
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007920722166499498,
+      "loss": 2.2385,
+      "theoretical_loss": 5.099029049925985,
+      "tokens_seen": 65404928
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007920561685055166,
+      "loss": 2.2834,
+      "theoretical_loss": 5.098282768078483,
+      "tokens_seen": 65470464
+    },
+    {
+      "epoch": 0.02,
+      "objective/train/docs_used": 28601,
+      "objective/train/instantaneous_batch_size": 32,
+      "objective/train/instantaneous_microbatch_size": 32768,
+      "objective/train/original_loss": 2.1837551593780518,
+      "objective/train/theoretical_loss": 5.097537441815887,
+      "objective/train/tokens_used": 85996000,
+      "theoretical_loss": 5.097537441815887,
+      "tokens_seen": 65536000
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007920401203610832,
+      "loss": 2.2508,
+      "theoretical_loss": 5.097537441815887,
+      "tokens_seen": 65536000
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.00079202407221665,
+      "loss": 2.2146,
+      "theoretical_loss": 5.096793068960857,
+      "tokens_seen": 65601536
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007920080240722166,
+      "loss": 2.2297,
+      "theoretical_loss": 5.096049647343182,
+      "tokens_seen": 65667072
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007919919759277834,
+      "loss": 2.2019,
+      "theoretical_loss": 5.095307174799752,
+      "tokens_seen": 65732608
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.00079197592778335,
+      "loss": 2.2517,
+      "theoretical_loss": 5.094565649174524,
+      "tokens_seen": 65798144
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007919598796389168,
+      "loss": 2.2072,
+      "theoretical_loss": 5.0938250683184965,
+      "tokens_seen": 65863680
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007919438314944834,
+      "loss": 2.1758,
+      "theoretical_loss": 5.093085430089674,
+      "tokens_seen": 65929216
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007919277833500502,
+      "loss": 2.0521,
+      "theoretical_loss": 5.092346732353046,
+      "tokens_seen": 65994752
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007919117352056168,
+      "loss": 2.2137,
+      "theoretical_loss": 5.091608972980545,
+      "tokens_seen": 66060288
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007918956870611836,
+      "loss": 2.1874,
+      "theoretical_loss": 5.090872149851029,
+      "tokens_seen": 66125824
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007918796389167502,
+      "loss": 2.1531,
+      "theoretical_loss": 5.090136260850245,
+      "tokens_seen": 66191360
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.000791863590772317,
+      "loss": 2.1465,
+      "theoretical_loss": 5.089401303870801,
+      "tokens_seen": 66256896
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007918475426278836,
+      "loss": 2.3518,
+      "theoretical_loss": 5.088667276812138,
+      "tokens_seen": 66322432
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007918314944834504,
+      "loss": 2.1216,
+      "theoretical_loss": 5.087934177580502,
+      "tokens_seen": 66387968
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.000791815446339017,
+      "loss": 2.2819,
+      "theoretical_loss": 5.087202004088914,
+      "tokens_seen": 66453504
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007917993981945838,
+      "loss": 2.1184,
+      "theoretical_loss": 5.086470754257139,
+      "tokens_seen": 66519040
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007917833500501504,
+      "loss": 2.1647,
+      "theoretical_loss": 5.0857404260116645,
+      "tokens_seen": 66584576
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007917673019057172,
+      "loss": 2.0028,
+      "theoretical_loss": 5.085011017285662,
+      "tokens_seen": 66650112
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007917512537612838,
+      "loss": 2.2067,
+      "theoretical_loss": 5.084282526018972,
+      "tokens_seen": 66715648
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007917352056168506,
+      "loss": 2.0789,
+      "theoretical_loss": 5.083554950158062,
+      "tokens_seen": 66781184
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007917191574724172,
+      "loss": 2.1791,
+      "theoretical_loss": 5.0828282876560085,
+      "tokens_seen": 66846720
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.000791703109327984,
+      "loss": 2.1499,
+      "theoretical_loss": 5.0821025364724655,
+      "tokens_seen": 66912256
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007916870611835506,
+      "loss": 2.1907,
+      "theoretical_loss": 5.081377694573638,
+      "tokens_seen": 66977792
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007916710130391174,
+      "loss": 2.2242,
+      "theoretical_loss": 5.080653759932255,
+      "tokens_seen": 67043328
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.000791654964894684,
+      "loss": 2.162,
+      "theoretical_loss": 5.079930730527538,
+      "tokens_seen": 67108864
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007916389167502508,
+      "loss": 2.2491,
+      "theoretical_loss": 5.079208604345181,
+      "tokens_seen": 67174400
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007916228686058174,
+      "loss": 2.3635,
+      "theoretical_loss": 5.078487379377318,
+      "tokens_seen": 67239936
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007916068204613842,
+      "loss": 2.1106,
+      "theoretical_loss": 5.0777670536225,
+      "tokens_seen": 67305472
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.000791590772316951,
+      "loss": 2.0667,
+      "theoretical_loss": 5.07704762508566,
+      "tokens_seen": 67371008
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007915747241725176,
+      "loss": 2.2173,
+      "theoretical_loss": 5.076329091778101,
+      "tokens_seen": 67436544
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007915586760280844,
+      "loss": 2.102,
+      "theoretical_loss": 5.075611451717457,
+      "tokens_seen": 67502080
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.000791542627883651,
+      "loss": 2.249,
+      "theoretical_loss": 5.0748947029276685,
+      "tokens_seen": 67567616
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007915265797392178,
+      "loss": 2.1799,
+      "theoretical_loss": 5.074178843438963,
+      "tokens_seen": 67633152
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007915105315947844,
+      "loss": 2.0891,
+      "theoretical_loss": 5.073463871287823,
+      "tokens_seen": 67698688
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007914944834503512,
+      "loss": 2.1138,
+      "theoretical_loss": 5.07274978451696,
+      "tokens_seen": 67764224
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007914784353059178,
+      "loss": 2.139,
+      "theoretical_loss": 5.072036581175293,
+      "tokens_seen": 67829760
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007914623871614846,
+      "loss": 2.1691,
+      "theoretical_loss": 5.071324259317921,
+      "tokens_seen": 67895296
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007914463390170512,
+      "loss": 2.067,
+      "theoretical_loss": 5.070612817006094,
+      "tokens_seen": 67960832
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.000791430290872618,
+      "loss": 2.1485,
+      "theoretical_loss": 5.06990225230719,
+      "tokens_seen": 68026368
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007914142427281846,
+      "loss": 1.9747,
+      "theoretical_loss": 5.069192563294694,
+      "tokens_seen": 68091904
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007913981945837514,
+      "loss": 2.1661,
+      "theoretical_loss": 5.068483748048166,
+      "tokens_seen": 68157440
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.000791382146439318,
+      "loss": 2.1057,
+      "theoretical_loss": 5.06777580465322,
+      "tokens_seen": 68222976
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007913660982948848,
+      "loss": 2.2941,
+      "theoretical_loss": 5.067068731201498,
+      "tokens_seen": 68288512
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007913500501504514,
+      "loss": 2.1956,
+      "theoretical_loss": 5.066362525790646,
+      "tokens_seen": 68354048
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007913340020060182,
+      "loss": 2.2727,
+      "theoretical_loss": 5.065657186524287,
+      "tokens_seen": 68419584
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007913179538615848,
+      "loss": 2.1148,
+      "theoretical_loss": 5.064952711512001,
+      "tokens_seen": 68485120
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007913019057171516,
+      "loss": 2.174,
+      "theoretical_loss": 5.064249098869295,
+      "tokens_seen": 68550656
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007912858575727182,
+      "loss": 2.2233,
+      "theoretical_loss": 5.063546346717585,
+      "tokens_seen": 68616192
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.000791269809428285,
+      "loss": 2.1508,
+      "theoretical_loss": 5.062844453184166,
+      "tokens_seen": 68681728
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007912537612838516,
+      "loss": 2.1924,
+      "theoretical_loss": 5.062143416402192,
+      "tokens_seen": 68747264
+    },
+    {
+      "epoch": 0.02,
+      "objective/train/docs_used": 29632,
+      "objective/train/instantaneous_batch_size": 32,
+      "objective/train/instantaneous_microbatch_size": 32768,
+      "objective/train/original_loss": 2.21848201751709,
+      "objective/train/theoretical_loss": 5.061443234510648,
+      "objective/train/tokens_used": 89272800,
+      "theoretical_loss": 5.061443234510648,
+      "tokens_seen": 68812800
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007912377131394184,
+      "loss": 2.2158,
+      "theoretical_loss": 5.061443234510648,
+      "tokens_seen": 68812800
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.000791221664994985,
+      "loss": 2.1701,
+      "theoretical_loss": 5.060743905654332,
+      "tokens_seen": 68878336
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007912056168505518,
+      "loss": 2.071,
+      "theoretical_loss": 5.0600454279838285,
+      "tokens_seen": 68943872
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007911895687061184,
+      "loss": 2.155,
+      "theoretical_loss": 5.059347799655483,
+      "tokens_seen": 69009408
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007911735205616852,
+      "loss": 2.2008,
+      "theoretical_loss": 5.05865101883138,
+      "tokens_seen": 69074944
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007911574724172518,
+      "loss": 2.2319,
+      "theoretical_loss": 5.057955083679321,
+      "tokens_seen": 69140480
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007911414242728186,
+      "loss": 2.2004,
+      "theoretical_loss": 5.0572599923728,
+      "tokens_seen": 69206016
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007911253761283852,
+      "loss": 2.1127,
+      "theoretical_loss": 5.056565743090982,
+      "tokens_seen": 69271552
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.000791109327983952,
+      "loss": 2.1262,
+      "theoretical_loss": 5.055872334018677,
+      "tokens_seen": 69337088
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007910932798395186,
+      "loss": 2.2264,
+      "theoretical_loss": 5.0551797633463185,
+      "tokens_seen": 69402624
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007910772316950853,
+      "loss": 2.1304,
+      "theoretical_loss": 5.054488029269946,
+      "tokens_seen": 69468160
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.000791061183550652,
+      "loss": 2.1443,
+      "theoretical_loss": 5.053797129991171,
+      "tokens_seen": 69533696
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007910451354062187,
+      "loss": 2.2802,
+      "theoretical_loss": 5.053107063717167,
+      "tokens_seen": 69599232
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007910290872617854,
+      "loss": 2.2566,
+      "theoretical_loss": 5.052417828660639,
+      "tokens_seen": 69664768
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007910130391173521,
+      "loss": 2.1684,
+      "theoretical_loss": 5.051729423039802,
+      "tokens_seen": 69730304
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007909969909729188,
+      "loss": 2.1875,
+      "theoretical_loss": 5.0510418450783625,
+      "tokens_seen": 69795840
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007909809428284855,
+      "loss": 2.3061,
+      "theoretical_loss": 5.050355093005495,
+      "tokens_seen": 69861376
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007909648946840522,
+      "loss": 2.1259,
+      "theoretical_loss": 5.049669165055816,
+      "tokens_seen": 69926912
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007909488465396189,
+      "loss": 2.1739,
+      "theoretical_loss": 5.048984059469369,
+      "tokens_seen": 69992448
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007909327983951855,
+      "loss": 2.1805,
+      "theoretical_loss": 5.048299774491598,
+      "tokens_seen": 70057984
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007909167502507523,
+      "loss": 2.1599,
+      "theoretical_loss": 5.047616308373328,
+      "tokens_seen": 70123520
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007909007021063189,
+      "loss": 2.1491,
+      "theoretical_loss": 5.046933659370742,
+      "tokens_seen": 70189056
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007908846539618857,
+      "loss": 2.2169,
+      "theoretical_loss": 5.046251825745361,
+      "tokens_seen": 70254592
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007908686058174523,
+      "loss": 2.2379,
+      "theoretical_loss": 5.0455708057640205,
+      "tokens_seen": 70320128
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007908525576730191,
+      "loss": 2.1031,
+      "theoretical_loss": 5.044890597698853,
+      "tokens_seen": 70385664
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007908365095285857,
+      "loss": 2.147,
+      "theoretical_loss": 5.044211199827263,
+      "tokens_seen": 70451200
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007908204613841525,
+      "loss": 2.2193,
+      "theoretical_loss": 5.04353261043191,
+      "tokens_seen": 70516736
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007908044132397191,
+      "loss": 2.1425,
+      "theoretical_loss": 5.0428548278006815,
+      "tokens_seen": 70582272
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007907883650952859,
+      "loss": 1.9801,
+      "theoretical_loss": 5.042177850226682,
+      "tokens_seen": 70647808
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007907723169508525,
+      "loss": 2.2487,
+      "theoretical_loss": 5.0415016760082,
+      "tokens_seen": 70713344
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007907562688064193,
+      "loss": 2.1081,
+      "theoretical_loss": 5.040826303448698,
+      "tokens_seen": 70778880
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007907402206619859,
+      "loss": 2.1463,
+      "theoretical_loss": 5.040151730856788,
+      "tokens_seen": 70844416
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007907241725175527,
+      "loss": 2.0669,
+      "theoretical_loss": 5.039477956546207,
+      "tokens_seen": 70909952
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007907081243731193,
+      "loss": 2.306,
+      "theoretical_loss": 5.038804978835808,
+      "tokens_seen": 70975488
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007906920762286861,
+      "loss": 2.261,
+      "theoretical_loss": 5.038132796049523,
+      "tokens_seen": 71041024
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007906760280842527,
+      "loss": 2.22,
+      "theoretical_loss": 5.037461406516362,
+      "tokens_seen": 71106560
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007906599799398195,
+      "loss": 2.1503,
+      "theoretical_loss": 5.036790808570377,
+      "tokens_seen": 71172096
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007906439317953861,
+      "loss": 2.0681,
+      "theoretical_loss": 5.036121000550651,
+      "tokens_seen": 71237632
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007906278836509529,
+      "loss": 2.2711,
+      "theoretical_loss": 5.035451980801276,
+      "tokens_seen": 71303168
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007906118355065195,
+      "loss": 2.1232,
+      "theoretical_loss": 5.034783747671332,
+      "tokens_seen": 71368704
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007905957873620863,
+      "loss": 2.2111,
+      "theoretical_loss": 5.034116299514871,
+      "tokens_seen": 71434240
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007905797392176529,
+      "loss": 2.0703,
+      "theoretical_loss": 5.033449634690893,
+      "tokens_seen": 71499776
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007905636910732197,
+      "loss": 2.0378,
+      "theoretical_loss": 5.032783751563327,
+      "tokens_seen": 71565312
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007905476429287863,
+      "loss": 2.1419,
+      "theoretical_loss": 5.032118648501017,
+      "tokens_seen": 71630848
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007905315947843531,
+      "loss": 2.159,
+      "theoretical_loss": 5.031454323877698,
+      "tokens_seen": 71696384
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007905155466399197,
+      "loss": 2.0934,
+      "theoretical_loss": 5.030790776071976,
+      "tokens_seen": 71761920
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007904994984954865,
+      "loss": 2.2547,
+      "theoretical_loss": 5.030128003467312,
+      "tokens_seen": 71827456
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007904834503510531,
+      "loss": 2.2213,
+      "theoretical_loss": 5.029466004452004,
+      "tokens_seen": 71892992
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007904674022066199,
+      "loss": 2.1576,
+      "theoretical_loss": 5.028804777419165,
+      "tokens_seen": 71958528
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007904513540621865,
+      "loss": 2.2725,
+      "theoretical_loss": 5.028144320766701,
+      "tokens_seen": 72024064
+    },
+    {
+      "epoch": 0.02,
+      "objective/train/docs_used": 30814,
+      "objective/train/instantaneous_batch_size": 32,
+      "objective/train/instantaneous_microbatch_size": 32768,
+      "objective/train/original_loss": 2.156130075454712,
+      "objective/train/theoretical_loss": 5.0274846328973055,
+      "objective/train/tokens_used": 92549600,
+      "theoretical_loss": 5.0274846328973055,
+      "tokens_seen": 72089600
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007904353059177533,
+      "loss": 2.0367,
+      "theoretical_loss": 5.0274846328973055,
+      "tokens_seen": 72089600
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007904192577733199,
+      "loss": 2.1309,
+      "theoretical_loss": 5.0268257122184234,
+      "tokens_seen": 72155136
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007904032096288867,
+      "loss": 2.1157,
+      "theoretical_loss": 5.026167557142247,
+      "tokens_seen": 72220672
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007903871614844534,
+      "loss": 2.2037,
+      "theoretical_loss": 5.02551016608569,
+      "tokens_seen": 72286208
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007903711133400201,
+      "loss": 2.0258,
+      "theoretical_loss": 5.024853537470369,
+      "tokens_seen": 72351744
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007903550651955868,
+      "loss": 2.225,
+      "theoretical_loss": 5.02419766972259,
+      "tokens_seen": 72417280
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007903390170511535,
+      "loss": 2.0335,
+      "theoretical_loss": 5.023542561273327,
+      "tokens_seen": 72482816
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007903229689067202,
+      "loss": 2.2431,
+      "theoretical_loss": 5.022888210558206,
+      "tokens_seen": 72548352
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007903069207622869,
+      "loss": 2.1291,
+      "theoretical_loss": 5.02223461601748,
+      "tokens_seen": 72613888
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007902908726178537,
+      "loss": 2.1585,
+      "theoretical_loss": 5.021581776096022,
+      "tokens_seen": 72679424
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007902748244734203,
+      "loss": 2.1885,
+      "theoretical_loss": 5.0209296892433,
+      "tokens_seen": 72744960
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.000790258776328987,
+      "loss": 2.0969,
+      "theoretical_loss": 5.020278353913362,
+      "tokens_seen": 72810496
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007902427281845537,
+      "loss": 2.0178,
+      "theoretical_loss": 5.019627768564816,
+      "tokens_seen": 72876032
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007902266800401205,
+      "loss": 2.1542,
+      "theoretical_loss": 5.018977931660814,
+      "tokens_seen": 72941568
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007902106318956871,
+      "loss": 2.1037,
+      "theoretical_loss": 5.018328841669037,
+      "tokens_seen": 73007104
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007901945837512539,
+      "loss": 2.103,
+      "theoretical_loss": 5.017680497061671,
+      "tokens_seen": 73072640
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007901785356068205,
+      "loss": 2.0543,
+      "theoretical_loss": 5.017032896315397,
+      "tokens_seen": 73138176
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007901624874623873,
+      "loss": 2.2007,
+      "theoretical_loss": 5.016386037911371,
+      "tokens_seen": 73203712
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007901464393179539,
+      "loss": 2.1312,
+      "theoretical_loss": 5.015739920335203,
+      "tokens_seen": 73269248
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007901303911735207,
+      "loss": 2.0943,
+      "theoretical_loss": 5.015094542076945,
+      "tokens_seen": 73334784
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007901143430290873,
+      "loss": 2.0704,
+      "theoretical_loss": 5.014449901631075,
+      "tokens_seen": 73400320
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007900982948846541,
+      "loss": 2.1444,
+      "theoretical_loss": 5.013805997496473,
+      "tokens_seen": 73465856
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007900822467402207,
+      "loss": 2.0357,
+      "theoretical_loss": 5.013162828176412,
+      "tokens_seen": 73531392
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007900661985957875,
+      "loss": 2.1537,
+      "theoretical_loss": 5.012520392178539,
+      "tokens_seen": 73596928
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007900501504513541,
+      "loss": 2.099,
+      "theoretical_loss": 5.011878688014852,
+      "tokens_seen": 73662464
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007900341023069209,
+      "loss": 2.0815,
+      "theoretical_loss": 5.011237714201694,
+      "tokens_seen": 73728000
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007900180541624875,
+      "loss": 2.0495,
+      "theoretical_loss": 5.010597469259729,
+      "tokens_seen": 73793536
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007900020060180543,
+      "loss": 2.1379,
+      "theoretical_loss": 5.009957951713927,
+      "tokens_seen": 73859072
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007899859578736209,
+      "loss": 2.0643,
+      "theoretical_loss": 5.009319160093551,
+      "tokens_seen": 73924608
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007899699097291877,
+      "loss": 1.9916,
+      "theoretical_loss": 5.008681092932137,
+      "tokens_seen": 73990144
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007899538615847543,
+      "loss": 2.0824,
+      "theoretical_loss": 5.008043748767477,
+      "tokens_seen": 74055680
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007899378134403211,
+      "loss": 2.2227,
+      "theoretical_loss": 5.007407126141608,
+      "tokens_seen": 74121216
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007899217652958877,
+      "loss": 2.082,
+      "theoretical_loss": 5.00677122360079,
+      "tokens_seen": 74186752
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007899057171514545,
+      "loss": 2.1113,
+      "theoretical_loss": 5.006136039695496,
+      "tokens_seen": 74252288
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007898896690070211,
+      "loss": 2.1242,
+      "theoretical_loss": 5.005501572980391,
+      "tokens_seen": 74317824
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007898736208625879,
+      "loss": 2.2854,
+      "theoretical_loss": 5.004867822014317,
+      "tokens_seen": 74383360
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007898575727181545,
+      "loss": 2.0454,
+      "theoretical_loss": 5.004234785360282,
+      "tokens_seen": 74448896
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007898415245737213,
+      "loss": 2.0465,
+      "theoretical_loss": 5.0036024615854355,
+      "tokens_seen": 74514432
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007898254764292879,
+      "loss": 1.9049,
+      "theoretical_loss": 5.002970849261065,
+      "tokens_seen": 74579968
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007898094282848547,
+      "loss": 2.0382,
+      "theoretical_loss": 5.002339946962568,
+      "tokens_seen": 74645504
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007897933801404213,
+      "loss": 2.1154,
+      "theoretical_loss": 5.001709753269443,
+      "tokens_seen": 74711040
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007897773319959881,
+      "loss": 2.1872,
+      "theoretical_loss": 5.001080266765275,
+      "tokens_seen": 74776576
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007897612838515547,
+      "loss": 2.2846,
+      "theoretical_loss": 5.0004514860377185,
+      "tokens_seen": 74842112
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007897452357071215,
+      "loss": 2.1182,
+      "theoretical_loss": 4.999823409678481,
+      "tokens_seen": 74907648
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007897291875626881,
+      "loss": 2.1742,
+      "theoretical_loss": 4.999196036283308,
+      "tokens_seen": 74973184
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007897131394182549,
+      "loss": 2.2014,
+      "theoretical_loss": 4.9985693644519715,
+      "tokens_seen": 75038720
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007896970912738215,
+      "loss": 2.1095,
+      "theoretical_loss": 4.997943392788251,
+      "tokens_seen": 75104256
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007896810431293882,
+      "loss": 2.1097,
+      "theoretical_loss": 4.997318119899922,
+      "tokens_seen": 75169792
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007896649949849549,
+      "loss": 2.1655,
+      "theoretical_loss": 4.996693544398734,
+      "tokens_seen": 75235328
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007896489468405216,
+      "loss": 2.0666,
+      "theoretical_loss": 4.996069664900406,
+      "tokens_seen": 75300864
+    },
+    {
+      "epoch": 0.02,
+      "objective/train/docs_used": 32008,
+      "objective/train/instantaneous_batch_size": 32,
+      "objective/train/instantaneous_microbatch_size": 32768,
+      "objective/train/original_loss": 2.099667549133301,
+      "objective/train/theoretical_loss": 4.9954464800246035,
+      "objective/train/tokens_used": 95826400,
+      "theoretical_loss": 4.9954464800246035,
+      "tokens_seen": 75366400
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007896328986960883,
+      "loss": 2.1507,
+      "theoretical_loss": 4.9954464800246035,
+      "tokens_seen": 75366400
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.000789616850551655,
+      "loss": 2.254,
+      "theoretical_loss": 4.994823988394928,
+      "tokens_seen": 75431936
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007896008024072217,
+      "loss": 2.0327,
+      "theoretical_loss": 4.994202188638901,
+      "tokens_seen": 75497472
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007895847542627884,
+      "loss": 2.1681,
+      "theoretical_loss": 4.9935810793879485,
+      "tokens_seen": 75563008
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007895687061183551,
+      "loss": 2.0672,
+      "theoretical_loss": 4.992960659277389,
+      "tokens_seen": 75628544
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007895526579739218,
+      "loss": 2.0648,
+      "theoretical_loss": 4.992340926946417,
+      "tokens_seen": 75694080
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007895366098294884,
+      "loss": 2.1304,
+      "theoretical_loss": 4.991721881038091,
+      "tokens_seen": 75759616
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007895205616850552,
+      "loss": 2.2517,
+      "theoretical_loss": 4.991103520199314,
+      "tokens_seen": 75825152
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007895045135406218,
+      "loss": 2.1684,
+      "theoretical_loss": 4.990485843080824,
+      "tokens_seen": 75890688
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007894884653961886,
+      "loss": 2.0915,
+      "theoretical_loss": 4.989868848337181,
+      "tokens_seen": 75956224
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007894724172517552,
+      "loss": 2.0643,
+      "theoretical_loss": 4.989252534626749,
+      "tokens_seen": 76021760
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.000789456369107322,
+      "loss": 2.0849,
+      "theoretical_loss": 4.988636900611683,
+      "tokens_seen": 76087296
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007894403209628886,
+      "loss": 1.8875,
+      "theoretical_loss": 4.988021944957915,
+      "tokens_seen": 76152832
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007894242728184554,
+      "loss": 2.1708,
+      "theoretical_loss": 4.987407666335143,
+      "tokens_seen": 76218368
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.000789408224674022,
+      "loss": 2.0622,
+      "theoretical_loss": 4.986794063416813,
+      "tokens_seen": 76283904
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007893921765295888,
+      "loss": 2.1683,
+      "theoretical_loss": 4.986181134880107,
+      "tokens_seen": 76349440
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007893761283851554,
+      "loss": 2.243,
+      "theoretical_loss": 4.98556887940593,
+      "tokens_seen": 76414976
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007893600802407222,
+      "loss": 1.9463,
+      "theoretical_loss": 4.984957295678894,
+      "tokens_seen": 76480512
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007893440320962888,
+      "loss": 2.093,
+      "theoretical_loss": 4.98434638238731,
+      "tokens_seen": 76546048
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007893279839518556,
+      "loss": 1.9806,
+      "theoretical_loss": 4.983736138223165,
+      "tokens_seen": 76611584
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007893119358074222,
+      "loss": 2.1226,
+      "theoretical_loss": 4.983126561882118,
+      "tokens_seen": 76677120
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.000789295887662989,
+      "loss": 2.0722,
+      "theoretical_loss": 4.982517652063482,
+      "tokens_seen": 76742656
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007892798395185556,
+      "loss": 2.1126,
+      "theoretical_loss": 4.981909407470209,
+      "tokens_seen": 76808192
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007892637913741224,
+      "loss": 2.1013,
+      "theoretical_loss": 4.981301826808882,
+      "tokens_seen": 76873728
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.000789247743229689,
+      "loss": 2.0177,
+      "theoretical_loss": 4.9806949087896975,
+      "tokens_seen": 76939264
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007892316950852558,
+      "loss": 2.0787,
+      "theoretical_loss": 4.980088652126453,
+      "tokens_seen": 77004800
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007892156469408224,
+      "loss": 2.0262,
+      "theoretical_loss": 4.9794830555365355,
+      "tokens_seen": 77070336
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007891995987963892,
+      "loss": 2.1339,
+      "theoretical_loss": 4.978878117740907,
+      "tokens_seen": 77135872
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007891835506519559,
+      "loss": 2.142,
+      "theoretical_loss": 4.978273837464091,
+      "tokens_seen": 77201408
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007891675025075226,
+      "loss": 2.0887,
+      "theoretical_loss": 4.977670213434161,
+      "tokens_seen": 77266944
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007891514543630893,
+      "loss": 1.9407,
+      "theoretical_loss": 4.977067244382729,
+      "tokens_seen": 77332480
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.000789135406218656,
+      "loss": 1.9978,
+      "theoretical_loss": 4.976464929044928,
+      "tokens_seen": 77398016
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007891193580742227,
+      "loss": 2.2369,
+      "theoretical_loss": 4.975863266159403,
+      "tokens_seen": 77463552
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007891033099297894,
+      "loss": 2.1298,
+      "theoretical_loss": 4.975262254468299,
+      "tokens_seen": 77529088
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007890872617853561,
+      "loss": 2.167,
+      "theoretical_loss": 4.9746618927172435,
+      "tokens_seen": 77594624
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007890712136409228,
+      "loss": 2.1512,
+      "theoretical_loss": 4.9740621796553395,
+      "tokens_seen": 77660160
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007890551654964895,
+      "loss": 1.991,
+      "theoretical_loss": 4.973463114035149,
+      "tokens_seen": 77725696
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007890391173520562,
+      "loss": 2.0322,
+      "theoretical_loss": 4.972864694612684,
+      "tokens_seen": 77791232
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.000789023069207623,
+      "loss": 2.1256,
+      "theoretical_loss": 4.972266920147391,
+      "tokens_seen": 77856768
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007890070210631896,
+      "loss": 2.1493,
+      "theoretical_loss": 4.971669789402139,
+      "tokens_seen": 77922304
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007889909729187563,
+      "loss": 2.034,
+      "theoretical_loss": 4.971073301143207,
+      "tokens_seen": 77987840
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.000788974924774323,
+      "loss": 2.1583,
+      "theoretical_loss": 4.970477454140278,
+      "tokens_seen": 78053376
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007889588766298897,
+      "loss": 2.0851,
+      "theoretical_loss": 4.9698822471664155,
+      "tokens_seen": 78118912
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007889428284854564,
+      "loss": 2.0552,
+      "theoretical_loss": 4.96928767899806,
+      "tokens_seen": 78184448
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007889267803410232,
+      "loss": 2.0769,
+      "theoretical_loss": 4.9686937484150135,
+      "tokens_seen": 78249984
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007889107321965898,
+      "loss": 2.0967,
+      "theoretical_loss": 4.96810045420043,
+      "tokens_seen": 78315520
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007888946840521566,
+      "loss": 2.0354,
+      "theoretical_loss": 4.967507795140797,
+      "tokens_seen": 78381056
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007888786359077232,
+      "loss": 2.1696,
+      "theoretical_loss": 4.966915770025935,
+      "tokens_seen": 78446592
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.00078886258776329,
+      "loss": 2.0989,
+      "theoretical_loss": 4.966324377648973,
+      "tokens_seen": 78512128
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007888465396188566,
+      "loss": 2.1939,
+      "theoretical_loss": 4.965733616806345,
+      "tokens_seen": 78577664
+    },
+    {
+      "epoch": 0.02,
+      "objective/train/docs_used": 32959,
+      "objective/train/instantaneous_batch_size": 32,
+      "objective/train/instantaneous_microbatch_size": 32768,
+      "objective/train/original_loss": 2.2311925888061523,
+      "objective/train/theoretical_loss": 4.965143486297777,
+      "objective/train/tokens_used": 99103200,
+      "theoretical_loss": 4.965143486297777,
+      "tokens_seen": 78643200
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007888304914744234,
+      "loss": 2.1223,
+      "theoretical_loss": 4.965143486297777,
+      "tokens_seen": 78643200
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.00078881444332999,
+      "loss": 1.9655,
+      "theoretical_loss": 4.964553984926271,
+      "tokens_seen": 78708736
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007887983951855568,
+      "loss": 2.1445,
+      "theoretical_loss": 4.963965111498098,
+      "tokens_seen": 78774272
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007887823470411234,
+      "loss": 2.0716,
+      "theoretical_loss": 4.963376864822784,
+      "tokens_seen": 78839808
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007887662988966902,
+      "loss": 2.1271,
+      "theoretical_loss": 4.962789243713102,
+      "tokens_seen": 78905344
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007887502507522568,
+      "loss": 2.0089,
+      "theoretical_loss": 4.962202246985054,
+      "tokens_seen": 78970880
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007887342026078236,
+      "loss": 2.0885,
+      "theoretical_loss": 4.9616158734578635,
+      "tokens_seen": 79036416
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007887181544633902,
+      "loss": 2.0025,
+      "theoretical_loss": 4.961030121953965,
+      "tokens_seen": 79101952
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.000788702106318957,
+      "loss": 2.108,
+      "theoretical_loss": 4.960444991298992,
+      "tokens_seen": 79167488
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007886860581745236,
+      "loss": 2.0414,
+      "theoretical_loss": 4.959860480321762,
+      "tokens_seen": 79233024
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007886700100300904,
+      "loss": 2.2156,
+      "theoretical_loss": 4.959276587854272,
+      "tokens_seen": 79298560
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.000788653961885657,
+      "loss": 2.1104,
+      "theoretical_loss": 4.958693312731681,
+      "tokens_seen": 79364096
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007886379137412238,
+      "loss": 2.1552,
+      "theoretical_loss": 4.9581106537923,
+      "tokens_seen": 79429632
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007886218655967904,
+      "loss": 2.0635,
+      "theoretical_loss": 4.957528609877587,
+      "tokens_seen": 79495168
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007886058174523572,
+      "loss": 2.1149,
+      "theoretical_loss": 4.9569471798321265,
+      "tokens_seen": 79560704
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007885897693079238,
+      "loss": 2.126,
+      "theoretical_loss": 4.9563663625036245,
+      "tokens_seen": 79626240
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007885737211634906,
+      "loss": 1.9757,
+      "theoretical_loss": 4.955786156742898,
+      "tokens_seen": 79691776
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007885576730190572,
+      "loss": 2.1144,
+      "theoretical_loss": 4.955206561403859,
+      "tokens_seen": 79757312
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.000788541624874624,
+      "loss": 1.974,
+      "theoretical_loss": 4.954627575343509,
+      "tokens_seen": 79822848
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007885255767301906,
+      "loss": 2.0177,
+      "theoretical_loss": 4.954049197421925,
+      "tokens_seen": 79888384
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007885095285857574,
+      "loss": 2.0737,
+      "theoretical_loss": 4.953471426502249,
+      "tokens_seen": 79953920
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.000788493480441324,
+      "loss": 2.0636,
+      "theoretical_loss": 4.952894261450679,
+      "tokens_seen": 80019456
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007884774322968908,
+      "loss": 2.1073,
+      "theoretical_loss": 4.952317701136457,
+      "tokens_seen": 80084992
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007884613841524574,
+      "loss": 2.153,
+      "theoretical_loss": 4.9517417444318585,
+      "tokens_seen": 80150528
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007884453360080242,
+      "loss": 2.0601,
+      "theoretical_loss": 4.951166390212181,
+      "tokens_seen": 80216064
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007884292878635908,
+      "loss": 2.073,
+      "theoretical_loss": 4.950591637355737,
+      "tokens_seen": 80281600
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007884132397191576,
+      "loss": 2.1266,
+      "theoretical_loss": 4.950017484743835,
+      "tokens_seen": 80347136
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007883971915747242,
+      "loss": 2.2302,
+      "theoretical_loss": 4.949443931260783,
+      "tokens_seen": 80412672
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.000788381143430291,
+      "loss": 2.0353,
+      "theoretical_loss": 4.948870975793863,
+      "tokens_seen": 80478208
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007883650952858576,
+      "loss": 2.0421,
+      "theoretical_loss": 4.9482986172333305,
+      "tokens_seen": 80543744
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007883490471414244,
+      "loss": 2.0759,
+      "theoretical_loss": 4.9477268544724,
+      "tokens_seen": 80609280
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.000788332998996991,
+      "loss": 1.9484,
+      "theoretical_loss": 4.947155686407239,
+      "tokens_seen": 80674816
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007883169508525578,
+      "loss": 2.0687,
+      "theoretical_loss": 4.946585111936951,
+      "tokens_seen": 80740352
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007883009027081244,
+      "loss": 2.0221,
+      "theoretical_loss": 4.94601512996357,
+      "tokens_seen": 80805888
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007882848545636911,
+      "loss": 2.0929,
+      "theoretical_loss": 4.945445739392049,
+      "tokens_seen": 80871424
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007882688064192578,
+      "loss": 2.0362,
+      "theoretical_loss": 4.944876939130252,
+      "tokens_seen": 80936960
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007882527582748245,
+      "loss": 1.9439,
+      "theoretical_loss": 4.94430872808894,
+      "tokens_seen": 81002496
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007882367101303912,
+      "loss": 2.0828,
+      "theoretical_loss": 4.943741105181765,
+      "tokens_seen": 81068032
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007882206619859579,
+      "loss": 2.079,
+      "theoretical_loss": 4.943174069325255,
+      "tokens_seen": 81133568
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007882046138415246,
+      "loss": 2.0084,
+      "theoretical_loss": 4.94260761943881,
+      "tokens_seen": 81199104
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007881885656970913,
+      "loss": 2.093,
+      "theoretical_loss": 4.942041754444688,
+      "tokens_seen": 81264640
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.000788172517552658,
+      "loss": 1.9797,
+      "theoretical_loss": 4.941476473267996,
+      "tokens_seen": 81330176
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007881564694082247,
+      "loss": 2.0787,
+      "theoretical_loss": 4.940911774836682,
+      "tokens_seen": 81395712
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007881404212637913,
+      "loss": 2.054,
+      "theoretical_loss": 4.940347658081521,
+      "tokens_seen": 81461248
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007881243731193581,
+      "loss": 2.045,
+      "theoretical_loss": 4.939784121936112,
+      "tokens_seen": 81526784
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007881083249749247,
+      "loss": 2.0953,
+      "theoretical_loss": 4.9392211653368605,
+      "tokens_seen": 81592320
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007880922768304915,
+      "loss": 2.0521,
+      "theoretical_loss": 4.938658787222975,
+      "tokens_seen": 81657856
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007880762286860581,
+      "loss": 2.0022,
+      "theoretical_loss": 4.938096986536452,
+      "tokens_seen": 81723392
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007880601805416249,
+      "loss": 2.1479,
+      "theoretical_loss": 4.937535762222075,
+      "tokens_seen": 81788928
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007880441323971915,
+      "loss": 2.0679,
+      "theoretical_loss": 4.936975113227394,
+      "tokens_seen": 81854464
+    },
+    {
+      "epoch": 0.02,
+      "objective/train/docs_used": 34278,
+      "objective/train/instantaneous_batch_size": 32,
+      "objective/train/instantaneous_microbatch_size": 32768,
+      "objective/train/original_loss": 2.2234320640563965,
+      "objective/train/theoretical_loss": 4.936415038502723,
+      "objective/train/tokens_used": 102380000,
+      "theoretical_loss": 4.936415038502723,
+      "tokens_seen": 81920000
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007880280842527583,
+      "loss": 2.1228,
+      "theoretical_loss": 4.936415038502723,
+      "tokens_seen": 81920000
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007880120361083249,
+      "loss": 2.0414,
+      "theoretical_loss": 4.93585553700113,
+      "tokens_seen": 81985536
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007879959879638917,
+      "loss": 1.9697,
+      "theoretical_loss": 4.935296607678426,
+      "tokens_seen": 82051072
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007879799398194584,
+      "loss": 2.1378,
+      "theoretical_loss": 4.934738249493156,
+      "tokens_seen": 82116608
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007879638916750251,
+      "loss": 2.0329,
+      "theoretical_loss": 4.934180461406591,
+      "tokens_seen": 82182144
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007879478435305918,
+      "loss": 1.9616,
+      "theoretical_loss": 4.933623242382714,
+      "tokens_seen": 82247680
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007879317953861585,
+      "loss": 1.9614,
+      "theoretical_loss": 4.9330665913882195,
+      "tokens_seen": 82313216
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007879157472417252,
+      "loss": 2.2081,
+      "theoretical_loss": 4.932510507392495,
+      "tokens_seen": 82378752
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0007878996990972919,
+      "loss": 2.1316,
+      "theoretical_loss": 4.931954989367618,
+      "tokens_seen": 82444288
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007878836509528586,
+      "loss": 2.0436,
+      "theoretical_loss": 4.931400036288343,
+      "tokens_seen": 82509824
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007878676028084253,
+      "loss": 2.0655,
+      "theoretical_loss": 4.930845647132097,
+      "tokens_seen": 82575360
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.000787851554663992,
+      "loss": 1.9909,
+      "theoretical_loss": 4.9302918208789634,
+      "tokens_seen": 82640896
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007878355065195587,
+      "loss": 2.0418,
+      "theoretical_loss": 4.929738556511681,
+      "tokens_seen": 82706432
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007878194583751254,
+      "loss": 1.9638,
+      "theoretical_loss": 4.929185853015629,
+      "tokens_seen": 82771968
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007878034102306921,
+      "loss": 2.1218,
+      "theoretical_loss": 4.928633709378822,
+      "tokens_seen": 82837504
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007877873620862588,
+      "loss": 2.0123,
+      "theoretical_loss": 4.9280821245918975,
+      "tokens_seen": 82903040
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007877713139418255,
+      "loss": 2.1631,
+      "theoretical_loss": 4.92753109764811,
+      "tokens_seen": 82968576
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007877552657973922,
+      "loss": 2.0317,
+      "theoretical_loss": 4.92698062754332,
+      "tokens_seen": 83034112
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007877392176529589,
+      "loss": 2.0362,
+      "theoretical_loss": 4.9264307132759875,
+      "tokens_seen": 83099648
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007877231695085256,
+      "loss": 2.0983,
+      "theoretical_loss": 4.92588135384716,
+      "tokens_seen": 83165184
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007877071213640923,
+      "loss": 2.1016,
+      "theoretical_loss": 4.925332548260469,
+      "tokens_seen": 83230720
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.000787691073219659,
+      "loss": 2.1058,
+      "theoretical_loss": 4.924784295522113,
+      "tokens_seen": 83296256
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007876750250752257,
+      "loss": 1.9915,
+      "theoretical_loss": 4.92423659464086,
+      "tokens_seen": 83361792
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007876589769307924,
+      "loss": 2.0509,
+      "theoretical_loss": 4.923689444628027,
+      "tokens_seen": 83427328
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007876429287863591,
+      "loss": 2.0901,
+      "theoretical_loss": 4.923142844497482,
+      "tokens_seen": 83492864
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007876268806419258,
+      "loss": 2.0036,
+      "theoretical_loss": 4.922596793265625,
+      "tokens_seen": 83558400
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007876108324974925,
+      "loss": 1.9617,
+      "theoretical_loss": 4.922051289951391,
+      "tokens_seen": 83623936
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007875947843530592,
+      "loss": 2.0941,
+      "theoretical_loss": 4.921506333576232,
+      "tokens_seen": 83689472
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007875787362086259,
+      "loss": 2.0417,
+      "theoretical_loss": 4.920961923164114,
+      "tokens_seen": 83755008
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007875626880641927,
+      "loss": 2.0198,
+      "theoretical_loss": 4.920418057741504,
+      "tokens_seen": 83820544
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007875466399197593,
+      "loss": 1.8703,
+      "theoretical_loss": 4.919874736337368,
+      "tokens_seen": 83886080
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.000787530591775326,
+      "loss": 2.1168,
+      "theoretical_loss": 4.9193319579831565,
+      "tokens_seen": 83951616
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007875145436308927,
+      "loss": 2.0664,
+      "theoretical_loss": 4.918789721712799,
+      "tokens_seen": 84017152
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007874984954864595,
+      "loss": 2.0419,
+      "theoretical_loss": 4.918248026562697,
+      "tokens_seen": 84082688
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007874824473420261,
+      "loss": 2.1125,
+      "theoretical_loss": 4.917706871571712,
+      "tokens_seen": 84148224
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007874663991975929,
+      "loss": 1.9691,
+      "theoretical_loss": 4.9171662557811615,
+      "tokens_seen": 84213760
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007874503510531595,
+      "loss": 2.1347,
+      "theoretical_loss": 4.916626178234807,
+      "tokens_seen": 84279296
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007874343029087263,
+      "loss": 2.0238,
+      "theoretical_loss": 4.916086637978851,
+      "tokens_seen": 84344832
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007874182547642929,
+      "loss": 2.039,
+      "theoretical_loss": 4.915547634061921,
+      "tokens_seen": 84410368
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007874022066198597,
+      "loss": 2.1473,
+      "theoretical_loss": 4.91500916553507,
+      "tokens_seen": 84475904
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007873861584754263,
+      "loss": 2.0671,
+      "theoretical_loss": 4.914471231451762,
+      "tokens_seen": 84541440
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007873701103309931,
+      "loss": 2.0727,
+      "theoretical_loss": 4.913933830867868,
+      "tokens_seen": 84606976
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007873540621865597,
+      "loss": 2.058,
+      "theoretical_loss": 4.913396962841655,
+      "tokens_seen": 84672512
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007873380140421265,
+      "loss": 1.9781,
+      "theoretical_loss": 4.9128606264337815,
+      "tokens_seen": 84738048
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007873219658976931,
+      "loss": 2.0159,
+      "theoretical_loss": 4.912324820707285,
+      "tokens_seen": 84803584
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007873059177532599,
+      "loss": 1.9919,
+      "theoretical_loss": 4.911789544727581,
+      "tokens_seen": 84869120
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007872898696088265,
+      "loss": 1.9598,
+      "theoretical_loss": 4.911254797562448,
+      "tokens_seen": 84934656
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007872738214643933,
+      "loss": 2.013,
+      "theoretical_loss": 4.910720578282021,
+      "tokens_seen": 85000192
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007872577733199599,
+      "loss": 1.9745,
+      "theoretical_loss": 4.91018688595879,
+      "tokens_seen": 85065728
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007872417251755267,
+      "loss": 2.0936,
+      "theoretical_loss": 4.909653719667584,
+      "tokens_seen": 85131264
+    },
+    {
+      "epoch": 0.03,
+      "objective/train/docs_used": 35080,
+      "objective/train/instantaneous_batch_size": 32,
+      "objective/train/instantaneous_microbatch_size": 32768,
+      "objective/train/original_loss": 2.1525585651397705,
+      "objective/train/theoretical_loss": 4.909121078485567,
+      "objective/train/tokens_used": 105656800,
+      "theoretical_loss": 4.909121078485567,
+      "tokens_seen": 85196800
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007872256770310933,
+      "loss": 2.1056,
+      "theoretical_loss": 4.909121078485567,
+      "tokens_seen": 85196800
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007872096288866601,
+      "loss": 2.0394,
+      "theoretical_loss": 4.908588961492235,
+      "tokens_seen": 85262336
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007871935807422267,
+      "loss": 1.947,
+      "theoretical_loss": 4.908057367769396,
+      "tokens_seen": 85327872
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007871775325977935,
+      "loss": 2.0141,
+      "theoretical_loss": 4.907526296401175,
+      "tokens_seen": 85393408
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007871614844533601,
+      "loss": 1.9779,
+      "theoretical_loss": 4.906995746474001,
+      "tokens_seen": 85458944
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007871454363089269,
+      "loss": 1.9658,
+      "theoretical_loss": 4.9064657170766,
+      "tokens_seen": 85524480
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007871293881644935,
+      "loss": 1.9654,
+      "theoretical_loss": 4.905936207299984,
+      "tokens_seen": 85590016
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007871133400200603,
+      "loss": 2.1111,
+      "theoretical_loss": 4.90540721623745,
+      "tokens_seen": 85655552
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007870972918756269,
+      "loss": 2.1011,
+      "theoretical_loss": 4.904878742984569,
+      "tokens_seen": 85721088
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007870812437311937,
+      "loss": 2.0838,
+      "theoretical_loss": 4.904350786639178,
+      "tokens_seen": 85786624
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007870651955867603,
+      "loss": 2.0206,
+      "theoretical_loss": 4.903823346301374,
+      "tokens_seen": 85852160
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007870491474423271,
+      "loss": 2.0783,
+      "theoretical_loss": 4.903296421073506,
+      "tokens_seen": 85917696
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007870330992978937,
+      "loss": 2.0444,
+      "theoretical_loss": 4.902770010060166,
+      "tokens_seen": 85983232
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007870170511534605,
+      "loss": 2.1048,
+      "theoretical_loss": 4.902244112368188,
+      "tokens_seen": 86048768
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007870010030090271,
+      "loss": 2.1213,
+      "theoretical_loss": 4.901718727106631,
+      "tokens_seen": 86114304
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007869849548645939,
+      "loss": 2.0449,
+      "theoretical_loss": 4.90119385338678,
+      "tokens_seen": 86179840
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007869689067201605,
+      "loss": 2.1075,
+      "theoretical_loss": 4.900669490322134,
+      "tokens_seen": 86245376
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007869528585757273,
+      "loss": 2.025,
+      "theoretical_loss": 4.900145637028402,
+      "tokens_seen": 86310912
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007869368104312939,
+      "loss": 2.1303,
+      "theoretical_loss": 4.899622292623493,
+      "tokens_seen": 86376448
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007869207622868607,
+      "loss": 2.0683,
+      "theoretical_loss": 4.8990994562275105,
+      "tokens_seen": 86441984
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007869047141424273,
+      "loss": 1.9725,
+      "theoretical_loss": 4.898577126962746,
+      "tokens_seen": 86507520
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007868886659979941,
+      "loss": 1.9686,
+      "theoretical_loss": 4.898055303953669,
+      "tokens_seen": 86573056
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007868726178535607,
+      "loss": 2.0449,
+      "theoretical_loss": 4.897533986326922,
+      "tokens_seen": 86638592
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007868565697091274,
+      "loss": 1.9079,
+      "theoretical_loss": 4.897013173211316,
+      "tokens_seen": 86704128
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007868405215646941,
+      "loss": 1.964,
+      "theoretical_loss": 4.896492863737818,
+      "tokens_seen": 86769664
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007868244734202608,
+      "loss": 2.0787,
+      "theoretical_loss": 4.895973057039548,
+      "tokens_seen": 86835200
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007868084252758275,
+      "loss": 2.0269,
+      "theoretical_loss": 4.89545375225177,
+      "tokens_seen": 86900736
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007867923771313942,
+      "loss": 2.016,
+      "theoretical_loss": 4.894934948511889,
+      "tokens_seen": 86966272
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007867763289869609,
+      "loss": 1.9923,
+      "theoretical_loss": 4.894416644959437,
+      "tokens_seen": 87031808
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007867602808425276,
+      "loss": 2.0054,
+      "theoretical_loss": 4.893898840736071,
+      "tokens_seen": 87097344
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007867442326980943,
+      "loss": 2.0477,
+      "theoretical_loss": 4.8933815349855685,
+      "tokens_seen": 87162880
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.000786728184553661,
+      "loss": 2.0445,
+      "theoretical_loss": 4.892864726853814,
+      "tokens_seen": 87228416
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007867121364092277,
+      "loss": 2.1697,
+      "theoretical_loss": 4.892348415488799,
+      "tokens_seen": 87293952
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007866960882647944,
+      "loss": 2.0416,
+      "theoretical_loss": 4.891832600040609,
+      "tokens_seen": 87359488
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007866800401203611,
+      "loss": 2.028,
+      "theoretical_loss": 4.891317279661422,
+      "tokens_seen": 87425024
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007866639919759278,
+      "loss": 2.0877,
+      "theoretical_loss": 4.890802453505498,
+      "tokens_seen": 87490560
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007866479438314945,
+      "loss": 1.9764,
+      "theoretical_loss": 4.890288120729174,
+      "tokens_seen": 87556096
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007866318956870612,
+      "loss": 1.9326,
+      "theoretical_loss": 4.88977428049086,
+      "tokens_seen": 87621632
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007866158475426279,
+      "loss": 2.0788,
+      "theoretical_loss": 4.889260931951026,
+      "tokens_seen": 87687168
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007865997993981946,
+      "loss": 2.1495,
+      "theoretical_loss": 4.888748074272201,
+      "tokens_seen": 87752704
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007865837512537613,
+      "loss": 2.0538,
+      "theoretical_loss": 4.888235706618964,
+      "tokens_seen": 87818240
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.000786567703109328,
+      "loss": 2.0681,
+      "theoretical_loss": 4.887723828157939,
+      "tokens_seen": 87883776
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007865516549648947,
+      "loss": 2.0325,
+      "theoretical_loss": 4.8872124380577855,
+      "tokens_seen": 87949312
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007865356068204614,
+      "loss": 1.9446,
+      "theoretical_loss": 4.886701535489195,
+      "tokens_seen": 88014848
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007865195586760281,
+      "loss": 2.0676,
+      "theoretical_loss": 4.886191119624883,
+      "tokens_seen": 88080384
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007865035105315948,
+      "loss": 2.0677,
+      "theoretical_loss": 4.885681189639584,
+      "tokens_seen": 88145920
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007864874623871615,
+      "loss": 2.0072,
+      "theoretical_loss": 4.885171744710044,
+      "tokens_seen": 88211456
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007864714142427282,
+      "loss": 2.0929,
+      "theoretical_loss": 4.884662784015012,
+      "tokens_seen": 88276992
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007864553660982949,
+      "loss": 2.0283,
+      "theoretical_loss": 4.884154306735239,
+      "tokens_seen": 88342528
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007864393179538616,
+      "loss": 2.0953,
+      "theoretical_loss": 4.8836463120534646,
+      "tokens_seen": 88408064
+    },
+    {
+      "epoch": 0.03,
+      "objective/train/docs_used": 36362,
+      "objective/train/instantaneous_batch_size": 32,
+      "objective/train/instantaneous_microbatch_size": 32768,
+      "objective/train/original_loss": 2.012448310852051,
+      "objective/train/theoretical_loss": 4.88313879915442,
+      "objective/train/tokens_used": 108933600,
+      "theoretical_loss": 4.88313879915442,
+      "tokens_seen": 88473600
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007864232698094283,
+      "loss": 2.0635,
+      "theoretical_loss": 4.88313879915442,
+      "tokens_seen": 88473600
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.000786407221664995,
+      "loss": 2.0915,
+      "theoretical_loss": 4.882631767224812,
+      "tokens_seen": 88539136
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007863911735205617,
+      "loss": 2.0535,
+      "theoretical_loss": 4.882125215453321,
+      "tokens_seen": 88604672
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007863751253761284,
+      "loss": 2.024,
+      "theoretical_loss": 4.881619143030598,
+      "tokens_seen": 88670208
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007863590772316951,
+      "loss": 1.9735,
+      "theoretical_loss": 4.8811135491492506,
+      "tokens_seen": 88735744
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007863430290872618,
+      "loss": 2.059,
+      "theoretical_loss": 4.880608433003846,
+      "tokens_seen": 88801280
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007863269809428285,
+      "loss": 1.9574,
+      "theoretical_loss": 4.880103793790896,
+      "tokens_seen": 88866816
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007863109327983952,
+      "loss": 2.108,
+      "theoretical_loss": 4.87959963070886,
+      "tokens_seen": 88932352
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.000786294884653962,
+      "loss": 2.0059,
+      "theoretical_loss": 4.879095942958127,
+      "tokens_seen": 88997888
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007862788365095286,
+      "loss": 1.9519,
+      "theoretical_loss": 4.878592729741024,
+      "tokens_seen": 89063424
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007862627883650953,
+      "loss": 1.9854,
+      "theoretical_loss": 4.878089990261797,
+      "tokens_seen": 89128960
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.000786246740220662,
+      "loss": 1.9926,
+      "theoretical_loss": 4.87758772372661,
+      "tokens_seen": 89194496
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007862306920762287,
+      "loss": 1.8898,
+      "theoretical_loss": 4.877085929343544,
+      "tokens_seen": 89260032
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007862146439317954,
+      "loss": 2.0137,
+      "theoretical_loss": 4.876584606322583,
+      "tokens_seen": 89325568
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007861985957873622,
+      "loss": 1.9756,
+      "theoretical_loss": 4.8760837538756086,
+      "tokens_seen": 89391104
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007861825476429288,
+      "loss": 2.127,
+      "theoretical_loss": 4.875583371216404,
+      "tokens_seen": 89456640
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007861664994984956,
+      "loss": 1.9945,
+      "theoretical_loss": 4.875083457560633,
+      "tokens_seen": 89522176
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007861504513540622,
+      "loss": 2.0463,
+      "theoretical_loss": 4.874584012125847,
+      "tokens_seen": 89587712
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.000786134403209629,
+      "loss": 1.8999,
+      "theoretical_loss": 4.874085034131472,
+      "tokens_seen": 89653248
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007861183550651956,
+      "loss": 2.0474,
+      "theoretical_loss": 4.873586522798803,
+      "tokens_seen": 89718784
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007861023069207624,
+      "loss": 2.0725,
+      "theoretical_loss": 4.873088477351005,
+      "tokens_seen": 89784320
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.000786086258776329,
+      "loss": 2.0274,
+      "theoretical_loss": 4.8725908970130964,
+      "tokens_seen": 89849856
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007860702106318958,
+      "loss": 2.0771,
+      "theoretical_loss": 4.872093781011952,
+      "tokens_seen": 89915392
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007860541624874624,
+      "loss": 1.9651,
+      "theoretical_loss": 4.871597128576292,
+      "tokens_seen": 89980928
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007860381143430292,
+      "loss": 1.9414,
+      "theoretical_loss": 4.8711009389366815,
+      "tokens_seen": 90046464
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007860220661985958,
+      "loss": 1.9186,
+      "theoretical_loss": 4.870605211325517,
+      "tokens_seen": 90112000
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007860060180541626,
+      "loss": 1.9044,
+      "theoretical_loss": 4.870109944977029,
+      "tokens_seen": 90177536
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007859899699097292,
+      "loss": 2.1207,
+      "theoretical_loss": 4.869615139127273,
+      "tokens_seen": 90243072
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.000785973921765296,
+      "loss": 2.039,
+      "theoretical_loss": 4.869120793014117,
+      "tokens_seen": 90308608
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007859578736208626,
+      "loss": 2.0347,
+      "theoretical_loss": 4.868626905877251,
+      "tokens_seen": 90374144
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007859418254764294,
+      "loss": 1.9877,
+      "theoretical_loss": 4.8681334769581674,
+      "tokens_seen": 90439680
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.000785925777331996,
+      "loss": 2.0885,
+      "theoretical_loss": 4.867640505500161,
+      "tokens_seen": 90505216
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007859097291875628,
+      "loss": 2.0793,
+      "theoretical_loss": 4.8671479907483235,
+      "tokens_seen": 90570752
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007858936810431294,
+      "loss": 2.091,
+      "theoretical_loss": 4.866655931949537,
+      "tokens_seen": 90636288
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007858776328986962,
+      "loss": 1.8927,
+      "theoretical_loss": 4.86616432835247,
+      "tokens_seen": 90701824
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007858615847542628,
+      "loss": 2.0022,
+      "theoretical_loss": 4.865673179207572,
+      "tokens_seen": 90767360
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007858455366098296,
+      "loss": 2.0913,
+      "theoretical_loss": 4.865182483767063,
+      "tokens_seen": 90832896
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007858294884653962,
+      "loss": 1.9703,
+      "theoretical_loss": 4.864692241284933,
+      "tokens_seen": 90898432
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.000785813440320963,
+      "loss": 1.9931,
+      "theoretical_loss": 4.864202451016939,
+      "tokens_seen": 90963968
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007857973921765296,
+      "loss": 1.9306,
+      "theoretical_loss": 4.863713112220592,
+      "tokens_seen": 91029504
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007857813440320964,
+      "loss": 1.9549,
+      "theoretical_loss": 4.8632242241551555,
+      "tokens_seen": 91095040
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.000785765295887663,
+      "loss": 1.9341,
+      "theoretical_loss": 4.862735786081642,
+      "tokens_seen": 91160576
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007857492477432298,
+      "loss": 2.0104,
+      "theoretical_loss": 4.862247797262806,
+      "tokens_seen": 91226112
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007857331995987964,
+      "loss": 1.9005,
+      "theoretical_loss": 4.861760256963136,
+      "tokens_seen": 91291648
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007857171514543632,
+      "loss": 1.9849,
+      "theoretical_loss": 4.861273164448854,
+      "tokens_seen": 91357184
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007857011033099298,
+      "loss": 2.1324,
+      "theoretical_loss": 4.860786518987906,
+      "tokens_seen": 91422720
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007856850551654966,
+      "loss": 1.985,
+      "theoretical_loss": 4.86030031984996,
+      "tokens_seen": 91488256
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007856690070210632,
+      "loss": 2.0782,
+      "theoretical_loss": 4.859814566306397,
+      "tokens_seen": 91553792
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.00078565295887663,
+      "loss": 1.9523,
+      "theoretical_loss": 4.859329257630311,
+      "tokens_seen": 91619328
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007856369107321966,
+      "loss": 2.0421,
+      "theoretical_loss": 4.858844393096497,
+      "tokens_seen": 91684864
+    },
+    {
+      "epoch": 0.03,
+      "objective/train/docs_used": 37324,
+      "objective/train/instantaneous_batch_size": 32,
+      "objective/train/instantaneous_microbatch_size": 32768,
+      "objective/train/original_loss": 2.0132102966308594,
+      "objective/train/theoretical_loss": 4.858359971981454,
+      "objective/train/tokens_used": 112210400,
+      "theoretical_loss": 4.858359971981454,
+      "tokens_seen": 91750400
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007856208625877634,
+      "loss": 2.0158,
+      "theoretical_loss": 4.858359971981454,
+      "tokens_seen": 91750400
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.00078560481444333,
+      "loss": 2.0729,
+      "theoretical_loss": 4.85787599356337,
+      "tokens_seen": 91815936
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007855887662988968,
+      "loss": 2.0301,
+      "theoretical_loss": 4.857392457122128,
+      "tokens_seen": 91881472
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007855727181544634,
+      "loss": 1.9507,
+      "theoretical_loss": 4.856909361939288,
+      "tokens_seen": 91947008
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007855566700100302,
+      "loss": 2.0377,
+      "theoretical_loss": 4.856426707298096,
+      "tokens_seen": 92012544
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007855406218655968,
+      "loss": 1.9297,
+      "theoretical_loss": 4.8559444924834665,
+      "tokens_seen": 92078080
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007855245737211636,
+      "loss": 1.9873,
+      "theoretical_loss": 4.855462716781985,
+      "tokens_seen": 92143616
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007855085255767302,
+      "loss": 1.9946,
+      "theoretical_loss": 4.854981379481901,
+      "tokens_seen": 92209152
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.000785492477432297,
+      "loss": 2.015,
+      "theoretical_loss": 4.854500479873119,
+      "tokens_seen": 92274688
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007854764292878636,
+      "loss": 2.0428,
+      "theoretical_loss": 4.854020017247203,
+      "tokens_seen": 92340224
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007854603811434303,
+      "loss": 2.1151,
+      "theoretical_loss": 4.853539990897358,
+      "tokens_seen": 92405760
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.000785444332998997,
+      "loss": 1.9446,
+      "theoretical_loss": 4.85306040011844,
+      "tokens_seen": 92471296
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007854282848545637,
+      "loss": 2.0156,
+      "theoretical_loss": 4.852581244206938,
+      "tokens_seen": 92536832
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007854122367101304,
+      "loss": 1.9775,
+      "theoretical_loss": 4.852102522460975,
+      "tokens_seen": 92602368
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007853961885656971,
+      "loss": 2.0026,
+      "theoretical_loss": 4.851624234180306,
+      "tokens_seen": 92667904
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007853801404212638,
+      "loss": 1.952,
+      "theoretical_loss": 4.851146378666305,
+      "tokens_seen": 92733440
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007853640922768305,
+      "loss": 1.9222,
+      "theoretical_loss": 4.85066895522197,
+      "tokens_seen": 92798976
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007853480441323972,
+      "loss": 1.927,
+      "theoretical_loss": 4.850191963151908,
+      "tokens_seen": 92864512
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007853319959879639,
+      "loss": 1.8673,
+      "theoretical_loss": 4.849715401762337,
+      "tokens_seen": 92930048
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007853159478435306,
+      "loss": 1.9885,
+      "theoretical_loss": 4.849239270361078,
+      "tokens_seen": 92995584
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007852998996990973,
+      "loss": 2.094,
+      "theoretical_loss": 4.848763568257554,
+      "tokens_seen": 93061120
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.000785283851554664,
+      "loss": 1.8801,
+      "theoretical_loss": 4.848288294762779,
+      "tokens_seen": 93126656
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007852678034102307,
+      "loss": 2.0318,
+      "theoretical_loss": 4.847813449189358,
+      "tokens_seen": 93192192
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007852517552657974,
+      "loss": 2.086,
+      "theoretical_loss": 4.847339030851482,
+      "tokens_seen": 93257728
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007852357071213641,
+      "loss": 1.9656,
+      "theoretical_loss": 4.84686503906492,
+      "tokens_seen": 93323264
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007852196589769308,
+      "loss": 2.0347,
+      "theoretical_loss": 4.846391473147019,
+      "tokens_seen": 93388800
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007852036108324975,
+      "loss": 2.0571,
+      "theoretical_loss": 4.8459183324166935,
+      "tokens_seen": 93454336
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007851875626880642,
+      "loss": 2.0381,
+      "theoretical_loss": 4.845445616194426,
+      "tokens_seen": 93519872
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007851715145436309,
+      "loss": 2.0005,
+      "theoretical_loss": 4.844973323802259,
+      "tokens_seen": 93585408
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007851554663991976,
+      "loss": 2.0188,
+      "theoretical_loss": 4.844501454563792,
+      "tokens_seen": 93650944
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007851394182547643,
+      "loss": 2.1137,
+      "theoretical_loss": 4.844030007804177,
+      "tokens_seen": 93716480
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.000785123370110331,
+      "loss": 1.8986,
+      "theoretical_loss": 4.843558982850113,
+      "tokens_seen": 93782016
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007851073219658977,
+      "loss": 2.0934,
+      "theoretical_loss": 4.84308837902984,
+      "tokens_seen": 93847552
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007850912738214644,
+      "loss": 1.9604,
+      "theoretical_loss": 4.842618195673138,
+      "tokens_seen": 93913088
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007850752256770311,
+      "loss": 1.9655,
+      "theoretical_loss": 4.84214843211132,
+      "tokens_seen": 93978624
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007850591775325978,
+      "loss": 2.0497,
+      "theoretical_loss": 4.841679087677226,
+      "tokens_seen": 94044160
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007850431293881645,
+      "loss": 2.0149,
+      "theoretical_loss": 4.841210161705223,
+      "tokens_seen": 94109696
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007850270812437312,
+      "loss": 2.0733,
+      "theoretical_loss": 4.840741653531195,
+      "tokens_seen": 94175232
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007850110330992979,
+      "loss": 1.9822,
+      "theoretical_loss": 4.840273562492545,
+      "tokens_seen": 94240768
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007849949849548646,
+      "loss": 2.0606,
+      "theoretical_loss": 4.839805887928181,
+      "tokens_seen": 94306304
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007849789368104313,
+      "loss": 2.0861,
+      "theoretical_loss": 4.839338629178522,
+      "tokens_seen": 94371840
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.000784962888665998,
+      "loss": 1.9923,
+      "theoretical_loss": 4.8388717855854875,
+      "tokens_seen": 94437376
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007849468405215647,
+      "loss": 1.9131,
+      "theoretical_loss": 4.838405356492494,
+      "tokens_seen": 94502912
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007849307923771314,
+      "loss": 1.9174,
+      "theoretical_loss": 4.83793934124445,
+      "tokens_seen": 94568448
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007849147442326981,
+      "loss": 1.9442,
+      "theoretical_loss": 4.837473739187754,
+      "tokens_seen": 94633984
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007848986960882648,
+      "loss": 1.9656,
+      "theoretical_loss": 4.837008549670285,
+      "tokens_seen": 94699520
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007848826479438315,
+      "loss": 2.0044,
+      "theoretical_loss": 4.8365437720414075,
+      "tokens_seen": 94765056
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007848665997993982,
+      "loss": 2.0683,
+      "theoretical_loss": 4.836079405651956,
+      "tokens_seen": 94830592
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007848505516549649,
+      "loss": 2.0223,
+      "theoretical_loss": 4.835615449854238,
+      "tokens_seen": 94896128
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007848345035105317,
+      "loss": 1.938,
+      "theoretical_loss": 4.8351519040020285,
+      "tokens_seen": 94961664
+    },
+    {
+      "epoch": 0.03,
+      "objective/train/docs_used": 38463,
+      "objective/train/instantaneous_batch_size": 32,
+      "objective/train/instantaneous_microbatch_size": 32768,
+      "objective/train/original_loss": 2.123032569885254,
+      "objective/train/theoretical_loss": 4.834688767450562,
+      "objective/train/tokens_used": 115487200,
+      "theoretical_loss": 4.834688767450562,
+      "tokens_seen": 95027200
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007848184553660983,
+      "loss": 2.1414,
+      "theoretical_loss": 4.834688767450562,
+      "tokens_seen": 95027200
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.000784802407221665,
+      "loss": 1.9367,
+      "theoretical_loss": 4.8342260395565315,
+      "tokens_seen": 95092736
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007847863590772317,
+      "loss": 1.9126,
+      "theoretical_loss": 4.833763719678085,
+      "tokens_seen": 95158272
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007847703109327985,
+      "loss": 1.9882,
+      "theoretical_loss": 4.833301807174822,
+      "tokens_seen": 95223808
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007847542627883651,
+      "loss": 2.0318,
+      "theoretical_loss": 4.83284030140778,
+      "tokens_seen": 95289344
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007847382146439319,
+      "loss": 1.8914,
+      "theoretical_loss": 4.832379201739442,
+      "tokens_seen": 95354880
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007847221664994985,
+      "loss": 2.0411,
+      "theoretical_loss": 4.831918507533728,
+      "tokens_seen": 95420416
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007847061183550653,
+      "loss": 1.9147,
+      "theoretical_loss": 4.831458218155989,
+      "tokens_seen": 95485952
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007846900702106319,
+      "loss": 1.85,
+      "theoretical_loss": 4.830998332973005,
+      "tokens_seen": 95551488
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007846740220661987,
+      "loss": 2.0202,
+      "theoretical_loss": 4.830538851352976,
+      "tokens_seen": 95617024
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007846579739217653,
+      "loss": 1.9628,
+      "theoretical_loss": 4.830079772665529,
+      "tokens_seen": 95682560
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007846419257773321,
+      "loss": 2.0797,
+      "theoretical_loss": 4.829621096281702,
+      "tokens_seen": 95748096
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007846258776328987,
+      "loss": 1.8946,
+      "theoretical_loss": 4.8291628215739415,
+      "tokens_seen": 95813632
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007846098294884655,
+      "loss": 2.0479,
+      "theoretical_loss": 4.828704947916108,
+      "tokens_seen": 95879168
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007845937813440321,
+      "loss": 2.0624,
+      "theoretical_loss": 4.82824747468346,
+      "tokens_seen": 95944704
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007845777331995989,
+      "loss": 1.9892,
+      "theoretical_loss": 4.827790401252658,
+      "tokens_seen": 96010240
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007845616850551655,
+      "loss": 1.9923,
+      "theoretical_loss": 4.827333727001756,
+      "tokens_seen": 96075776
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007845456369107323,
+      "loss": 2.0063,
+      "theoretical_loss": 4.8268774513101995,
+      "tokens_seen": 96141312
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007845295887662989,
+      "loss": 2.0631,
+      "theoretical_loss": 4.82642157355882,
+      "tokens_seen": 96206848
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007845135406218657,
+      "loss": 1.996,
+      "theoretical_loss": 4.825966093129834,
+      "tokens_seen": 96272384
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007844974924774323,
+      "loss": 1.9977,
+      "theoretical_loss": 4.825511009406833,
+      "tokens_seen": 96337920
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007844814443329991,
+      "loss": 1.9286,
+      "theoretical_loss": 4.825056321774788,
+      "tokens_seen": 96403456
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007844653961885657,
+      "loss": 1.8181,
+      "theoretical_loss": 4.8246020296200385,
+      "tokens_seen": 96468992
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007844493480441325,
+      "loss": 1.8653,
+      "theoretical_loss": 4.82414813233029,
+      "tokens_seen": 96534528
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007844332998996991,
+      "loss": 1.9037,
+      "theoretical_loss": 4.823694629294609,
+      "tokens_seen": 96600064
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007844172517552659,
+      "loss": 2.0652,
+      "theoretical_loss": 4.823241519903428,
+      "tokens_seen": 96665600
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007844012036108325,
+      "loss": 1.7936,
+      "theoretical_loss": 4.8227888035485265,
+      "tokens_seen": 96731136
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007843851554663993,
+      "loss": 1.966,
+      "theoretical_loss": 4.82233647962304,
+      "tokens_seen": 96796672
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.000784369107321966,
+      "loss": 2.0758,
+      "theoretical_loss": 4.821884547521449,
+      "tokens_seen": 96862208
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007843530591775327,
+      "loss": 1.8621,
+      "theoretical_loss": 4.821433006639578,
+      "tokens_seen": 96927744
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007843370110330994,
+      "loss": 1.9484,
+      "theoretical_loss": 4.820981856374589,
+      "tokens_seen": 96993280
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007843209628886661,
+      "loss": 1.8707,
+      "theoretical_loss": 4.8205310961249825,
+      "tokens_seen": 97058816
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007843049147442328,
+      "loss": 1.8772,
+      "theoretical_loss": 4.820080725290589,
+      "tokens_seen": 97124352
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007842888665997995,
+      "loss": 1.947,
+      "theoretical_loss": 4.819630743272566,
+      "tokens_seen": 97189888
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007842728184553662,
+      "loss": 1.8908,
+      "theoretical_loss": 4.819181149473396,
+      "tokens_seen": 97255424
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007842567703109329,
+      "loss": 1.9488,
+      "theoretical_loss": 4.81873194329688,
+      "tokens_seen": 97320960
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007842407221664996,
+      "loss": 2.0333,
+      "theoretical_loss": 4.81828312414814,
+      "tokens_seen": 97386496
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007842246740220663,
+      "loss": 1.9136,
+      "theoretical_loss": 4.817834691433602,
+      "tokens_seen": 97452032
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007842086258776329,
+      "loss": 1.9076,
+      "theoretical_loss": 4.81738664456101,
+      "tokens_seen": 97517568
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007841925777331997,
+      "loss": 1.8764,
+      "theoretical_loss": 4.816938982939407,
+      "tokens_seen": 97583104
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007841765295887663,
+      "loss": 1.8929,
+      "theoretical_loss": 4.816491705979138,
+      "tokens_seen": 97648640
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007841604814443331,
+      "loss": 1.9527,
+      "theoretical_loss": 4.816044813091848,
+      "tokens_seen": 97714176
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007841444332998997,
+      "loss": 1.8418,
+      "theoretical_loss": 4.815598303690473,
+      "tokens_seen": 97779712
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007841283851554665,
+      "loss": 1.8748,
+      "theoretical_loss": 4.8151521771892405,
+      "tokens_seen": 97845248
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007841123370110331,
+      "loss": 1.8331,
+      "theoretical_loss": 4.814706433003665,
+      "tokens_seen": 97910784
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007840962888665999,
+      "loss": 1.9921,
+      "theoretical_loss": 4.8142610705505415,
+      "tokens_seen": 97976320
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007840802407221665,
+      "loss": 1.8227,
+      "theoretical_loss": 4.813816089247945,
+      "tokens_seen": 98041856
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007840641925777332,
+      "loss": 1.9452,
+      "theoretical_loss": 4.813371488515227,
+      "tokens_seen": 98107392
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007840481444332999,
+      "loss": 1.8985,
+      "theoretical_loss": 4.812927267773008,
+      "tokens_seen": 98172928
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007840320962888666,
+      "loss": 2.1108,
+      "theoretical_loss": 4.812483426443181,
+      "tokens_seen": 98238464
+    },
+    {
+      "epoch": 0.03,
+      "objective/train/docs_used": 39401,
+      "objective/train/instantaneous_batch_size": 32,
+      "objective/train/instantaneous_microbatch_size": 32768,
+      "objective/train/original_loss": 1.9625983238220215,
+      "objective/train/theoretical_loss": 4.812039963948898,
+      "objective/train/tokens_used": 118764000,
+      "theoretical_loss": 4.812039963948898,
+      "tokens_seen": 98304000
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007840160481444333,
+      "loss": 2.0361,
+      "theoretical_loss": 4.812039963948898,
+      "tokens_seen": 98304000
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.000784,
+      "loss": 1.7952,
+      "theoretical_loss": 4.811596879714575,
+      "tokens_seen": 98369536
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007839839518555667,
+      "loss": 1.972,
+      "theoretical_loss": 4.811154173165886,
+      "tokens_seen": 98435072
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007839679037111334,
+      "loss": 1.9437,
+      "theoretical_loss": 4.810711843729758,
+      "tokens_seen": 98500608
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007839518555667001,
+      "loss": 2.0114,
+      "theoretical_loss": 4.810269890834364,
+      "tokens_seen": 98566144
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007839358074222668,
+      "loss": 2.0227,
+      "theoretical_loss": 4.809828313909129,
+      "tokens_seen": 98631680
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007839197592778335,
+      "loss": 1.9338,
+      "theoretical_loss": 4.809387112384721,
+      "tokens_seen": 98697216
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007839037111334002,
+      "loss": 1.736,
+      "theoretical_loss": 4.808946285693043,
+      "tokens_seen": 98762752
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007838876629889669,
+      "loss": 1.96,
+      "theoretical_loss": 4.808505833267237,
+      "tokens_seen": 98828288
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007838716148445336,
+      "loss": 2.0351,
+      "theoretical_loss": 4.808065754541676,
+      "tokens_seen": 98893824
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007838555667001003,
+      "loss": 2.1194,
+      "theoretical_loss": 4.807626048951965,
+      "tokens_seen": 98959360
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.000783839518555667,
+      "loss": 1.9028,
+      "theoretical_loss": 4.807186715934931,
+      "tokens_seen": 99024896
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007838234704112337,
+      "loss": 1.9041,
+      "theoretical_loss": 4.806747754928622,
+      "tokens_seen": 99090432
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007838074222668004,
+      "loss": 1.8704,
+      "theoretical_loss": 4.806309165372311,
+      "tokens_seen": 99155968
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007837913741223671,
+      "loss": 2.0163,
+      "theoretical_loss": 4.805870946706479,
+      "tokens_seen": 99221504
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007837753259779338,
+      "loss": 1.9428,
+      "theoretical_loss": 4.805433098372822,
+      "tokens_seen": 99287040
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007837592778335005,
+      "loss": 1.9543,
+      "theoretical_loss": 4.8049956198142425,
+      "tokens_seen": 99352576
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007837432296890672,
+      "loss": 1.9419,
+      "theoretical_loss": 4.804558510474852,
+      "tokens_seen": 99418112
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007837271815446339,
+      "loss": 1.9263,
+      "theoretical_loss": 4.804121769799959,
+      "tokens_seen": 99483648
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007837111334002006,
+      "loss": 1.9272,
+      "theoretical_loss": 4.8036853972360705,
+      "tokens_seen": 99549184
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007836950852557673,
+      "loss": 1.9411,
+      "theoretical_loss": 4.80324939223089,
+      "tokens_seen": 99614720
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.000783679037111334,
+      "loss": 1.8515,
+      "theoretical_loss": 4.802813754233311,
+      "tokens_seen": 99680256
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007836629889669007,
+      "loss": 2.0173,
+      "theoretical_loss": 4.802378482693417,
+      "tokens_seen": 99745792
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007836469408224674,
+      "loss": 1.9552,
+      "theoretical_loss": 4.801943577062472,
+      "tokens_seen": 99811328
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007836308926780341,
+      "loss": 1.9646,
+      "theoretical_loss": 4.801509036792925,
+      "tokens_seen": 99876864
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007836148445336008,
+      "loss": 1.9256,
+      "theoretical_loss": 4.801074861338401,
+      "tokens_seen": 99942400
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007835987963891675,
+      "loss": 1.9346,
+      "theoretical_loss": 4.800641050153703,
+      "tokens_seen": 100007936
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007835827482447342,
+      "loss": 1.9672,
+      "theoretical_loss": 4.800207602694801,
+      "tokens_seen": 100073472
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.000783566700100301,
+      "loss": 1.9447,
+      "theoretical_loss": 4.7997745184188325,
+      "tokens_seen": 100139008
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007835506519558676,
+      "loss": 1.9428,
+      "theoretical_loss": 4.799341796784106,
+      "tokens_seen": 100204544
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007835346038114343,
+      "loss": 1.9955,
+      "theoretical_loss": 4.798909437250086,
+      "tokens_seen": 100270080
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.000783518555667001,
+      "loss": 1.9506,
+      "theoretical_loss": 4.798477439277397,
+      "tokens_seen": 100335616
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007835025075225677,
+      "loss": 1.9319,
+      "theoretical_loss": 4.7980458023278185,
+      "tokens_seen": 100401152
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007834864593781344,
+      "loss": 1.8792,
+      "theoretical_loss": 4.797614525864282,
+      "tokens_seen": 100466688
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007834704112337011,
+      "loss": 1.9857,
+      "theoretical_loss": 4.797183609350869,
+      "tokens_seen": 100532224
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007834543630892678,
+      "loss": 1.936,
+      "theoretical_loss": 4.796753052252802,
+      "tokens_seen": 100597760
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007834383149448346,
+      "loss": 2.0787,
+      "theoretical_loss": 4.79632285403645,
+      "tokens_seen": 100663296
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007834222668004012,
+      "loss": 1.9013,
+      "theoretical_loss": 4.795893014169323,
+      "tokens_seen": 100728832
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.000783406218655968,
+      "loss": 1.9283,
+      "theoretical_loss": 4.795463532120058,
+      "tokens_seen": 100794368
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007833901705115346,
+      "loss": 1.8996,
+      "theoretical_loss": 4.795034407358435,
+      "tokens_seen": 100859904
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007833741223671014,
+      "loss": 1.8647,
+      "theoretical_loss": 4.794605639355357,
+      "tokens_seen": 100925440
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.000783358074222668,
+      "loss": 1.947,
+      "theoretical_loss": 4.794177227582855,
+      "tokens_seen": 100990976
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007833420260782348,
+      "loss": 1.7844,
+      "theoretical_loss": 4.793749171514085,
+      "tokens_seen": 101056512
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007833259779338014,
+      "loss": 2.0191,
+      "theoretical_loss": 4.793321470623321,
+      "tokens_seen": 101122048
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007833099297893682,
+      "loss": 1.9303,
+      "theoretical_loss": 4.792894124385955,
+      "tokens_seen": 101187584
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007832938816449348,
+      "loss": 1.9049,
+      "theoretical_loss": 4.792467132278494,
+      "tokens_seen": 101253120
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007832778335005016,
+      "loss": 1.9418,
+      "theoretical_loss": 4.792040493778553,
+      "tokens_seen": 101318656
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007832617853560682,
+      "loss": 1.8472,
+      "theoretical_loss": 4.791614208364859,
+      "tokens_seen": 101384192
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.000783245737211635,
+      "loss": 2.0587,
+      "theoretical_loss": 4.791188275517239,
+      "tokens_seen": 101449728
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007832296890672016,
+      "loss": 1.932,
+      "theoretical_loss": 4.7907626947166255,
+      "tokens_seen": 101515264
+    },
+    {
+      "epoch": 0.03,
+      "objective/train/docs_used": 40675,
+      "objective/train/instantaneous_batch_size": 32,
+      "objective/train/instantaneous_microbatch_size": 32768,
+      "objective/train/original_loss": 1.7645740509033203,
+      "objective/train/theoretical_loss": 4.790337465445049,
+      "objective/train/tokens_used": 122040800,
+      "theoretical_loss": 4.790337465445049,
+      "tokens_seen": 101580800
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007832136409227684,
+      "loss": 1.7755,
+      "theoretical_loss": 4.790337465445049,
+      "tokens_seen": 101580800
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.000783197592778335,
+      "loss": 1.9279,
+      "theoretical_loss": 4.7899125871856345,
+      "tokens_seen": 101646336
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007831815446339018,
+      "loss": 1.7947,
+      "theoretical_loss": 4.789488059422599,
+      "tokens_seen": 101711872
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007831654964894685,
+      "loss": 2.0236,
+      "theoretical_loss": 4.789063881641251,
+      "tokens_seen": 101777408
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007831494483450352,
+      "loss": 1.9807,
+      "theoretical_loss": 4.7886400533279865,
+      "tokens_seen": 101842944
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007831334002006019,
+      "loss": 1.9176,
+      "theoretical_loss": 4.788216573970281,
+      "tokens_seen": 101908480
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007831173520561686,
+      "loss": 1.9392,
+      "theoretical_loss": 4.787793443056694,
+      "tokens_seen": 101974016
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007831013039117353,
+      "loss": 1.9071,
+      "theoretical_loss": 4.787370660076862,
+      "tokens_seen": 102039552
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.000783085255767302,
+      "loss": 1.8304,
+      "theoretical_loss": 4.786948224521495,
+      "tokens_seen": 102105088
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007830692076228687,
+      "loss": 2.0425,
+      "theoretical_loss": 4.786526135882378,
+      "tokens_seen": 102170624
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007830531594784354,
+      "loss": 1.8017,
+      "theoretical_loss": 4.786104393652359,
+      "tokens_seen": 102236160
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007830371113340021,
+      "loss": 1.9887,
+      "theoretical_loss": 4.785682997325358,
+      "tokens_seen": 102301696
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007830210631895688,
+      "loss": 1.8868,
+      "theoretical_loss": 4.7852619463963535,
+      "tokens_seen": 102367232
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007830050150451355,
+      "loss": 2.0222,
+      "theoretical_loss": 4.784841240361389,
+      "tokens_seen": 102432768
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007829889669007022,
+      "loss": 1.9609,
+      "theoretical_loss": 4.784420878717558,
+      "tokens_seen": 102498304
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007829729187562689,
+      "loss": 2.0194,
+      "theoretical_loss": 4.7840008609630145,
+      "tokens_seen": 102563840
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007829568706118356,
+      "loss": 1.9684,
+      "theoretical_loss": 4.783581186596962,
+      "tokens_seen": 102629376
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007829408224674023,
+      "loss": 1.9301,
+      "theoretical_loss": 4.78316185511965,
+      "tokens_seen": 102694912
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.000782924774322969,
+      "loss": 1.9289,
+      "theoretical_loss": 4.782742866032379,
+      "tokens_seen": 102760448
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007829087261785357,
+      "loss": 1.8827,
+      "theoretical_loss": 4.782324218837486,
+      "tokens_seen": 102825984
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007828926780341024,
+      "loss": 1.9958,
+      "theoretical_loss": 4.781905913038351,
+      "tokens_seen": 102891520
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007828766298896691,
+      "loss": 1.9792,
+      "theoretical_loss": 4.781487948139393,
+      "tokens_seen": 102957056
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007828605817452358,
+      "loss": 1.8882,
+      "theoretical_loss": 4.7810703236460625,
+      "tokens_seen": 103022592
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007828445336008025,
+      "loss": 1.9826,
+      "theoretical_loss": 4.780653039064843,
+      "tokens_seen": 103088128
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007828284854563692,
+      "loss": 1.906,
+      "theoretical_loss": 4.780236093903245,
+      "tokens_seen": 103153664
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007828124373119358,
+      "loss": 1.9079,
+      "theoretical_loss": 4.779819487669805,
+      "tokens_seen": 103219200
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007827963891675026,
+      "loss": 1.8466,
+      "theoretical_loss": 4.779403219874085,
+      "tokens_seen": 103284736
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007827803410230692,
+      "loss": 1.9385,
+      "theoretical_loss": 4.778987290026665,
+      "tokens_seen": 103350272
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.000782764292878636,
+      "loss": 1.9471,
+      "theoretical_loss": 4.778571697639142,
+      "tokens_seen": 103415808
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007827482447342026,
+      "loss": 1.9158,
+      "theoretical_loss": 4.778156442224132,
+      "tokens_seen": 103481344
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007827321965897694,
+      "loss": 1.808,
+      "theoretical_loss": 4.7777415232952585,
+      "tokens_seen": 103546880
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.000782716148445336,
+      "loss": 1.9538,
+      "theoretical_loss": 4.777326940367155,
+      "tokens_seen": 103612416
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007827001003009028,
+      "loss": 2.0013,
+      "theoretical_loss": 4.776912692955463,
+      "tokens_seen": 103677952
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007826840521564694,
+      "loss": 1.8926,
+      "theoretical_loss": 4.776498780576826,
+      "tokens_seen": 103743488
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007826680040120361,
+      "loss": 1.8803,
+      "theoretical_loss": 4.776085202748893,
+      "tokens_seen": 103809024
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007826519558676028,
+      "loss": 1.8783,
+      "theoretical_loss": 4.775671958990305,
+      "tokens_seen": 103874560
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007826359077231695,
+      "loss": 1.9359,
+      "theoretical_loss": 4.775259048820702,
+      "tokens_seen": 103940096
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007826198595787362,
+      "loss": 1.7915,
+      "theoretical_loss": 4.7748464717607195,
+      "tokens_seen": 104005632
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007826038114343029,
+      "loss": 1.8613,
+      "theoretical_loss": 4.774434227331979,
+      "tokens_seen": 104071168
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007825877632898696,
+      "loss": 1.8973,
+      "theoretical_loss": 4.774022315057092,
+      "tokens_seen": 104136704
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007825717151454363,
+      "loss": 1.9505,
+      "theoretical_loss": 4.773610734459654,
+      "tokens_seen": 104202240
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.000782555667001003,
+      "loss": 1.9035,
+      "theoretical_loss": 4.773199485064243,
+      "tokens_seen": 104267776
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007825396188565697,
+      "loss": 2.0313,
+      "theoretical_loss": 4.772788566396418,
+      "tokens_seen": 104333312
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007825235707121364,
+      "loss": 1.9059,
+      "theoretical_loss": 4.772377977982714,
+      "tokens_seen": 104398848
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007825075225677031,
+      "loss": 1.9743,
+      "theoretical_loss": 4.771967719350641,
+      "tokens_seen": 104464384
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007824914744232698,
+      "loss": 1.9425,
+      "theoretical_loss": 4.77155779002868,
+      "tokens_seen": 104529920
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007824754262788365,
+      "loss": 2.0226,
+      "theoretical_loss": 4.771148189546282,
+      "tokens_seen": 104595456
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007824593781344032,
+      "loss": 1.9521,
+      "theoretical_loss": 4.770738917433864,
+      "tokens_seen": 104660992
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007824433299899699,
+      "loss": 1.951,
+      "theoretical_loss": 4.770329973222809,
+      "tokens_seen": 104726528
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007824272818455366,
+      "loss": 1.9467,
+      "theoretical_loss": 4.769921356445458,
+      "tokens_seen": 104792064
+    },
+    {
+      "epoch": 0.03,
+      "objective/train/docs_used": 41997,
+      "objective/train/instantaneous_batch_size": 32,
+      "objective/train/instantaneous_microbatch_size": 32768,
+      "objective/train/original_loss": 1.8928864002227783,
+      "objective/train/theoretical_loss": 4.769513066635114,
+      "objective/train/tokens_used": 125317600,
+      "theoretical_loss": 4.769513066635114,
+      "tokens_seen": 104857600
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007824112337011033,
+      "loss": 1.8939,
+      "theoretical_loss": 4.769513066635114,
+      "tokens_seen": 104857600
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.00078239518555667,
+      "loss": 1.9826,
+      "theoretical_loss": 4.769105103326036,
+      "tokens_seen": 104923136
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007823791374122367,
+      "loss": 1.9868,
+      "theoretical_loss": 4.768697466053435,
+      "tokens_seen": 104988672
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007823630892678034,
+      "loss": 1.8314,
+      "theoretical_loss": 4.768290154353474,
+      "tokens_seen": 105054208
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007823470411233701,
+      "loss": 2.0056,
+      "theoretical_loss": 4.767883167763267,
+      "tokens_seen": 105119744
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007823309929789368,
+      "loss": 1.8825,
+      "theoretical_loss": 4.76747650582087,
+      "tokens_seen": 105185280
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007823149448345035,
+      "loss": 1.9932,
+      "theoretical_loss": 4.767070168065285,
+      "tokens_seen": 105250816
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007822988966900702,
+      "loss": 2.0225,
+      "theoretical_loss": 4.766664154036456,
+      "tokens_seen": 105316352
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007822828485456369,
+      "loss": 1.9272,
+      "theoretical_loss": 4.766258463275265,
+      "tokens_seen": 105381888
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007822668004012036,
+      "loss": 1.8662,
+      "theoretical_loss": 4.7658530953235285,
+      "tokens_seen": 105447424
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007822507522567703,
+      "loss": 1.9259,
+      "theoretical_loss": 4.765448049723998,
+      "tokens_seen": 105512960
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.000782234704112337,
+      "loss": 2.0227,
+      "theoretical_loss": 4.765043326020358,
+      "tokens_seen": 105578496
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007822186559679037,
+      "loss": 1.8364,
+      "theoretical_loss": 4.764638923757217,
+      "tokens_seen": 105644032
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007822026078234704,
+      "loss": 1.9143,
+      "theoretical_loss": 4.764234842480114,
+      "tokens_seen": 105709568
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007821865596790371,
+      "loss": 1.9753,
+      "theoretical_loss": 4.7638310817355105,
+      "tokens_seen": 105775104
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007821705115346038,
+      "loss": 1.991,
+      "theoretical_loss": 4.763427641070789,
+      "tokens_seen": 105840640
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007821544633901705,
+      "loss": 1.9054,
+      "theoretical_loss": 4.76302452003425,
+      "tokens_seen": 105906176
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007821384152457372,
+      "loss": 1.9555,
+      "theoretical_loss": 4.7626217181751125,
+      "tokens_seen": 105971712
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007821223671013039,
+      "loss": 1.9969,
+      "theoretical_loss": 4.762219235043508,
+      "tokens_seen": 106037248
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007821063189568706,
+      "loss": 1.9379,
+      "theoretical_loss": 4.761817070190481,
+      "tokens_seen": 106102784
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007820902708124373,
+      "loss": 1.9104,
+      "theoretical_loss": 4.761415223167982,
+      "tokens_seen": 106168320
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.000782074222668004,
+      "loss": 2.0642,
+      "theoretical_loss": 4.761013693528874,
+      "tokens_seen": 106233856
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007820581745235707,
+      "loss": 1.9026,
+      "theoretical_loss": 4.760612480826917,
+      "tokens_seen": 106299392
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007820421263791375,
+      "loss": 1.809,
+      "theoretical_loss": 4.76021158461678,
+      "tokens_seen": 106364928
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007820260782347041,
+      "loss": 1.8463,
+      "theoretical_loss": 4.759811004454027,
+      "tokens_seen": 106430464
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007820100300902709,
+      "loss": 1.8905,
+      "theoretical_loss": 4.759410739895122,
+      "tokens_seen": 106496000
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007819939819458375,
+      "loss": 2.0088,
+      "theoretical_loss": 4.759010790497422,
+      "tokens_seen": 106561536
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007819779338014043,
+      "loss": 1.9447,
+      "theoretical_loss": 4.758611155819178,
+      "tokens_seen": 106627072
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.000781961885656971,
+      "loss": 1.9734,
+      "theoretical_loss": 4.758211835419531,
+      "tokens_seen": 106692608
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007819458375125377,
+      "loss": 1.9726,
+      "theoretical_loss": 4.757812828858508,
+      "tokens_seen": 106758144
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007819297893681044,
+      "loss": 1.7108,
+      "theoretical_loss": 4.757414135697024,
+      "tokens_seen": 106823680
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007819137412236711,
+      "loss": 1.9816,
+      "theoretical_loss": 4.757015755496877,
+      "tokens_seen": 106889216
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007818976930792378,
+      "loss": 1.8987,
+      "theoretical_loss": 4.756617687820743,
+      "tokens_seen": 106954752
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007818816449348045,
+      "loss": 1.8319,
+      "theoretical_loss": 4.756219932232181,
+      "tokens_seen": 107020288
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007818655967903712,
+      "loss": 1.8272,
+      "theoretical_loss": 4.755822488295621,
+      "tokens_seen": 107085824
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007818495486459379,
+      "loss": 1.9908,
+      "theoretical_loss": 4.755425355576373,
+      "tokens_seen": 107151360
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007818335005015046,
+      "loss": 2.0564,
+      "theoretical_loss": 4.755028533640614,
+      "tokens_seen": 107216896
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007818174523570713,
+      "loss": 1.933,
+      "theoretical_loss": 4.754632022055392,
+      "tokens_seen": 107282432
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.000781801404212638,
+      "loss": 1.8667,
+      "theoretical_loss": 4.754235820388624,
+      "tokens_seen": 107347968
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007817853560682047,
+      "loss": 2.0636,
+      "theoretical_loss": 4.7538399282090875,
+      "tokens_seen": 107413504
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007817693079237714,
+      "loss": 1.8853,
+      "theoretical_loss": 4.753444345086428,
+      "tokens_seen": 107479040
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007817532597793381,
+      "loss": 1.7999,
+      "theoretical_loss": 4.753049070591146,
+      "tokens_seen": 107544576
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007817372116349048,
+      "loss": 1.8508,
+      "theoretical_loss": 4.752654104294605,
+      "tokens_seen": 107610112
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007817211634904715,
+      "loss": 1.9497,
+      "theoretical_loss": 4.752259445769022,
+      "tokens_seen": 107675648
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007817051153460382,
+      "loss": 1.9291,
+      "theoretical_loss": 4.751865094587465,
+      "tokens_seen": 107741184
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007816890672016049,
+      "loss": 1.8917,
+      "theoretical_loss": 4.75147105032386,
+      "tokens_seen": 107806720
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007816730190571716,
+      "loss": 1.9627,
+      "theoretical_loss": 4.751077312552976,
+      "tokens_seen": 107872256
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007816569709127383,
+      "loss": 1.9369,
+      "theoretical_loss": 4.750683880850433,
+      "tokens_seen": 107937792
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.000781640922768305,
+      "loss": 1.9078,
+      "theoretical_loss": 4.750290754792694,
+      "tokens_seen": 108003328
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007816248746238717,
+      "loss": 2.0577,
+      "theoretical_loss": 4.749897933957065,
+      "tokens_seen": 108068864
+    },
+    {
+      "epoch": 0.03,
+      "objective/train/docs_used": 42847,
+      "objective/train/instantaneous_batch_size": 32,
+      "objective/train/instantaneous_microbatch_size": 32768,
+      "objective/train/original_loss": 2.111037254333496,
+      "objective/train/theoretical_loss": 4.749505417921692,
+      "objective/train/tokens_used": 128594400,
+      "theoretical_loss": 4.749505417921692,
+      "tokens_seen": 108134400
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007816088264794384,
+      "loss": 1.9969,
+      "theoretical_loss": 4.749505417921692,
+      "tokens_seen": 108134400
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007815927783350051,
+      "loss": 1.8179,
+      "theoretical_loss": 4.7491132062655605,
+      "tokens_seen": 108199936
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007815767301905718,
+      "loss": 1.8487,
+      "theoretical_loss": 4.748721298568491,
+      "tokens_seen": 108265472
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007815606820461385,
+      "loss": 1.9754,
+      "theoretical_loss": 4.748329694411137,
+      "tokens_seen": 108331008
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007815446339017052,
+      "loss": 1.7678,
+      "theoretical_loss": 4.747938393374987,
+      "tokens_seen": 108396544
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007815285857572719,
+      "loss": 1.8025,
+      "theoretical_loss": 4.747547395042355,
+      "tokens_seen": 108462080
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007815125376128386,
+      "loss": 1.9422,
+      "theoretical_loss": 4.747156698996388,
+      "tokens_seen": 108527616
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007814964894684053,
+      "loss": 1.8459,
+      "theoretical_loss": 4.746766304821053,
+      "tokens_seen": 108593152
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.000781480441323972,
+      "loss": 1.8893,
+      "theoretical_loss": 4.746376212101142,
+      "tokens_seen": 108658688
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007814643931795387,
+      "loss": 1.8665,
+      "theoretical_loss": 4.745986420422267,
+      "tokens_seen": 108724224
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007814483450351054,
+      "loss": 1.7588,
+      "theoretical_loss": 4.745596929370863,
+      "tokens_seen": 108789760
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007814322968906721,
+      "loss": 1.7836,
+      "theoretical_loss": 4.745207738534177,
+      "tokens_seen": 108855296
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007814162487462388,
+      "loss": 1.8065,
+      "theoretical_loss": 4.744818847500274,
+      "tokens_seen": 108920832
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007814002006018055,
+      "loss": 1.8137,
+      "theoretical_loss": 4.744430255858029,
+      "tokens_seen": 108986368
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007813841524573721,
+      "loss": 1.7859,
+      "theoretical_loss": 4.744041963197132,
+      "tokens_seen": 109051904
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007813681043129389,
+      "loss": 1.832,
+      "theoretical_loss": 4.743653969108076,
+      "tokens_seen": 109117440
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007813520561685055,
+      "loss": 1.8956,
+      "theoretical_loss": 4.743266273182163,
+      "tokens_seen": 109182976
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007813360080240723,
+      "loss": 1.7972,
+      "theoretical_loss": 4.742878875011501,
+      "tokens_seen": 109248512
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007813199598796389,
+      "loss": 1.8974,
+      "theoretical_loss": 4.742491774188997,
+      "tokens_seen": 109314048
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007813039117352057,
+      "loss": 1.9126,
+      "theoretical_loss": 4.742104970308359,
+      "tokens_seen": 109379584
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007812878635907723,
+      "loss": 1.9241,
+      "theoretical_loss": 4.741718462964095,
+      "tokens_seen": 109445120
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.000781271815446339,
+      "loss": 1.9692,
+      "theoretical_loss": 4.741332251751509,
+      "tokens_seen": 109510656
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007812557673019057,
+      "loss": 1.8587,
+      "theoretical_loss": 4.740946336266697,
+      "tokens_seen": 109576192
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007812397191574724,
+      "loss": 1.8201,
+      "theoretical_loss": 4.740560716106549,
+      "tokens_seen": 109641728
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007812236710130391,
+      "loss": 1.8934,
+      "theoretical_loss": 4.740175390868742,
+      "tokens_seen": 109707264
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007812076228686058,
+      "loss": 1.8603,
+      "theoretical_loss": 4.739790360151744,
+      "tokens_seen": 109772800
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007811915747241725,
+      "loss": 1.9489,
+      "theoretical_loss": 4.73940562355481,
+      "tokens_seen": 109838336
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007811755265797392,
+      "loss": 1.9317,
+      "theoretical_loss": 4.739021180677976,
+      "tokens_seen": 109903872
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007811594784353059,
+      "loss": 1.8175,
+      "theoretical_loss": 4.73863703112206,
+      "tokens_seen": 109969408
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007811434302908726,
+      "loss": 1.8667,
+      "theoretical_loss": 4.738253174488662,
+      "tokens_seen": 110034944
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007811273821464393,
+      "loss": 1.8779,
+      "theoretical_loss": 4.73786961038016,
+      "tokens_seen": 110100480
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.000781111334002006,
+      "loss": 1.9613,
+      "theoretical_loss": 4.737486338399704,
+      "tokens_seen": 110166016
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007810952858575727,
+      "loss": 1.8601,
+      "theoretical_loss": 4.737103358151225,
+      "tokens_seen": 110231552
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007810792377131394,
+      "loss": 1.9503,
+      "theoretical_loss": 4.7367206692394195,
+      "tokens_seen": 110297088
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007810631895687061,
+      "loss": 1.9074,
+      "theoretical_loss": 4.736338271269759,
+      "tokens_seen": 110362624
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007810471414242728,
+      "loss": 1.9128,
+      "theoretical_loss": 4.735956163848478,
+      "tokens_seen": 110428160
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007810310932798395,
+      "loss": 1.8355,
+      "theoretical_loss": 4.735574346582582,
+      "tokens_seen": 110493696
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007810150451354062,
+      "loss": 1.8716,
+      "theoretical_loss": 4.735192819079838,
+      "tokens_seen": 110559232
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007809989969909729,
+      "loss": 1.9584,
+      "theoretical_loss": 4.734811580948779,
+      "tokens_seen": 110624768
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007809829488465396,
+      "loss": 1.9324,
+      "theoretical_loss": 4.734430631798692,
+      "tokens_seen": 110690304
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007809669007021063,
+      "loss": 1.8178,
+      "theoretical_loss": 4.734049971239628,
+      "tokens_seen": 110755840
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.000780950852557673,
+      "loss": 1.9168,
+      "theoretical_loss": 4.7336695988823925,
+      "tokens_seen": 110821376
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007809348044132397,
+      "loss": 1.8893,
+      "theoretical_loss": 4.733289514338546,
+      "tokens_seen": 110886912
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007809187562688064,
+      "loss": 1.8867,
+      "theoretical_loss": 4.7329097172204,
+      "tokens_seen": 110952448
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007809027081243731,
+      "loss": 1.8564,
+      "theoretical_loss": 4.73253020714102,
+      "tokens_seen": 111017984
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007808866599799398,
+      "loss": 1.8978,
+      "theoretical_loss": 4.732150983714217,
+      "tokens_seen": 111083520
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007808706118355065,
+      "loss": 1.741,
+      "theoretical_loss": 4.731772046554552,
+      "tokens_seen": 111149056
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007808545636910732,
+      "loss": 1.8705,
+      "theoretical_loss": 4.731393395277331,
+      "tokens_seen": 111214592
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007808385155466399,
+      "loss": 1.7457,
+      "theoretical_loss": 4.731015029498598,
+      "tokens_seen": 111280128
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007808224674022066,
+      "loss": 1.9939,
+      "theoretical_loss": 4.730636948835148,
+      "tokens_seen": 111345664
+    },
+    {
+      "epoch": 0.03,
+      "objective/train/docs_used": 43986,
+      "objective/train/instantaneous_batch_size": 32,
+      "objective/train/instantaneous_microbatch_size": 32768,
+      "objective/train/original_loss": 2.0052099227905273,
+      "objective/train/theoretical_loss": 4.730259152904507,
+      "objective/train/tokens_used": 131871200,
+      "theoretical_loss": 4.730259152904507,
+      "tokens_seen": 111411200
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007808064192577733,
+      "loss": 1.9258,
+      "theoretical_loss": 4.730259152904507,
+      "tokens_seen": 111411200
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.00078079037111334,
+      "loss": 1.8683,
+      "theoretical_loss": 4.7298816413249405,
+      "tokens_seen": 111476736
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007807743229689067,
+      "loss": 1.9213,
+      "theoretical_loss": 4.729504413715454,
+      "tokens_seen": 111542272
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007807582748244735,
+      "loss": 1.8681,
+      "theoretical_loss": 4.729127469695783,
+      "tokens_seen": 111607808
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007807422266800401,
+      "loss": 1.8414,
+      "theoretical_loss": 4.728750808886394,
+      "tokens_seen": 111673344
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007807261785356069,
+      "loss": 1.9041,
+      "theoretical_loss": 4.72837443090849,
+      "tokens_seen": 111738880
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007807101303911736,
+      "loss": 1.811,
+      "theoretical_loss": 4.727998335383996,
+      "tokens_seen": 111804416
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007806940822467403,
+      "loss": 1.7952,
+      "theoretical_loss": 4.727622521935563,
+      "tokens_seen": 111869952
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.000780678034102307,
+      "loss": 1.9115,
+      "theoretical_loss": 4.7272469901865755,
+      "tokens_seen": 111935488
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007806619859578737,
+      "loss": 1.9802,
+      "theoretical_loss": 4.726871739761132,
+      "tokens_seen": 112001024
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007806459378134404,
+      "loss": 1.9238,
+      "theoretical_loss": 4.726496770284056,
+      "tokens_seen": 112066560
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007806298896690071,
+      "loss": 1.9923,
+      "theoretical_loss": 4.726122081380891,
+      "tokens_seen": 112132096
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007806138415245738,
+      "loss": 1.8348,
+      "theoretical_loss": 4.725747672677894,
+      "tokens_seen": 112197632
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007805977933801405,
+      "loss": 1.8965,
+      "theoretical_loss": 4.725373543802043,
+      "tokens_seen": 112263168
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007805817452357072,
+      "loss": 1.871,
+      "theoretical_loss": 4.724999694381027,
+      "tokens_seen": 112328704
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007805656970912739,
+      "loss": 1.8715,
+      "theoretical_loss": 4.724626124043247,
+      "tokens_seen": 112394240
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007805496489468406,
+      "loss": 1.8391,
+      "theoretical_loss": 4.724252832417817,
+      "tokens_seen": 112459776
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007805336008024073,
+      "loss": 1.9707,
+      "theoretical_loss": 4.723879819134558,
+      "tokens_seen": 112525312
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.000780517552657974,
+      "loss": 1.9352,
+      "theoretical_loss": 4.7235070838239945,
+      "tokens_seen": 112590848
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007805015045135407,
+      "loss": 1.7698,
+      "theoretical_loss": 4.723134626117363,
+      "tokens_seen": 112656384
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007804854563691074,
+      "loss": 1.9468,
+      "theoretical_loss": 4.722762445646598,
+      "tokens_seen": 112721920
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007804694082246741,
+      "loss": 1.9002,
+      "theoretical_loss": 4.722390542044339,
+      "tokens_seen": 112787456
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007804533600802408,
+      "loss": 1.8566,
+      "theoretical_loss": 4.722018914943924,
+      "tokens_seen": 112852992
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007804373119358075,
+      "loss": 1.8656,
+      "theoretical_loss": 4.721647563979385,
+      "tokens_seen": 112918528
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007804212637913742,
+      "loss": 1.8733,
+      "theoretical_loss": 4.72127648878546,
+      "tokens_seen": 112984064
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007804052156469409,
+      "loss": 1.8284,
+      "theoretical_loss": 4.720905688997572,
+      "tokens_seen": 113049600
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007803891675025076,
+      "loss": 1.8965,
+      "theoretical_loss": 4.720535164251842,
+      "tokens_seen": 113115136
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007803731193580743,
+      "loss": 1.8138,
+      "theoretical_loss": 4.720164914185084,
+      "tokens_seen": 113180672
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.000780357071213641,
+      "loss": 1.8462,
+      "theoretical_loss": 4.719794938434794,
+      "tokens_seen": 113246208
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007803410230692077,
+      "loss": 1.9124,
+      "theoretical_loss": 4.7194252366391645,
+      "tokens_seen": 113311744
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007803249749247744,
+      "loss": 1.8613,
+      "theoretical_loss": 4.719055808437068,
+      "tokens_seen": 113377280
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007803089267803411,
+      "loss": 1.8895,
+      "theoretical_loss": 4.718686653468065,
+      "tokens_seen": 113442816
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007802928786359078,
+      "loss": 1.883,
+      "theoretical_loss": 4.718317771372398,
+      "tokens_seen": 113508352
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007802768304914745,
+      "loss": 1.8539,
+      "theoretical_loss": 4.7179491617909886,
+      "tokens_seen": 113573888
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007802607823470412,
+      "loss": 1.7646,
+      "theoretical_loss": 4.717580824365439,
+      "tokens_seen": 113639424
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007802447342026079,
+      "loss": 1.8556,
+      "theoretical_loss": 4.71721275873803,
+      "tokens_seen": 113704960
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007802286860581746,
+      "loss": 1.8328,
+      "theoretical_loss": 4.716844964551717,
+      "tokens_seen": 113770496
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007802126379137413,
+      "loss": 1.8493,
+      "theoretical_loss": 4.716477441450131,
+      "tokens_seen": 113836032
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.000780196589769308,
+      "loss": 1.8481,
+      "theoretical_loss": 4.716110189077575,
+      "tokens_seen": 113901568
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007801805416248747,
+      "loss": 1.8687,
+      "theoretical_loss": 4.715743207079022,
+      "tokens_seen": 113967104
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007801644934804414,
+      "loss": 1.8578,
+      "theoretical_loss": 4.715376495100118,
+      "tokens_seen": 114032640
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007801484453360081,
+      "loss": 1.7646,
+      "theoretical_loss": 4.7150100527871714,
+      "tokens_seen": 114098176
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007801323971915748,
+      "loss": 1.8426,
+      "theoretical_loss": 4.714643879787161,
+      "tokens_seen": 114163712
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007801163490471415,
+      "loss": 1.9294,
+      "theoretical_loss": 4.714277975747729,
+      "tokens_seen": 114229248
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007801003009027082,
+      "loss": 1.7953,
+      "theoretical_loss": 4.7139123403171785,
+      "tokens_seen": 114294784
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007800842527582749,
+      "loss": 1.869,
+      "theoretical_loss": 4.713546973144476,
+      "tokens_seen": 114360320
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007800682046138416,
+      "loss": 1.9534,
+      "theoretical_loss": 4.713181873879248,
+      "tokens_seen": 114425856
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007800521564694083,
+      "loss": 1.9403,
+      "theoretical_loss": 4.712817042171776,
+      "tokens_seen": 114491392
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.000780036108324975,
+      "loss": 1.8532,
+      "theoretical_loss": 4.712452477673001,
+      "tokens_seen": 114556928
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007800200601805417,
+      "loss": 1.852,
+      "theoretical_loss": 4.712088180034517,
+      "tokens_seen": 114622464
+    },
+    {
+      "epoch": 0.03,
+      "objective/train/docs_used": 44900,
+      "objective/train/instantaneous_batch_size": 32,
+      "objective/train/instantaneous_microbatch_size": 32768,
+      "objective/train/original_loss": 2.0067858695983887,
+      "objective/train/theoretical_loss": 4.711724148908571,
+      "objective/train/tokens_used": 135148000,
+      "theoretical_loss": 4.711724148908571,
+      "tokens_seen": 114688000
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007800040120361084,
+      "loss": 1.9495,
+      "theoretical_loss": 4.711724148908571,
+      "tokens_seen": 114688000
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.000779987963891675,
+      "loss": 1.7656,
+      "theoretical_loss": 4.711360383948064,
+      "tokens_seen": 114753536
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007799719157472418,
+      "loss": 1.8776,
+      "theoretical_loss": 4.710996884806542,
+      "tokens_seen": 114819072
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007799558676028084,
+      "loss": 1.8349,
+      "theoretical_loss": 4.710633651138206,
+      "tokens_seen": 114884608
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007799398194583752,
+      "loss": 1.9441,
+      "theoretical_loss": 4.710270682597898,
+      "tokens_seen": 114950144
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007799237713139418,
+      "loss": 1.9338,
+      "theoretical_loss": 4.709907978841109,
+      "tokens_seen": 115015680
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007799077231695086,
+      "loss": 1.7752,
+      "theoretical_loss": 4.709545539523971,
+      "tokens_seen": 115081216
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007798916750250752,
+      "loss": 1.8558,
+      "theoretical_loss": 4.709183364303258,
+      "tokens_seen": 115146752
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.000779875626880642,
+      "loss": 1.8422,
+      "theoretical_loss": 4.708821452836388,
+      "tokens_seen": 115212288
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007798595787362086,
+      "loss": 1.8582,
+      "theoretical_loss": 4.708459804781414,
+      "tokens_seen": 115277824
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007798435305917753,
+      "loss": 1.9525,
+      "theoretical_loss": 4.708098419797031,
+      "tokens_seen": 115343360
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.000779827482447342,
+      "loss": 1.762,
+      "theoretical_loss": 4.707737297542563,
+      "tokens_seen": 115408896
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0007798114343029087,
+      "loss": 1.8618,
+      "theoretical_loss": 4.707376437677974,
+      "tokens_seen": 115474432
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007797953861584754,
+      "loss": 1.9417,
+      "theoretical_loss": 4.707015839863859,
+      "tokens_seen": 115539968
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007797793380140421,
+      "loss": 1.8223,
+      "theoretical_loss": 4.706655503761443,
+      "tokens_seen": 115605504
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007797632898696088,
+      "loss": 1.8191,
+      "theoretical_loss": 4.706295429032584,
+      "tokens_seen": 115671040
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007797472417251755,
+      "loss": 1.7398,
+      "theoretical_loss": 4.705935615339763,
+      "tokens_seen": 115736576
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007797311935807422,
+      "loss": 1.8098,
+      "theoretical_loss": 4.705576062346094,
+      "tokens_seen": 115802112
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007797151454363089,
+      "loss": 1.7037,
+      "theoretical_loss": 4.70521676971531,
+      "tokens_seen": 115867648
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007796990972918756,
+      "loss": 1.8367,
+      "theoretical_loss": 4.704857737111773,
+      "tokens_seen": 115933184
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007796830491474423,
+      "loss": 1.9333,
+      "theoretical_loss": 4.704498964200463,
+      "tokens_seen": 115998720
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.000779667001003009,
+      "loss": 1.8915,
+      "theoretical_loss": 4.70414045064698,
+      "tokens_seen": 116064256
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007796509528585757,
+      "loss": 1.9717,
+      "theoretical_loss": 4.703782196117549,
+      "tokens_seen": 116129792
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007796349047141424,
+      "loss": 1.8622,
+      "theoretical_loss": 4.703424200279007,
+      "tokens_seen": 116195328
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007796188565697091,
+      "loss": 1.7389,
+      "theoretical_loss": 4.703066462798809,
+      "tokens_seen": 116260864
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007796028084252758,
+      "loss": 1.7881,
+      "theoretical_loss": 4.7027089833450235,
+      "tokens_seen": 116326400
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007795867602808425,
+      "loss": 1.7667,
+      "theoretical_loss": 4.7023517615863355,
+      "tokens_seen": 116391936
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007795707121364092,
+      "loss": 1.8272,
+      "theoretical_loss": 4.701994797192038,
+      "tokens_seen": 116457472
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.000779554663991976,
+      "loss": 1.9811,
+      "theoretical_loss": 4.701638089832034,
+      "tokens_seen": 116523008
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007795386158475426,
+      "loss": 1.8322,
+      "theoretical_loss": 4.701281639176839,
+      "tokens_seen": 116588544
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007795225677031094,
+      "loss": 1.9583,
+      "theoretical_loss": 4.700925444897575,
+      "tokens_seen": 116654080
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.000779506519558676,
+      "loss": 1.795,
+      "theoretical_loss": 4.700569506665965,
+      "tokens_seen": 116719616
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007794904714142428,
+      "loss": 1.8679,
+      "theoretical_loss": 4.700213824154341,
+      "tokens_seen": 116785152
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007794744232698094,
+      "loss": 1.8274,
+      "theoretical_loss": 4.699858397035637,
+      "tokens_seen": 116850688
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007794583751253762,
+      "loss": 1.834,
+      "theoretical_loss": 4.699503224983388,
+      "tokens_seen": 116916224
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007794423269809428,
+      "loss": 1.9068,
+      "theoretical_loss": 4.699148307671729,
+      "tokens_seen": 116981760
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007794262788365096,
+      "loss": 1.8584,
+      "theoretical_loss": 4.698793644775394,
+      "tokens_seen": 117047296
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007794102306920762,
+      "loss": 1.7628,
+      "theoretical_loss": 4.698439235969713,
+      "tokens_seen": 117112832
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.000779394182547643,
+      "loss": 1.7451,
+      "theoretical_loss": 4.698085080930612,
+      "tokens_seen": 117178368
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007793781344032096,
+      "loss": 1.7782,
+      "theoretical_loss": 4.697731179334614,
+      "tokens_seen": 117243904
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007793620862587764,
+      "loss": 1.9503,
+      "theoretical_loss": 4.69737753085883,
+      "tokens_seen": 117309440
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.000779346038114343,
+      "loss": 1.7033,
+      "theoretical_loss": 4.697024135180968,
+      "tokens_seen": 117374976
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007793299899699098,
+      "loss": 1.8659,
+      "theoretical_loss": 4.696670991979321,
+      "tokens_seen": 117440512
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007793139418254765,
+      "loss": 1.9028,
+      "theoretical_loss": 4.696318100932773,
+      "tokens_seen": 117506048
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007792978936810432,
+      "loss": 1.797,
+      "theoretical_loss": 4.695965461720796,
+      "tokens_seen": 117571584
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007792818455366099,
+      "loss": 1.804,
+      "theoretical_loss": 4.695613074023446,
+      "tokens_seen": 117637120
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007792657973921766,
+      "loss": 1.906,
+      "theoretical_loss": 4.695260937521364,
+      "tokens_seen": 117702656
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007792497492477433,
+      "loss": 1.7889,
+      "theoretical_loss": 4.694909051895775,
+      "tokens_seen": 117768192
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00077923370110331,
+      "loss": 1.8768,
+      "theoretical_loss": 4.694557416828484,
+      "tokens_seen": 117833728
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007792176529588767,
+      "loss": 1.8065,
+      "theoretical_loss": 4.694206032001879,
+      "tokens_seen": 117899264
+    },
+    {
+      "epoch": 0.04,
+      "objective/train/docs_used": 46098,
+      "objective/train/instantaneous_batch_size": 32,
+      "objective/train/instantaneous_microbatch_size": 32768,
+      "objective/train/original_loss": 1.9403468370437622,
+      "objective/train/theoretical_loss": 4.693854897098923,
+      "objective/train/tokens_used": 138424800,
+      "theoretical_loss": 4.693854897098923,
+      "tokens_seen": 117964800
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007792016048144434,
+      "loss": 1.8638,
+      "theoretical_loss": 4.693854897098923,
+      "tokens_seen": 117964800
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007791855566700101,
+      "loss": 1.7358,
+      "theoretical_loss": 4.693504011803159,
+      "tokens_seen": 118030336
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007791695085255768,
+      "loss": 1.8575,
+      "theoretical_loss": 4.6931533757987065,
+      "tokens_seen": 118095872
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007791534603811435,
+      "loss": 1.8339,
+      "theoretical_loss": 4.692802988770258,
+      "tokens_seen": 118161408
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007791374122367102,
+      "loss": 1.7278,
+      "theoretical_loss": 4.692452850403081,
+      "tokens_seen": 118226944
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007791213640922769,
+      "loss": 1.8402,
+      "theoretical_loss": 4.692102960383014,
+      "tokens_seen": 118292480
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007791053159478436,
+      "loss": 1.8852,
+      "theoretical_loss": 4.691753318396467,
+      "tokens_seen": 118358016
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007790892678034103,
+      "loss": 1.7678,
+      "theoretical_loss": 4.691403924130418,
+      "tokens_seen": 118423552
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.000779073219658977,
+      "loss": 1.8664,
+      "theoretical_loss": 4.691054777272415,
+      "tokens_seen": 118489088
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007790571715145437,
+      "loss": 1.9721,
+      "theoretical_loss": 4.69070587751057,
+      "tokens_seen": 118554624
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007790411233701104,
+      "loss": 1.8822,
+      "theoretical_loss": 4.690357224533564,
+      "tokens_seen": 118620160
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007790250752256771,
+      "loss": 1.8383,
+      "theoretical_loss": 4.690008818030638,
+      "tokens_seen": 118685696
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007790090270812438,
+      "loss": 1.9328,
+      "theoretical_loss": 4.689660657691598,
+      "tokens_seen": 118751232
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007789929789368105,
+      "loss": 1.7851,
+      "theoretical_loss": 4.689312743206811,
+      "tokens_seen": 118816768
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007789769307923772,
+      "loss": 1.8794,
+      "theoretical_loss": 4.688965074267203,
+      "tokens_seen": 118882304
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007789608826479439,
+      "loss": 1.8132,
+      "theoretical_loss": 4.688617650564259,
+      "tokens_seen": 118947840
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007789448345035106,
+      "loss": 1.8665,
+      "theoretical_loss": 4.688270471790023,
+      "tokens_seen": 119013376
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007789287863590773,
+      "loss": 1.784,
+      "theoretical_loss": 4.687923537637094,
+      "tokens_seen": 119078912
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.000778912738214644,
+      "loss": 1.8039,
+      "theoretical_loss": 4.687576847798622,
+      "tokens_seen": 119144448
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007788966900702107,
+      "loss": 1.9227,
+      "theoretical_loss": 4.687230401968319,
+      "tokens_seen": 119209984
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007788806419257774,
+      "loss": 1.825,
+      "theoretical_loss": 4.68688419984044,
+      "tokens_seen": 119275520
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007788645937813441,
+      "loss": 1.7368,
+      "theoretical_loss": 4.686538241109796,
+      "tokens_seen": 119341056
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007788485456369108,
+      "loss": 1.9235,
+      "theoretical_loss": 4.686192525471746,
+      "tokens_seen": 119406592
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007788324974924775,
+      "loss": 1.9492,
+      "theoretical_loss": 4.685847052622199,
+      "tokens_seen": 119472128
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007788164493480442,
+      "loss": 1.7735,
+      "theoretical_loss": 4.685501822257608,
+      "tokens_seen": 119537664
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007788004012036109,
+      "loss": 1.9409,
+      "theoretical_loss": 4.685156834074972,
+      "tokens_seen": 119603200
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007787843530591776,
+      "loss": 1.8524,
+      "theoretical_loss": 4.684812087771839,
+      "tokens_seen": 119668736
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007787683049147443,
+      "loss": 1.8636,
+      "theoretical_loss": 4.684467583046292,
+      "tokens_seen": 119734272
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.000778752256770311,
+      "loss": 1.7902,
+      "theoretical_loss": 4.6841233195969645,
+      "tokens_seen": 119799808
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007787362086258777,
+      "loss": 1.8824,
+      "theoretical_loss": 4.683779297123024,
+      "tokens_seen": 119865344
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007787201604814444,
+      "loss": 1.9832,
+      "theoretical_loss": 4.683435515324181,
+      "tokens_seen": 119930880
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007787041123370111,
+      "loss": 1.7237,
+      "theoretical_loss": 4.683091973900682,
+      "tokens_seen": 119996416
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007786880641925778,
+      "loss": 1.7697,
+      "theoretical_loss": 4.682748672553313,
+      "tokens_seen": 120061952
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007786720160481445,
+      "loss": 1.8179,
+      "theoretical_loss": 4.68240561098339,
+      "tokens_seen": 120127488
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007786559679037112,
+      "loss": 1.9435,
+      "theoretical_loss": 4.68206278889277,
+      "tokens_seen": 120193024
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007786399197592779,
+      "loss": 1.785,
+      "theoretical_loss": 4.681720205983839,
+      "tokens_seen": 120258560
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007786238716148446,
+      "loss": 1.7739,
+      "theoretical_loss": 4.681377861959516,
+      "tokens_seen": 120324096
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007786078234704113,
+      "loss": 1.8369,
+      "theoretical_loss": 4.681035756523249,
+      "tokens_seen": 120389632
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007785917753259779,
+      "loss": 1.8283,
+      "theoretical_loss": 4.680693889379018,
+      "tokens_seen": 120455168
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007785757271815447,
+      "loss": 1.8239,
+      "theoretical_loss": 4.680352260231329,
+      "tokens_seen": 120520704
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007785596790371113,
+      "loss": 1.8973,
+      "theoretical_loss": 4.680010868785216,
+      "tokens_seen": 120586240
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007785436308926781,
+      "loss": 1.8416,
+      "theoretical_loss": 4.679669714746238,
+      "tokens_seen": 120651776
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007785275827482447,
+      "loss": 1.8488,
+      "theoretical_loss": 4.679328797820478,
+      "tokens_seen": 120717312
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007785115346038115,
+      "loss": 1.7723,
+      "theoretical_loss": 4.678988117714544,
+      "tokens_seen": 120782848
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007784954864593781,
+      "loss": 1.7675,
+      "theoretical_loss": 4.678647674135563,
+      "tokens_seen": 120848384
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007784794383149449,
+      "loss": 1.7905,
+      "theoretical_loss": 4.6783074667911855,
+      "tokens_seen": 120913920
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007784633901705115,
+      "loss": 1.7957,
+      "theoretical_loss": 4.677967495389581,
+      "tokens_seen": 120979456
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007784473420260782,
+      "loss": 1.8591,
+      "theoretical_loss": 4.677627759639435,
+      "tokens_seen": 121044992
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007784312938816449,
+      "loss": 1.6745,
+      "theoretical_loss": 4.677288259249954,
+      "tokens_seen": 121110528
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007784152457372116,
+      "loss": 1.9251,
+      "theoretical_loss": 4.676948993930857,
+      "tokens_seen": 121176064
+    },
+    {
+      "epoch": 0.04,
+      "objective/train/docs_used": 47228,
+      "objective/train/instantaneous_batch_size": 32,
+      "objective/train/instantaneous_microbatch_size": 32768,
+      "objective/train/original_loss": 1.9976223707199097,
+      "objective/train/theoretical_loss": 4.67660996339238,
+      "objective/train/tokens_used": 141701600,
+      "theoretical_loss": 4.67660996339238,
+      "tokens_seen": 121241600
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007783991975927783,
+      "loss": 1.9199,
+      "theoretical_loss": 4.67660996339238,
+      "tokens_seen": 121241600
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.000778383149448345,
+      "loss": 1.7332,
+      "theoretical_loss": 4.676271167345272,
+      "tokens_seen": 121307136
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007783671013039117,
+      "loss": 1.8232,
+      "theoretical_loss": 4.675932605500794,
+      "tokens_seen": 121372672
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007783510531594785,
+      "loss": 1.8272,
+      "theoretical_loss": 4.675594277570718,
+      "tokens_seen": 121438208
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007783350050150451,
+      "loss": 1.8456,
+      "theoretical_loss": 4.675256183267327,
+      "tokens_seen": 121503744
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007783189568706119,
+      "loss": 1.753,
+      "theoretical_loss": 4.674918322303411,
+      "tokens_seen": 121569280
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007783029087261785,
+      "loss": 1.9959,
+      "theoretical_loss": 4.6745806943922705,
+      "tokens_seen": 121634816
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007782868605817453,
+      "loss": 1.8123,
+      "theoretical_loss": 4.67424329924771,
+      "tokens_seen": 121700352
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007782708124373119,
+      "loss": 1.8314,
+      "theoretical_loss": 4.673906136584041,
+      "tokens_seen": 121765888
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007782547642928787,
+      "loss": 1.9434,
+      "theoretical_loss": 4.673569206116078,
+      "tokens_seen": 121831424
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007782387161484453,
+      "loss": 1.7416,
+      "theoretical_loss": 4.6732325075591366,
+      "tokens_seen": 121896960
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007782226680040121,
+      "loss": 1.741,
+      "theoretical_loss": 4.67289604062904,
+      "tokens_seen": 121962496
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007782066198595787,
+      "loss": 1.9697,
+      "theoretical_loss": 4.672559805042107,
+      "tokens_seen": 122028032
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007781905717151455,
+      "loss": 1.8553,
+      "theoretical_loss": 4.672223800515159,
+      "tokens_seen": 122093568
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007781745235707121,
+      "loss": 1.8653,
+      "theoretical_loss": 4.671888026765512,
+      "tokens_seen": 122159104
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007781584754262789,
+      "loss": 1.919,
+      "theoretical_loss": 4.6715524835109825,
+      "tokens_seen": 122224640
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007781424272818455,
+      "loss": 1.7892,
+      "theoretical_loss": 4.671217170469884,
+      "tokens_seen": 122290176
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007781263791374123,
+      "loss": 1.807,
+      "theoretical_loss": 4.67088208736102,
+      "tokens_seen": 122355712
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007781103309929789,
+      "loss": 1.8688,
+      "theoretical_loss": 4.670547233903694,
+      "tokens_seen": 122421248
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007780942828485457,
+      "loss": 1.9215,
+      "theoretical_loss": 4.670212609817698,
+      "tokens_seen": 122486784
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007780782347041123,
+      "loss": 1.8097,
+      "theoretical_loss": 4.669878214823316,
+      "tokens_seen": 122552320
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007780621865596791,
+      "loss": 1.8815,
+      "theoretical_loss": 4.669544048641325,
+      "tokens_seen": 122617856
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007780461384152457,
+      "loss": 1.8586,
+      "theoretical_loss": 4.669210110992989,
+      "tokens_seen": 122683392
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007780300902708125,
+      "loss": 1.8749,
+      "theoretical_loss": 4.668876401600059,
+      "tokens_seen": 122748928
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007780140421263791,
+      "loss": 1.8212,
+      "theoretical_loss": 4.668542920184779,
+      "tokens_seen": 122814464
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007779979939819459,
+      "loss": 1.7877,
+      "theoretical_loss": 4.6682096664698705,
+      "tokens_seen": 122880000
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007779819458375126,
+      "loss": 1.893,
+      "theoretical_loss": 4.667876640178546,
+      "tokens_seen": 122945536
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007779658976930793,
+      "loss": 1.7865,
+      "theoretical_loss": 4.667543841034502,
+      "tokens_seen": 123011072
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.000777949849548646,
+      "loss": 1.8616,
+      "theoretical_loss": 4.667211268761914,
+      "tokens_seen": 123076608
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007779338014042127,
+      "loss": 1.8971,
+      "theoretical_loss": 4.66687892308544,
+      "tokens_seen": 123142144
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007779177532597794,
+      "loss": 1.7553,
+      "theoretical_loss": 4.6665468037302205,
+      "tokens_seen": 123207680
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007779017051153461,
+      "loss": 1.8428,
+      "theoretical_loss": 4.666214910421875,
+      "tokens_seen": 123273216
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007778856569709128,
+      "loss": 1.7548,
+      "theoretical_loss": 4.665883242886498,
+      "tokens_seen": 123338752
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007778696088264795,
+      "loss": 1.8123,
+      "theoretical_loss": 4.6655518008506665,
+      "tokens_seen": 123404288
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007778535606820462,
+      "loss": 1.8484,
+      "theoretical_loss": 4.665220584041428,
+      "tokens_seen": 123469824
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007778375125376129,
+      "loss": 1.8273,
+      "theoretical_loss": 4.664889592186309,
+      "tokens_seen": 123535360
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007778214643931796,
+      "loss": 1.7669,
+      "theoretical_loss": 4.6645588250133105,
+      "tokens_seen": 123600896
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007778054162487463,
+      "loss": 1.7995,
+      "theoretical_loss": 4.664228282250902,
+      "tokens_seen": 123666432
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.000777789368104313,
+      "loss": 1.9218,
+      "theoretical_loss": 4.663897963628029,
+      "tokens_seen": 123731968
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007777733199598797,
+      "loss": 1.8008,
+      "theoretical_loss": 4.6635678688741065,
+      "tokens_seen": 123797504
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007777572718154464,
+      "loss": 1.8049,
+      "theoretical_loss": 4.663237997719017,
+      "tokens_seen": 123863040
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007777412236710131,
+      "loss": 1.9144,
+      "theoretical_loss": 4.662908349893116,
+      "tokens_seen": 123928576
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007777251755265798,
+      "loss": 1.7767,
+      "theoretical_loss": 4.662578925127223,
+      "tokens_seen": 123994112
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007777091273821465,
+      "loss": 1.943,
+      "theoretical_loss": 4.662249723152624,
+      "tokens_seen": 124059648
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007776930792377132,
+      "loss": 1.81,
+      "theoretical_loss": 4.661920743701073,
+      "tokens_seen": 124125184
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007776770310932799,
+      "loss": 1.9506,
+      "theoretical_loss": 4.661591986504786,
+      "tokens_seen": 124190720
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007776609829488466,
+      "loss": 1.7081,
+      "theoretical_loss": 4.661263451296444,
+      "tokens_seen": 124256256
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007776449348044133,
+      "loss": 1.8282,
+      "theoretical_loss": 4.660935137809188,
+      "tokens_seen": 124321792
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00077762888665998,
+      "loss": 1.8308,
+      "theoretical_loss": 4.660607045776624,
+      "tokens_seen": 124387328
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007776128385155467,
+      "loss": 1.7525,
+      "theoretical_loss": 4.660279174932814,
+      "tokens_seen": 124452864
+    },
+    {
+      "epoch": 0.04,
+      "objective/train/docs_used": 48149,
+      "objective/train/instantaneous_batch_size": 32,
+      "objective/train/instantaneous_microbatch_size": 32768,
+      "objective/train/original_loss": 1.6645214557647705,
+      "objective/train/theoretical_loss": 4.659951525012284,
+      "objective/train/tokens_used": 144978400,
+      "theoretical_loss": 4.659951525012284,
+      "tokens_seen": 124518400
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007775967903711134,
+      "loss": 1.7816,
+      "theoretical_loss": 4.659951525012284,
+      "tokens_seen": 124518400
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007775807422266801,
+      "loss": 1.7902,
+      "theoretical_loss": 4.6596240957500115,
+      "tokens_seen": 124583936
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007775646940822468,
+      "loss": 1.7569,
+      "theoretical_loss": 4.659296886881438,
+      "tokens_seen": 124649472
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007775486459378135,
+      "loss": 1.8885,
+      "theoretical_loss": 4.658969898142456,
+      "tokens_seen": 124715008
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007775325977933802,
+      "loss": 1.8672,
+      "theoretical_loss": 4.658643129269416,
+      "tokens_seen": 124780544
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007775165496489469,
+      "loss": 1.8886,
+      "theoretical_loss": 4.658316579999122,
+      "tokens_seen": 124846080
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007775005015045136,
+      "loss": 1.8843,
+      "theoretical_loss": 4.65799025006883,
+      "tokens_seen": 124911616
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007774844533600803,
+      "loss": 1.8544,
+      "theoretical_loss": 4.65766413921625,
+      "tokens_seen": 124977152
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.000777468405215647,
+      "loss": 1.7525,
+      "theoretical_loss": 4.657338247179538,
+      "tokens_seen": 125042688
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007774523570712137,
+      "loss": 1.8075,
+      "theoretical_loss": 4.657012573697307,
+      "tokens_seen": 125108224
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007774363089267804,
+      "loss": 1.7625,
+      "theoretical_loss": 4.656687118508616,
+      "tokens_seen": 125173760
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007774202607823471,
+      "loss": 1.8404,
+      "theoretical_loss": 4.656361881352968,
+      "tokens_seen": 125239296
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007774042126379138,
+      "loss": 1.8472,
+      "theoretical_loss": 4.656036861970319,
+      "tokens_seen": 125304832
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007773881644934805,
+      "loss": 1.7599,
+      "theoretical_loss": 4.655712060101068,
+      "tokens_seen": 125370368
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007773721163490472,
+      "loss": 1.7566,
+      "theoretical_loss": 4.65538747548606,
+      "tokens_seen": 125435904
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007773560682046139,
+      "loss": 1.7796,
+      "theoretical_loss": 4.655063107866582,
+      "tokens_seen": 125501440
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007773400200601806,
+      "loss": 1.8812,
+      "theoretical_loss": 4.654738956984366,
+      "tokens_seen": 125566976
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007773239719157473,
+      "loss": 1.8837,
+      "theoretical_loss": 4.654415022581585,
+      "tokens_seen": 125632512
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.000777307923771314,
+      "loss": 1.8805,
+      "theoretical_loss": 4.654091304400853,
+      "tokens_seen": 125698048
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007772918756268807,
+      "loss": 1.8101,
+      "theoretical_loss": 4.653767802185225,
+      "tokens_seen": 125763584
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007772758274824474,
+      "loss": 1.8291,
+      "theoretical_loss": 4.653444515678195,
+      "tokens_seen": 125829120
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007772597793380141,
+      "loss": 1.8134,
+      "theoretical_loss": 4.653121444623693,
+      "tokens_seen": 125894656
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007772437311935808,
+      "loss": 1.9031,
+      "theoretical_loss": 4.652798588766088,
+      "tokens_seen": 125960192
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007772276830491475,
+      "loss": 1.7734,
+      "theoretical_loss": 4.652475947850185,
+      "tokens_seen": 126025728
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007772116349047142,
+      "loss": 1.8418,
+      "theoretical_loss": 4.652153521621223,
+      "tokens_seen": 126091264
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007771955867602808,
+      "loss": 1.7352,
+      "theoretical_loss": 4.651831309824875,
+      "tokens_seen": 126156800
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007771795386158476,
+      "loss": 1.9202,
+      "theoretical_loss": 4.65150931220725,
+      "tokens_seen": 126222336
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007771634904714142,
+      "loss": 1.8301,
+      "theoretical_loss": 4.651187528514887,
+      "tokens_seen": 126287872
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.000777147442326981,
+      "loss": 1.7024,
+      "theoretical_loss": 4.650865958494755,
+      "tokens_seen": 126353408
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007771313941825476,
+      "loss": 1.8496,
+      "theoretical_loss": 4.650544601894257,
+      "tokens_seen": 126418944
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007771153460381144,
+      "loss": 1.9233,
+      "theoretical_loss": 4.650223458461221,
+      "tokens_seen": 126484480
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.000777099297893681,
+      "loss": 1.9058,
+      "theoretical_loss": 4.649902527943908,
+      "tokens_seen": 126550016
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007770832497492478,
+      "loss": 1.7415,
+      "theoretical_loss": 4.649581810091003,
+      "tokens_seen": 126615552
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007770672016048144,
+      "loss": 1.8208,
+      "theoretical_loss": 4.649261304651619,
+      "tokens_seen": 126681088
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007770511534603812,
+      "loss": 1.7909,
+      "theoretical_loss": 4.6489410113752925,
+      "tokens_seen": 126746624
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007770351053159478,
+      "loss": 1.8102,
+      "theoretical_loss": 4.6486209300119885,
+      "tokens_seen": 126812160
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007770190571715146,
+      "loss": 1.8493,
+      "theoretical_loss": 4.648301060312093,
+      "tokens_seen": 126877696
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007770030090270812,
+      "loss": 1.7896,
+      "theoretical_loss": 4.647981402026417,
+      "tokens_seen": 126943232
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.000776986960882648,
+      "loss": 1.8832,
+      "theoretical_loss": 4.647661954906188,
+      "tokens_seen": 127008768
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007769709127382146,
+      "loss": 1.7515,
+      "theoretical_loss": 4.647342718703062,
+      "tokens_seen": 127074304
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007769548645937814,
+      "loss": 1.9025,
+      "theoretical_loss": 4.647023693169107,
+      "tokens_seen": 127139840
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.000776938816449348,
+      "loss": 1.763,
+      "theoretical_loss": 4.646704878056816,
+      "tokens_seen": 127205376
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007769227683049148,
+      "loss": 1.7669,
+      "theoretical_loss": 4.6463862731191,
+      "tokens_seen": 127270912
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007769067201604814,
+      "loss": 1.8073,
+      "theoretical_loss": 4.6460678781092835,
+      "tokens_seen": 127336448
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007768906720160482,
+      "loss": 1.8357,
+      "theoretical_loss": 4.6457496927811075,
+      "tokens_seen": 127401984
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007768746238716148,
+      "loss": 1.7812,
+      "theoretical_loss": 4.645431716888734,
+      "tokens_seen": 127467520
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007768585757271816,
+      "loss": 1.7667,
+      "theoretical_loss": 4.645113950186732,
+      "tokens_seen": 127533056
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007768425275827482,
+      "loss": 1.9436,
+      "theoretical_loss": 4.64479639243009,
+      "tokens_seen": 127598592
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.000776826479438315,
+      "loss": 1.8954,
+      "theoretical_loss": 4.644479043374204,
+      "tokens_seen": 127664128
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007768104312938816,
+      "loss": 1.7894,
+      "theoretical_loss": 4.644161902774886,
+      "tokens_seen": 127729664
+    },
+    {
+      "epoch": 0.04,
+      "objective/train/docs_used": 49397,
+      "objective/train/instantaneous_batch_size": 32,
+      "objective/train/instantaneous_microbatch_size": 32768,
+      "objective/train/original_loss": 1.994436264038086,
+      "objective/train/theoretical_loss": 4.643844970388358,
+      "objective/train/tokens_used": 148255200,
+      "theoretical_loss": 4.643844970388358,
+      "tokens_seen": 127795200
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007767943831494484,
+      "loss": 1.9433,
+      "theoretical_loss": 4.643844970388358,
+      "tokens_seen": 127795200
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.000776778335005015,
+      "loss": 1.8353,
+      "theoretical_loss": 4.6435282459712495,
+      "tokens_seen": 127860736
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007767622868605818,
+      "loss": 1.8332,
+      "theoretical_loss": 4.643211729280601,
+      "tokens_seen": 127926272
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007767462387161484,
+      "loss": 1.9271,
+      "theoretical_loss": 4.64289542007386,
+      "tokens_seen": 127991808
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007767301905717152,
+      "loss": 1.8337,
+      "theoretical_loss": 4.642579318108883,
+      "tokens_seen": 128057344
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007767141424272818,
+      "loss": 1.8747,
+      "theoretical_loss": 4.642263423143932,
+      "tokens_seen": 128122880
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007766980942828486,
+      "loss": 1.8821,
+      "theoretical_loss": 4.641947734937673,
+      "tokens_seen": 128188416
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007766820461384152,
+      "loss": 1.8517,
+      "theoretical_loss": 4.641632253249179,
+      "tokens_seen": 128253952
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.000776665997993982,
+      "loss": 1.7698,
+      "theoretical_loss": 4.641316977837924,
+      "tokens_seen": 128319488
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007766499498495486,
+      "loss": 1.8586,
+      "theoretical_loss": 4.641001908463787,
+      "tokens_seen": 128385024
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007766339017051154,
+      "loss": 1.7673,
+      "theoretical_loss": 4.6406870448870485,
+      "tokens_seen": 128450560
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.000776617853560682,
+      "loss": 1.8098,
+      "theoretical_loss": 4.64037238686839,
+      "tokens_seen": 128516096
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007766018054162488,
+      "loss": 1.8943,
+      "theoretical_loss": 4.640057934168891,
+      "tokens_seen": 128581632
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007765857572718155,
+      "loss": 1.8456,
+      "theoretical_loss": 4.639743686550034,
+      "tokens_seen": 128647168
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007765697091273822,
+      "loss": 1.96,
+      "theoretical_loss": 4.639429643773697,
+      "tokens_seen": 128712704
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007765536609829489,
+      "loss": 1.9007,
+      "theoretical_loss": 4.639115805602156,
+      "tokens_seen": 128778240
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007765376128385156,
+      "loss": 1.9059,
+      "theoretical_loss": 4.638802171798086,
+      "tokens_seen": 128843776
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007765215646940823,
+      "loss": 1.7999,
+      "theoretical_loss": 4.638488742124554,
+      "tokens_seen": 128909312
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.000776505516549649,
+      "loss": 1.7901,
+      "theoretical_loss": 4.638175516345027,
+      "tokens_seen": 128974848
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007764894684052157,
+      "loss": 1.8626,
+      "theoretical_loss": 4.637862494223359,
+      "tokens_seen": 129040384
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007764734202607824,
+      "loss": 1.8844,
+      "theoretical_loss": 4.637549675523806,
+      "tokens_seen": 129105920
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007764573721163491,
+      "loss": 1.7921,
+      "theoretical_loss": 4.637237060011007,
+      "tokens_seen": 129171456
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007764413239719158,
+      "loss": 1.7432,
+      "theoretical_loss": 4.636924647450002,
+      "tokens_seen": 129236992
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007764252758274825,
+      "loss": 1.9541,
+      "theoretical_loss": 4.636612437606212,
+      "tokens_seen": 129302528
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007764092276830492,
+      "loss": 1.7851,
+      "theoretical_loss": 4.636300430245457,
+      "tokens_seen": 129368064
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007763931795386159,
+      "loss": 1.8196,
+      "theoretical_loss": 4.63598862513394,
+      "tokens_seen": 129433600
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007763771313941826,
+      "loss": 1.8,
+      "theoretical_loss": 4.635677022038254,
+      "tokens_seen": 129499136
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007763610832497493,
+      "loss": 1.8628,
+      "theoretical_loss": 4.63536562072538,
+      "tokens_seen": 129564672
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.000776345035105316,
+      "loss": 1.719,
+      "theoretical_loss": 4.635054420962684,
+      "tokens_seen": 129630208
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007763289869608827,
+      "loss": 1.8873,
+      "theoretical_loss": 4.634743422517918,
+      "tokens_seen": 129695744
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007763129388164494,
+      "loss": 1.7792,
+      "theoretical_loss": 4.634432625159221,
+      "tokens_seen": 129761280
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007762968906720161,
+      "loss": 1.8217,
+      "theoretical_loss": 4.634122028655112,
+      "tokens_seen": 129826816
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007762808425275828,
+      "loss": 1.844,
+      "theoretical_loss": 4.633811632774496,
+      "tokens_seen": 129892352
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007762647943831495,
+      "loss": 1.8314,
+      "theoretical_loss": 4.633501437286659,
+      "tokens_seen": 129957888
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007762487462387162,
+      "loss": 1.8211,
+      "theoretical_loss": 4.633191441961272,
+      "tokens_seen": 130023424
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007762326980942829,
+      "loss": 1.7701,
+      "theoretical_loss": 4.63288164656838,
+      "tokens_seen": 130088960
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007762166499498496,
+      "loss": 1.86,
+      "theoretical_loss": 4.632572050878415,
+      "tokens_seen": 130154496
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007762006018054163,
+      "loss": 1.6954,
+      "theoretical_loss": 4.632262654662181,
+      "tokens_seen": 130220032
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.000776184553660983,
+      "loss": 1.7993,
+      "theoretical_loss": 4.631953457690868,
+      "tokens_seen": 130285568
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007761685055165497,
+      "loss": 1.8045,
+      "theoretical_loss": 4.631644459736036,
+      "tokens_seen": 130351104
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007761524573721164,
+      "loss": 1.7381,
+      "theoretical_loss": 4.631335660569626,
+      "tokens_seen": 130416640
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007761364092276831,
+      "loss": 1.7095,
+      "theoretical_loss": 4.631027059963953,
+      "tokens_seen": 130482176
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007761203610832498,
+      "loss": 1.7498,
+      "theoretical_loss": 4.630718657691709,
+      "tokens_seen": 130547712
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007761043129388165,
+      "loss": 1.7518,
+      "theoretical_loss": 4.630410453525958,
+      "tokens_seen": 130613248
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007760882647943832,
+      "loss": 1.8563,
+      "theoretical_loss": 4.630102447240138,
+      "tokens_seen": 130678784
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007760722166499499,
+      "loss": 1.955,
+      "theoretical_loss": 4.629794638608059,
+      "tokens_seen": 130744320
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007760561685055166,
+      "loss": 1.8386,
+      "theoretical_loss": 4.629487027403905,
+      "tokens_seen": 130809856
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007760401203610833,
+      "loss": 1.8178,
+      "theoretical_loss": 4.629179613402228,
+      "tokens_seen": 130875392
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00077602407221665,
+      "loss": 1.8286,
+      "theoretical_loss": 4.628872396377953,
+      "tokens_seen": 130940928
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007760080240722167,
+      "loss": 1.6955,
+      "theoretical_loss": 4.628565376106371,
+      "tokens_seen": 131006464
+    },
+    {
+      "epoch": 0.04,
+      "objective/train/docs_used": 50322,
+      "objective/train/instantaneous_batch_size": 32,
+      "objective/train/instantaneous_microbatch_size": 32768,
+      "objective/train/original_loss": 1.6800458431243896,
+      "objective/train/theoretical_loss": 4.628258552363146,
+      "objective/train/tokens_used": 151532000,
+      "theoretical_loss": 4.628258552363146,
+      "tokens_seen": 131072000
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007759919759277834,
+      "loss": 1.8156,
+      "theoretical_loss": 4.628258552363146,
+      "tokens_seen": 131072000
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007759759277833502,
+      "loss": 1.8918,
+      "theoretical_loss": 4.627951924924305,
+      "tokens_seen": 131137536
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007759598796389168,
+      "loss": 1.8027,
+      "theoretical_loss": 4.627645493566245,
+      "tokens_seen": 131203072
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007759438314944836,
+      "loss": 1.845,
+      "theoretical_loss": 4.62733925806573,
+      "tokens_seen": 131268608
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007759277833500502,
+      "loss": 1.8806,
+      "theoretical_loss": 4.627033218199886,
+      "tokens_seen": 131334144
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007759117352056169,
+      "loss": 1.9016,
+      "theoretical_loss": 4.626727373746206,
+      "tokens_seen": 131399680
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007758956870611836,
+      "loss": 1.7789,
+      "theoretical_loss": 4.626421724482549,
+      "tokens_seen": 131465216
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007758796389167503,
+      "loss": 1.7614,
+      "theoretical_loss": 4.626116270187131,
+      "tokens_seen": 131530752
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.000775863590772317,
+      "loss": 1.8855,
+      "theoretical_loss": 4.625811010638538,
+      "tokens_seen": 131596288
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007758475426278837,
+      "loss": 1.875,
+      "theoretical_loss": 4.62550594561571,
+      "tokens_seen": 131661824
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007758314944834504,
+      "loss": 1.7714,
+      "theoretical_loss": 4.625201074897952,
+      "tokens_seen": 131727360
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007758154463390171,
+      "loss": 1.8235,
+      "theoretical_loss": 4.62489639826493,
+      "tokens_seen": 131792896
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007757993981945837,
+      "loss": 1.7658,
+      "theoretical_loss": 4.6245919154966675,
+      "tokens_seen": 131858432
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007757833500501505,
+      "loss": 1.8592,
+      "theoretical_loss": 4.624287626373545,
+      "tokens_seen": 131923968
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007757673019057171,
+      "loss": 1.9013,
+      "theoretical_loss": 4.623983530676304,
+      "tokens_seen": 131989504
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007757512537612839,
+      "loss": 1.8966,
+      "theoretical_loss": 4.62367962818604,
+      "tokens_seen": 132055040
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007757352056168505,
+      "loss": 1.7468,
+      "theoretical_loss": 4.623375918684207,
+      "tokens_seen": 132120576
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007757191574724173,
+      "loss": 1.7464,
+      "theoretical_loss": 4.623072401952614,
+      "tokens_seen": 132186112
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007757031093279839,
+      "loss": 1.679,
+      "theoretical_loss": 4.622769077773423,
+      "tokens_seen": 132251648
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007756870611835507,
+      "loss": 1.8434,
+      "theoretical_loss": 4.622465945929152,
+      "tokens_seen": 132317184
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007756710130391173,
+      "loss": 1.9906,
+      "theoretical_loss": 4.622163006202671,
+      "tokens_seen": 132382720
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007756549648946841,
+      "loss": 1.8441,
+      "theoretical_loss": 4.621860258377203,
+      "tokens_seen": 132448256
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007756389167502507,
+      "loss": 1.8535,
+      "theoretical_loss": 4.621557702236323,
+      "tokens_seen": 132513792
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007756228686058175,
+      "loss": 1.7977,
+      "theoretical_loss": 4.621255337563955,
+      "tokens_seen": 132579328
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007756068204613841,
+      "loss": 2.0245,
+      "theoretical_loss": 4.620953164144376,
+      "tokens_seen": 132644864
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007755907723169509,
+      "loss": 1.8917,
+      "theoretical_loss": 4.620651181762212,
+      "tokens_seen": 132710400
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007755747241725175,
+      "loss": 1.8085,
+      "theoretical_loss": 4.620349390202436,
+      "tokens_seen": 132775936
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007755586760280843,
+      "loss": 1.7517,
+      "theoretical_loss": 4.6200477892503695,
+      "tokens_seen": 132841472
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007755426278836509,
+      "loss": 1.9267,
+      "theoretical_loss": 4.619746378691682,
+      "tokens_seen": 132907008
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007755265797392177,
+      "loss": 1.8224,
+      "theoretical_loss": 4.61944515831239,
+      "tokens_seen": 132972544
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007755105315947843,
+      "loss": 1.8742,
+      "theoretical_loss": 4.6191441278988545,
+      "tokens_seen": 133038080
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007754944834503511,
+      "loss": 1.6607,
+      "theoretical_loss": 4.618843287237782,
+      "tokens_seen": 133103616
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007754784353059177,
+      "loss": 1.8491,
+      "theoretical_loss": 4.618542636116224,
+      "tokens_seen": 133169152
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007754623871614845,
+      "loss": 1.7973,
+      "theoretical_loss": 4.618242174321575,
+      "tokens_seen": 133234688
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007754463390170511,
+      "loss": 1.8659,
+      "theoretical_loss": 4.617941901641573,
+      "tokens_seen": 133300224
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007754302908726179,
+      "loss": 1.6716,
+      "theoretical_loss": 4.617641817864296,
+      "tokens_seen": 133365760
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007754142427281845,
+      "loss": 1.783,
+      "theoretical_loss": 4.617341922778168,
+      "tokens_seen": 133431296
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007753981945837513,
+      "loss": 1.7202,
+      "theoretical_loss": 4.617042216171949,
+      "tokens_seen": 133496832
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007753821464393179,
+      "loss": 1.8185,
+      "theoretical_loss": 4.616742697834742,
+      "tokens_seen": 133562368
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007753660982948847,
+      "loss": 1.6321,
+      "theoretical_loss": 4.616443367555988,
+      "tokens_seen": 133627904
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007753500501504513,
+      "loss": 1.8057,
+      "theoretical_loss": 4.616144225125465,
+      "tokens_seen": 133693440
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007753340020060181,
+      "loss": 1.7431,
+      "theoretical_loss": 4.615845270333294,
+      "tokens_seen": 133758976
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007753179538615847,
+      "loss": 1.8242,
+      "theoretical_loss": 4.615546502969929,
+      "tokens_seen": 133824512
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007753019057171515,
+      "loss": 1.7991,
+      "theoretical_loss": 4.61524792282616,
+      "tokens_seen": 133890048
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007752858575727181,
+      "loss": 1.7559,
+      "theoretical_loss": 4.614949529693115,
+      "tokens_seen": 133955584
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007752698094282849,
+      "loss": 1.889,
+      "theoretical_loss": 4.614651323362257,
+      "tokens_seen": 134021120
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007752537612838515,
+      "loss": 1.7772,
+      "theoretical_loss": 4.614353303625382,
+      "tokens_seen": 134086656
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007752377131394183,
+      "loss": 1.7549,
+      "theoretical_loss": 4.6140554702746215,
+      "tokens_seen": 134152192
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.000775221664994985,
+      "loss": 1.89,
+      "theoretical_loss": 4.613757823102437,
+      "tokens_seen": 134217728
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007752056168505517,
+      "loss": 1.8581,
+      "theoretical_loss": 4.613460361901627,
+      "tokens_seen": 134283264
+    },
+    {
+      "epoch": 0.04,
+      "objective/train/docs_used": 51516,
+      "objective/train/instantaneous_batch_size": 32,
+      "objective/train/instantaneous_microbatch_size": 32768,
+      "objective/train/original_loss": 1.8509464263916016,
+      "objective/train/theoretical_loss": 4.613163086465315,
+      "objective/train/tokens_used": 154808800,
+      "theoretical_loss": 4.613163086465315,
+      "tokens_seen": 134348800
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007751895687061184,
+      "loss": 1.8096,
+      "theoretical_loss": 4.613163086465315,
+      "tokens_seen": 134348800
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007751735205616851,
+      "loss": 1.6757,
+      "theoretical_loss": 4.612865996586962,
+      "tokens_seen": 134414336
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007751574724172518,
+      "loss": 1.7133,
+      "theoretical_loss": 4.612569092060355,
+      "tokens_seen": 134479872
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007751414242728185,
+      "loss": 1.8462,
+      "theoretical_loss": 4.612272372679611,
+      "tokens_seen": 134545408
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007751253761283852,
+      "loss": 1.7826,
+      "theoretical_loss": 4.611975838239179,
+      "tokens_seen": 134610944
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007751093279839519,
+      "loss": 1.8349,
+      "theoretical_loss": 4.611679488533832,
+      "tokens_seen": 134676480
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007750932798395186,
+      "loss": 1.7926,
+      "theoretical_loss": 4.611383323358671,
+      "tokens_seen": 134742016
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007750772316950853,
+      "loss": 1.8531,
+      "theoretical_loss": 4.61108734250913,
+      "tokens_seen": 134807552
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.000775061183550652,
+      "loss": 1.9454,
+      "theoretical_loss": 4.610791545780959,
+      "tokens_seen": 134873088
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007750451354062187,
+      "loss": 1.889,
+      "theoretical_loss": 4.610495932970241,
+      "tokens_seen": 134938624
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007750290872617854,
+      "loss": 1.7593,
+      "theoretical_loss": 4.61020050387338,
+      "tokens_seen": 135004160
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007750130391173521,
+      "loss": 1.8416,
+      "theoretical_loss": 4.609905258287107,
+      "tokens_seen": 135069696
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007749969909729188,
+      "loss": 1.8243,
+      "theoretical_loss": 4.609610196008473,
+      "tokens_seen": 135135232
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007749809428284855,
+      "loss": 1.856,
+      "theoretical_loss": 4.609315316834855,
+      "tokens_seen": 135200768
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007749648946840522,
+      "loss": 1.7466,
+      "theoretical_loss": 4.60902062056395,
+      "tokens_seen": 135266304
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007749488465396189,
+      "loss": 1.6934,
+      "theoretical_loss": 4.608726106993776,
+      "tokens_seen": 135331840
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007749327983951856,
+      "loss": 1.8517,
+      "theoretical_loss": 4.608431775922673,
+      "tokens_seen": 135397376
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007749167502507523,
+      "loss": 1.9445,
+      "theoretical_loss": 4.608137627149302,
+      "tokens_seen": 135462912
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.000774900702106319,
+      "loss": 1.8379,
+      "theoretical_loss": 4.607843660472642,
+      "tokens_seen": 135528448
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007748846539618857,
+      "loss": 1.8525,
+      "theoretical_loss": 4.607549875691989,
+      "tokens_seen": 135593984
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007748686058174524,
+      "loss": 1.8739,
+      "theoretical_loss": 4.607256272606961,
+      "tokens_seen": 135659520
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007748525576730191,
+      "loss": 1.6829,
+      "theoretical_loss": 4.606962851017489,
+      "tokens_seen": 135725056
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007748365095285858,
+      "loss": 1.8055,
+      "theoretical_loss": 4.606669610723825,
+      "tokens_seen": 135790592
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007748204613841525,
+      "loss": 1.7795,
+      "theoretical_loss": 4.606376551526536,
+      "tokens_seen": 135856128
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007748044132397192,
+      "loss": 1.8209,
+      "theoretical_loss": 4.6060836732265,
+      "tokens_seen": 135921664
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007747883650952859,
+      "loss": 1.7812,
+      "theoretical_loss": 4.605790975624917,
+      "tokens_seen": 135987200
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007747723169508527,
+      "loss": 1.844,
+      "theoretical_loss": 4.605498458523296,
+      "tokens_seen": 136052736
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007747562688064193,
+      "loss": 1.7985,
+      "theoretical_loss": 4.605206121723461,
+      "tokens_seen": 136118272
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007747402206619861,
+      "loss": 1.766,
+      "theoretical_loss": 4.604913965027551,
+      "tokens_seen": 136183808
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007747241725175527,
+      "loss": 1.8559,
+      "theoretical_loss": 4.604621988238012,
+      "tokens_seen": 136249344
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007747081243731195,
+      "loss": 1.7859,
+      "theoretical_loss": 4.604330191157608,
+      "tokens_seen": 136314880
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007746920762286861,
+      "loss": 1.7933,
+      "theoretical_loss": 4.604038573589408,
+      "tokens_seen": 136380416
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007746760280842529,
+      "loss": 1.7707,
+      "theoretical_loss": 4.603747135336796,
+      "tokens_seen": 136445952
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007746599799398195,
+      "loss": 1.9309,
+      "theoretical_loss": 4.603455876203463,
+      "tokens_seen": 136511488
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007746439317953863,
+      "loss": 1.8149,
+      "theoretical_loss": 4.603164795993409,
+      "tokens_seen": 136577024
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007746278836509529,
+      "loss": 1.8279,
+      "theoretical_loss": 4.602873894510945,
+      "tokens_seen": 136642560
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007746118355065197,
+      "loss": 1.7564,
+      "theoretical_loss": 4.602583171560686,
+      "tokens_seen": 136708096
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007745957873620863,
+      "loss": 1.6954,
+      "theoretical_loss": 4.602292626947556,
+      "tokens_seen": 136773632
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007745797392176531,
+      "loss": 1.8599,
+      "theoretical_loss": 4.602002260476787,
+      "tokens_seen": 136839168
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007745636910732197,
+      "loss": 1.853,
+      "theoretical_loss": 4.601712071953914,
+      "tokens_seen": 136904704
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007745476429287865,
+      "loss": 1.7501,
+      "theoretical_loss": 4.601422061184778,
+      "tokens_seen": 136970240
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007745315947843531,
+      "loss": 1.6276,
+      "theoretical_loss": 4.601132227975527,
+      "tokens_seen": 137035776
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007745155466399198,
+      "loss": 1.8525,
+      "theoretical_loss": 4.60084257213261,
+      "tokens_seen": 137101312
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007744994984954865,
+      "loss": 1.778,
+      "theoretical_loss": 4.600553093462781,
+      "tokens_seen": 137166848
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007744834503510532,
+      "loss": 1.9005,
+      "theoretical_loss": 4.600263791773097,
+      "tokens_seen": 137232384
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007744674022066199,
+      "loss": 1.8085,
+      "theoretical_loss": 4.599974666870914,
+      "tokens_seen": 137297920
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007744513540621866,
+      "loss": 1.7596,
+      "theoretical_loss": 4.599685718563894,
+      "tokens_seen": 137363456
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007744353059177533,
+      "loss": 1.8408,
+      "theoretical_loss": 4.599396946659997,
+      "tokens_seen": 137428992
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00077441925777332,
+      "loss": 1.8655,
+      "theoretical_loss": 4.5991083509674855,
+      "tokens_seen": 137494528
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007744032096288867,
+      "loss": 1.7595,
+      "theoretical_loss": 4.598819931294919,
+      "tokens_seen": 137560064
+    },
+    {
+      "epoch": 0.04,
+      "objective/train/docs_used": 52430,
+      "objective/train/instantaneous_batch_size": 32,
+      "objective/train/instantaneous_microbatch_size": 32768,
+      "objective/train/original_loss": 1.8565382957458496,
+      "objective/train/theoretical_loss": 4.598531687451157,
+      "objective/train/tokens_used": 158085600,
+      "theoretical_loss": 4.598531687451157,
+      "tokens_seen": 137625600
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007743871614844534,
+      "loss": 1.8098,
+      "theoretical_loss": 4.598531687451157,
+      "tokens_seen": 137625600
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00077437111334002,
+      "loss": 1.7782,
+      "theoretical_loss": 4.598243619245359,
+      "tokens_seen": 137691136
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007743550651955868,
+      "loss": 1.7704,
+      "theoretical_loss": 4.59795572648698,
+      "tokens_seen": 137756672
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007743390170511534,
+      "loss": 1.8489,
+      "theoretical_loss": 4.597668008985774,
+      "tokens_seen": 137822208
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007743229689067202,
+      "loss": 1.8287,
+      "theoretical_loss": 4.59738046655179,
+      "tokens_seen": 137887744
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007743069207622868,
+      "loss": 1.8362,
+      "theoretical_loss": 4.597093098995377,
+      "tokens_seen": 137953280
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007742908726178536,
+      "loss": 1.8108,
+      "theoretical_loss": 4.596805906127173,
+      "tokens_seen": 138018816
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007742748244734202,
+      "loss": 1.857,
+      "theoretical_loss": 4.596518887758117,
+      "tokens_seen": 138084352
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.000774258776328987,
+      "loss": 1.8529,
+      "theoretical_loss": 4.596232043699438,
+      "tokens_seen": 138149888
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007742427281845536,
+      "loss": 1.9016,
+      "theoretical_loss": 4.59594537376266,
+      "tokens_seen": 138215424
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007742266800401204,
+      "loss": 1.7711,
+      "theoretical_loss": 4.595658877759602,
+      "tokens_seen": 138280960
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.000774210631895687,
+      "loss": 1.7571,
+      "theoretical_loss": 4.595372555502371,
+      "tokens_seen": 138346496
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007741945837512538,
+      "loss": 1.8428,
+      "theoretical_loss": 4.59508640680337,
+      "tokens_seen": 138412032
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007741785356068204,
+      "loss": 1.8147,
+      "theoretical_loss": 4.594800431475292,
+      "tokens_seen": 138477568
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007741624874623872,
+      "loss": 1.7274,
+      "theoretical_loss": 4.594514629331121,
+      "tokens_seen": 138543104
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007741464393179538,
+      "loss": 1.8693,
+      "theoretical_loss": 4.594229000184128,
+      "tokens_seen": 138608640
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007741303911735206,
+      "loss": 1.8598,
+      "theoretical_loss": 4.593943543847878,
+      "tokens_seen": 138674176
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007741143430290872,
+      "loss": 1.9131,
+      "theoretical_loss": 4.5936582601362215,
+      "tokens_seen": 138739712
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.000774098294884654,
+      "loss": 1.8231,
+      "theoretical_loss": 4.593373148863301,
+      "tokens_seen": 138805248
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007740822467402206,
+      "loss": 1.7716,
+      "theoretical_loss": 4.593088209843543,
+      "tokens_seen": 138870784
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007740661985957874,
+      "loss": 1.7785,
+      "theoretical_loss": 4.592803442891663,
+      "tokens_seen": 138936320
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.000774050150451354,
+      "loss": 1.6731,
+      "theoretical_loss": 4.592518847822661,
+      "tokens_seen": 139001856
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007740341023069208,
+      "loss": 1.8166,
+      "theoretical_loss": 4.592234424451828,
+      "tokens_seen": 139067392
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007740180541624874,
+      "loss": 1.9211,
+      "theoretical_loss": 4.591950172594734,
+      "tokens_seen": 139132928
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007740020060180542,
+      "loss": 1.8133,
+      "theoretical_loss": 4.591666092067238,
+      "tokens_seen": 139198464
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007739859578736208,
+      "loss": 1.8798,
+      "theoretical_loss": 4.591382182685484,
+      "tokens_seen": 139264000
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007739699097291876,
+      "loss": 1.8957,
+      "theoretical_loss": 4.591098444265896,
+      "tokens_seen": 139329536
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007739538615847542,
+      "loss": 1.8194,
+      "theoretical_loss": 4.590814876625183,
+      "tokens_seen": 139395072
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.000773937813440321,
+      "loss": 1.7456,
+      "theoretical_loss": 4.590531479580338,
+      "tokens_seen": 139460608
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007739217652958876,
+      "loss": 1.7113,
+      "theoretical_loss": 4.5902482529486335,
+      "tokens_seen": 139526144
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007739057171514544,
+      "loss": 1.759,
+      "theoretical_loss": 4.589965196547625,
+      "tokens_seen": 139591680
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.000773889669007021,
+      "loss": 1.7587,
+      "theoretical_loss": 4.5896823101951485,
+      "tokens_seen": 139657216
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007738736208625878,
+      "loss": 1.8341,
+      "theoretical_loss": 4.58939959370932,
+      "tokens_seen": 139722752
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007738575727181545,
+      "loss": 1.7848,
+      "theoretical_loss": 4.5891170469085365,
+      "tokens_seen": 139788288
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007738415245737212,
+      "loss": 1.7989,
+      "theoretical_loss": 4.588834669611472,
+      "tokens_seen": 139853824
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007738254764292879,
+      "loss": 1.7692,
+      "theoretical_loss": 4.5885524616370805,
+      "tokens_seen": 139919360
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007738094282848546,
+      "loss": 1.7518,
+      "theoretical_loss": 4.5882704228045945,
+      "tokens_seen": 139984896
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007737933801404213,
+      "loss": 1.7858,
+      "theoretical_loss": 4.587988552933523,
+      "tokens_seen": 140050432
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.000773777331995988,
+      "loss": 1.8491,
+      "theoretical_loss": 4.587706851843652,
+      "tokens_seen": 140115968
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007737612838515547,
+      "loss": 1.7993,
+      "theoretical_loss": 4.587425319355044,
+      "tokens_seen": 140181504
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007737452357071214,
+      "loss": 1.8233,
+      "theoretical_loss": 4.587143955288038,
+      "tokens_seen": 140247040
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007737291875626881,
+      "loss": 1.849,
+      "theoretical_loss": 4.586862759463248,
+      "tokens_seen": 140312576
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007737131394182548,
+      "loss": 1.9398,
+      "theoretical_loss": 4.586581731701562,
+      "tokens_seen": 140378112
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007736970912738215,
+      "loss": 1.8538,
+      "theoretical_loss": 4.586300871824143,
+      "tokens_seen": 140443648
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007736810431293882,
+      "loss": 1.8352,
+      "theoretical_loss": 4.586020179652427,
+      "tokens_seen": 140509184
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007736649949849549,
+      "loss": 1.8762,
+      "theoretical_loss": 4.585739655008123,
+      "tokens_seen": 140574720
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007736489468405216,
+      "loss": 1.8606,
+      "theoretical_loss": 4.585459297713215,
+      "tokens_seen": 140640256
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007736328986960883,
+      "loss": 1.8666,
+      "theoretical_loss": 4.585179107589955,
+      "tokens_seen": 140705792
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.000773616850551655,
+      "loss": 1.7146,
+      "theoretical_loss": 4.58489908446087,
+      "tokens_seen": 140771328
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007736008024072217,
+      "loss": 1.6647,
+      "theoretical_loss": 4.584619228148755,
+      "tokens_seen": 140836864
+    },
+    {
+      "epoch": 0.04,
+      "objective/train/docs_used": 53599,
+      "objective/train/instantaneous_batch_size": 32,
+      "objective/train/instantaneous_microbatch_size": 32768,
+      "objective/train/original_loss": 1.8628878593444824,
+      "objective/train/theoretical_loss": 4.584339538476678,
+      "objective/train/tokens_used": 161362400,
+      "theoretical_loss": 4.584339538476678,
+      "tokens_seen": 140902400
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007735847542627884,
+      "loss": 1.8575,
+      "theoretical_loss": 4.584339538476678,
+      "tokens_seen": 140902400
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007735687061183552,
+      "loss": 1.7988,
+      "theoretical_loss": 4.5840600152679745,
+      "tokens_seen": 140967936
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007735526579739218,
+      "loss": 1.8108,
+      "theoretical_loss": 4.583780658346252,
+      "tokens_seen": 141033472
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007735366098294886,
+      "loss": 1.8225,
+      "theoretical_loss": 4.583501467535383,
+      "tokens_seen": 141099008
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007735205616850552,
+      "loss": 1.7739,
+      "theoretical_loss": 4.583222442659514,
+      "tokens_seen": 141164544
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.000773504513540622,
+      "loss": 1.7832,
+      "theoretical_loss": 4.5829435835430505,
+      "tokens_seen": 141230080
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007734884653961886,
+      "loss": 1.7594,
+      "theoretical_loss": 4.582664890010674,
+      "tokens_seen": 141295616
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007734724172517554,
+      "loss": 1.8773,
+      "theoretical_loss": 4.582386361887329,
+      "tokens_seen": 141361152
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.000773456369107322,
+      "loss": 1.7994,
+      "theoretical_loss": 4.582107998998225,
+      "tokens_seen": 141426688
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007734403209628888,
+      "loss": 1.8104,
+      "theoretical_loss": 4.581829801168839,
+      "tokens_seen": 141492224
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007734242728184554,
+      "loss": 1.8975,
+      "theoretical_loss": 4.5815517682249105,
+      "tokens_seen": 141557760
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007734082246740222,
+      "loss": 1.8208,
+      "theoretical_loss": 4.581273899992447,
+      "tokens_seen": 141623296
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007733921765295888,
+      "loss": 1.7032,
+      "theoretical_loss": 4.580996196297718,
+      "tokens_seen": 141688832
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007733761283851556,
+      "loss": 1.7215,
+      "theoretical_loss": 4.580718656967257,
+      "tokens_seen": 141754368
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007733600802407222,
+      "loss": 1.7209,
+      "theoretical_loss": 4.580441281827858,
+      "tokens_seen": 141819904
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.000773344032096289,
+      "loss": 1.7961,
+      "theoretical_loss": 4.580164070706584,
+      "tokens_seen": 141885440
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007733279839518556,
+      "loss": 1.8012,
+      "theoretical_loss": 4.579887023430752,
+      "tokens_seen": 141950976
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007733119358074224,
+      "loss": 1.7978,
+      "theoretical_loss": 4.579610139827945,
+      "tokens_seen": 142016512
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.000773295887662989,
+      "loss": 1.792,
+      "theoretical_loss": 4.579333419726007,
+      "tokens_seen": 142082048
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007732798395185558,
+      "loss": 1.7991,
+      "theoretical_loss": 4.579056862953039,
+      "tokens_seen": 142147584
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007732637913741224,
+      "loss": 1.8455,
+      "theoretical_loss": 4.578780469337407,
+      "tokens_seen": 142213120
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007732477432296892,
+      "loss": 1.7209,
+      "theoretical_loss": 4.5785042387077315,
+      "tokens_seen": 142278656
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007732316950852558,
+      "loss": 1.7165,
+      "theoretical_loss": 4.578228170892896,
+      "tokens_seen": 142344192
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007732156469408226,
+      "loss": 1.8058,
+      "theoretical_loss": 4.577952265722038,
+      "tokens_seen": 142409728
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007731995987963892,
+      "loss": 1.8097,
+      "theoretical_loss": 4.5776765230245555,
+      "tokens_seen": 142475264
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.000773183550651956,
+      "loss": 1.9091,
+      "theoretical_loss": 4.5774009426301046,
+      "tokens_seen": 142540800
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007731675025075226,
+      "loss": 1.7356,
+      "theoretical_loss": 4.577125524368595,
+      "tokens_seen": 142606336
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007731514543630894,
+      "loss": 1.9817,
+      "theoretical_loss": 4.5768502680701975,
+      "tokens_seen": 142671872
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.000773135406218656,
+      "loss": 1.7755,
+      "theoretical_loss": 4.576575173565334,
+      "tokens_seen": 142737408
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007731193580742227,
+      "loss": 1.8315,
+      "theoretical_loss": 4.5763002406846836,
+      "tokens_seen": 142802944
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007731033099297894,
+      "loss": 1.8265,
+      "theoretical_loss": 4.57602546925918,
+      "tokens_seen": 142868480
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007730872617853561,
+      "loss": 1.8341,
+      "theoretical_loss": 4.575750859120014,
+      "tokens_seen": 142934016
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007730712136409228,
+      "loss": 1.7674,
+      "theoretical_loss": 4.575476410098624,
+      "tokens_seen": 142999552
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007730551654964895,
+      "loss": 1.8959,
+      "theoretical_loss": 4.575202122026706,
+      "tokens_seen": 143065088
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007730391173520562,
+      "loss": 1.8057,
+      "theoretical_loss": 4.5749279947362105,
+      "tokens_seen": 143130624
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007730230692076229,
+      "loss": 1.7849,
+      "theoretical_loss": 4.574654028059336,
+      "tokens_seen": 143196160
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007730070210631896,
+      "loss": 1.8779,
+      "theoretical_loss": 4.574380221828535,
+      "tokens_seen": 143261696
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007729909729187563,
+      "loss": 1.7999,
+      "theoretical_loss": 4.574106575876511,
+      "tokens_seen": 143327232
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007729749247743229,
+      "loss": 1.8704,
+      "theoretical_loss": 4.573833090036219,
+      "tokens_seen": 143392768
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007729588766298897,
+      "loss": 1.7967,
+      "theoretical_loss": 4.573559764140864,
+      "tokens_seen": 143458304
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007729428284854563,
+      "loss": 1.8476,
+      "theoretical_loss": 4.573286598023898,
+      "tokens_seen": 143523840
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007729267803410231,
+      "loss": 1.7928,
+      "theoretical_loss": 4.573013591519029,
+      "tokens_seen": 143589376
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007729107321965897,
+      "loss": 1.7736,
+      "theoretical_loss": 4.572740744460207,
+      "tokens_seen": 143654912
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007728946840521565,
+      "loss": 1.8692,
+      "theoretical_loss": 4.572468056681634,
+      "tokens_seen": 143720448
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007728786359077231,
+      "loss": 1.6882,
+      "theoretical_loss": 4.57219552801776,
+      "tokens_seen": 143785984
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007728625877632899,
+      "loss": 1.8081,
+      "theoretical_loss": 4.57192315830328,
+      "tokens_seen": 143851520
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007728465396188565,
+      "loss": 1.8358,
+      "theoretical_loss": 4.571650947373138,
+      "tokens_seen": 143917056
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007728304914744233,
+      "loss": 1.8131,
+      "theoretical_loss": 4.571378895062525,
+      "tokens_seen": 143982592
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007728144433299899,
+      "loss": 1.8788,
+      "theoretical_loss": 4.571107001206875,
+      "tokens_seen": 144048128
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007727983951855567,
+      "loss": 1.8819,
+      "theoretical_loss": 4.570835265641872,
+      "tokens_seen": 144113664
+    },
+    {
+      "epoch": 0.04,
+      "objective/train/docs_used": 54784,
+      "objective/train/instantaneous_batch_size": 32,
+      "objective/train/instantaneous_microbatch_size": 32768,
+      "objective/train/original_loss": 1.7508742809295654,
+      "objective/train/theoretical_loss": 4.57056368820344,
+      "objective/train/tokens_used": 164639200,
+      "theoretical_loss": 4.57056368820344,
+      "tokens_seen": 144179200
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007727823470411233,
+      "loss": 1.8309,
+      "theoretical_loss": 4.57056368820344,
+      "tokens_seen": 144179200
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007727662988966901,
+      "loss": 1.6846,
+      "theoretical_loss": 4.570292268727751,
+      "tokens_seen": 144244736
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007727502507522567,
+      "loss": 1.9016,
+      "theoretical_loss": 4.570021007051221,
+      "tokens_seen": 144310272
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007727342026078235,
+      "loss": 1.7859,
+      "theoretical_loss": 4.569749903010507,
+      "tokens_seen": 144375808
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007727181544633901,
+      "loss": 1.7926,
+      "theoretical_loss": 4.569478956442513,
+      "tokens_seen": 144441344
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007727021063189569,
+      "loss": 1.7525,
+      "theoretical_loss": 4.569208167184382,
+      "tokens_seen": 144506880
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007726860581745235,
+      "loss": 1.8334,
+      "theoretical_loss": 4.5689375350735,
+      "tokens_seen": 144572416
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007726700100300903,
+      "loss": 1.743,
+      "theoretical_loss": 4.568667059947496,
+      "tokens_seen": 144637952
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007726539618856569,
+      "loss": 1.8145,
+      "theoretical_loss": 4.568396741644241,
+      "tokens_seen": 144703488
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007726379137412237,
+      "loss": 1.8106,
+      "theoretical_loss": 4.568126580001843,
+      "tokens_seen": 144769024
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007726218655967903,
+      "loss": 1.7527,
+      "theoretical_loss": 4.567856574858656,
+      "tokens_seen": 144834560
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007726058174523571,
+      "loss": 1.7625,
+      "theoretical_loss": 4.567586726053266,
+      "tokens_seen": 144900096
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007725897693079237,
+      "loss": 1.8854,
+      "theoretical_loss": 4.567317033424507,
+      "tokens_seen": 144965632
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007725737211634905,
+      "loss": 1.854,
+      "theoretical_loss": 4.567047496811445,
+      "tokens_seen": 145031168
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007725576730190571,
+      "loss": 1.7072,
+      "theoretical_loss": 4.566778116053389,
+      "tokens_seen": 145096704
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007725416248746239,
+      "loss": 1.68,
+      "theoretical_loss": 4.566508890989883,
+      "tokens_seen": 145162240
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007725255767301905,
+      "loss": 1.7653,
+      "theoretical_loss": 4.566239821460712,
+      "tokens_seen": 145227776
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007725095285857573,
+      "loss": 1.819,
+      "theoretical_loss": 4.5659709073058945,
+      "tokens_seen": 145293312
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.000772493480441324,
+      "loss": 1.7384,
+      "theoretical_loss": 4.565702148365686,
+      "tokens_seen": 145358848
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007724774322968907,
+      "loss": 1.7723,
+      "theoretical_loss": 4.56543354448058,
+      "tokens_seen": 145424384
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007724613841524574,
+      "loss": 1.7699,
+      "theoretical_loss": 4.565165095491306,
+      "tokens_seen": 145489920
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007724453360080241,
+      "loss": 1.7046,
+      "theoretical_loss": 4.5648968012388265,
+      "tokens_seen": 145555456
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007724292878635908,
+      "loss": 1.7736,
+      "theoretical_loss": 4.56462866156434,
+      "tokens_seen": 145620992
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007724132397191575,
+      "loss": 2.0358,
+      "theoretical_loss": 4.564360676309279,
+      "tokens_seen": 145686528
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007723971915747242,
+      "loss": 1.9187,
+      "theoretical_loss": 4.564092845315312,
+      "tokens_seen": 145752064
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007723811434302909,
+      "loss": 1.7898,
+      "theoretical_loss": 4.563825168424337,
+      "tokens_seen": 145817600
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007723650952858577,
+      "loss": 1.8226,
+      "theoretical_loss": 4.563557645478488,
+      "tokens_seen": 145883136
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007723490471414243,
+      "loss": 1.7962,
+      "theoretical_loss": 4.56329027632013,
+      "tokens_seen": 145948672
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007723329989969911,
+      "loss": 1.7818,
+      "theoretical_loss": 4.563023060791862,
+      "tokens_seen": 146014208
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007723169508525577,
+      "loss": 1.7425,
+      "theoretical_loss": 4.562755998736511,
+      "tokens_seen": 146079744
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007723009027081245,
+      "loss": 1.6183,
+      "theoretical_loss": 4.562489089997141,
+      "tokens_seen": 146145280
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007722848545636911,
+      "loss": 1.821,
+      "theoretical_loss": 4.562222334417041,
+      "tokens_seen": 146210816
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007722688064192579,
+      "loss": 1.8382,
+      "theoretical_loss": 4.561955731839734,
+      "tokens_seen": 146276352
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007722527582748245,
+      "loss": 1.8122,
+      "theoretical_loss": 4.561689282108971,
+      "tokens_seen": 146341888
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007722367101303913,
+      "loss": 1.7589,
+      "theoretical_loss": 4.561422985068733,
+      "tokens_seen": 146407424
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007722206619859579,
+      "loss": 1.8358,
+      "theoretical_loss": 4.561156840563231,
+      "tokens_seen": 146472960
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007722046138415247,
+      "loss": 1.8378,
+      "theoretical_loss": 4.560890848436902,
+      "tokens_seen": 146538496
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007721885656970913,
+      "loss": 1.8638,
+      "theoretical_loss": 4.560625008534414,
+      "tokens_seen": 146604032
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007721725175526581,
+      "loss": 1.8072,
+      "theoretical_loss": 4.560359320700662,
+      "tokens_seen": 146669568
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007721564694082247,
+      "loss": 1.8276,
+      "theoretical_loss": 4.560093784780766,
+      "tokens_seen": 146735104
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007721404212637915,
+      "loss": 1.773,
+      "theoretical_loss": 4.559828400620075,
+      "tokens_seen": 146800640
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007721243731193581,
+      "loss": 1.931,
+      "theoretical_loss": 4.559563168064165,
+      "tokens_seen": 146866176
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007721083249749249,
+      "loss": 1.7954,
+      "theoretical_loss": 4.559298086958837,
+      "tokens_seen": 146931712
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007720922768304915,
+      "loss": 1.7741,
+      "theoretical_loss": 4.559033157150115,
+      "tokens_seen": 146997248
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007720762286860583,
+      "loss": 1.7928,
+      "theoretical_loss": 4.5587683784842525,
+      "tokens_seen": 147062784
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007720601805416249,
+      "loss": 1.7564,
+      "theoretical_loss": 4.558503750807725,
+      "tokens_seen": 147128320
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007720441323971917,
+      "loss": 1.8492,
+      "theoretical_loss": 4.558239273967234,
+      "tokens_seen": 147193856
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007720280842527583,
+      "loss": 1.7341,
+      "theoretical_loss": 4.5579749478097025,
+      "tokens_seen": 147259392
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007720120361083251,
+      "loss": 1.6948,
+      "theoretical_loss": 4.557710772182278,
+      "tokens_seen": 147324928
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007719959879638917,
+      "loss": 1.8954,
+      "theoretical_loss": 4.5574467469323325,
+      "tokens_seen": 147390464
+    },
+    {
+      "epoch": 0.04,
+      "objective/train/docs_used": 55745,
+      "objective/train/instantaneous_batch_size": 32,
+      "objective/train/instantaneous_microbatch_size": 32768,
+      "objective/train/original_loss": 1.8895684480667114,
+      "objective/train/theoretical_loss": 4.557182871907457,
+      "objective/train/tokens_used": 167916000,
+      "theoretical_loss": 4.557182871907457,
+      "tokens_seen": 147456000
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007719799398194585,
+      "loss": 1.8234,
+      "theoretical_loss": 4.557182871907457,
+      "tokens_seen": 147456000
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007719638916750251,
+      "loss": 1.7556,
+      "theoretical_loss": 4.556919146955469,
+      "tokens_seen": 147521536
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007719478435305919,
+      "loss": 1.8363,
+      "theoretical_loss": 4.556655571924404,
+      "tokens_seen": 147587072
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007719317953861585,
+      "loss": 1.7651,
+      "theoretical_loss": 4.556392146662521,
+      "tokens_seen": 147652608
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007719157472417253,
+      "loss": 1.6694,
+      "theoretical_loss": 4.556128871018297,
+      "tokens_seen": 147718144
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007718996990972919,
+      "loss": 1.7767,
+      "theoretical_loss": 4.555865744840434,
+      "tokens_seen": 147783680
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007718836509528587,
+      "loss": 1.7228,
+      "theoretical_loss": 4.55560276797785,
+      "tokens_seen": 147849216
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007718676028084253,
+      "loss": 1.8281,
+      "theoretical_loss": 4.5553399402796835,
+      "tokens_seen": 147914752
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007718515546639921,
+      "loss": 1.9202,
+      "theoretical_loss": 4.5550772615952955,
+      "tokens_seen": 147980288
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007718355065195587,
+      "loss": 1.8705,
+      "theoretical_loss": 4.554814731774259,
+      "tokens_seen": 148045824
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007718194583751255,
+      "loss": 1.8085,
+      "theoretical_loss": 4.554552350666373,
+      "tokens_seen": 148111360
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007718034102306921,
+      "loss": 1.8387,
+      "theoretical_loss": 4.554290118121649,
+      "tokens_seen": 148176896
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007717873620862589,
+      "loss": 1.6868,
+      "theoretical_loss": 4.554028033990319,
+      "tokens_seen": 148242432
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007717713139418255,
+      "loss": 1.7188,
+      "theoretical_loss": 4.5537660981228285,
+      "tokens_seen": 148307968
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007717552657973923,
+      "loss": 1.7805,
+      "theoretical_loss": 4.553504310369844,
+      "tokens_seen": 148373504
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0007717392176529589,
+      "loss": 1.7819,
+      "theoretical_loss": 4.553242670582245,
+      "tokens_seen": 148439040
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007717231695085256,
+      "loss": 1.6758,
+      "theoretical_loss": 4.55298117861113,
+      "tokens_seen": 148504576
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007717071213640923,
+      "loss": 1.749,
+      "theoretical_loss": 4.55271983430781,
+      "tokens_seen": 148570112
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.000771691073219659,
+      "loss": 1.7768,
+      "theoretical_loss": 4.552458637523815,
+      "tokens_seen": 148635648
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007716750250752257,
+      "loss": 1.8353,
+      "theoretical_loss": 4.552197588110882,
+      "tokens_seen": 148701184
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007716589769307924,
+      "loss": 1.7761,
+      "theoretical_loss": 4.551936685920971,
+      "tokens_seen": 148766720
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007716429287863591,
+      "loss": 1.788,
+      "theoretical_loss": 4.551675930806251,
+      "tokens_seen": 148832256
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007716268806419258,
+      "loss": 1.7663,
+      "theoretical_loss": 4.551415322619107,
+      "tokens_seen": 148897792
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007716108324974925,
+      "loss": 1.8699,
+      "theoretical_loss": 4.551154861212133,
+      "tokens_seen": 148963328
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007715947843530592,
+      "loss": 1.7655,
+      "theoretical_loss": 4.550894546438139,
+      "tokens_seen": 149028864
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007715787362086258,
+      "loss": 1.8114,
+      "theoretical_loss": 4.550634378150148,
+      "tokens_seen": 149094400
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007715626880641926,
+      "loss": 1.781,
+      "theoretical_loss": 4.550374356201392,
+      "tokens_seen": 149159936
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007715466399197592,
+      "loss": 1.8267,
+      "theoretical_loss": 4.550114480445316,
+      "tokens_seen": 149225472
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.000771530591775326,
+      "loss": 1.8981,
+      "theoretical_loss": 4.549854750735575,
+      "tokens_seen": 149291008
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007715145436308926,
+      "loss": 1.8099,
+      "theoretical_loss": 4.5495951669260375,
+      "tokens_seen": 149356544
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007714984954864594,
+      "loss": 1.7378,
+      "theoretical_loss": 4.54933572887078,
+      "tokens_seen": 149422080
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.000771482447342026,
+      "loss": 1.7544,
+      "theoretical_loss": 4.5490764364240865,
+      "tokens_seen": 149487616
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007714663991975928,
+      "loss": 1.7501,
+      "theoretical_loss": 4.548817289440455,
+      "tokens_seen": 149553152
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007714503510531594,
+      "loss": 1.9403,
+      "theoretical_loss": 4.548558287774592,
+      "tokens_seen": 149618688
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007714343029087262,
+      "loss": 1.6857,
+      "theoretical_loss": 4.548299431281409,
+      "tokens_seen": 149684224
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007714182547642928,
+      "loss": 1.7335,
+      "theoretical_loss": 4.548040719816029,
+      "tokens_seen": 149749760
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007714022066198596,
+      "loss": 1.888,
+      "theoretical_loss": 4.547782153233783,
+      "tokens_seen": 149815296
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007713861584754262,
+      "loss": 1.8103,
+      "theoretical_loss": 4.547523731390206,
+      "tokens_seen": 149880832
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.000771370110330993,
+      "loss": 1.9674,
+      "theoretical_loss": 4.547265454141046,
+      "tokens_seen": 149946368
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007713540621865596,
+      "loss": 1.9512,
+      "theoretical_loss": 4.547007321342251,
+      "tokens_seen": 150011904
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007713380140421264,
+      "loss": 1.7966,
+      "theoretical_loss": 4.546749332849982,
+      "tokens_seen": 150077440
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.000771321965897693,
+      "loss": 1.7437,
+      "theoretical_loss": 4.546491488520601,
+      "tokens_seen": 150142976
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007713059177532598,
+      "loss": 1.8675,
+      "theoretical_loss": 4.546233788210679,
+      "tokens_seen": 150208512
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007712898696088264,
+      "loss": 1.7287,
+      "theoretical_loss": 4.545976231776988,
+      "tokens_seen": 150274048
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007712738214643932,
+      "loss": 1.8823,
+      "theoretical_loss": 4.545718819076509,
+      "tokens_seen": 150339584
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007712577733199598,
+      "loss": 1.8053,
+      "theoretical_loss": 4.545461549966426,
+      "tokens_seen": 150405120
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007712417251755266,
+      "loss": 1.9149,
+      "theoretical_loss": 4.545204424304126,
+      "tokens_seen": 150470656
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007712256770310932,
+      "loss": 1.8979,
+      "theoretical_loss": 4.544947441947201,
+      "tokens_seen": 150536192
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00077120962888666,
+      "loss": 1.7907,
+      "theoretical_loss": 4.544690602753445,
+      "tokens_seen": 150601728
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007711935807422266,
+      "loss": 1.8044,
+      "theoretical_loss": 4.544433906580857,
+      "tokens_seen": 150667264
+    },
+    {
+      "epoch": 0.05,
+      "objective/train/docs_used": 56887,
+      "objective/train/instantaneous_batch_size": 32,
+      "objective/train/instantaneous_microbatch_size": 32768,
+      "objective/train/original_loss": 1.63490891456604,
+      "objective/train/theoretical_loss": 4.544177353287637,
+      "objective/train/tokens_used": 171192800,
+      "theoretical_loss": 4.544177353287637,
+      "tokens_seen": 150732800
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007711775325977934,
+      "loss": 1.6508,
+      "theoretical_loss": 4.544177353287637,
+      "tokens_seen": 150732800
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007711614844533602,
+      "loss": 1.6864,
+      "theoretical_loss": 4.543920942732187,
+      "tokens_seen": 150798336
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007711454363089268,
+      "loss": 1.9748,
+      "theoretical_loss": 4.543664674773112,
+      "tokens_seen": 150863872
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007711293881644936,
+      "loss": 1.7271,
+      "theoretical_loss": 4.543408549269218,
+      "tokens_seen": 150929408
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007711133400200602,
+      "loss": 1.6813,
+      "theoretical_loss": 4.54315256607951,
+      "tokens_seen": 150994944
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.000771097291875627,
+      "loss": 1.8142,
+      "theoretical_loss": 4.542896725063195,
+      "tokens_seen": 151060480
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007710812437311936,
+      "loss": 1.8094,
+      "theoretical_loss": 4.542641026079683,
+      "tokens_seen": 151126016
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007710651955867604,
+      "loss": 1.7599,
+      "theoretical_loss": 4.542385468988581,
+      "tokens_seen": 151191552
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.000771049147442327,
+      "loss": 1.8046,
+      "theoretical_loss": 4.542130053649695,
+      "tokens_seen": 151257088
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007710330992978938,
+      "loss": 1.7616,
+      "theoretical_loss": 4.541874779923031,
+      "tokens_seen": 151322624
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007710170511534604,
+      "loss": 1.8387,
+      "theoretical_loss": 4.541619647668796,
+      "tokens_seen": 151388160
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007710010030090272,
+      "loss": 1.7802,
+      "theoretical_loss": 4.541364656747392,
+      "tokens_seen": 151453696
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007709849548645938,
+      "loss": 1.7625,
+      "theoretical_loss": 4.541109807019422,
+      "tokens_seen": 151519232
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007709689067201606,
+      "loss": 1.9998,
+      "theoretical_loss": 4.540855098345683,
+      "tokens_seen": 151584768
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007709528585757272,
+      "loss": 1.7149,
+      "theoretical_loss": 4.540600530587175,
+      "tokens_seen": 151650304
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.000770936810431294,
+      "loss": 1.776,
+      "theoretical_loss": 4.5403461036050885,
+      "tokens_seen": 151715840
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007709207622868606,
+      "loss": 1.7425,
+      "theoretical_loss": 4.540091817260815,
+      "tokens_seen": 151781376
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007709047141424274,
+      "loss": 1.7865,
+      "theoretical_loss": 4.539837671415941,
+      "tokens_seen": 151846912
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.000770888665997994,
+      "loss": 1.9128,
+      "theoretical_loss": 4.539583665932249,
+      "tokens_seen": 151912448
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007708726178535608,
+      "loss": 1.7491,
+      "theoretical_loss": 4.539329800671718,
+      "tokens_seen": 151977984
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007708565697091274,
+      "loss": 1.7516,
+      "theoretical_loss": 4.539076075496519,
+      "tokens_seen": 152043520
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007708405215646942,
+      "loss": 1.8449,
+      "theoretical_loss": 4.538822490269021,
+      "tokens_seen": 152109056
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007708244734202608,
+      "loss": 1.713,
+      "theoretical_loss": 4.538569044851785,
+      "tokens_seen": 152174592
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007708084252758276,
+      "loss": 1.7693,
+      "theoretical_loss": 4.538315739107571,
+      "tokens_seen": 152240128
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007707923771313942,
+      "loss": 1.7236,
+      "theoretical_loss": 4.5380625728993245,
+      "tokens_seen": 152305664
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.000770776328986961,
+      "loss": 1.6802,
+      "theoretical_loss": 4.537809546090193,
+      "tokens_seen": 152371200
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007707602808425276,
+      "loss": 1.7341,
+      "theoretical_loss": 4.53755665854351,
+      "tokens_seen": 152436736
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007707442326980944,
+      "loss": 1.8303,
+      "theoretical_loss": 4.537303910122807,
+      "tokens_seen": 152502272
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.000770728184553661,
+      "loss": 1.7807,
+      "theoretical_loss": 4.537051300691805,
+      "tokens_seen": 152567808
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007707121364092278,
+      "loss": 1.8008,
+      "theoretical_loss": 4.5367988301144155,
+      "tokens_seen": 152633344
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007706960882647944,
+      "loss": 1.7131,
+      "theoretical_loss": 4.536546498254746,
+      "tokens_seen": 152698880
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007706800401203612,
+      "loss": 1.9089,
+      "theoretical_loss": 4.536294304977092,
+      "tokens_seen": 152764416
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007706639919759278,
+      "loss": 1.6296,
+      "theoretical_loss": 4.536042250145941,
+      "tokens_seen": 152829952
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007706479438314946,
+      "loss": 1.8343,
+      "theoretical_loss": 4.53579033362597,
+      "tokens_seen": 152895488
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007706318956870612,
+      "loss": 1.8083,
+      "theoretical_loss": 4.535538555282048,
+      "tokens_seen": 152961024
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.000770615847542628,
+      "loss": 1.7174,
+      "theoretical_loss": 4.535286914979232,
+      "tokens_seen": 153026560
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007705997993981946,
+      "loss": 1.8334,
+      "theoretical_loss": 4.5350354125827685,
+      "tokens_seen": 153092096
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007705837512537614,
+      "loss": 1.868,
+      "theoretical_loss": 4.534784047958096,
+      "tokens_seen": 153157632
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.000770567703109328,
+      "loss": 1.91,
+      "theoretical_loss": 4.534532820970839,
+      "tokens_seen": 153223168
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007705516549648948,
+      "loss": 1.7686,
+      "theoretical_loss": 4.534281731486812,
+      "tokens_seen": 153288704
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007705356068204614,
+      "loss": 1.8133,
+      "theoretical_loss": 4.5340307793720145,
+      "tokens_seen": 153354240
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007705195586760282,
+      "loss": 1.6424,
+      "theoretical_loss": 4.533779964492638,
+      "tokens_seen": 153419776
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007705035105315948,
+      "loss": 1.6773,
+      "theoretical_loss": 4.533529286715059,
+      "tokens_seen": 153485312
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007704874623871616,
+      "loss": 1.7532,
+      "theoretical_loss": 4.533278745905843,
+      "tokens_seen": 153550848
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007704714142427282,
+      "loss": 1.7867,
+      "theoretical_loss": 4.533028341931738,
+      "tokens_seen": 153616384
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.000770455366098295,
+      "loss": 1.8078,
+      "theoretical_loss": 4.532778074659685,
+      "tokens_seen": 153681920
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007704393179538616,
+      "loss": 1.7447,
+      "theoretical_loss": 4.532527943956804,
+      "tokens_seen": 153747456
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007704232698094284,
+      "loss": 1.8068,
+      "theoretical_loss": 4.532277949690407,
+      "tokens_seen": 153812992
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.000770407221664995,
+      "loss": 1.8632,
+      "theoretical_loss": 4.532028091727986,
+      "tokens_seen": 153878528
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007703911735205618,
+      "loss": 1.5535,
+      "theoretical_loss": 4.531778369937222,
+      "tokens_seen": 153944064
+    },
+    {
+      "epoch": 0.05,
+      "objective/train/docs_used": 57715,
+      "objective/train/instantaneous_batch_size": 32,
+      "objective/train/instantaneous_microbatch_size": 32768,
+      "objective/train/original_loss": 1.7412933111190796,
+      "objective/train/theoretical_loss": 4.53152878418598,
+      "objective/train/tokens_used": 174469600,
+      "theoretical_loss": 4.53152878418598,
+      "tokens_seen": 154009600
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007703751253761284,
+      "loss": 1.7334,
+      "theoretical_loss": 4.53152878418598,
+      "tokens_seen": 154009600
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007703590772316952,
+      "loss": 1.7489,
+      "theoretical_loss": 4.531279334342306,
+      "tokens_seen": 154075136
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007703430290872618,
+      "loss": 1.6676,
+      "theoretical_loss": 4.531030020274436,
+      "tokens_seen": 154140672
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007703269809428285,
+      "loss": 1.7444,
+      "theoretical_loss": 4.530780841850783,
+      "tokens_seen": 154206208
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007703109327983952,
+      "loss": 1.7422,
+      "theoretical_loss": 4.5305317989399505,
+      "tokens_seen": 154271744
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007702948846539619,
+      "loss": 1.7678,
+      "theoretical_loss": 4.530282891410717,
+      "tokens_seen": 154337280
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007702788365095286,
+      "loss": 1.9158,
+      "theoretical_loss": 4.530034119132052,
+      "tokens_seen": 154402816
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007702627883650953,
+      "loss": 1.7104,
+      "theoretical_loss": 4.529785481973102,
+      "tokens_seen": 154468352
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.000770246740220662,
+      "loss": 1.7653,
+      "theoretical_loss": 4.529536979803195,
+      "tokens_seen": 154533888
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007702306920762287,
+      "loss": 1.7803,
+      "theoretical_loss": 4.529288612491846,
+      "tokens_seen": 154599424
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007702146439317954,
+      "loss": 1.8489,
+      "theoretical_loss": 4.5290403799087455,
+      "tokens_seen": 154664960
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007701985957873621,
+      "loss": 1.7535,
+      "theoretical_loss": 4.528792281923769,
+      "tokens_seen": 154730496
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007701825476429287,
+      "loss": 1.657,
+      "theoretical_loss": 4.52854431840697,
+      "tokens_seen": 154796032
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007701664994984955,
+      "loss": 1.8202,
+      "theoretical_loss": 4.528296489228585,
+      "tokens_seen": 154861568
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007701504513540621,
+      "loss": 1.8586,
+      "theoretical_loss": 4.528048794259028,
+      "tokens_seen": 154927104
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007701344032096289,
+      "loss": 1.7151,
+      "theoretical_loss": 4.527801233368896,
+      "tokens_seen": 154992640
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007701183550651955,
+      "loss": 1.8635,
+      "theoretical_loss": 4.527553806428962,
+      "tokens_seen": 155058176
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007701023069207623,
+      "loss": 1.7421,
+      "theoretical_loss": 4.5273065133101795,
+      "tokens_seen": 155123712
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007700862587763289,
+      "loss": 1.7428,
+      "theoretical_loss": 4.527059353883682,
+      "tokens_seen": 155189248
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007700702106318957,
+      "loss": 1.7279,
+      "theoretical_loss": 4.526812328020778,
+      "tokens_seen": 155254784
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007700541624874623,
+      "loss": 1.5764,
+      "theoretical_loss": 4.52656543559296,
+      "tokens_seen": 155320320
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007700381143430291,
+      "loss": 1.9455,
+      "theoretical_loss": 4.526318676471891,
+      "tokens_seen": 155385856
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007700220661985957,
+      "loss": 1.8923,
+      "theoretical_loss": 4.526072050529417,
+      "tokens_seen": 155451392
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007700060180541625,
+      "loss": 1.8616,
+      "theoretical_loss": 4.52582555763756,
+      "tokens_seen": 155516928
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007699899699097291,
+      "loss": 1.7906,
+      "theoretical_loss": 4.525579197668517,
+      "tokens_seen": 155582464
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007699739217652959,
+      "loss": 1.8154,
+      "theoretical_loss": 4.525332970494663,
+      "tokens_seen": 155648000
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007699578736208627,
+      "loss": 1.7713,
+      "theoretical_loss": 4.525086875988549,
+      "tokens_seen": 155713536
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007699418254764293,
+      "loss": 1.7506,
+      "theoretical_loss": 4.524840914022902,
+      "tokens_seen": 155779072
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.000769925777331996,
+      "loss": 1.887,
+      "theoretical_loss": 4.524595084470625,
+      "tokens_seen": 155844608
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007699097291875627,
+      "loss": 1.7664,
+      "theoretical_loss": 4.524349387204794,
+      "tokens_seen": 155910144
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007698936810431295,
+      "loss": 1.7646,
+      "theoretical_loss": 4.524103822098664,
+      "tokens_seen": 155975680
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007698776328986961,
+      "loss": 1.7573,
+      "theoretical_loss": 4.52385838902566,
+      "tokens_seen": 156041216
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007698615847542629,
+      "loss": 1.8096,
+      "theoretical_loss": 4.523613087859386,
+      "tokens_seen": 156106752
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007698455366098295,
+      "loss": 1.7087,
+      "theoretical_loss": 4.523367918473614,
+      "tokens_seen": 156172288
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007698294884653963,
+      "loss": 1.824,
+      "theoretical_loss": 4.523122880742298,
+      "tokens_seen": 156237824
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007698134403209629,
+      "loss": 1.7668,
+      "theoretical_loss": 4.522877974539556,
+      "tokens_seen": 156303360
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007697973921765297,
+      "loss": 1.6875,
+      "theoretical_loss": 4.522633199739687,
+      "tokens_seen": 156368896
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007697813440320963,
+      "loss": 1.7105,
+      "theoretical_loss": 4.522388556217157,
+      "tokens_seen": 156434432
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007697652958876631,
+      "loss": 1.7198,
+      "theoretical_loss": 4.522144043846609,
+      "tokens_seen": 156499968
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007697492477432297,
+      "loss": 1.6635,
+      "theoretical_loss": 4.521899662502856,
+      "tokens_seen": 156565504
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007697331995987965,
+      "loss": 1.7064,
+      "theoretical_loss": 4.5216554120608805,
+      "tokens_seen": 156631040
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007697171514543631,
+      "loss": 1.8732,
+      "theoretical_loss": 4.521411292395841,
+      "tokens_seen": 156696576
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007697011033099299,
+      "loss": 1.6125,
+      "theoretical_loss": 4.521167303383065,
+      "tokens_seen": 156762112
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007696850551654965,
+      "loss": 1.8425,
+      "theoretical_loss": 4.520923444898051,
+      "tokens_seen": 156827648
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007696690070210633,
+      "loss": 1.6217,
+      "theoretical_loss": 4.520679716816467,
+      "tokens_seen": 156893184
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007696529588766299,
+      "loss": 1.706,
+      "theoretical_loss": 4.520436119014155,
+      "tokens_seen": 156958720
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007696369107321967,
+      "loss": 1.7967,
+      "theoretical_loss": 4.520192651367121,
+      "tokens_seen": 157024256
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007696208625877633,
+      "loss": 1.9293,
+      "theoretical_loss": 4.519949313751547,
+      "tokens_seen": 157089792
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007696048144433301,
+      "loss": 1.7454,
+      "theoretical_loss": 4.519706106043779,
+      "tokens_seen": 157155328
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007695887662988967,
+      "loss": 1.8863,
+      "theoretical_loss": 4.519463028120338,
+      "tokens_seen": 157220864
+    },
+    {
+      "epoch": 0.05,
+      "objective/train/docs_used": 59019,
+      "objective/train/instantaneous_batch_size": 32,
+      "objective/train/instantaneous_microbatch_size": 32768,
+      "objective/train/original_loss": 1.6308947801589966,
+      "objective/train/theoretical_loss": 4.519220079857908,
+      "objective/train/tokens_used": 177746400,
+      "theoretical_loss": 4.519220079857908,
+      "tokens_seen": 157286400
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007695727181544635,
+      "loss": 1.6509,
+      "theoretical_loss": 4.519220079857908,
+      "tokens_seen": 157286400
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007695566700100301,
+      "loss": 1.7408,
+      "theoretical_loss": 4.518977261133343,
+      "tokens_seen": 157351936
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007695406218655969,
+      "loss": 1.8094,
+      "theoretical_loss": 4.518734571823668,
+      "tokens_seen": 157417472
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007695245737211635,
+      "loss": 1.6798,
+      "theoretical_loss": 4.518492011806073,
+      "tokens_seen": 157483008
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007695085255767303,
+      "loss": 1.8061,
+      "theoretical_loss": 4.518249580957917,
+      "tokens_seen": 157548544
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007694924774322969,
+      "loss": 1.7263,
+      "theoretical_loss": 4.518007279156725,
+      "tokens_seen": 157614080
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007694764292878637,
+      "loss": 1.8322,
+      "theoretical_loss": 4.5177651062801925,
+      "tokens_seen": 157679616
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007694603811434303,
+      "loss": 1.7504,
+      "theoretical_loss": 4.517523062206175,
+      "tokens_seen": 157745152
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007694443329989971,
+      "loss": 1.8138,
+      "theoretical_loss": 4.517281146812701,
+      "tokens_seen": 157810688
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007694282848545637,
+      "loss": 1.7472,
+      "theoretical_loss": 4.517039359977961,
+      "tokens_seen": 157876224
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007694122367101305,
+      "loss": 1.7232,
+      "theoretical_loss": 4.516797701580314,
+      "tokens_seen": 157941760
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007693961885656971,
+      "loss": 1.7443,
+      "theoretical_loss": 4.516556171498284,
+      "tokens_seen": 158007296
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007693801404212639,
+      "loss": 1.6531,
+      "theoretical_loss": 4.516314769610557,
+      "tokens_seen": 158072832
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007693640922768305,
+      "loss": 1.6818,
+      "theoretical_loss": 4.516073495795988,
+      "tokens_seen": 158138368
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007693480441323973,
+      "loss": 1.7311,
+      "theoretical_loss": 4.5158323499335955,
+      "tokens_seen": 158203904
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007693319959879639,
+      "loss": 1.7534,
+      "theoretical_loss": 4.515591331902561,
+      "tokens_seen": 158269440
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007693159478435307,
+      "loss": 1.7351,
+      "theoretical_loss": 4.515350441582231,
+      "tokens_seen": 158334976
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007692998996990973,
+      "loss": 1.7799,
+      "theoretical_loss": 4.515109678852117,
+      "tokens_seen": 158400512
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007692838515546641,
+      "loss": 1.8024,
+      "theoretical_loss": 4.514869043591891,
+      "tokens_seen": 158466048
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007692678034102307,
+      "loss": 1.7493,
+      "theoretical_loss": 4.514628535681391,
+      "tokens_seen": 158531584
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007692517552657975,
+      "loss": 1.7613,
+      "theoretical_loss": 4.514388155000615,
+      "tokens_seen": 158597120
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007692357071213641,
+      "loss": 1.8053,
+      "theoretical_loss": 4.5141479014297285,
+      "tokens_seen": 158662656
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007692196589769309,
+      "loss": 1.8065,
+      "theoretical_loss": 4.513907774849054,
+      "tokens_seen": 158728192
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007692036108324975,
+      "loss": 1.8002,
+      "theoretical_loss": 4.513667775139078,
+      "tokens_seen": 158793728
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007691875626880643,
+      "loss": 1.749,
+      "theoretical_loss": 4.5134279021804495,
+      "tokens_seen": 158859264
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007691715145436309,
+      "loss": 1.7745,
+      "theoretical_loss": 4.513188155853978,
+      "tokens_seen": 158924800
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007691554663991977,
+      "loss": 1.66,
+      "theoretical_loss": 4.512948536040636,
+      "tokens_seen": 158990336
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007691394182547643,
+      "loss": 1.7401,
+      "theoretical_loss": 4.512709042621554,
+      "tokens_seen": 159055872
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007691233701103311,
+      "loss": 1.7178,
+      "theoretical_loss": 4.512469675478026,
+      "tokens_seen": 159121408
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007691073219658977,
+      "loss": 1.7648,
+      "theoretical_loss": 4.512230434491503,
+      "tokens_seen": 159186944
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007690912738214645,
+      "loss": 1.9136,
+      "theoretical_loss": 4.511991319543599,
+      "tokens_seen": 159252480
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007690752256770311,
+      "loss": 1.7195,
+      "theoretical_loss": 4.511752330516086,
+      "tokens_seen": 159318016
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007690591775325979,
+      "loss": 1.8116,
+      "theoretical_loss": 4.5115134672908965,
+      "tokens_seen": 159383552
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007690431293881645,
+      "loss": 1.7999,
+      "theoretical_loss": 4.511274729750123,
+      "tokens_seen": 159449088
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007690270812437313,
+      "loss": 1.7108,
+      "theoretical_loss": 4.5110361177760145,
+      "tokens_seen": 159514624
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007690110330992979,
+      "loss": 1.7656,
+      "theoretical_loss": 4.5107976312509805,
+      "tokens_seen": 159580160
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007689949849548647,
+      "loss": 1.8134,
+      "theoretical_loss": 4.510559270057587,
+      "tokens_seen": 159645696
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007689789368104313,
+      "loss": 1.771,
+      "theoretical_loss": 4.51032103407856,
+      "tokens_seen": 159711232
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007689628886659981,
+      "loss": 1.6666,
+      "theoretical_loss": 4.510082923196784,
+      "tokens_seen": 159776768
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007689468405215647,
+      "loss": 1.6879,
+      "theoretical_loss": 4.5098449372952985,
+      "tokens_seen": 159842304
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007689307923771315,
+      "loss": 1.6481,
+      "theoretical_loss": 4.5096070762573,
+      "tokens_seen": 159907840
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007689147442326981,
+      "loss": 1.631,
+      "theoretical_loss": 4.509369339966146,
+      "tokens_seen": 159973376
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007688986960882648,
+      "loss": 1.7176,
+      "theoretical_loss": 4.509131728305347,
+      "tokens_seen": 160038912
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007688826479438315,
+      "loss": 1.7804,
+      "theoretical_loss": 4.50889424115857,
+      "tokens_seen": 160104448
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007688665997993982,
+      "loss": 1.83,
+      "theoretical_loss": 4.50865687840964,
+      "tokens_seen": 160169984
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007688505516549649,
+      "loss": 1.7308,
+      "theoretical_loss": 4.508419639942538,
+      "tokens_seen": 160235520
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007688345035105316,
+      "loss": 1.7693,
+      "theoretical_loss": 4.5081825256413985,
+      "tokens_seen": 160301056
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007688184553660983,
+      "loss": 1.7288,
+      "theoretical_loss": 4.507945535390512,
+      "tokens_seen": 160366592
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.000768802407221665,
+      "loss": 1.7572,
+      "theoretical_loss": 4.507708669074326,
+      "tokens_seen": 160432128
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007687863590772316,
+      "loss": 1.7183,
+      "theoretical_loss": 4.5074719265774394,
+      "tokens_seen": 160497664
+    },
+    {
+      "epoch": 0.05,
+      "objective/train/docs_used": 59900,
+      "objective/train/instantaneous_batch_size": 32,
+      "objective/train/instantaneous_microbatch_size": 32768,
+      "objective/train/original_loss": 1.7780349254608154,
+      "objective/train/theoretical_loss": 4.507235307784608,
+      "objective/train/tokens_used": 181023200,
+      "theoretical_loss": 4.507235307784608,
+      "tokens_seen": 160563200
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007687703109327984,
+      "loss": 1.674,
+      "theoretical_loss": 4.507235307784608,
+      "tokens_seen": 160563200
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007687542627883651,
+      "loss": 1.9229,
+      "theoretical_loss": 4.5069988125807425,
+      "tokens_seen": 160628736
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007687382146439318,
+      "loss": 1.7718,
+      "theoretical_loss": 4.506762440850906,
+      "tokens_seen": 160694272
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007687221664994985,
+      "loss": 1.6868,
+      "theoretical_loss": 4.506526192480315,
+      "tokens_seen": 160759808
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007687061183550652,
+      "loss": 1.7526,
+      "theoretical_loss": 4.506290067354341,
+      "tokens_seen": 160825344
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.000768690070210632,
+      "loss": 1.6493,
+      "theoretical_loss": 4.506054065358508,
+      "tokens_seen": 160890880
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007686740220661986,
+      "loss": 1.6333,
+      "theoretical_loss": 4.505818186378491,
+      "tokens_seen": 160956416
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007686579739217653,
+      "loss": 1.8546,
+      "theoretical_loss": 4.505582430300121,
+      "tokens_seen": 161021952
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.000768641925777332,
+      "loss": 1.8297,
+      "theoretical_loss": 4.505346797009381,
+      "tokens_seen": 161087488
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007686258776328987,
+      "loss": 1.7726,
+      "theoretical_loss": 4.505111286392402,
+      "tokens_seen": 161153024
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007686098294884654,
+      "loss": 1.8979,
+      "theoretical_loss": 4.50487589833547,
+      "tokens_seen": 161218560
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007685937813440322,
+      "loss": 1.7368,
+      "theoretical_loss": 4.504640632725025,
+      "tokens_seen": 161284096
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007685777331995988,
+      "loss": 1.714,
+      "theoretical_loss": 4.504405489447654,
+      "tokens_seen": 161349632
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007685616850551656,
+      "loss": 1.7229,
+      "theoretical_loss": 4.504170468390096,
+      "tokens_seen": 161415168
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007685456369107322,
+      "loss": 1.7624,
+      "theoretical_loss": 4.503935569439242,
+      "tokens_seen": 161480704
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.000768529588766299,
+      "loss": 1.8142,
+      "theoretical_loss": 4.5037007924821335,
+      "tokens_seen": 161546240
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007685135406218656,
+      "loss": 1.654,
+      "theoretical_loss": 4.503466137405961,
+      "tokens_seen": 161611776
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007684974924774324,
+      "loss": 1.7044,
+      "theoretical_loss": 4.503231604098065,
+      "tokens_seen": 161677312
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.000768481444332999,
+      "loss": 1.8339,
+      "theoretical_loss": 4.50299719244594,
+      "tokens_seen": 161742848
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007684653961885658,
+      "loss": 1.7165,
+      "theoretical_loss": 4.502762902337222,
+      "tokens_seen": 161808384
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007684493480441324,
+      "loss": 1.793,
+      "theoretical_loss": 4.502528733659704,
+      "tokens_seen": 161873920
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007684332998996992,
+      "loss": 1.7393,
+      "theoretical_loss": 4.502294686301323,
+      "tokens_seen": 161939456
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007684172517552658,
+      "loss": 1.9238,
+      "theoretical_loss": 4.502060760150167,
+      "tokens_seen": 162004992
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007684012036108326,
+      "loss": 1.7235,
+      "theoretical_loss": 4.501826955094472,
+      "tokens_seen": 162070528
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007683851554663992,
+      "loss": 1.6647,
+      "theoretical_loss": 4.501593271022623,
+      "tokens_seen": 162136064
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.000768369107321966,
+      "loss": 1.7966,
+      "theoretical_loss": 4.501359707823149,
+      "tokens_seen": 162201600
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007683530591775326,
+      "loss": 1.7503,
+      "theoretical_loss": 4.501126265384732,
+      "tokens_seen": 162267136
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007683370110330994,
+      "loss": 1.6422,
+      "theoretical_loss": 4.500892943596199,
+      "tokens_seen": 162332672
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.000768320962888666,
+      "loss": 1.7443,
+      "theoretical_loss": 4.500659742346524,
+      "tokens_seen": 162398208
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007683049147442328,
+      "loss": 1.8113,
+      "theoretical_loss": 4.500426661524828,
+      "tokens_seen": 162463744
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007682888665997994,
+      "loss": 1.8531,
+      "theoretical_loss": 4.500193701020379,
+      "tokens_seen": 162529280
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007682728184553662,
+      "loss": 1.8051,
+      "theoretical_loss": 4.499960860722592,
+      "tokens_seen": 162594816
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007682567703109328,
+      "loss": 1.8076,
+      "theoretical_loss": 4.499728140521025,
+      "tokens_seen": 162660352
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007682407221664996,
+      "loss": 1.6289,
+      "theoretical_loss": 4.499495540305388,
+      "tokens_seen": 162725888
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007682246740220662,
+      "loss": 1.8266,
+      "theoretical_loss": 4.49926305996553,
+      "tokens_seen": 162791424
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.000768208625877633,
+      "loss": 1.7347,
+      "theoretical_loss": 4.499030699391449,
+      "tokens_seen": 162856960
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007681925777331996,
+      "loss": 1.7979,
+      "theoretical_loss": 4.49879845847329,
+      "tokens_seen": 162922496
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007681765295887664,
+      "loss": 1.6315,
+      "theoretical_loss": 4.498566337101337,
+      "tokens_seen": 162988032
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.000768160481444333,
+      "loss": 1.8132,
+      "theoretical_loss": 4.4983343351660245,
+      "tokens_seen": 163053568
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007681444332998998,
+      "loss": 1.7402,
+      "theoretical_loss": 4.498102452557928,
+      "tokens_seen": 163119104
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007681283851554664,
+      "loss": 1.7068,
+      "theoretical_loss": 4.497870689167771,
+      "tokens_seen": 163184640
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007681123370110332,
+      "loss": 1.6993,
+      "theoretical_loss": 4.497639044886412,
+      "tokens_seen": 163250176
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007680962888665998,
+      "loss": 1.7059,
+      "theoretical_loss": 4.497407519604866,
+      "tokens_seen": 163315712
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007680802407221666,
+      "loss": 1.6622,
+      "theoretical_loss": 4.49717611321428,
+      "tokens_seen": 163381248
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007680641925777332,
+      "loss": 1.7537,
+      "theoretical_loss": 4.496944825605951,
+      "tokens_seen": 163446784
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007680481444333,
+      "loss": 1.7066,
+      "theoretical_loss": 4.496713656671317,
+      "tokens_seen": 163512320
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007680320962888666,
+      "loss": 1.6718,
+      "theoretical_loss": 4.496482606301957,
+      "tokens_seen": 163577856
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007680160481444334,
+      "loss": 1.9074,
+      "theoretical_loss": 4.496251674389596,
+      "tokens_seen": 163643392
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.000768,
+      "loss": 1.709,
+      "theoretical_loss": 4.496020860826098,
+      "tokens_seen": 163708928
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007679839518555668,
+      "loss": 1.6631,
+      "theoretical_loss": 4.49579016550347,
+      "tokens_seen": 163774464
+    },
+    {
+      "epoch": 0.05,
+      "objective/train/docs_used": 60960,
+      "objective/train/instantaneous_batch_size": 32,
+      "objective/train/instantaneous_microbatch_size": 32768,
+      "objective/train/original_loss": 1.8355538845062256,
+      "objective/train/theoretical_loss": 4.49555958831386,
+      "objective/train/tokens_used": 184300000,
+      "theoretical_loss": 4.49555958831386,
+      "tokens_seen": 163840000
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007679679037111334,
+      "loss": 1.7909,
+      "theoretical_loss": 4.49555958831386,
+      "tokens_seen": 163840000
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007679518555667002,
+      "loss": 1.7424,
+      "theoretical_loss": 4.49532912914956,
+      "tokens_seen": 163905536
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007679358074222668,
+      "loss": 1.7327,
+      "theoretical_loss": 4.495098787903,
+      "tokens_seen": 163971072
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007679197592778336,
+      "loss": 1.7795,
+      "theoretical_loss": 4.4948685644667545,
+      "tokens_seen": 164036608
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007679037111334002,
+      "loss": 1.7746,
+      "theoretical_loss": 4.494638458733535,
+      "tokens_seen": 164102144
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.000767887662988967,
+      "loss": 1.8402,
+      "theoretical_loss": 4.494408470596195,
+      "tokens_seen": 164167680
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007678716148445336,
+      "loss": 1.6975,
+      "theoretical_loss": 4.494178599947729,
+      "tokens_seen": 164233216
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007678555667001004,
+      "loss": 1.5993,
+      "theoretical_loss": 4.4939488466812705,
+      "tokens_seen": 164298752
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.000767839518555667,
+      "loss": 1.7182,
+      "theoretical_loss": 4.493719210690093,
+      "tokens_seen": 164364288
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007678234704112338,
+      "loss": 1.8341,
+      "theoretical_loss": 4.493489691867612,
+      "tokens_seen": 164429824
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007678074222668004,
+      "loss": 1.6518,
+      "theoretical_loss": 4.493260290107379,
+      "tokens_seen": 164495360
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007677913741223672,
+      "loss": 1.6636,
+      "theoretical_loss": 4.493031005303084,
+      "tokens_seen": 164560896
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007677753259779338,
+      "loss": 1.7213,
+      "theoretical_loss": 4.49280183734856,
+      "tokens_seen": 164626432
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007677592778335006,
+      "loss": 1.7418,
+      "theoretical_loss": 4.492572786137774,
+      "tokens_seen": 164691968
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007677432296890672,
+      "loss": 1.6195,
+      "theoretical_loss": 4.492343851564835,
+      "tokens_seen": 164757504
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.000767727181544634,
+      "loss": 1.6928,
+      "theoretical_loss": 4.492115033523987,
+      "tokens_seen": 164823040
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007677111334002006,
+      "loss": 1.7838,
+      "theoretical_loss": 4.491886331909615,
+      "tokens_seen": 164888576
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007676950852557674,
+      "loss": 1.8192,
+      "theoretical_loss": 4.491657746616241,
+      "tokens_seen": 164954112
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.000767679037111334,
+      "loss": 1.7331,
+      "theoretical_loss": 4.49142927753852,
+      "tokens_seen": 165019648
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007676629889669008,
+      "loss": 1.6931,
+      "theoretical_loss": 4.491200924571251,
+      "tokens_seen": 165085184
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007676469408224674,
+      "loss": 1.6643,
+      "theoretical_loss": 4.490972687609364,
+      "tokens_seen": 165150720
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007676308926780342,
+      "loss": 1.6989,
+      "theoretical_loss": 4.4907445665479315,
+      "tokens_seen": 165216256
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007676148445336008,
+      "loss": 1.7728,
+      "theoretical_loss": 4.490516561282157,
+      "tokens_seen": 165281792
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007675987963891676,
+      "loss": 1.7324,
+      "theoretical_loss": 4.4902886717073835,
+      "tokens_seen": 165347328
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007675827482447342,
+      "loss": 1.6209,
+      "theoretical_loss": 4.49006089771909,
+      "tokens_seen": 165412864
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.000767566700100301,
+      "loss": 1.8131,
+      "theoretical_loss": 4.489833239212888,
+      "tokens_seen": 165478400
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007675506519558676,
+      "loss": 1.8885,
+      "theoretical_loss": 4.48960569608453,
+      "tokens_seen": 165543936
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007675346038114344,
+      "loss": 1.7492,
+      "theoretical_loss": 4.489378268229897,
+      "tokens_seen": 165609472
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.000767518555667001,
+      "loss": 1.7735,
+      "theoretical_loss": 4.489150955545012,
+      "tokens_seen": 165675008
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007675025075225677,
+      "loss": 1.6713,
+      "theoretical_loss": 4.488923757926031,
+      "tokens_seen": 165740544
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007674864593781344,
+      "loss": 1.6972,
+      "theoretical_loss": 4.4886966752692405,
+      "tokens_seen": 165806080
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007674704112337011,
+      "loss": 1.8043,
+      "theoretical_loss": 4.488469707471066,
+      "tokens_seen": 165871616
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007674543630892678,
+      "loss": 1.7284,
+      "theoretical_loss": 4.488242854428064,
+      "tokens_seen": 165937152
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007674383149448345,
+      "loss": 1.7696,
+      "theoretical_loss": 4.488016116036929,
+      "tokens_seen": 166002688
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007674222668004012,
+      "loss": 1.6841,
+      "theoretical_loss": 4.487789492194485,
+      "tokens_seen": 166068224
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007674062186559679,
+      "loss": 1.6543,
+      "theoretical_loss": 4.487562982797693,
+      "tokens_seen": 166133760
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007673901705115346,
+      "loss": 1.6708,
+      "theoretical_loss": 4.487336587743643,
+      "tokens_seen": 166199296
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007673741223671013,
+      "loss": 1.7559,
+      "theoretical_loss": 4.487110306929562,
+      "tokens_seen": 166264832
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.000767358074222668,
+      "loss": 1.7378,
+      "theoretical_loss": 4.48688414025281,
+      "tokens_seen": 166330368
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007673420260782347,
+      "loss": 1.6406,
+      "theoretical_loss": 4.486658087610875,
+      "tokens_seen": 166395904
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007673259779338014,
+      "loss": 1.6805,
+      "theoretical_loss": 4.486432148901384,
+      "tokens_seen": 166461440
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007673099297893681,
+      "loss": 1.6788,
+      "theoretical_loss": 4.48620632402209,
+      "tokens_seen": 166526976
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007672938816449348,
+      "loss": 1.7499,
+      "theoretical_loss": 4.485980612870883,
+      "tokens_seen": 166592512
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007672778335005015,
+      "loss": 1.7161,
+      "theoretical_loss": 4.48575501534578,
+      "tokens_seen": 166658048
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007672617853560682,
+      "loss": 1.669,
+      "theoretical_loss": 4.485529531344932,
+      "tokens_seen": 166723584
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007672457372116349,
+      "loss": 1.7296,
+      "theoretical_loss": 4.485304160766624,
+      "tokens_seen": 166789120
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007672296890672016,
+      "loss": 1.733,
+      "theoretical_loss": 4.485078903509266,
+      "tokens_seen": 166854656
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007672136409227683,
+      "loss": 1.8134,
+      "theoretical_loss": 4.484853759471403,
+      "tokens_seen": 166920192
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.000767197592778335,
+      "loss": 1.8388,
+      "theoretical_loss": 4.484628728551711,
+      "tokens_seen": 166985728
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007671815446339017,
+      "loss": 1.7632,
+      "theoretical_loss": 4.484403810648994,
+      "tokens_seen": 167051264
+    },
+    {
+      "epoch": 0.05,
+      "objective/train/docs_used": 62225,
+      "objective/train/instantaneous_batch_size": 32,
+      "objective/train/instantaneous_microbatch_size": 32768,
+      "objective/train/original_loss": 1.8262088298797607,
+      "objective/train/theoretical_loss": 4.484179005662186,
+      "objective/train/tokens_used": 187576800,
+      "theoretical_loss": 4.484179005662186,
+      "tokens_seen": 167116800
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007671654964894685,
+      "loss": 1.7725,
+      "theoretical_loss": 4.484179005662186,
+      "tokens_seen": 167116800
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007671494483450351,
+      "loss": 1.6502,
+      "theoretical_loss": 4.483954313490354,
+      "tokens_seen": 167182336
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007671334002006019,
+      "loss": 1.6791,
+      "theoretical_loss": 4.483729734032694,
+      "tokens_seen": 167247872
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007671173520561685,
+      "loss": 1.693,
+      "theoretical_loss": 4.483505267188528,
+      "tokens_seen": 167313408
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007671013039117353,
+      "loss": 1.7342,
+      "theoretical_loss": 4.48328091285731,
+      "tokens_seen": 167378944
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007670852557673019,
+      "loss": 1.7493,
+      "theoretical_loss": 4.483056670938625,
+      "tokens_seen": 167444480
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007670692076228687,
+      "loss": 1.7194,
+      "theoretical_loss": 4.482832541332183,
+      "tokens_seen": 167510016
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007670531594784353,
+      "loss": 1.6792,
+      "theoretical_loss": 4.482608523937824,
+      "tokens_seen": 167575552
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007670371113340021,
+      "loss": 1.8132,
+      "theoretical_loss": 4.4823846186555185,
+      "tokens_seen": 167641088
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007670210631895687,
+      "loss": 1.7539,
+      "theoretical_loss": 4.482160825385363,
+      "tokens_seen": 167706624
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007670050150451355,
+      "loss": 1.7526,
+      "theoretical_loss": 4.481937144027583,
+      "tokens_seen": 167772160
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007669889669007021,
+      "loss": 1.7945,
+      "theoretical_loss": 4.48171357448253,
+      "tokens_seen": 167837696
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007669729187562689,
+      "loss": 1.6467,
+      "theoretical_loss": 4.481490116650686,
+      "tokens_seen": 167903232
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007669568706118355,
+      "loss": 1.7569,
+      "theoretical_loss": 4.48126677043266,
+      "tokens_seen": 167968768
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007669408224674023,
+      "loss": 1.666,
+      "theoretical_loss": 4.481043535729183,
+      "tokens_seen": 168034304
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007669247743229689,
+      "loss": 1.761,
+      "theoretical_loss": 4.480820412441122,
+      "tokens_seen": 168099840
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007669087261785357,
+      "loss": 1.6113,
+      "theoretical_loss": 4.4805974004694615,
+      "tokens_seen": 168165376
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007668926780341023,
+      "loss": 1.74,
+      "theoretical_loss": 4.48037449971532,
+      "tokens_seen": 168230912
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007668766298896691,
+      "loss": 1.8286,
+      "theoretical_loss": 4.480151710079937,
+      "tokens_seen": 168296448
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007668605817452357,
+      "loss": 1.7082,
+      "theoretical_loss": 4.479929031464682,
+      "tokens_seen": 168361984
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007668445336008025,
+      "loss": 1.7087,
+      "theoretical_loss": 4.479706463771047,
+      "tokens_seen": 168427520
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007668284854563691,
+      "loss": 1.685,
+      "theoretical_loss": 4.479484006900652,
+      "tokens_seen": 168493056
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007668124373119359,
+      "loss": 1.7437,
+      "theoretical_loss": 4.479261660755242,
+      "tokens_seen": 168558592
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007667963891675025,
+      "loss": 1.7892,
+      "theoretical_loss": 4.479039425236687,
+      "tokens_seen": 168624128
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007667803410230693,
+      "loss": 1.8271,
+      "theoretical_loss": 4.478817300246982,
+      "tokens_seen": 168689664
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007667642928786359,
+      "loss": 1.7136,
+      "theoretical_loss": 4.478595285688247,
+      "tokens_seen": 168755200
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007667482447342027,
+      "loss": 1.7083,
+      "theoretical_loss": 4.478373381462728,
+      "tokens_seen": 168820736
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007667321965897693,
+      "loss": 1.6812,
+      "theoretical_loss": 4.478151587472791,
+      "tokens_seen": 168886272
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007667161484453361,
+      "loss": 1.6949,
+      "theoretical_loss": 4.477929903620931,
+      "tokens_seen": 168951808
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007667001003009027,
+      "loss": 1.7211,
+      "theoretical_loss": 4.477708329809766,
+      "tokens_seen": 169017344
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007666840521564695,
+      "loss": 1.8296,
+      "theoretical_loss": 4.477486865942036,
+      "tokens_seen": 169082880
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007666680040120361,
+      "loss": 1.7783,
+      "theoretical_loss": 4.477265511920607,
+      "tokens_seen": 169148416
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007666519558676029,
+      "loss": 1.7958,
+      "theoretical_loss": 4.477044267648466,
+      "tokens_seen": 169213952
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007666359077231695,
+      "loss": 1.7416,
+      "theoretical_loss": 4.476823133028725,
+      "tokens_seen": 169279488
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007666198595787363,
+      "loss": 1.7753,
+      "theoretical_loss": 4.47660210796462,
+      "tokens_seen": 169345024
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007666038114343029,
+      "loss": 1.681,
+      "theoretical_loss": 4.476381192359504,
+      "tokens_seen": 169410560
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007665877632898697,
+      "loss": 1.7031,
+      "theoretical_loss": 4.476160386116861,
+      "tokens_seen": 169476096
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007665717151454363,
+      "loss": 1.6248,
+      "theoretical_loss": 4.475939689140291,
+      "tokens_seen": 169541632
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007665556670010031,
+      "loss": 1.6146,
+      "theoretical_loss": 4.475719101333519,
+      "tokens_seen": 169607168
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007665396188565697,
+      "loss": 1.768,
+      "theoretical_loss": 4.475498622600391,
+      "tokens_seen": 169672704
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007665235707121365,
+      "loss": 1.6983,
+      "theoretical_loss": 4.475278252844875,
+      "tokens_seen": 169738240
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007665075225677031,
+      "loss": 1.7969,
+      "theoretical_loss": 4.475057991971062,
+      "tokens_seen": 169803776
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007664914744232699,
+      "loss": 1.8226,
+      "theoretical_loss": 4.474837839883162,
+      "tokens_seen": 169869312
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007664754262788365,
+      "loss": 1.7586,
+      "theoretical_loss": 4.474617796485507,
+      "tokens_seen": 169934848
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007664593781344033,
+      "loss": 1.7622,
+      "theoretical_loss": 4.474397861682552,
+      "tokens_seen": 170000384
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007664433299899699,
+      "loss": 1.8242,
+      "theoretical_loss": 4.474178035378869,
+      "tokens_seen": 170065920
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007664272818455367,
+      "loss": 1.8125,
+      "theoretical_loss": 4.473958317479154,
+      "tokens_seen": 170131456
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007664112337011033,
+      "loss": 1.6376,
+      "theoretical_loss": 4.473738707888221,
+      "tokens_seen": 170196992
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007663951855566701,
+      "loss": 1.8528,
+      "theoretical_loss": 4.473519206511006,
+      "tokens_seen": 170262528
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007663791374122367,
+      "loss": 1.8236,
+      "theoretical_loss": 4.473299813252565,
+      "tokens_seen": 170328064
+    },
+    {
+      "epoch": 0.05,
+      "objective/train/docs_used": 63071,
+      "objective/train/instantaneous_batch_size": 32,
+      "objective/train/instantaneous_microbatch_size": 32768,
+      "objective/train/original_loss": 1.8780279159545898,
+      "objective/train/theoretical_loss": 4.473080528018071,
+      "objective/train/tokens_used": 190853600,
+      "theoretical_loss": 4.473080528018071,
+      "tokens_seen": 170393600
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007663630892678035,
+      "loss": 1.8163,
+      "theoretical_loss": 4.473080528018071,
+      "tokens_seen": 170393600
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007663470411233702,
+      "loss": 1.7009,
+      "theoretical_loss": 4.4728613507128205,
+      "tokens_seen": 170459136
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007663309929789369,
+      "loss": 1.7468,
+      "theoretical_loss": 4.472642281242226,
+      "tokens_seen": 170524672
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007663149448345036,
+      "loss": 1.7318,
+      "theoretical_loss": 4.472423319511822,
+      "tokens_seen": 170590208
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007662988966900703,
+      "loss": 1.6851,
+      "theoretical_loss": 4.472204465427261,
+      "tokens_seen": 170655744
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.000766282848545637,
+      "loss": 1.8008,
+      "theoretical_loss": 4.471985718894312,
+      "tokens_seen": 170721280
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007662668004012037,
+      "loss": 1.8603,
+      "theoretical_loss": 4.471767079818866,
+      "tokens_seen": 170786816
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007662507522567703,
+      "loss": 1.6043,
+      "theoretical_loss": 4.471548548106932,
+      "tokens_seen": 170852352
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007662347041123371,
+      "loss": 1.6812,
+      "theoretical_loss": 4.471330123664634,
+      "tokens_seen": 170917888
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007662186559679037,
+      "loss": 1.8008,
+      "theoretical_loss": 4.471111806398218,
+      "tokens_seen": 170983424
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007662026078234705,
+      "loss": 1.807,
+      "theoretical_loss": 4.4708935962140455,
+      "tokens_seen": 171048960
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007661865596790371,
+      "loss": 1.8099,
+      "theoretical_loss": 4.4706754930185975,
+      "tokens_seen": 171114496
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007661705115346039,
+      "loss": 1.7755,
+      "theoretical_loss": 4.47045749671847,
+      "tokens_seen": 171180032
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007661544633901705,
+      "loss": 1.6996,
+      "theoretical_loss": 4.470239607220377,
+      "tokens_seen": 171245568
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007661384152457373,
+      "loss": 1.6756,
+      "theoretical_loss": 4.470021824431152,
+      "tokens_seen": 171311104
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007661223671013039,
+      "loss": 1.8663,
+      "theoretical_loss": 4.469804148257742,
+      "tokens_seen": 171376640
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007661063189568706,
+      "loss": 1.7221,
+      "theoretical_loss": 4.469586578607213,
+      "tokens_seen": 171442176
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007660902708124373,
+      "loss": 1.6569,
+      "theoretical_loss": 4.469369115386746,
+      "tokens_seen": 171507712
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.000766074222668004,
+      "loss": 1.6757,
+      "theoretical_loss": 4.46915175850364,
+      "tokens_seen": 171573248
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007660581745235707,
+      "loss": 1.6542,
+      "theoretical_loss": 4.468934507865308,
+      "tokens_seen": 171638784
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007660421263791374,
+      "loss": 1.7006,
+      "theoretical_loss": 4.46871736337928,
+      "tokens_seen": 171704320
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007660260782347041,
+      "loss": 1.7182,
+      "theoretical_loss": 4.468500324953202,
+      "tokens_seen": 171769856
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007660100300902708,
+      "loss": 1.7916,
+      "theoretical_loss": 4.468283392494836,
+      "tokens_seen": 171835392
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007659939819458375,
+      "loss": 1.613,
+      "theoretical_loss": 4.468066565912057,
+      "tokens_seen": 171900928
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007659779338014042,
+      "loss": 1.6786,
+      "theoretical_loss": 4.467849845112859,
+      "tokens_seen": 171966464
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.000765961885656971,
+      "loss": 1.7226,
+      "theoretical_loss": 4.467633230005347,
+      "tokens_seen": 172032000
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007659458375125376,
+      "loss": 1.7142,
+      "theoretical_loss": 4.467416720497743,
+      "tokens_seen": 172097536
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007659297893681043,
+      "loss": 1.579,
+      "theoretical_loss": 4.467200316498385,
+      "tokens_seen": 172163072
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.000765913741223671,
+      "loss": 1.633,
+      "theoretical_loss": 4.4669840179157205,
+      "tokens_seen": 172228608
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007658976930792377,
+      "loss": 1.7211,
+      "theoretical_loss": 4.466767824658317,
+      "tokens_seen": 172294144
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007658816449348044,
+      "loss": 1.697,
+      "theoretical_loss": 4.466551736634852,
+      "tokens_seen": 172359680
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007658655967903711,
+      "loss": 1.6743,
+      "theoretical_loss": 4.466335753754118,
+      "tokens_seen": 172425216
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007658495486459378,
+      "loss": 1.7834,
+      "theoretical_loss": 4.466119875925022,
+      "tokens_seen": 172490752
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007658335005015046,
+      "loss": 1.6828,
+      "theoretical_loss": 4.465904103056586,
+      "tokens_seen": 172556288
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007658174523570712,
+      "loss": 1.7718,
+      "theoretical_loss": 4.465688435057938,
+      "tokens_seen": 172621824
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.000765801404212638,
+      "loss": 1.593,
+      "theoretical_loss": 4.46547287183833,
+      "tokens_seen": 172687360
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007657853560682046,
+      "loss": 1.7453,
+      "theoretical_loss": 4.465257413307119,
+      "tokens_seen": 172752896
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007657693079237714,
+      "loss": 1.6048,
+      "theoretical_loss": 4.465042059373776,
+      "tokens_seen": 172818432
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.000765753259779338,
+      "loss": 1.6982,
+      "theoretical_loss": 4.464826809947887,
+      "tokens_seen": 172883968
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007657372116349048,
+      "loss": 1.6822,
+      "theoretical_loss": 4.464611664939149,
+      "tokens_seen": 172949504
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007657211634904714,
+      "loss": 1.6698,
+      "theoretical_loss": 4.46439662425737,
+      "tokens_seen": 173015040
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007657051153460382,
+      "loss": 1.6845,
+      "theoretical_loss": 4.464181687812474,
+      "tokens_seen": 173080576
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007656890672016048,
+      "loss": 1.6059,
+      "theoretical_loss": 4.4639668555144905,
+      "tokens_seen": 173146112
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007656730190571716,
+      "loss": 1.7757,
+      "theoretical_loss": 4.463752127273567,
+      "tokens_seen": 173211648
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007656569709127382,
+      "loss": 1.6976,
+      "theoretical_loss": 4.463537502999959,
+      "tokens_seen": 173277184
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.000765640922768305,
+      "loss": 1.7552,
+      "theoretical_loss": 4.4633229826040335,
+      "tokens_seen": 173342720
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007656248746238716,
+      "loss": 1.586,
+      "theoretical_loss": 4.46310856599627,
+      "tokens_seen": 173408256
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007656088264794384,
+      "loss": 1.7207,
+      "theoretical_loss": 4.462894253087258,
+      "tokens_seen": 173473792
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.000765592778335005,
+      "loss": 1.7036,
+      "theoretical_loss": 4.462680043787699,
+      "tokens_seen": 173539328
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007655767301905718,
+      "loss": 1.6616,
+      "theoretical_loss": 4.4624659380084015,
+      "tokens_seen": 173604864
+    },
+    {
+      "epoch": 0.05,
+      "objective/train/docs_used": 64224,
+      "objective/train/instantaneous_batch_size": 32,
+      "objective/train/instantaneous_microbatch_size": 32768,
+      "objective/train/original_loss": 1.7511711120605469,
+      "objective/train/theoretical_loss": 4.462251935660287,
+      "objective/train/tokens_used": 194130400,
+      "theoretical_loss": 4.462251935660287,
+      "tokens_seen": 173670400
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007655606820461384,
+      "loss": 1.8024,
+      "theoretical_loss": 4.462251935660287,
+      "tokens_seen": 173670400
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007655446339017052,
+      "loss": 1.7087,
+      "theoretical_loss": 4.46203803665439,
+      "tokens_seen": 173735936
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007655285857572718,
+      "loss": 1.6677,
+      "theoretical_loss": 4.4618242409018505,
+      "tokens_seen": 173801472
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007655125376128386,
+      "loss": 1.67,
+      "theoretical_loss": 4.461610548313919,
+      "tokens_seen": 173867008
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007654964894684052,
+      "loss": 1.7889,
+      "theoretical_loss": 4.461396958801959,
+      "tokens_seen": 173932544
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.000765480441323972,
+      "loss": 1.7825,
+      "theoretical_loss": 4.461183472277439,
+      "tokens_seen": 173998080
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007654643931795386,
+      "loss": 1.7596,
+      "theoretical_loss": 4.460970088651941,
+      "tokens_seen": 174063616
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007654483450351054,
+      "loss": 1.7615,
+      "theoretical_loss": 4.460756807837154,
+      "tokens_seen": 174129152
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.000765432296890672,
+      "loss": 1.7388,
+      "theoretical_loss": 4.460543629744874,
+      "tokens_seen": 174194688
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007654162487462388,
+      "loss": 1.721,
+      "theoretical_loss": 4.460330554287011,
+      "tokens_seen": 174260224
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007654002006018054,
+      "loss": 1.7317,
+      "theoretical_loss": 4.460117581375579,
+      "tokens_seen": 174325760
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007653841524573722,
+      "loss": 1.7083,
+      "theoretical_loss": 4.459904710922704,
+      "tokens_seen": 174391296
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007653681043129388,
+      "loss": 1.7169,
+      "theoretical_loss": 4.459691942840616,
+      "tokens_seen": 174456832
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007653520561685056,
+      "loss": 1.6035,
+      "theoretical_loss": 4.459479277041657,
+      "tokens_seen": 174522368
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007653360080240722,
+      "loss": 1.7568,
+      "theoretical_loss": 4.459266713438277,
+      "tokens_seen": 174587904
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.000765319959879639,
+      "loss": 1.7212,
+      "theoretical_loss": 4.459054251943029,
+      "tokens_seen": 174653440
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007653039117352056,
+      "loss": 1.6524,
+      "theoretical_loss": 4.4588418924685795,
+      "tokens_seen": 174718976
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007652878635907724,
+      "loss": 1.7311,
+      "theoretical_loss": 4.458629634927698,
+      "tokens_seen": 174784512
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.000765271815446339,
+      "loss": 1.7316,
+      "theoretical_loss": 4.458417479233265,
+      "tokens_seen": 174850048
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007652557673019058,
+      "loss": 1.8556,
+      "theoretical_loss": 4.4582054252982655,
+      "tokens_seen": 174915584
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007652397191574724,
+      "loss": 1.7294,
+      "theoretical_loss": 4.457993473035792,
+      "tokens_seen": 174981120
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007652236710130392,
+      "loss": 1.6293,
+      "theoretical_loss": 4.457781622359043,
+      "tokens_seen": 175046656
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007652076228686058,
+      "loss": 1.7797,
+      "theoretical_loss": 4.457569873181326,
+      "tokens_seen": 175112192
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007651915747241726,
+      "loss": 1.5975,
+      "theoretical_loss": 4.457358225416052,
+      "tokens_seen": 175177728
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007651755265797392,
+      "loss": 1.8424,
+      "theoretical_loss": 4.4571466789767396,
+      "tokens_seen": 175243264
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.000765159478435306,
+      "loss": 1.8261,
+      "theoretical_loss": 4.456935233777013,
+      "tokens_seen": 175308800
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007651434302908727,
+      "loss": 1.8127,
+      "theoretical_loss": 4.456723889730605,
+      "tokens_seen": 175374336
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007651273821464394,
+      "loss": 1.7676,
+      "theoretical_loss": 4.456512646751349,
+      "tokens_seen": 175439872
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007651113340020061,
+      "loss": 1.7524,
+      "theoretical_loss": 4.456301504753188,
+      "tokens_seen": 175505408
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007650952858575728,
+      "loss": 1.6736,
+      "theoretical_loss": 4.456090463650169,
+      "tokens_seen": 175570944
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007650792377131395,
+      "loss": 1.6814,
+      "theoretical_loss": 4.455879523356444,
+      "tokens_seen": 175636480
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007650631895687062,
+      "loss": 1.6477,
+      "theoretical_loss": 4.4556686837862705,
+      "tokens_seen": 175702016
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007650471414242729,
+      "loss": 1.6564,
+      "theoretical_loss": 4.455457944854011,
+      "tokens_seen": 175767552
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007650310932798396,
+      "loss": 1.7469,
+      "theoretical_loss": 4.455247306474131,
+      "tokens_seen": 175833088
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007650150451354063,
+      "loss": 1.6944,
+      "theoretical_loss": 4.455036768561204,
+      "tokens_seen": 175898624
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.000764998996990973,
+      "loss": 1.7011,
+      "theoretical_loss": 4.4548263310299046,
+      "tokens_seen": 175964160
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007649829488465397,
+      "loss": 1.5012,
+      "theoretical_loss": 4.454615993795015,
+      "tokens_seen": 176029696
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007649669007021064,
+      "loss": 1.6227,
+      "theoretical_loss": 4.454405756771415,
+      "tokens_seen": 176095232
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007649508525576731,
+      "loss": 1.6004,
+      "theoretical_loss": 4.454195619874096,
+      "tokens_seen": 176160768
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007649348044132398,
+      "loss": 1.7566,
+      "theoretical_loss": 4.453985583018149,
+      "tokens_seen": 176226304
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007649187562688065,
+      "loss": 1.7557,
+      "theoretical_loss": 4.4537756461187685,
+      "tokens_seen": 176291840
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007649027081243732,
+      "loss": 1.7158,
+      "theoretical_loss": 4.453565809091254,
+      "tokens_seen": 176357376
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007648866599799399,
+      "loss": 1.6165,
+      "theoretical_loss": 4.453356071851006,
+      "tokens_seen": 176422912
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007648706118355066,
+      "loss": 1.7583,
+      "theoretical_loss": 4.453146434313531,
+      "tokens_seen": 176488448
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007648545636910732,
+      "loss": 1.649,
+      "theoretical_loss": 4.452936896394435,
+      "tokens_seen": 176553984
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00076483851554664,
+      "loss": 1.6462,
+      "theoretical_loss": 4.452727458009431,
+      "tokens_seen": 176619520
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007648224674022066,
+      "loss": 1.7599,
+      "theoretical_loss": 4.45251811907433,
+      "tokens_seen": 176685056
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007648064192577734,
+      "loss": 1.7308,
+      "theoretical_loss": 4.452308879505047,
+      "tokens_seen": 176750592
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00076479037111334,
+      "loss": 1.8075,
+      "theoretical_loss": 4.452099739217601,
+      "tokens_seen": 176816128
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007647743229689068,
+      "loss": 1.6846,
+      "theoretical_loss": 4.451890698128112,
+      "tokens_seen": 176881664
+    },
+    {
+      "epoch": 0.05,
+      "objective/train/docs_used": 65176,
+      "objective/train/instantaneous_batch_size": 32,
+      "objective/train/instantaneous_microbatch_size": 32768,
+      "objective/train/original_loss": 1.6887078285217285,
+      "objective/train/theoretical_loss": 4.4516817561528015,
+      "objective/train/tokens_used": 197407200,
+      "theoretical_loss": 4.4516817561528015,
+      "tokens_seen": 176947200
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007647582748244734,
+      "loss": 1.6268,
+      "theoretical_loss": 4.4516817561528015,
+      "tokens_seen": 176947200
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007647422266800402,
+      "loss": 1.6461,
+      "theoretical_loss": 4.451472913207992,
+      "tokens_seen": 177012736
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007647261785356068,
+      "loss": 1.7003,
+      "theoretical_loss": 4.451264169210107,
+      "tokens_seen": 177078272
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007647101303911735,
+      "loss": 1.649,
+      "theoretical_loss": 4.451055524075677,
+      "tokens_seen": 177143808
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007646940822467402,
+      "loss": 1.548,
+      "theoretical_loss": 4.450846977721326,
+      "tokens_seen": 177209344
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007646780341023069,
+      "loss": 1.726,
+      "theoretical_loss": 4.450638530063786,
+      "tokens_seen": 177274880
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007646619859578736,
+      "loss": 1.7409,
+      "theoretical_loss": 4.450430181019884,
+      "tokens_seen": 177340416
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007646459378134403,
+      "loss": 1.7246,
+      "theoretical_loss": 4.450221930506553,
+      "tokens_seen": 177405952
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.000764629889669007,
+      "loss": 1.7443,
+      "theoretical_loss": 4.450013778440822,
+      "tokens_seen": 177471488
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007646138415245737,
+      "loss": 1.7073,
+      "theoretical_loss": 4.449805724739824,
+      "tokens_seen": 177537024
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007645977933801404,
+      "loss": 1.6598,
+      "theoretical_loss": 4.449597769320791,
+      "tokens_seen": 177602560
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007645817452357071,
+      "loss": 1.7131,
+      "theoretical_loss": 4.4493899121010525,
+      "tokens_seen": 177668096
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007645656970912738,
+      "loss": 1.6204,
+      "theoretical_loss": 4.449182152998044,
+      "tokens_seen": 177733632
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007645496489468405,
+      "loss": 1.6482,
+      "theoretical_loss": 4.448974491929297,
+      "tokens_seen": 177799168
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007645336008024072,
+      "loss": 1.597,
+      "theoretical_loss": 4.448766928812441,
+      "tokens_seen": 177864704
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007645175526579739,
+      "loss": 1.7089,
+      "theoretical_loss": 4.448559463565209,
+      "tokens_seen": 177930240
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007645015045135406,
+      "loss": 1.6069,
+      "theoretical_loss": 4.44835209610543,
+      "tokens_seen": 177995776
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007644854563691073,
+      "loss": 1.6326,
+      "theoretical_loss": 4.448144826351036,
+      "tokens_seen": 178061312
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.000764469408224674,
+      "loss": 1.788,
+      "theoretical_loss": 4.447937654220054,
+      "tokens_seen": 178126848
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007644533600802407,
+      "loss": 1.7535,
+      "theoretical_loss": 4.447730579630612,
+      "tokens_seen": 178192384
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007644373119358075,
+      "loss": 1.8256,
+      "theoretical_loss": 4.447523602500936,
+      "tokens_seen": 178257920
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007644212637913741,
+      "loss": 1.7735,
+      "theoretical_loss": 4.447316722749351,
+      "tokens_seen": 178323456
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007644052156469409,
+      "loss": 1.76,
+      "theoretical_loss": 4.447109940294283,
+      "tokens_seen": 178388992
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007643891675025075,
+      "loss": 1.6671,
+      "theoretical_loss": 4.44690325505425,
+      "tokens_seen": 178454528
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007643731193580743,
+      "loss": 1.6877,
+      "theoretical_loss": 4.446696666947874,
+      "tokens_seen": 178520064
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007643570712136409,
+      "loss": 1.7155,
+      "theoretical_loss": 4.446490175893872,
+      "tokens_seen": 178585600
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007643410230692077,
+      "loss": 1.6927,
+      "theoretical_loss": 4.44628378181106,
+      "tokens_seen": 178651136
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007643249749247743,
+      "loss": 1.6744,
+      "theoretical_loss": 4.4460774846183515,
+      "tokens_seen": 178716672
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007643089267803411,
+      "loss": 1.6351,
+      "theoretical_loss": 4.445871284234757,
+      "tokens_seen": 178782208
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007642928786359077,
+      "loss": 1.7762,
+      "theoretical_loss": 4.445665180579383,
+      "tokens_seen": 178847744
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007642768304914745,
+      "loss": 1.5467,
+      "theoretical_loss": 4.445459173571438,
+      "tokens_seen": 178913280
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007642607823470411,
+      "loss": 1.6636,
+      "theoretical_loss": 4.445253263130223,
+      "tokens_seen": 178978816
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007642447342026079,
+      "loss": 1.652,
+      "theoretical_loss": 4.445047449175136,
+      "tokens_seen": 179044352
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007642286860581745,
+      "loss": 1.6698,
+      "theoretical_loss": 4.444841731625674,
+      "tokens_seen": 179109888
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007642126379137413,
+      "loss": 1.6116,
+      "theoretical_loss": 4.444636110401431,
+      "tokens_seen": 179175424
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007641965897693079,
+      "loss": 1.8467,
+      "theoretical_loss": 4.444430585422094,
+      "tokens_seen": 179240960
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007641805416248747,
+      "loss": 1.75,
+      "theoretical_loss": 4.444225156607449,
+      "tokens_seen": 179306496
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007641644934804413,
+      "loss": 1.8194,
+      "theoretical_loss": 4.444019823877378,
+      "tokens_seen": 179372032
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007641484453360081,
+      "loss": 1.7522,
+      "theoretical_loss": 4.4438145871518575,
+      "tokens_seen": 179437568
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007641323971915747,
+      "loss": 1.6433,
+      "theoretical_loss": 4.443609446350962,
+      "tokens_seen": 179503104
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007641163490471415,
+      "loss": 1.7147,
+      "theoretical_loss": 4.44340440139486,
+      "tokens_seen": 179568640
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007641003009027081,
+      "loss": 1.6794,
+      "theoretical_loss": 4.443199452203816,
+      "tokens_seen": 179634176
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007640842527582749,
+      "loss": 1.5769,
+      "theoretical_loss": 4.442994598698189,
+      "tokens_seen": 179699712
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007640682046138415,
+      "loss": 1.6097,
+      "theoretical_loss": 4.4427898407984365,
+      "tokens_seen": 179765248
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007640521564694083,
+      "loss": 1.6779,
+      "theoretical_loss": 4.442585178425107,
+      "tokens_seen": 179830784
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007640361083249749,
+      "loss": 1.6772,
+      "theoretical_loss": 4.442380611498846,
+      "tokens_seen": 179896320
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007640200601805417,
+      "loss": 1.6807,
+      "theoretical_loss": 4.442176139940393,
+      "tokens_seen": 179961856
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007640040120361083,
+      "loss": 1.6869,
+      "theoretical_loss": 4.441971763670585,
+      "tokens_seen": 180027392
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007639879638916751,
+      "loss": 1.6977,
+      "theoretical_loss": 4.441767482610349,
+      "tokens_seen": 180092928
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007639719157472417,
+      "loss": 1.6518,
+      "theoretical_loss": 4.441563296680709,
+      "tokens_seen": 180158464
+    },
+    {
+      "epoch": 0.05,
+      "objective/train/docs_used": 66469,
+      "objective/train/instantaneous_batch_size": 32,
+      "objective/train/instantaneous_microbatch_size": 32768,
+      "objective/train/original_loss": 1.672188639640808,
+      "objective/train/theoretical_loss": 4.441359205802783,
+      "objective/train/tokens_used": 200684000,
+      "theoretical_loss": 4.441359205802783,
+      "tokens_seen": 180224000
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007639558676028085,
+      "loss": 1.6723,
+      "theoretical_loss": 4.441359205802783,
+      "tokens_seen": 180224000
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007639398194583752,
+      "loss": 1.7121,
+      "theoretical_loss": 4.441155209897782,
+      "tokens_seen": 180289536
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007639237713139419,
+      "loss": 1.6871,
+      "theoretical_loss": 4.440951308887014,
+      "tokens_seen": 180355072
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007639077231695086,
+      "loss": 1.8032,
+      "theoretical_loss": 4.440747502691876,
+      "tokens_seen": 180420608
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007638916750250753,
+      "loss": 1.6908,
+      "theoretical_loss": 4.440543791233862,
+      "tokens_seen": 180486144
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.000763875626880642,
+      "loss": 1.7624,
+      "theoretical_loss": 4.440340174434558,
+      "tokens_seen": 180551680
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007638595787362087,
+      "loss": 1.6142,
+      "theoretical_loss": 4.440136652215646,
+      "tokens_seen": 180617216
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007638435305917754,
+      "loss": 1.6298,
+      "theoretical_loss": 4.439933224498898,
+      "tokens_seen": 180682752
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007638274824473421,
+      "loss": 1.667,
+      "theoretical_loss": 4.43972989120618,
+      "tokens_seen": 180748288
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007638114343029088,
+      "loss": 1.7213,
+      "theoretical_loss": 4.439526652259451,
+      "tokens_seen": 180813824
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007637953861584755,
+      "loss": 1.6179,
+      "theoretical_loss": 4.439323507580761,
+      "tokens_seen": 180879360
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007637793380140422,
+      "loss": 1.7025,
+      "theoretical_loss": 4.439120457092258,
+      "tokens_seen": 180944896
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007637632898696089,
+      "loss": 1.6234,
+      "theoretical_loss": 4.438917500716178,
+      "tokens_seen": 181010432
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007637472417251756,
+      "loss": 1.631,
+      "theoretical_loss": 4.438714638374849,
+      "tokens_seen": 181075968
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007637311935807423,
+      "loss": 1.638,
+      "theoretical_loss": 4.438511869990693,
+      "tokens_seen": 181141504
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.000763715145436309,
+      "loss": 1.8766,
+      "theoretical_loss": 4.438309195486223,
+      "tokens_seen": 181207040
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007636990972918757,
+      "loss": 1.7718,
+      "theoretical_loss": 4.438106614784045,
+      "tokens_seen": 181272576
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007636830491474424,
+      "loss": 1.6277,
+      "theoretical_loss": 4.437904127806856,
+      "tokens_seen": 181338112
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007636670010030091,
+      "loss": 1.6041,
+      "theoretical_loss": 4.437701734477445,
+      "tokens_seen": 181403648
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0007636509528585758,
+      "loss": 1.6715,
+      "theoretical_loss": 4.437499434718692,
+      "tokens_seen": 181469184
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007636349047141425,
+      "loss": 1.7125,
+      "theoretical_loss": 4.437297228453567,
+      "tokens_seen": 181534720
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007636188565697092,
+      "loss": 1.6455,
+      "theoretical_loss": 4.4370951156051355,
+      "tokens_seen": 181600256
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007636028084252759,
+      "loss": 1.6288,
+      "theoretical_loss": 4.43689309609655,
+      "tokens_seen": 181665792
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007635867602808426,
+      "loss": 1.5784,
+      "theoretical_loss": 4.436691169851055,
+      "tokens_seen": 181731328
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007635707121364093,
+      "loss": 1.5495,
+      "theoretical_loss": 4.436489336791985,
+      "tokens_seen": 181796864
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.000763554663991976,
+      "loss": 1.6357,
+      "theoretical_loss": 4.436287596842769,
+      "tokens_seen": 181862400
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007635386158475427,
+      "loss": 1.7203,
+      "theoretical_loss": 4.4360859499269205,
+      "tokens_seen": 181927936
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007635225677031094,
+      "loss": 1.6698,
+      "theoretical_loss": 4.435884395968047,
+      "tokens_seen": 181993472
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007635065195586761,
+      "loss": 1.6639,
+      "theoretical_loss": 4.435682934889847,
+      "tokens_seen": 182059008
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007634904714142428,
+      "loss": 1.7783,
+      "theoretical_loss": 4.435481566616108,
+      "tokens_seen": 182124544
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007634744232698095,
+      "loss": 1.6566,
+      "theoretical_loss": 4.435280291070705,
+      "tokens_seen": 182190080
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007634583751253762,
+      "loss": 1.7183,
+      "theoretical_loss": 4.435079108177606,
+      "tokens_seen": 182255616
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007634423269809429,
+      "loss": 1.7325,
+      "theoretical_loss": 4.434878017860869,
+      "tokens_seen": 182321152
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007634262788365095,
+      "loss": 1.6528,
+      "theoretical_loss": 4.434677020044637,
+      "tokens_seen": 182386688
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007634102306920763,
+      "loss": 1.7373,
+      "theoretical_loss": 4.4344761146531475,
+      "tokens_seen": 182452224
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007633941825476429,
+      "loss": 1.649,
+      "theoretical_loss": 4.434275301610725,
+      "tokens_seen": 182517760
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007633781344032097,
+      "loss": 1.4904,
+      "theoretical_loss": 4.434074580841784,
+      "tokens_seen": 182583296
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007633620862587763,
+      "loss": 1.6541,
+      "theoretical_loss": 4.433873952270826,
+      "tokens_seen": 182648832
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007633460381143431,
+      "loss": 1.8746,
+      "theoretical_loss": 4.433673415822443,
+      "tokens_seen": 182714368
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007633299899699097,
+      "loss": 1.5972,
+      "theoretical_loss": 4.433472971421315,
+      "tokens_seen": 182779904
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007633139418254764,
+      "loss": 1.7089,
+      "theoretical_loss": 4.433272618992213,
+      "tokens_seen": 182845440
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007632978936810431,
+      "loss": 1.5085,
+      "theoretical_loss": 4.433072358459991,
+      "tokens_seen": 182910976
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007632818455366098,
+      "loss": 1.7725,
+      "theoretical_loss": 4.432872189749596,
+      "tokens_seen": 182976512
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007632657973921765,
+      "loss": 1.6167,
+      "theoretical_loss": 4.432672112786063,
+      "tokens_seen": 183042048
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007632497492477432,
+      "loss": 1.6598,
+      "theoretical_loss": 4.4324721274945125,
+      "tokens_seen": 183107584
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007632337011033099,
+      "loss": 1.6746,
+      "theoretical_loss": 4.432272233800154,
+      "tokens_seen": 183173120
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007632176529588766,
+      "loss": 1.7455,
+      "theoretical_loss": 4.432072431628286,
+      "tokens_seen": 183238656
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007632016048144433,
+      "loss": 1.6502,
+      "theoretical_loss": 4.431872720904292,
+      "tokens_seen": 183304192
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.00076318555667001,
+      "loss": 1.709,
+      "theoretical_loss": 4.4316731015536455,
+      "tokens_seen": 183369728
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007631695085255767,
+      "loss": 1.5837,
+      "theoretical_loss": 4.431473573501906,
+      "tokens_seen": 183435264
+    },
+    {
+      "epoch": 0.06,
+      "objective/train/docs_used": 67339,
+      "objective/train/instantaneous_batch_size": 32,
+      "objective/train/instantaneous_microbatch_size": 32768,
+      "objective/train/original_loss": 1.8913339376449585,
+      "objective/train/theoretical_loss": 4.431274136674721,
+      "objective/train/tokens_used": 203960800,
+      "theoretical_loss": 4.431274136674721,
+      "tokens_seen": 183500800
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007631534603811434,
+      "loss": 1.8362,
+      "theoretical_loss": 4.431274136674721,
+      "tokens_seen": 183500800
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007631374122367101,
+      "loss": 1.7904,
+      "theoretical_loss": 4.431074790997823,
+      "tokens_seen": 183566336
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007631213640922768,
+      "loss": 1.8053,
+      "theoretical_loss": 4.430875536397036,
+      "tokens_seen": 183631872
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007631053159478436,
+      "loss": 1.632,
+      "theoretical_loss": 4.430676372798265,
+      "tokens_seen": 183697408
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007630892678034102,
+      "loss": 1.7575,
+      "theoretical_loss": 4.430477300127507,
+      "tokens_seen": 183762944
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.000763073219658977,
+      "loss": 1.5416,
+      "theoretical_loss": 4.43027831831084,
+      "tokens_seen": 183828480
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007630571715145436,
+      "loss": 1.817,
+      "theoretical_loss": 4.430079427274435,
+      "tokens_seen": 183894016
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007630411233701104,
+      "loss": 1.7328,
+      "theoretical_loss": 4.429880626944543,
+      "tokens_seen": 183959552
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.000763025075225677,
+      "loss": 1.6868,
+      "theoretical_loss": 4.429681917247506,
+      "tokens_seen": 184025088
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007630090270812438,
+      "loss": 1.5732,
+      "theoretical_loss": 4.429483298109749,
+      "tokens_seen": 184090624
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007629929789368104,
+      "loss": 1.6259,
+      "theoretical_loss": 4.429284769457784,
+      "tokens_seen": 184156160
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007629769307923772,
+      "loss": 1.8012,
+      "theoretical_loss": 4.429086331218208,
+      "tokens_seen": 184221696
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007629608826479438,
+      "loss": 1.7037,
+      "theoretical_loss": 4.428887983317706,
+      "tokens_seen": 184287232
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007629448345035106,
+      "loss": 1.6343,
+      "theoretical_loss": 4.4286897256830455,
+      "tokens_seen": 184352768
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007629287863590772,
+      "loss": 1.6312,
+      "theoretical_loss": 4.428491558241081,
+      "tokens_seen": 184418304
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.000762912738214644,
+      "loss": 1.5467,
+      "theoretical_loss": 4.428293480918751,
+      "tokens_seen": 184483840
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007628966900702106,
+      "loss": 1.5543,
+      "theoretical_loss": 4.428095493643081,
+      "tokens_seen": 184549376
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007628806419257774,
+      "loss": 1.6001,
+      "theoretical_loss": 4.427897596341181,
+      "tokens_seen": 184614912
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.000762864593781344,
+      "loss": 1.6384,
+      "theoretical_loss": 4.427699788940243,
+      "tokens_seen": 184680448
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007628485456369108,
+      "loss": 1.598,
+      "theoretical_loss": 4.427502071367549,
+      "tokens_seen": 184745984
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007628324974924774,
+      "loss": 1.6392,
+      "theoretical_loss": 4.42730444355046,
+      "tokens_seen": 184811520
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007628164493480442,
+      "loss": 1.6979,
+      "theoretical_loss": 4.4271069054164265,
+      "tokens_seen": 184877056
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007628004012036108,
+      "loss": 1.6848,
+      "theoretical_loss": 4.426909456892979,
+      "tokens_seen": 184942592
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007627843530591776,
+      "loss": 1.6445,
+      "theoretical_loss": 4.426712097907735,
+      "tokens_seen": 185008128
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007627683049147442,
+      "loss": 1.7925,
+      "theoretical_loss": 4.426514828388394,
+      "tokens_seen": 185073664
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.000762752256770311,
+      "loss": 1.5444,
+      "theoretical_loss": 4.426317648262743,
+      "tokens_seen": 185139200
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007627362086258777,
+      "loss": 1.769,
+      "theoretical_loss": 4.426120557458649,
+      "tokens_seen": 185204736
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007627201604814444,
+      "loss": 1.7283,
+      "theoretical_loss": 4.425923555904065,
+      "tokens_seen": 185270272
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007627041123370111,
+      "loss": 1.6861,
+      "theoretical_loss": 4.4257266435270255,
+      "tokens_seen": 185335808
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007626880641925778,
+      "loss": 1.719,
+      "theoretical_loss": 4.425529820255651,
+      "tokens_seen": 185401344
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007626720160481445,
+      "loss": 1.4554,
+      "theoretical_loss": 4.425333086018144,
+      "tokens_seen": 185466880
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007626559679037112,
+      "loss": 1.747,
+      "theoretical_loss": 4.425136440742788,
+      "tokens_seen": 185532416
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007626399197592779,
+      "loss": 1.6811,
+      "theoretical_loss": 4.424939884357956,
+      "tokens_seen": 185597952
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007626238716148446,
+      "loss": 1.6326,
+      "theoretical_loss": 4.424743416792096,
+      "tokens_seen": 185663488
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007626078234704113,
+      "loss": 1.8225,
+      "theoretical_loss": 4.424547037973745,
+      "tokens_seen": 185729024
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.000762591775325978,
+      "loss": 1.6163,
+      "theoretical_loss": 4.424350747831518,
+      "tokens_seen": 185794560
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007625757271815447,
+      "loss": 1.6868,
+      "theoretical_loss": 4.424154546294117,
+      "tokens_seen": 185860096
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007625596790371114,
+      "loss": 1.6912,
+      "theoretical_loss": 4.4239584332903235,
+      "tokens_seen": 185925632
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007625436308926781,
+      "loss": 1.7262,
+      "theoretical_loss": 4.423762408749001,
+      "tokens_seen": 185991168
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007625275827482448,
+      "loss": 1.7826,
+      "theoretical_loss": 4.423566472599099,
+      "tokens_seen": 186056704
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007625115346038115,
+      "loss": 1.6683,
+      "theoretical_loss": 4.423370624769643,
+      "tokens_seen": 186122240
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007624954864593782,
+      "loss": 1.6865,
+      "theoretical_loss": 4.423174865189747,
+      "tokens_seen": 186187776
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007624794383149449,
+      "loss": 1.6972,
+      "theoretical_loss": 4.422979193788601,
+      "tokens_seen": 186253312
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007624633901705116,
+      "loss": 1.723,
+      "theoretical_loss": 4.422783610495481,
+      "tokens_seen": 186318848
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007624473420260783,
+      "loss": 1.6847,
+      "theoretical_loss": 4.422588115239742,
+      "tokens_seen": 186384384
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.000762431293881645,
+      "loss": 1.6419,
+      "theoretical_loss": 4.422392707950824,
+      "tokens_seen": 186449920
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007624152457372117,
+      "loss": 1.6232,
+      "theoretical_loss": 4.42219738855824,
+      "tokens_seen": 186515456
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007623991975927784,
+      "loss": 1.691,
+      "theoretical_loss": 4.422002156991597,
+      "tokens_seen": 186580992
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007623831494483451,
+      "loss": 1.6773,
+      "theoretical_loss": 4.421807013180571,
+      "tokens_seen": 186646528
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007623671013039118,
+      "loss": 1.6533,
+      "theoretical_loss": 4.421611957054925,
+      "tokens_seen": 186712064
+    },
+    {
+      "epoch": 0.06,
+      "objective/train/docs_used": 68658,
+      "objective/train/instantaneous_batch_size": 32,
+      "objective/train/instantaneous_microbatch_size": 32768,
+      "objective/train/original_loss": 1.6083440780639648,
+      "objective/train/theoretical_loss": 4.421416988544503,
+      "objective/train/tokens_used": 207237600,
+      "theoretical_loss": 4.421416988544503,
+      "tokens_seen": 186777600
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007623510531594785,
+      "loss": 1.6078,
+      "theoretical_loss": 4.421416988544503,
+      "tokens_seen": 186777600
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007623350050150452,
+      "loss": 1.5592,
+      "theoretical_loss": 4.4212221075792275,
+      "tokens_seen": 186843136
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007623189568706119,
+      "loss": 1.6722,
+      "theoretical_loss": 4.421027314089103,
+      "tokens_seen": 186908672
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007623029087261786,
+      "loss": 1.7075,
+      "theoretical_loss": 4.420832608004215,
+      "tokens_seen": 186974208
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007622868605817453,
+      "loss": 1.7427,
+      "theoretical_loss": 4.420637989254726,
+      "tokens_seen": 187039744
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.000762270812437312,
+      "loss": 1.6546,
+      "theoretical_loss": 4.420443457770885,
+      "tokens_seen": 187105280
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007622547642928787,
+      "loss": 1.6579,
+      "theoretical_loss": 4.420249013483014,
+      "tokens_seen": 187170816
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007622387161484454,
+      "loss": 1.4673,
+      "theoretical_loss": 4.42005465632152,
+      "tokens_seen": 187236352
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007622226680040121,
+      "loss": 1.6929,
+      "theoretical_loss": 4.419860386216888,
+      "tokens_seen": 187301888
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007622066198595788,
+      "loss": 1.6732,
+      "theoretical_loss": 4.419666203099683,
+      "tokens_seen": 187367424
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007621905717151455,
+      "loss": 1.6573,
+      "theoretical_loss": 4.41947210690055,
+      "tokens_seen": 187432960
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007621745235707122,
+      "loss": 1.6647,
+      "theoretical_loss": 4.419278097550212,
+      "tokens_seen": 187498496
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007621584754262789,
+      "loss": 1.6223,
+      "theoretical_loss": 4.419084174979473,
+      "tokens_seen": 187564032
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007621424272818456,
+      "loss": 1.6381,
+      "theoretical_loss": 4.4188903391192165,
+      "tokens_seen": 187629568
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007621263791374123,
+      "loss": 1.6911,
+      "theoretical_loss": 4.418696589900405,
+      "tokens_seen": 187695104
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.000762110330992979,
+      "loss": 1.7589,
+      "theoretical_loss": 4.418502927254078,
+      "tokens_seen": 187760640
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007620942828485457,
+      "loss": 1.7452,
+      "theoretical_loss": 4.418309351111356,
+      "tokens_seen": 187826176
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007620782347041124,
+      "loss": 1.7065,
+      "theoretical_loss": 4.418115861403439,
+      "tokens_seen": 187891712
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007620621865596791,
+      "loss": 1.6495,
+      "theoretical_loss": 4.417922458061603,
+      "tokens_seen": 187957248
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007620461384152458,
+      "loss": 1.6708,
+      "theoretical_loss": 4.417729141017205,
+      "tokens_seen": 188022784
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007620300902708124,
+      "loss": 1.6741,
+      "theoretical_loss": 4.4175359102016785,
+      "tokens_seen": 188088320
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007620140421263792,
+      "loss": 1.8191,
+      "theoretical_loss": 4.417342765546538,
+      "tokens_seen": 188153856
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007619979939819458,
+      "loss": 1.6297,
+      "theoretical_loss": 4.417149706983373,
+      "tokens_seen": 188219392
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007619819458375126,
+      "loss": 1.7377,
+      "theoretical_loss": 4.416956734443854,
+      "tokens_seen": 188284928
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007619658976930792,
+      "loss": 1.7455,
+      "theoretical_loss": 4.416763847859728,
+      "tokens_seen": 188350464
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.000761949849548646,
+      "loss": 1.5804,
+      "theoretical_loss": 4.41657104716282,
+      "tokens_seen": 188416000
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007619338014042126,
+      "loss": 1.705,
+      "theoretical_loss": 4.416378332285031,
+      "tokens_seen": 188481536
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007619177532597794,
+      "loss": 1.7445,
+      "theoretical_loss": 4.4161857031583445,
+      "tokens_seen": 188547072
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.000761901705115346,
+      "loss": 1.5133,
+      "theoretical_loss": 4.415993159714818,
+      "tokens_seen": 188612608
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007618856569709127,
+      "loss": 1.6943,
+      "theoretical_loss": 4.415800701886584,
+      "tokens_seen": 188678144
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007618696088264794,
+      "loss": 1.6854,
+      "theoretical_loss": 4.415608329605859,
+      "tokens_seen": 188743680
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007618535606820461,
+      "loss": 1.7005,
+      "theoretical_loss": 4.415416042804931,
+      "tokens_seen": 188809216
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007618375125376128,
+      "loss": 1.7116,
+      "theoretical_loss": 4.415223841416167,
+      "tokens_seen": 188874752
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007618214643931795,
+      "loss": 1.6836,
+      "theoretical_loss": 4.415031725372011,
+      "tokens_seen": 188940288
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007618054162487462,
+      "loss": 1.66,
+      "theoretical_loss": 4.414839694604985,
+      "tokens_seen": 189005824
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007617893681043129,
+      "loss": 1.7335,
+      "theoretical_loss": 4.414647749047685,
+      "tokens_seen": 189071360
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007617733199598796,
+      "loss": 1.6927,
+      "theoretical_loss": 4.414455888632786,
+      "tokens_seen": 189136896
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007617572718154463,
+      "loss": 1.6226,
+      "theoretical_loss": 4.414264113293038,
+      "tokens_seen": 189202432
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.000761741223671013,
+      "loss": 1.7233,
+      "theoretical_loss": 4.41407242296127,
+      "tokens_seen": 189267968
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007617251755265797,
+      "loss": 1.6426,
+      "theoretical_loss": 4.413880817570382,
+      "tokens_seen": 189333504
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007617091273821465,
+      "loss": 1.5316,
+      "theoretical_loss": 4.413689297053356,
+      "tokens_seen": 189399040
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007616930792377131,
+      "loss": 1.7907,
+      "theoretical_loss": 4.413497861343247,
+      "tokens_seen": 189464576
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007616770310932799,
+      "loss": 1.6265,
+      "theoretical_loss": 4.413306510373185,
+      "tokens_seen": 189530112
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007616609829488465,
+      "loss": 1.7281,
+      "theoretical_loss": 4.41311524407638,
+      "tokens_seen": 189595648
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007616449348044133,
+      "loss": 1.5898,
+      "theoretical_loss": 4.412924062386112,
+      "tokens_seen": 189661184
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007616288866599799,
+      "loss": 1.7076,
+      "theoretical_loss": 4.412732965235742,
+      "tokens_seen": 189726720
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007616128385155467,
+      "loss": 1.66,
+      "theoretical_loss": 4.412541952558703,
+      "tokens_seen": 189792256
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007615967903711133,
+      "loss": 1.7227,
+      "theoretical_loss": 4.412351024288504,
+      "tokens_seen": 189857792
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007615807422266801,
+      "loss": 1.6849,
+      "theoretical_loss": 4.412160180358731,
+      "tokens_seen": 189923328
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007615646940822467,
+      "loss": 1.7016,
+      "theoretical_loss": 4.411969420703042,
+      "tokens_seen": 189988864
+    },
+    {
+      "epoch": 0.06,
+      "objective/train/docs_used": 69573,
+      "objective/train/instantaneous_batch_size": 32,
+      "objective/train/instantaneous_microbatch_size": 32768,
+      "objective/train/original_loss": 1.7352272272109985,
+      "objective/train/theoretical_loss": 4.411778745255174,
+      "objective/train/tokens_used": 210514400,
+      "theoretical_loss": 4.411778745255174,
+      "tokens_seen": 190054400
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007615486459378135,
+      "loss": 1.7127,
+      "theoretical_loss": 4.411778745255174,
+      "tokens_seen": 190054400
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007615325977933802,
+      "loss": 1.7479,
+      "theoretical_loss": 4.411588153948935,
+      "tokens_seen": 190119936
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007615165496489469,
+      "loss": 1.6213,
+      "theoretical_loss": 4.411397646718211,
+      "tokens_seen": 190185472
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007615005015045136,
+      "loss": 1.7912,
+      "theoretical_loss": 4.411207223496962,
+      "tokens_seen": 190251008
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007614844533600803,
+      "loss": 1.7089,
+      "theoretical_loss": 4.41101688421922,
+      "tokens_seen": 190316544
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.000761468405215647,
+      "loss": 1.6525,
+      "theoretical_loss": 4.410826628819094,
+      "tokens_seen": 190382080
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007614523570712137,
+      "loss": 1.6185,
+      "theoretical_loss": 4.410636457230767,
+      "tokens_seen": 190447616
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007614363089267804,
+      "loss": 1.6048,
+      "theoretical_loss": 4.410446369388497,
+      "tokens_seen": 190513152
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007614202607823471,
+      "loss": 1.6883,
+      "theoretical_loss": 4.410256365226614,
+      "tokens_seen": 190578688
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007614042126379138,
+      "loss": 1.6795,
+      "theoretical_loss": 4.410066444679524,
+      "tokens_seen": 190644224
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007613881644934805,
+      "loss": 1.5914,
+      "theoretical_loss": 4.409876607681706,
+      "tokens_seen": 190709760
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007613721163490472,
+      "loss": 1.7626,
+      "theoretical_loss": 4.409686854167714,
+      "tokens_seen": 190775296
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007613560682046139,
+      "loss": 1.7654,
+      "theoretical_loss": 4.4094971840721735,
+      "tokens_seen": 190840832
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007613400200601806,
+      "loss": 1.6914,
+      "theoretical_loss": 4.409307597329786,
+      "tokens_seen": 190906368
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007613239719157473,
+      "loss": 1.6707,
+      "theoretical_loss": 4.409118093875324,
+      "tokens_seen": 190971904
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.000761307923771314,
+      "loss": 1.6284,
+      "theoretical_loss": 4.408928673643636,
+      "tokens_seen": 191037440
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007612918756268807,
+      "loss": 1.6022,
+      "theoretical_loss": 4.408739336569644,
+      "tokens_seen": 191102976
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007612758274824474,
+      "loss": 1.7029,
+      "theoretical_loss": 4.408550082588339,
+      "tokens_seen": 191168512
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007612597793380141,
+      "loss": 1.659,
+      "theoretical_loss": 4.408360911634789,
+      "tokens_seen": 191234048
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007612437311935808,
+      "loss": 1.6809,
+      "theoretical_loss": 4.408171823644135,
+      "tokens_seen": 191299584
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007612276830491475,
+      "loss": 1.7004,
+      "theoretical_loss": 4.407982818551589,
+      "tokens_seen": 191365120
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007612116349047142,
+      "loss": 1.6978,
+      "theoretical_loss": 4.407793896292437,
+      "tokens_seen": 191430656
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007611955867602809,
+      "loss": 1.6428,
+      "theoretical_loss": 4.407605056802035,
+      "tokens_seen": 191496192
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007611795386158476,
+      "loss": 1.7133,
+      "theoretical_loss": 4.407416300015816,
+      "tokens_seen": 191561728
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007611634904714143,
+      "loss": 1.5496,
+      "theoretical_loss": 4.407227625869283,
+      "tokens_seen": 191627264
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.000761147442326981,
+      "loss": 1.4881,
+      "theoretical_loss": 4.407039034298011,
+      "tokens_seen": 191692800
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007611313941825477,
+      "loss": 1.671,
+      "theoretical_loss": 4.4068505252376475,
+      "tokens_seen": 191758336
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007611153460381144,
+      "loss": 1.652,
+      "theoretical_loss": 4.406662098623913,
+      "tokens_seen": 191823872
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007610992978936811,
+      "loss": 1.6884,
+      "theoretical_loss": 4.406473754392598,
+      "tokens_seen": 191889408
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007610832497492478,
+      "loss": 1.5689,
+      "theoretical_loss": 4.406285492479569,
+      "tokens_seen": 191954944
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007610672016048145,
+      "loss": 1.6187,
+      "theoretical_loss": 4.406097312820759,
+      "tokens_seen": 192020480
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007610511534603812,
+      "loss": 1.6176,
+      "theoretical_loss": 4.4059092153521755,
+      "tokens_seen": 192086016
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007610351053159479,
+      "loss": 1.6055,
+      "theoretical_loss": 4.405721200009898,
+      "tokens_seen": 192151552
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007610190571715146,
+      "loss": 1.8174,
+      "theoretical_loss": 4.405533266730077,
+      "tokens_seen": 192217088
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007610030090270813,
+      "loss": 1.765,
+      "theoretical_loss": 4.405345415448934,
+      "tokens_seen": 192282624
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.000760986960882648,
+      "loss": 1.6827,
+      "theoretical_loss": 4.405157646102762,
+      "tokens_seen": 192348160
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007609709127382147,
+      "loss": 1.647,
+      "theoretical_loss": 4.404969958627927,
+      "tokens_seen": 192413696
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007609548645937814,
+      "loss": 1.5969,
+      "theoretical_loss": 4.4047823529608605,
+      "tokens_seen": 192479232
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007609388164493481,
+      "loss": 1.6968,
+      "theoretical_loss": 4.404594829038071,
+      "tokens_seen": 192544768
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007609227683049148,
+      "loss": 1.6337,
+      "theoretical_loss": 4.404407386796137,
+      "tokens_seen": 192610304
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007609067201604815,
+      "loss": 1.6884,
+      "theoretical_loss": 4.404220026171704,
+      "tokens_seen": 192675840
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007608906720160482,
+      "loss": 1.7427,
+      "theoretical_loss": 4.404032747101493,
+      "tokens_seen": 192741376
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007608746238716149,
+      "loss": 1.7303,
+      "theoretical_loss": 4.4038455495222895,
+      "tokens_seen": 192806912
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007608585757271816,
+      "loss": 1.6699,
+      "theoretical_loss": 4.403658433370957,
+      "tokens_seen": 192872448
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007608425275827483,
+      "loss": 1.6285,
+      "theoretical_loss": 4.403471398584422,
+      "tokens_seen": 192937984
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.000760826479438315,
+      "loss": 1.7095,
+      "theoretical_loss": 4.403284445099688,
+      "tokens_seen": 193003520
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007608104312938817,
+      "loss": 1.7003,
+      "theoretical_loss": 4.403097572853823,
+      "tokens_seen": 193069056
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007607943831494484,
+      "loss": 1.6738,
+      "theoretical_loss": 4.402910781783968,
+      "tokens_seen": 193134592
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007607783350050151,
+      "loss": 1.7029,
+      "theoretical_loss": 4.4027240718273335,
+      "tokens_seen": 193200128
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007607622868605818,
+      "loss": 1.6215,
+      "theoretical_loss": 4.4025374429212,
+      "tokens_seen": 193265664
+    },
+    {
+      "epoch": 0.06,
+      "objective/train/docs_used": 70720,
+      "objective/train/instantaneous_batch_size": 32,
+      "objective/train/instantaneous_microbatch_size": 32768,
+      "objective/train/original_loss": 1.8479371070861816,
+      "objective/train/theoretical_loss": 4.402350895002916,
+      "objective/train/tokens_used": 213791200,
+      "theoretical_loss": 4.402350895002916,
+      "tokens_seen": 193331200
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007607462387161485,
+      "loss": 1.7445,
+      "theoretical_loss": 4.402350895002916,
+      "tokens_seen": 193331200
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007607301905717152,
+      "loss": 1.6507,
+      "theoretical_loss": 4.402164428009904,
+      "tokens_seen": 193396736
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007607141424272819,
+      "loss": 1.7054,
+      "theoretical_loss": 4.40197804187965,
+      "tokens_seen": 193462272
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007606980942828486,
+      "loss": 1.6454,
+      "theoretical_loss": 4.401791736549714,
+      "tokens_seen": 193527808
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007606820461384153,
+      "loss": 1.7356,
+      "theoretical_loss": 4.401605511957724,
+      "tokens_seen": 193593344
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.000760665997993982,
+      "loss": 1.6621,
+      "theoretical_loss": 4.401419368041377,
+      "tokens_seen": 193658880
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007606499498495487,
+      "loss": 1.707,
+      "theoretical_loss": 4.401233304738438,
+      "tokens_seen": 193724416
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007606339017051153,
+      "loss": 1.7116,
+      "theoretical_loss": 4.401047321986745,
+      "tokens_seen": 193789952
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007606178535606821,
+      "loss": 1.6979,
+      "theoretical_loss": 4.4008614197242,
+      "tokens_seen": 193855488
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007606018054162487,
+      "loss": 1.6398,
+      "theoretical_loss": 4.400675597888777,
+      "tokens_seen": 193921024
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007605857572718155,
+      "loss": 1.6676,
+      "theoretical_loss": 4.400489856418518,
+      "tokens_seen": 193986560
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007605697091273821,
+      "loss": 1.7465,
+      "theoretical_loss": 4.4003041952515325,
+      "tokens_seen": 194052096
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007605536609829489,
+      "loss": 1.6486,
+      "theoretical_loss": 4.400118614326001,
+      "tokens_seen": 194117632
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007605376128385155,
+      "loss": 1.7039,
+      "theoretical_loss": 4.39993311358017,
+      "tokens_seen": 194183168
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007605215646940823,
+      "loss": 1.7368,
+      "theoretical_loss": 4.399747692952355,
+      "tokens_seen": 194248704
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007605055165496489,
+      "loss": 1.6376,
+      "theoretical_loss": 4.399562352380941,
+      "tokens_seen": 194314240
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007604894684052156,
+      "loss": 1.7371,
+      "theoretical_loss": 4.399377091804379,
+      "tokens_seen": 194379776
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007604734202607823,
+      "loss": 1.6241,
+      "theoretical_loss": 4.399191911161191,
+      "tokens_seen": 194445312
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.000760457372116349,
+      "loss": 1.5524,
+      "theoretical_loss": 4.399006810389963,
+      "tokens_seen": 194510848
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007604413239719157,
+      "loss": 1.685,
+      "theoretical_loss": 4.398821789429354,
+      "tokens_seen": 194576384
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007604252758274824,
+      "loss": 1.8051,
+      "theoretical_loss": 4.398636848218084,
+      "tokens_seen": 194641920
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007604092276830491,
+      "loss": 1.5873,
+      "theoretical_loss": 4.398451986694948,
+      "tokens_seen": 194707456
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007603931795386158,
+      "loss": 1.5808,
+      "theoretical_loss": 4.398267204798803,
+      "tokens_seen": 194772992
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007603771313941825,
+      "loss": 1.6304,
+      "theoretical_loss": 4.398082502468577,
+      "tokens_seen": 194838528
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007603610832497492,
+      "loss": 1.5813,
+      "theoretical_loss": 4.397897879643262,
+      "tokens_seen": 194904064
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.000760345035105316,
+      "loss": 1.6571,
+      "theoretical_loss": 4.39771333626192,
+      "tokens_seen": 194969600
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007603289869608827,
+      "loss": 1.6452,
+      "theoretical_loss": 4.397528872263679,
+      "tokens_seen": 195035136
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007603129388164494,
+      "loss": 1.7315,
+      "theoretical_loss": 4.397344487587736,
+      "tokens_seen": 195100672
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007602968906720161,
+      "loss": 1.7477,
+      "theoretical_loss": 4.3971601821733515,
+      "tokens_seen": 195166208
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007602808425275828,
+      "loss": 1.7163,
+      "theoretical_loss": 4.396975955959856,
+      "tokens_seen": 195231744
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007602647943831495,
+      "loss": 1.7856,
+      "theoretical_loss": 4.396791808886643,
+      "tokens_seen": 195297280
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007602487462387162,
+      "loss": 1.6566,
+      "theoretical_loss": 4.396607740893179,
+      "tokens_seen": 195362816
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007602326980942829,
+      "loss": 1.6452,
+      "theoretical_loss": 4.396423751918991,
+      "tokens_seen": 195428352
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007602166499498496,
+      "loss": 1.7718,
+      "theoretical_loss": 4.396239841903674,
+      "tokens_seen": 195493888
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007602006018054163,
+      "loss": 1.7305,
+      "theoretical_loss": 4.396056010786893,
+      "tokens_seen": 195559424
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.000760184553660983,
+      "loss": 1.6121,
+      "theoretical_loss": 4.3958722585083745,
+      "tokens_seen": 195624960
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007601685055165497,
+      "loss": 1.7438,
+      "theoretical_loss": 4.395688585007913,
+      "tokens_seen": 195690496
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007601524573721164,
+      "loss": 1.6999,
+      "theoretical_loss": 4.395504990225371,
+      "tokens_seen": 195756032
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007601364092276831,
+      "loss": 1.6127,
+      "theoretical_loss": 4.395321474100673,
+      "tokens_seen": 195821568
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007601203610832498,
+      "loss": 1.6554,
+      "theoretical_loss": 4.395138036573814,
+      "tokens_seen": 195887104
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007601043129388165,
+      "loss": 1.7556,
+      "theoretical_loss": 4.3949546775848525,
+      "tokens_seen": 195952640
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007600882647943832,
+      "loss": 1.7985,
+      "theoretical_loss": 4.394771397073912,
+      "tokens_seen": 196018176
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007600722166499499,
+      "loss": 1.6537,
+      "theoretical_loss": 4.394588194981182,
+      "tokens_seen": 196083712
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007600561685055166,
+      "loss": 1.7197,
+      "theoretical_loss": 4.394405071246919,
+      "tokens_seen": 196149248
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007600401203610833,
+      "loss": 1.663,
+      "theoretical_loss": 4.394222025811445,
+      "tokens_seen": 196214784
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.00076002407221665,
+      "loss": 1.6842,
+      "theoretical_loss": 4.394039058615144,
+      "tokens_seen": 196280320
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007600080240722167,
+      "loss": 1.6186,
+      "theoretical_loss": 4.39385616959847,
+      "tokens_seen": 196345856
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007599919759277834,
+      "loss": 1.7185,
+      "theoretical_loss": 4.393673358701939,
+      "tokens_seen": 196411392
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007599759277833501,
+      "loss": 1.6769,
+      "theoretical_loss": 4.393490625866132,
+      "tokens_seen": 196476928
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007599598796389168,
+      "loss": 1.5546,
+      "theoretical_loss": 4.393307971031697,
+      "tokens_seen": 196542464
+    },
+    {
+      "epoch": 0.06,
+      "objective/train/docs_used": 71923,
+      "objective/train/instantaneous_batch_size": 32,
+      "objective/train/instantaneous_microbatch_size": 32768,
+      "objective/train/original_loss": 1.8782007694244385,
+      "objective/train/theoretical_loss": 4.3931253941393456,
+      "objective/train/tokens_used": 217068000,
+      "theoretical_loss": 4.3931253941393456,
+      "tokens_seen": 196608000
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007599438314944835,
+      "loss": 1.815,
+      "theoretical_loss": 4.3931253941393456,
+      "tokens_seen": 196608000
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007599277833500502,
+      "loss": 1.5262,
+      "theoretical_loss": 4.392942895129854,
+      "tokens_seen": 196673536
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007599117352056169,
+      "loss": 1.6844,
+      "theoretical_loss": 4.392760473944064,
+      "tokens_seen": 196739072
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007598956870611836,
+      "loss": 1.7263,
+      "theoretical_loss": 4.392578130522883,
+      "tokens_seen": 196804608
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007598796389167503,
+      "loss": 1.7445,
+      "theoretical_loss": 4.392395864807279,
+      "tokens_seen": 196870144
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.000759863590772317,
+      "loss": 1.7149,
+      "theoretical_loss": 4.392213676738288,
+      "tokens_seen": 196935680
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007598475426278837,
+      "loss": 1.6839,
+      "theoretical_loss": 4.3920315662570095,
+      "tokens_seen": 197001216
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007598314944834504,
+      "loss": 1.7113,
+      "theoretical_loss": 4.391849533304607,
+      "tokens_seen": 197066752
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007598154463390171,
+      "loss": 1.8717,
+      "theoretical_loss": 4.391667577822308,
+      "tokens_seen": 197132288
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007597993981945838,
+      "loss": 1.6624,
+      "theoretical_loss": 4.391485699751406,
+      "tokens_seen": 197197824
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007597833500501505,
+      "loss": 1.6145,
+      "theoretical_loss": 4.391303899033253,
+      "tokens_seen": 197263360
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007597673019057172,
+      "loss": 1.7104,
+      "theoretical_loss": 4.391122175609272,
+      "tokens_seen": 197328896
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007597512537612839,
+      "loss": 1.7074,
+      "theoretical_loss": 4.390940529420946,
+      "tokens_seen": 197394432
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007597352056168506,
+      "loss": 1.8087,
+      "theoretical_loss": 4.39075896040982,
+      "tokens_seen": 197459968
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007597191574724173,
+      "loss": 1.6558,
+      "theoretical_loss": 4.390577468517508,
+      "tokens_seen": 197525504
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.000759703109327984,
+      "loss": 1.7509,
+      "theoretical_loss": 4.390396053685682,
+      "tokens_seen": 197591040
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007596870611835507,
+      "loss": 1.8001,
+      "theoretical_loss": 4.39021471585608,
+      "tokens_seen": 197656576
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007596710130391174,
+      "loss": 1.8231,
+      "theoretical_loss": 4.390033454970504,
+      "tokens_seen": 197722112
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007596549648946841,
+      "loss": 1.6138,
+      "theoretical_loss": 4.389852270970818,
+      "tokens_seen": 197787648
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007596389167502508,
+      "loss": 1.698,
+      "theoretical_loss": 4.389671163798949,
+      "tokens_seen": 197853184
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007596228686058175,
+      "loss": 1.8845,
+      "theoretical_loss": 4.3894901333968885,
+      "tokens_seen": 197918720
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007596068204613842,
+      "loss": 1.9291,
+      "theoretical_loss": 4.38930917970669,
+      "tokens_seen": 197984256
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007595907723169509,
+      "loss": 1.7239,
+      "theoretical_loss": 4.3891283026704695,
+      "tokens_seen": 198049792
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007595747241725176,
+      "loss": 1.7376,
+      "theoretical_loss": 4.388947502230407,
+      "tokens_seen": 198115328
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007595586760280843,
+      "loss": 1.8912,
+      "theoretical_loss": 4.388766778328744,
+      "tokens_seen": 198180864
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.000759542627883651,
+      "loss": 1.7471,
+      "theoretical_loss": 4.388586130907786,
+      "tokens_seen": 198246400
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007595265797392177,
+      "loss": 1.7832,
+      "theoretical_loss": 4.388405559909899,
+      "tokens_seen": 198311936
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007595105315947844,
+      "loss": 1.6903,
+      "theoretical_loss": 4.3882250652775125,
+      "tokens_seen": 198377472
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007594944834503511,
+      "loss": 1.7915,
+      "theoretical_loss": 4.38804464695312,
+      "tokens_seen": 198443008
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007594784353059178,
+      "loss": 1.8154,
+      "theoretical_loss": 4.387864304879275,
+      "tokens_seen": 198508544
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007594623871614845,
+      "loss": 1.8244,
+      "theoretical_loss": 4.387684038998595,
+      "tokens_seen": 198574080
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007594463390170512,
+      "loss": 1.6945,
+      "theoretical_loss": 4.387503849253757,
+      "tokens_seen": 198639616
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007594302908726179,
+      "loss": 1.7162,
+      "theoretical_loss": 4.387323735587502,
+      "tokens_seen": 198705152
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007594142427281846,
+      "loss": 1.7068,
+      "theoretical_loss": 4.387143697942634,
+      "tokens_seen": 198770688
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007593981945837513,
+      "loss": 1.6999,
+      "theoretical_loss": 4.386963736262016,
+      "tokens_seen": 198836224
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.000759382146439318,
+      "loss": 1.8626,
+      "theoretical_loss": 4.386783850488575,
+      "tokens_seen": 198901760
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007593660982948847,
+      "loss": 1.6224,
+      "theoretical_loss": 4.386604040565299,
+      "tokens_seen": 198967296
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007593500501504514,
+      "loss": 1.7606,
+      "theoretical_loss": 4.386424306435236,
+      "tokens_seen": 199032832
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007593340020060181,
+      "loss": 1.7761,
+      "theoretical_loss": 4.386244648041498,
+      "tokens_seen": 199098368
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007593179538615848,
+      "loss": 1.7878,
+      "theoretical_loss": 4.386065065327258,
+      "tokens_seen": 199163904
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007593019057171515,
+      "loss": 1.817,
+      "theoretical_loss": 4.385885558235747,
+      "tokens_seen": 199229440
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007592858575727182,
+      "loss": 1.6564,
+      "theoretical_loss": 4.385706126710264,
+      "tokens_seen": 199294976
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.000759269809428285,
+      "loss": 1.6816,
+      "theoretical_loss": 4.385526770694161,
+      "tokens_seen": 199360512
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007592537612838516,
+      "loss": 1.7888,
+      "theoretical_loss": 4.385347490130858,
+      "tokens_seen": 199426048
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007592377131394182,
+      "loss": 1.6447,
+      "theoretical_loss": 4.385168284963832,
+      "tokens_seen": 199491584
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.000759221664994985,
+      "loss": 1.7855,
+      "theoretical_loss": 4.384989155136621,
+      "tokens_seen": 199557120
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007592056168505516,
+      "loss": 1.7398,
+      "theoretical_loss": 4.384810100592826,
+      "tokens_seen": 199622656
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007591895687061184,
+      "loss": 1.6494,
+      "theoretical_loss": 4.384631121276108,
+      "tokens_seen": 199688192
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.000759173520561685,
+      "loss": 1.8777,
+      "theoretical_loss": 4.384452217130187,
+      "tokens_seen": 199753728
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007591574724172518,
+      "loss": 1.6866,
+      "theoretical_loss": 4.384273388098845,
+      "tokens_seen": 199819264
+    },
+    {
+      "epoch": 0.06,
+      "objective/train/docs_used": 72845,
+      "objective/train/instantaneous_batch_size": 32,
+      "objective/train/instantaneous_microbatch_size": 32768,
+      "objective/train/original_loss": 1.7771557569503784,
+      "objective/train/theoretical_loss": 4.384094634125924,
+      "objective/train/tokens_used": 220344800,
+      "theoretical_loss": 4.384094634125924,
+      "tokens_seen": 199884800
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007591414242728184,
+      "loss": 1.8039,
+      "theoretical_loss": 4.384094634125924,
+      "tokens_seen": 199884800
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007591253761283852,
+      "loss": 1.6788,
+      "theoretical_loss": 4.383915955155326,
+      "tokens_seen": 199950336
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007591093279839518,
+      "loss": 1.8913,
+      "theoretical_loss": 4.383737351131016,
+      "tokens_seen": 200015872
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007590932798395186,
+      "loss": 1.7216,
+      "theoretical_loss": 4.383558821997014,
+      "tokens_seen": 200081408
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007590772316950852,
+      "loss": 1.645,
+      "theoretical_loss": 4.383380367697406,
+      "tokens_seen": 200146944
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.000759061183550652,
+      "loss": 1.7902,
+      "theoretical_loss": 4.383201988176333,
+      "tokens_seen": 200212480
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007590451354062186,
+      "loss": 1.7366,
+      "theoretical_loss": 4.383023683377999,
+      "tokens_seen": 200278016
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007590290872617854,
+      "loss": 1.7718,
+      "theoretical_loss": 4.382845453246668,
+      "tokens_seen": 200343552
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.000759013039117352,
+      "loss": 1.7315,
+      "theoretical_loss": 4.382667297726661,
+      "tokens_seen": 200409088
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007589969909729188,
+      "loss": 1.878,
+      "theoretical_loss": 4.3824892167623615,
+      "tokens_seen": 200474624
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007589809428284855,
+      "loss": 1.7494,
+      "theoretical_loss": 4.382311210298212,
+      "tokens_seen": 200540160
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007589648946840522,
+      "loss": 1.6989,
+      "theoretical_loss": 4.3821332782787135,
+      "tokens_seen": 200605696
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007589488465396189,
+      "loss": 1.728,
+      "theoretical_loss": 4.381955420648428,
+      "tokens_seen": 200671232
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007589327983951856,
+      "loss": 1.7805,
+      "theoretical_loss": 4.381777637351974,
+      "tokens_seen": 200736768
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007589167502507523,
+      "loss": 1.8564,
+      "theoretical_loss": 4.381599928334033,
+      "tokens_seen": 200802304
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.000758900702106319,
+      "loss": 1.7495,
+      "theoretical_loss": 4.381422293539345,
+      "tokens_seen": 200867840
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007588846539618857,
+      "loss": 1.7009,
+      "theoretical_loss": 4.381244732912706,
+      "tokens_seen": 200933376
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007588686058174524,
+      "loss": 1.586,
+      "theoretical_loss": 4.381067246398975,
+      "tokens_seen": 200998912
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007588525576730191,
+      "loss": 1.8278,
+      "theoretical_loss": 4.380889833943066,
+      "tokens_seen": 201064448
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007588365095285858,
+      "loss": 1.6234,
+      "theoretical_loss": 4.380712495489957,
+      "tokens_seen": 201129984
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007588204613841525,
+      "loss": 1.7723,
+      "theoretical_loss": 4.380535230984679,
+      "tokens_seen": 201195520
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007588044132397192,
+      "loss": 1.6197,
+      "theoretical_loss": 4.380358040372325,
+      "tokens_seen": 201261056
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007587883650952859,
+      "loss": 1.725,
+      "theoretical_loss": 4.380180923598047,
+      "tokens_seen": 201326592
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007587723169508526,
+      "loss": 1.6964,
+      "theoretical_loss": 4.380003880607055,
+      "tokens_seen": 201392128
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007587562688064193,
+      "loss": 1.8181,
+      "theoretical_loss": 4.379826911344617,
+      "tokens_seen": 201457664
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.000758740220661986,
+      "loss": 1.6887,
+      "theoretical_loss": 4.3796500157560585,
+      "tokens_seen": 201523200
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007587241725175527,
+      "loss": 1.6997,
+      "theoretical_loss": 4.379473193786765,
+      "tokens_seen": 201588736
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007587081243731194,
+      "loss": 1.6867,
+      "theoretical_loss": 4.37929644538218,
+      "tokens_seen": 201654272
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007586920762286861,
+      "loss": 1.7476,
+      "theoretical_loss": 4.379119770487805,
+      "tokens_seen": 201719808
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007586760280842528,
+      "loss": 1.8003,
+      "theoretical_loss": 4.378943169049198,
+      "tokens_seen": 201785344
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007586599799398195,
+      "loss": 1.7035,
+      "theoretical_loss": 4.378766641011978,
+      "tokens_seen": 201850880
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007586439317953862,
+      "loss": 1.7468,
+      "theoretical_loss": 4.378590186321819,
+      "tokens_seen": 201916416
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007586278836509529,
+      "loss": 1.7194,
+      "theoretical_loss": 4.378413804924454,
+      "tokens_seen": 201981952
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007586118355065196,
+      "loss": 1.8233,
+      "theoretical_loss": 4.378237496765674,
+      "tokens_seen": 202047488
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007585957873620863,
+      "loss": 1.5598,
+      "theoretical_loss": 4.378061261791328,
+      "tokens_seen": 202113024
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.000758579739217653,
+      "loss": 1.6599,
+      "theoretical_loss": 4.377885099947321,
+      "tokens_seen": 202178560
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007585636910732197,
+      "loss": 1.6418,
+      "theoretical_loss": 4.3777090111796175,
+      "tokens_seen": 202244096
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007585476429287864,
+      "loss": 1.8023,
+      "theoretical_loss": 4.377532995434238,
+      "tokens_seen": 202309632
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007585315947843531,
+      "loss": 1.639,
+      "theoretical_loss": 4.37735705265726,
+      "tokens_seen": 202375168
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007585155466399198,
+      "loss": 1.6263,
+      "theoretical_loss": 4.3771811827948195,
+      "tokens_seen": 202440704
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007584994984954865,
+      "loss": 1.6996,
+      "theoretical_loss": 4.37700538579311,
+      "tokens_seen": 202506240
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007584834503510532,
+      "loss": 1.5977,
+      "theoretical_loss": 4.37682966159838,
+      "tokens_seen": 202571776
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007584674022066199,
+      "loss": 1.7018,
+      "theoretical_loss": 4.376654010156939,
+      "tokens_seen": 202637312
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007584513540621866,
+      "loss": 1.7452,
+      "theoretical_loss": 4.376478431415148,
+      "tokens_seen": 202702848
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007584353059177533,
+      "loss": 1.7746,
+      "theoretical_loss": 4.376302925319427,
+      "tokens_seen": 202768384
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.00075841925777332,
+      "loss": 1.6853,
+      "theoretical_loss": 4.3761274918162565,
+      "tokens_seen": 202833920
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007584032096288867,
+      "loss": 1.6854,
+      "theoretical_loss": 4.375952130852169,
+      "tokens_seen": 202899456
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007583871614844534,
+      "loss": 1.6129,
+      "theoretical_loss": 4.375776842373755,
+      "tokens_seen": 202964992
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007583711133400201,
+      "loss": 1.6554,
+      "theoretical_loss": 4.375601626327662,
+      "tokens_seen": 203030528
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007583550651955868,
+      "loss": 1.6618,
+      "theoretical_loss": 4.375426482660595,
+      "tokens_seen": 203096064
+    },
+    {
+      "epoch": 0.06,
+      "objective/train/docs_used": 74020,
+      "objective/train/instantaneous_batch_size": 32,
+      "objective/train/instantaneous_microbatch_size": 32768,
+      "objective/train/original_loss": 1.6644554138183594,
+      "objective/train/theoretical_loss": 4.375251411319312,
+      "objective/train/tokens_used": 223621600,
+      "theoretical_loss": 4.375251411319312,
+      "tokens_seen": 203161600
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007583390170511535,
+      "loss": 1.6488,
+      "theoretical_loss": 4.375251411319312,
+      "tokens_seen": 203161600
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007583229689067202,
+      "loss": 1.6196,
+      "theoretical_loss": 4.375076412250632,
+      "tokens_seen": 203227136
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007583069207622869,
+      "loss": 1.6985,
+      "theoretical_loss": 4.374901485401426,
+      "tokens_seen": 203292672
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007582908726178536,
+      "loss": 1.7379,
+      "theoretical_loss": 4.374726630718624,
+      "tokens_seen": 203358208
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007582748244734203,
+      "loss": 1.7147,
+      "theoretical_loss": 4.37455184814921,
+      "tokens_seen": 203423744
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.000758258776328987,
+      "loss": 1.6941,
+      "theoretical_loss": 4.374377137640225,
+      "tokens_seen": 203489280
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007582427281845537,
+      "loss": 1.6603,
+      "theoretical_loss": 4.374202499138768,
+      "tokens_seen": 203554816
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007582266800401204,
+      "loss": 1.6174,
+      "theoretical_loss": 4.374027932591989,
+      "tokens_seen": 203620352
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007582106318956871,
+      "loss": 1.6624,
+      "theoretical_loss": 4.373853437947099,
+      "tokens_seen": 203685888
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007581945837512538,
+      "loss": 1.7826,
+      "theoretical_loss": 4.37367901515136,
+      "tokens_seen": 203751424
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007581785356068205,
+      "loss": 1.6085,
+      "theoretical_loss": 4.373504664152094,
+      "tokens_seen": 203816960
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007581624874623872,
+      "loss": 1.7104,
+      "theoretical_loss": 4.373330384896676,
+      "tokens_seen": 203882496
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007581464393179539,
+      "loss": 1.7447,
+      "theoretical_loss": 4.373156177332536,
+      "tokens_seen": 203948032
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007581303911735206,
+      "loss": 1.683,
+      "theoretical_loss": 4.372982041407161,
+      "tokens_seen": 204013568
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007581143430290873,
+      "loss": 1.7584,
+      "theoretical_loss": 4.372807977068092,
+      "tokens_seen": 204079104
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.000758098294884654,
+      "loss": 1.703,
+      "theoretical_loss": 4.372633984262928,
+      "tokens_seen": 204144640
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007580822467402207,
+      "loss": 1.7222,
+      "theoretical_loss": 4.372460062939318,
+      "tokens_seen": 204210176
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007580661985957874,
+      "loss": 1.6514,
+      "theoretical_loss": 4.372286213044972,
+      "tokens_seen": 204275712
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007580501504513541,
+      "loss": 1.6188,
+      "theoretical_loss": 4.37211243452765,
+      "tokens_seen": 204341248
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007580341023069208,
+      "loss": 1.6626,
+      "theoretical_loss": 4.37193872733517,
+      "tokens_seen": 204406784
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007580180541624875,
+      "loss": 1.6699,
+      "theoretical_loss": 4.371765091415404,
+      "tokens_seen": 204472320
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007580020060180542,
+      "loss": 1.5978,
+      "theoretical_loss": 4.371591526716279,
+      "tokens_seen": 204537856
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007579859578736209,
+      "loss": 1.7566,
+      "theoretical_loss": 4.371418033185777,
+      "tokens_seen": 204603392
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007579699097291876,
+      "loss": 1.5674,
+      "theoretical_loss": 4.3712446107719325,
+      "tokens_seen": 204668928
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007579538615847543,
+      "loss": 1.5815,
+      "theoretical_loss": 4.371071259422838,
+      "tokens_seen": 204734464
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.000757937813440321,
+      "loss": 1.6574,
+      "theoretical_loss": 4.370897979086636,
+      "tokens_seen": 204800000
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007579217652958877,
+      "loss": 1.7342,
+      "theoretical_loss": 4.370724769711529,
+      "tokens_seen": 204865536
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007579057171514544,
+      "loss": 1.7287,
+      "theoretical_loss": 4.370551631245769,
+      "tokens_seen": 204931072
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007578896690070211,
+      "loss": 1.6055,
+      "theoretical_loss": 4.370378563637667,
+      "tokens_seen": 204996608
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007578736208625878,
+      "loss": 1.7417,
+      "theoretical_loss": 4.3702055668355815,
+      "tokens_seen": 205062144
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007578575727181545,
+      "loss": 1.7055,
+      "theoretical_loss": 4.370032640787931,
+      "tokens_seen": 205127680
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007578415245737211,
+      "loss": 1.6784,
+      "theoretical_loss": 4.369859785443188,
+      "tokens_seen": 205193216
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007578254764292879,
+      "loss": 1.6829,
+      "theoretical_loss": 4.369687000749873,
+      "tokens_seen": 205258752
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007578094282848545,
+      "loss": 1.7588,
+      "theoretical_loss": 4.3695142866565675,
+      "tokens_seen": 205324288
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007577933801404213,
+      "loss": 1.64,
+      "theoretical_loss": 4.369341643111902,
+      "tokens_seen": 205389824
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007577773319959879,
+      "loss": 1.6531,
+      "theoretical_loss": 4.369169070064563,
+      "tokens_seen": 205455360
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007577612838515547,
+      "loss": 1.6938,
+      "theoretical_loss": 4.368996567463292,
+      "tokens_seen": 205520896
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007577452357071213,
+      "loss": 1.6523,
+      "theoretical_loss": 4.3688241352568795,
+      "tokens_seen": 205586432
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007577291875626881,
+      "loss": 1.725,
+      "theoretical_loss": 4.368651773394173,
+      "tokens_seen": 205651968
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007577131394182547,
+      "loss": 1.8234,
+      "theoretical_loss": 4.368479481824075,
+      "tokens_seen": 205717504
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007576970912738215,
+      "loss": 1.7173,
+      "theoretical_loss": 4.368307260495536,
+      "tokens_seen": 205783040
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007576810431293881,
+      "loss": 1.7403,
+      "theoretical_loss": 4.368135109357564,
+      "tokens_seen": 205848576
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007576649949849549,
+      "loss": 1.6939,
+      "theoretical_loss": 4.36796302835922,
+      "tokens_seen": 205914112
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007576489468405215,
+      "loss": 1.6724,
+      "theoretical_loss": 4.367791017449618,
+      "tokens_seen": 205979648
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007576328986960883,
+      "loss": 1.7243,
+      "theoretical_loss": 4.367619076577923,
+      "tokens_seen": 206045184
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.000757616850551655,
+      "loss": 1.628,
+      "theoretical_loss": 4.367447205693354,
+      "tokens_seen": 206110720
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007576008024072217,
+      "loss": 1.7714,
+      "theoretical_loss": 4.367275404745186,
+      "tokens_seen": 206176256
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007575847542627884,
+      "loss": 1.6331,
+      "theoretical_loss": 4.367103673682742,
+      "tokens_seen": 206241792
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007575687061183551,
+      "loss": 1.6633,
+      "theoretical_loss": 4.3669320124554,
+      "tokens_seen": 206307328
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007575526579739218,
+      "loss": 1.81,
+      "theoretical_loss": 4.366760421012592,
+      "tokens_seen": 206372864
+    },
+    {
+      "epoch": 0.06,
+      "objective/train/docs_used": 74982,
+      "objective/train/instantaneous_batch_size": 32,
+      "objective/train/instantaneous_microbatch_size": 32768,
+      "objective/train/original_loss": 1.5668761730194092,
+      "objective/train/theoretical_loss": 4.366588899303801,
+      "objective/train/tokens_used": 226898400,
+      "theoretical_loss": 4.366588899303801,
+      "tokens_seen": 206438400
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007575366098294885,
+      "loss": 1.646,
+      "theoretical_loss": 4.366588899303801,
+      "tokens_seen": 206438400
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007575205616850552,
+      "loss": 1.658,
+      "theoretical_loss": 4.366417447278565,
+      "tokens_seen": 206503936
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007575045135406219,
+      "loss": 1.6617,
+      "theoretical_loss": 4.366246064886468,
+      "tokens_seen": 206569472
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007574884653961886,
+      "loss": 1.665,
+      "theoretical_loss": 4.3660747520771555,
+      "tokens_seen": 206635008
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007574724172517553,
+      "loss": 1.6843,
+      "theoretical_loss": 4.36590350880032,
+      "tokens_seen": 206700544
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.000757456369107322,
+      "loss": 1.6879,
+      "theoretical_loss": 4.365732335005706,
+      "tokens_seen": 206766080
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007574403209628887,
+      "loss": 1.6198,
+      "theoretical_loss": 4.365561230643112,
+      "tokens_seen": 206831616
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007574242728184554,
+      "loss": 1.6828,
+      "theoretical_loss": 4.365390195662389,
+      "tokens_seen": 206897152
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007574082246740221,
+      "loss": 1.6962,
+      "theoretical_loss": 4.365219230013438,
+      "tokens_seen": 206962688
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007573921765295888,
+      "loss": 1.6293,
+      "theoretical_loss": 4.3650483336462145,
+      "tokens_seen": 207028224
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007573761283851555,
+      "loss": 1.6155,
+      "theoretical_loss": 4.364877506510723,
+      "tokens_seen": 207093760
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007573600802407222,
+      "loss": 1.6634,
+      "theoretical_loss": 4.3647067485570235,
+      "tokens_seen": 207159296
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007573440320962889,
+      "loss": 1.7377,
+      "theoretical_loss": 4.364536059735224,
+      "tokens_seen": 207224832
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007573279839518556,
+      "loss": 1.6982,
+      "theoretical_loss": 4.364365439995488,
+      "tokens_seen": 207290368
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007573119358074223,
+      "loss": 1.6499,
+      "theoretical_loss": 4.3641948892880285,
+      "tokens_seen": 207355904
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.000757295887662989,
+      "loss": 1.5874,
+      "theoretical_loss": 4.364024407563109,
+      "tokens_seen": 207421440
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007572798395185557,
+      "loss": 1.6287,
+      "theoretical_loss": 4.363853994771048,
+      "tokens_seen": 207486976
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007572637913741224,
+      "loss": 1.7731,
+      "theoretical_loss": 4.363683650862212,
+      "tokens_seen": 207552512
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007572477432296891,
+      "loss": 1.7701,
+      "theoretical_loss": 4.36351337578702,
+      "tokens_seen": 207618048
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007572316950852558,
+      "loss": 1.7821,
+      "theoretical_loss": 4.363343169495945,
+      "tokens_seen": 207683584
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007572156469408225,
+      "loss": 1.7471,
+      "theoretical_loss": 4.363173031939506,
+      "tokens_seen": 207749120
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007571995987963892,
+      "loss": 1.5793,
+      "theoretical_loss": 4.363002963068277,
+      "tokens_seen": 207814656
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007571835506519559,
+      "loss": 1.603,
+      "theoretical_loss": 4.362832962832883,
+      "tokens_seen": 207880192
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007571675025075226,
+      "loss": 1.6413,
+      "theoretical_loss": 4.362663031183999,
+      "tokens_seen": 207945728
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007571514543630893,
+      "loss": 1.6473,
+      "theoretical_loss": 4.36249316807235,
+      "tokens_seen": 208011264
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.000757135406218656,
+      "loss": 1.6029,
+      "theoretical_loss": 4.3623233734487155,
+      "tokens_seen": 208076800
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007571193580742227,
+      "loss": 1.6679,
+      "theoretical_loss": 4.362153647263921,
+      "tokens_seen": 208142336
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007571033099297894,
+      "loss": 1.6077,
+      "theoretical_loss": 4.361983989468847,
+      "tokens_seen": 208207872
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007570872617853561,
+      "loss": 1.5238,
+      "theoretical_loss": 4.3618144000144214,
+      "tokens_seen": 208273408
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007570712136409228,
+      "loss": 1.5603,
+      "theoretical_loss": 4.361644878851626,
+      "tokens_seen": 208338944
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007570551654964895,
+      "loss": 1.6313,
+      "theoretical_loss": 4.36147542593149,
+      "tokens_seen": 208404480
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007570391173520562,
+      "loss": 1.6174,
+      "theoretical_loss": 4.361306041205095,
+      "tokens_seen": 208470016
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007570230692076229,
+      "loss": 1.6451,
+      "theoretical_loss": 4.361136724623573,
+      "tokens_seen": 208535552
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007570070210631896,
+      "loss": 1.7059,
+      "theoretical_loss": 4.360967476138105,
+      "tokens_seen": 208601088
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007569909729187563,
+      "loss": 1.6309,
+      "theoretical_loss": 4.360798295699925,
+      "tokens_seen": 208666624
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.000756974924774323,
+      "loss": 1.722,
+      "theoretical_loss": 4.360629183260313,
+      "tokens_seen": 208732160
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007569588766298897,
+      "loss": 1.4713,
+      "theoretical_loss": 4.360460138770604,
+      "tokens_seen": 208797696
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007569428284854564,
+      "loss": 1.7233,
+      "theoretical_loss": 4.360291162182179,
+      "tokens_seen": 208863232
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007569267803410231,
+      "loss": 1.7762,
+      "theoretical_loss": 4.360122253446472,
+      "tokens_seen": 208928768
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007569107321965898,
+      "loss": 1.62,
+      "theoretical_loss": 4.359953412514965,
+      "tokens_seen": 208994304
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007568946840521565,
+      "loss": 1.6154,
+      "theoretical_loss": 4.359784639339191,
+      "tokens_seen": 209059840
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007568786359077232,
+      "loss": 1.7554,
+      "theoretical_loss": 4.359615933870732,
+      "tokens_seen": 209125376
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007568625877632899,
+      "loss": 1.7173,
+      "theoretical_loss": 4.359447296061219,
+      "tokens_seen": 209190912
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007568465396188566,
+      "loss": 1.711,
+      "theoretical_loss": 4.359278725862336,
+      "tokens_seen": 209256448
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007568304914744233,
+      "loss": 1.5769,
+      "theoretical_loss": 4.359110223225814,
+      "tokens_seen": 209321984
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.00075681444332999,
+      "loss": 1.7187,
+      "theoretical_loss": 4.358941788103433,
+      "tokens_seen": 209387520
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007567983951855567,
+      "loss": 1.6829,
+      "theoretical_loss": 4.358773420447024,
+      "tokens_seen": 209453056
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007567823470411234,
+      "loss": 1.682,
+      "theoretical_loss": 4.358605120208469,
+      "tokens_seen": 209518592
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007567662988966901,
+      "loss": 1.6175,
+      "theoretical_loss": 4.358436887339694,
+      "tokens_seen": 209584128
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007567502507522569,
+      "loss": 1.7561,
+      "theoretical_loss": 4.35826872179268,
+      "tokens_seen": 209649664
+    },
+    {
+      "epoch": 0.06,
+      "objective/train/docs_used": 76132,
+      "objective/train/instantaneous_batch_size": 32,
+      "objective/train/instantaneous_microbatch_size": 32768,
+      "objective/train/original_loss": 1.7629177570343018,
+      "objective/train/theoretical_loss": 4.358100623519453,
+      "objective/train/tokens_used": 230175200,
+      "theoretical_loss": 4.358100623519453,
+      "tokens_seen": 209715200
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007567342026078235,
+      "loss": 1.682,
+      "theoretical_loss": 4.358100623519453,
+      "tokens_seen": 209715200
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007567181544633903,
+      "loss": 1.6279,
+      "theoretical_loss": 4.357932592472092,
+      "tokens_seen": 209780736
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007567021063189569,
+      "loss": 1.6896,
+      "theoretical_loss": 4.357764628602721,
+      "tokens_seen": 209846272
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007566860581745237,
+      "loss": 1.6235,
+      "theoretical_loss": 4.357596731863517,
+      "tokens_seen": 209911808
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007566700100300903,
+      "loss": 1.7687,
+      "theoretical_loss": 4.357428902206704,
+      "tokens_seen": 209977344
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007566539618856571,
+      "loss": 1.7629,
+      "theoretical_loss": 4.357261139584553,
+      "tokens_seen": 210042880
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007566379137412237,
+      "loss": 1.7007,
+      "theoretical_loss": 4.357093443949388,
+      "tokens_seen": 210108416
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007566218655967905,
+      "loss": 1.7062,
+      "theoretical_loss": 4.356925815253579,
+      "tokens_seen": 210173952
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007566058174523571,
+      "loss": 1.6921,
+      "theoretical_loss": 4.356758253449544,
+      "tokens_seen": 210239488
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007565897693079239,
+      "loss": 1.6388,
+      "theoretical_loss": 4.356590758489753,
+      "tokens_seen": 210305024
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007565737211634905,
+      "loss": 1.5737,
+      "theoretical_loss": 4.356423330326722,
+      "tokens_seen": 210370560
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007565576730190572,
+      "loss": 1.7739,
+      "theoretical_loss": 4.356255968913015,
+      "tokens_seen": 210436096
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.000756541624874624,
+      "loss": 1.6745,
+      "theoretical_loss": 4.356088674201246,
+      "tokens_seen": 210501632
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007565255767301906,
+      "loss": 1.7266,
+      "theoretical_loss": 4.355921446144077,
+      "tokens_seen": 210567168
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007565095285857573,
+      "loss": 1.5708,
+      "theoretical_loss": 4.355754284694218,
+      "tokens_seen": 210632704
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.000756493480441324,
+      "loss": 1.7609,
+      "theoretical_loss": 4.355587189804427,
+      "tokens_seen": 210698240
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007564774322968907,
+      "loss": 1.7054,
+      "theoretical_loss": 4.355420161427512,
+      "tokens_seen": 210763776
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007564613841524574,
+      "loss": 1.6645,
+      "theoretical_loss": 4.355253199516326,
+      "tokens_seen": 210829312
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007564453360080242,
+      "loss": 1.7689,
+      "theoretical_loss": 4.355086304023773,
+      "tokens_seen": 210894848
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007564292878635908,
+      "loss": 1.6643,
+      "theoretical_loss": 4.354919474902804,
+      "tokens_seen": 210960384
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007564132397191574,
+      "loss": 1.7519,
+      "theoretical_loss": 4.354752712106417,
+      "tokens_seen": 211025920
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007563971915747242,
+      "loss": 1.7519,
+      "theoretical_loss": 4.354586015587659,
+      "tokens_seen": 211091456
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007563811434302908,
+      "loss": 1.6904,
+      "theoretical_loss": 4.354419385299623,
+      "tokens_seen": 211156992
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007563650952858576,
+      "loss": 1.5502,
+      "theoretical_loss": 4.354252821195453,
+      "tokens_seen": 211222528
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007563490471414242,
+      "loss": 1.703,
+      "theoretical_loss": 4.354086323228338,
+      "tokens_seen": 211288064
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.000756332998996991,
+      "loss": 1.7125,
+      "theoretical_loss": 4.353919891351516,
+      "tokens_seen": 211353600
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007563169508525576,
+      "loss": 1.7125,
+      "theoretical_loss": 4.35375352551827,
+      "tokens_seen": 211419136
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007563009027081244,
+      "loss": 1.6203,
+      "theoretical_loss": 4.353587225681935,
+      "tokens_seen": 211484672
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.000756284854563691,
+      "loss": 1.7624,
+      "theoretical_loss": 4.35342099179589,
+      "tokens_seen": 211550208
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007562688064192578,
+      "loss": 1.7121,
+      "theoretical_loss": 4.353254823813561,
+      "tokens_seen": 211615744
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007562527582748245,
+      "loss": 1.757,
+      "theoretical_loss": 4.353088721688424,
+      "tokens_seen": 211681280
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007562367101303912,
+      "loss": 1.7457,
+      "theoretical_loss": 4.352922685374001,
+      "tokens_seen": 211746816
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007562206619859579,
+      "loss": 1.5599,
+      "theoretical_loss": 4.352756714823859,
+      "tokens_seen": 211812352
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007562046138415246,
+      "loss": 1.6563,
+      "theoretical_loss": 4.352590809991616,
+      "tokens_seen": 211877888
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007561885656970913,
+      "loss": 1.6943,
+      "theoretical_loss": 4.352424970830935,
+      "tokens_seen": 211943424
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.000756172517552658,
+      "loss": 1.7164,
+      "theoretical_loss": 4.352259197295525,
+      "tokens_seen": 212008960
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007561564694082247,
+      "loss": 1.6345,
+      "theoretical_loss": 4.352093489339143,
+      "tokens_seen": 212074496
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007561404212637914,
+      "loss": 1.7932,
+      "theoretical_loss": 4.351927846915595,
+      "tokens_seen": 212140032
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007561243731193581,
+      "loss": 1.7008,
+      "theoretical_loss": 4.35176226997873,
+      "tokens_seen": 212205568
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007561083249749248,
+      "loss": 1.7164,
+      "theoretical_loss": 4.351596758482445,
+      "tokens_seen": 212271104
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007560922768304915,
+      "loss": 1.7722,
+      "theoretical_loss": 4.351431312380684,
+      "tokens_seen": 212336640
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007560762286860582,
+      "loss": 1.6708,
+      "theoretical_loss": 4.351265931627439,
+      "tokens_seen": 212402176
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007560601805416249,
+      "loss": 1.6154,
+      "theoretical_loss": 4.351100616176747,
+      "tokens_seen": 212467712
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007560441323971916,
+      "loss": 1.6389,
+      "theoretical_loss": 4.350935365982691,
+      "tokens_seen": 212533248
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007560280842527583,
+      "loss": 1.6767,
+      "theoretical_loss": 4.350770180999402,
+      "tokens_seen": 212598784
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.000756012036108325,
+      "loss": 1.5465,
+      "theoretical_loss": 4.3506050611810565,
+      "tokens_seen": 212664320
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007559959879638917,
+      "loss": 1.6647,
+      "theoretical_loss": 4.350440006481877,
+      "tokens_seen": 212729856
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007559799398194584,
+      "loss": 1.7883,
+      "theoretical_loss": 4.350275016856132,
+      "tokens_seen": 212795392
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007559638916750251,
+      "loss": 1.7281,
+      "theoretical_loss": 4.350110092258138,
+      "tokens_seen": 212860928
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007559478435305918,
+      "loss": 1.6009,
+      "theoretical_loss": 4.349945232642256,
+      "tokens_seen": 212926464
+    },
+    {
+      "epoch": 0.06,
+      "objective/train/docs_used": 77051,
+      "objective/train/instantaneous_batch_size": 32,
+      "objective/train/instantaneous_microbatch_size": 32768,
+      "objective/train/original_loss": 1.6532769203186035,
+      "objective/train/theoretical_loss": 4.3497804379628935,
+      "objective/train/tokens_used": 233452000,
+      "theoretical_loss": 4.3497804379628935,
+      "tokens_seen": 212992000
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007559317953861585,
+      "loss": 1.7022,
+      "theoretical_loss": 4.3497804379628935,
+      "tokens_seen": 212992000
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007559157472417252,
+      "loss": 1.7101,
+      "theoretical_loss": 4.349615708174504,
+      "tokens_seen": 213057536
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007558996990972919,
+      "loss": 1.5849,
+      "theoretical_loss": 4.3494510432315865,
+      "tokens_seen": 213123072
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007558836509528586,
+      "loss": 1.6844,
+      "theoretical_loss": 4.349286443088687,
+      "tokens_seen": 213188608
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007558676028084253,
+      "loss": 1.6763,
+      "theoretical_loss": 4.349121907700396,
+      "tokens_seen": 213254144
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.000755851554663992,
+      "loss": 1.7083,
+      "theoretical_loss": 4.348957437021351,
+      "tokens_seen": 213319680
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007558355065195587,
+      "loss": 1.7361,
+      "theoretical_loss": 4.348793031006235,
+      "tokens_seen": 213385216
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007558194583751254,
+      "loss": 1.6701,
+      "theoretical_loss": 4.348628689609775,
+      "tokens_seen": 213450752
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007558034102306921,
+      "loss": 1.6713,
+      "theoretical_loss": 4.348464412786746,
+      "tokens_seen": 213516288
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007557873620862588,
+      "loss": 1.6983,
+      "theoretical_loss": 4.348300200491966,
+      "tokens_seen": 213581824
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007557713139418255,
+      "loss": 1.6225,
+      "theoretical_loss": 4.348136052680301,
+      "tokens_seen": 213647360
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007557552657973922,
+      "loss": 1.6256,
+      "theoretical_loss": 4.347971969306662,
+      "tokens_seen": 213712896
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007557392176529589,
+      "loss": 1.6923,
+      "theoretical_loss": 4.347807950326002,
+      "tokens_seen": 213778432
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007557231695085256,
+      "loss": 1.6672,
+      "theoretical_loss": 4.347643995693325,
+      "tokens_seen": 213843968
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007557071213640923,
+      "loss": 1.5255,
+      "theoretical_loss": 4.347480105363674,
+      "tokens_seen": 213909504
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.000755691073219659,
+      "loss": 1.6764,
+      "theoretical_loss": 4.3473162792921425,
+      "tokens_seen": 213975040
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007556750250752257,
+      "loss": 1.6176,
+      "theoretical_loss": 4.347152517433866,
+      "tokens_seen": 214040576
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007556589769307924,
+      "loss": 1.6105,
+      "theoretical_loss": 4.346988819744026,
+      "tokens_seen": 214106112
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007556429287863591,
+      "loss": 1.7308,
+      "theoretical_loss": 4.346825186177848,
+      "tokens_seen": 214171648
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007556268806419258,
+      "loss": 1.4866,
+      "theoretical_loss": 4.346661616690605,
+      "tokens_seen": 214237184
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007556108324974925,
+      "loss": 1.6574,
+      "theoretical_loss": 4.346498111237612,
+      "tokens_seen": 214302720
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007555947843530592,
+      "loss": 1.568,
+      "theoretical_loss": 4.346334669774231,
+      "tokens_seen": 214368256
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007555787362086259,
+      "loss": 1.5096,
+      "theoretical_loss": 4.346171292255867,
+      "tokens_seen": 214433792
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0007555626880641926,
+      "loss": 1.694,
+      "theoretical_loss": 4.34600797863797,
+      "tokens_seen": 214499328
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007555466399197594,
+      "loss": 1.604,
+      "theoretical_loss": 4.345844728876036,
+      "tokens_seen": 214564864
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.000755530591775326,
+      "loss": 1.5796,
+      "theoretical_loss": 4.345681542925604,
+      "tokens_seen": 214630400
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007555145436308928,
+      "loss": 1.6733,
+      "theoretical_loss": 4.345518420742259,
+      "tokens_seen": 214695936
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007554984954864594,
+      "loss": 1.6053,
+      "theoretical_loss": 4.345355362281628,
+      "tokens_seen": 214761472
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007554824473420262,
+      "loss": 1.6213,
+      "theoretical_loss": 4.345192367499386,
+      "tokens_seen": 214827008
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007554663991975928,
+      "loss": 1.6248,
+      "theoretical_loss": 4.3450294363512505,
+      "tokens_seen": 214892544
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007554503510531596,
+      "loss": 1.7046,
+      "theoretical_loss": 4.34486656879298,
+      "tokens_seen": 214958080
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007554343029087262,
+      "loss": 1.6248,
+      "theoretical_loss": 4.344703764780386,
+      "tokens_seen": 215023616
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.000755418254764293,
+      "loss": 1.6069,
+      "theoretical_loss": 4.344541024269312,
+      "tokens_seen": 215089152
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007554022066198596,
+      "loss": 1.6682,
+      "theoretical_loss": 4.344378347215657,
+      "tokens_seen": 215154688
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007553861584754264,
+      "loss": 1.6614,
+      "theoretical_loss": 4.344215733575357,
+      "tokens_seen": 215220224
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.000755370110330993,
+      "loss": 1.5608,
+      "theoretical_loss": 4.344053183304396,
+      "tokens_seen": 215285760
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007553540621865598,
+      "loss": 1.7457,
+      "theoretical_loss": 4.343890696358798,
+      "tokens_seen": 215351296
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007553380140421264,
+      "loss": 1.6441,
+      "theoretical_loss": 4.343728272694635,
+      "tokens_seen": 215416832
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007553219658976932,
+      "loss": 1.597,
+      "theoretical_loss": 4.34356591226802,
+      "tokens_seen": 215482368
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007553059177532598,
+      "loss": 1.6342,
+      "theoretical_loss": 4.343403615035111,
+      "tokens_seen": 215547904
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007552898696088266,
+      "loss": 1.6817,
+      "theoretical_loss": 4.343241380952109,
+      "tokens_seen": 215613440
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007552738214643932,
+      "loss": 1.6443,
+      "theoretical_loss": 4.34307920997526,
+      "tokens_seen": 215678976
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00075525777331996,
+      "loss": 1.7266,
+      "theoretical_loss": 4.34291710206085,
+      "tokens_seen": 215744512
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007552417251755266,
+      "loss": 1.5673,
+      "theoretical_loss": 4.342755057165214,
+      "tokens_seen": 215810048
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007552256770310934,
+      "loss": 1.7022,
+      "theoretical_loss": 4.342593075244728,
+      "tokens_seen": 215875584
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00075520962888666,
+      "loss": 1.5824,
+      "theoretical_loss": 4.342431156255809,
+      "tokens_seen": 215941120
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007551935807422268,
+      "loss": 1.6162,
+      "theoretical_loss": 4.3422693001549195,
+      "tokens_seen": 216006656
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007551775325977934,
+      "loss": 1.6574,
+      "theoretical_loss": 4.342107506898567,
+      "tokens_seen": 216072192
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007551614844533601,
+      "loss": 1.6443,
+      "theoretical_loss": 4.3419457764433,
+      "tokens_seen": 216137728
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007551454363089268,
+      "loss": 1.6301,
+      "theoretical_loss": 4.34178410874571,
+      "tokens_seen": 216203264
+    },
+    {
+      "epoch": 0.07,
+      "objective/train/docs_used": 78201,
+      "objective/train/instantaneous_batch_size": 32,
+      "objective/train/instantaneous_microbatch_size": 32768,
+      "objective/train/original_loss": 1.6742914915084839,
+      "objective/train/theoretical_loss": 4.341622503762434,
+      "objective/train/tokens_used": 236728800,
+      "theoretical_loss": 4.341622503762434,
+      "tokens_seen": 216268800
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007551293881644935,
+      "loss": 1.7051,
+      "theoretical_loss": 4.341622503762434,
+      "tokens_seen": 216268800
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007551133400200602,
+      "loss": 1.6294,
+      "theoretical_loss": 4.341460961450148,
+      "tokens_seen": 216334336
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007550972918756269,
+      "loss": 1.5905,
+      "theoretical_loss": 4.341299481765575,
+      "tokens_seen": 216399872
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007550812437311937,
+      "loss": 1.7096,
+      "theoretical_loss": 4.34113806466548,
+      "tokens_seen": 216465408
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007550651955867603,
+      "loss": 1.6635,
+      "theoretical_loss": 4.340976710106671,
+      "tokens_seen": 216530944
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.000755049147442327,
+      "loss": 1.5918,
+      "theoretical_loss": 4.340815418045995,
+      "tokens_seen": 216596480
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007550330992978937,
+      "loss": 1.501,
+      "theoretical_loss": 4.340654188440349,
+      "tokens_seen": 216662016
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007550170511534603,
+      "loss": 1.5726,
+      "theoretical_loss": 4.340493021246668,
+      "tokens_seen": 216727552
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007550010030090271,
+      "loss": 1.68,
+      "theoretical_loss": 4.3403319164219285,
+      "tokens_seen": 216793088
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007549849548645937,
+      "loss": 1.5825,
+      "theoretical_loss": 4.340170873923155,
+      "tokens_seen": 216858624
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007549689067201605,
+      "loss": 1.5164,
+      "theoretical_loss": 4.340009893707408,
+      "tokens_seen": 216924160
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007549528585757271,
+      "loss": 1.6415,
+      "theoretical_loss": 4.339848975731796,
+      "tokens_seen": 216989696
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007549368104312939,
+      "loss": 1.6807,
+      "theoretical_loss": 4.3396881199534665,
+      "tokens_seen": 217055232
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007549207622868605,
+      "loss": 1.5347,
+      "theoretical_loss": 4.339527326329613,
+      "tokens_seen": 217120768
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007549047141424273,
+      "loss": 1.5583,
+      "theoretical_loss": 4.3393665948174664,
+      "tokens_seen": 217186304
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.000754888665997994,
+      "loss": 1.6724,
+      "theoretical_loss": 4.339205925374305,
+      "tokens_seen": 217251840
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007548726178535607,
+      "loss": 1.7399,
+      "theoretical_loss": 4.339045317957446,
+      "tokens_seen": 217317376
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007548565697091274,
+      "loss": 1.5985,
+      "theoretical_loss": 4.33888477252425,
+      "tokens_seen": 217382912
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007548405215646941,
+      "loss": 1.6879,
+      "theoretical_loss": 4.33872428903212,
+      "tokens_seen": 217448448
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007548244734202608,
+      "loss": 1.5445,
+      "theoretical_loss": 4.338563867438499,
+      "tokens_seen": 217513984
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007548084252758275,
+      "loss": 1.7851,
+      "theoretical_loss": 4.338403507700877,
+      "tokens_seen": 217579520
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007547923771313942,
+      "loss": 1.6267,
+      "theoretical_loss": 4.33824320977678,
+      "tokens_seen": 217645056
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007547763289869609,
+      "loss": 1.6144,
+      "theoretical_loss": 4.33808297362378,
+      "tokens_seen": 217710592
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007547602808425276,
+      "loss": 1.6337,
+      "theoretical_loss": 4.337922799199489,
+      "tokens_seen": 217776128
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007547442326980943,
+      "loss": 1.6475,
+      "theoretical_loss": 4.337762686461561,
+      "tokens_seen": 217841664
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.000754728184553661,
+      "loss": 1.5541,
+      "theoretical_loss": 4.337602635367694,
+      "tokens_seen": 217907200
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007547121364092277,
+      "loss": 1.5994,
+      "theoretical_loss": 4.337442645875624,
+      "tokens_seen": 217972736
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007546960882647944,
+      "loss": 1.6685,
+      "theoretical_loss": 4.337282717943132,
+      "tokens_seen": 218038272
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007546800401203611,
+      "loss": 1.691,
+      "theoretical_loss": 4.337122851528037,
+      "tokens_seen": 218103808
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007546639919759278,
+      "loss": 1.6155,
+      "theoretical_loss": 4.336963046588204,
+      "tokens_seen": 218169344
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007546479438314945,
+      "loss": 1.6102,
+      "theoretical_loss": 4.336803303081538,
+      "tokens_seen": 218234880
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007546318956870612,
+      "loss": 1.625,
+      "theoretical_loss": 4.336643620965982,
+      "tokens_seen": 218300416
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007546158475426279,
+      "loss": 1.7399,
+      "theoretical_loss": 4.336484000199524,
+      "tokens_seen": 218365952
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007545997993981946,
+      "loss": 1.6097,
+      "theoretical_loss": 4.336324440740195,
+      "tokens_seen": 218431488
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007545837512537613,
+      "loss": 1.644,
+      "theoretical_loss": 4.336164942546061,
+      "tokens_seen": 218497024
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.000754567703109328,
+      "loss": 1.6141,
+      "theoretical_loss": 4.336005505575235,
+      "tokens_seen": 218562560
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007545516549648947,
+      "loss": 1.6087,
+      "theoretical_loss": 4.335846129785869,
+      "tokens_seen": 218628096
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007545356068204614,
+      "loss": 1.6509,
+      "theoretical_loss": 4.335686815136157,
+      "tokens_seen": 218693632
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007545195586760281,
+      "loss": 1.7906,
+      "theoretical_loss": 4.335527561584333,
+      "tokens_seen": 218759168
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007545035105315948,
+      "loss": 1.5998,
+      "theoretical_loss": 4.3353683690886715,
+      "tokens_seen": 218824704
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007544874623871615,
+      "loss": 1.5911,
+      "theoretical_loss": 4.33520923760749,
+      "tokens_seen": 218890240
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007544714142427282,
+      "loss": 1.6318,
+      "theoretical_loss": 4.335050167099145,
+      "tokens_seen": 218955776
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007544553660982949,
+      "loss": 1.6194,
+      "theoretical_loss": 4.334891157522035,
+      "tokens_seen": 219021312
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007544393179538616,
+      "loss": 1.7399,
+      "theoretical_loss": 4.3347322088346,
+      "tokens_seen": 219086848
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007544232698094283,
+      "loss": 1.5408,
+      "theoretical_loss": 4.33457332099532,
+      "tokens_seen": 219152384
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.000754407221664995,
+      "loss": 1.6205,
+      "theoretical_loss": 4.334414493962714,
+      "tokens_seen": 219217920
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007543911735205617,
+      "loss": 1.6574,
+      "theoretical_loss": 4.334255727695344,
+      "tokens_seen": 219283456
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007543751253761284,
+      "loss": 1.6661,
+      "theoretical_loss": 4.334097022151812,
+      "tokens_seen": 219348992
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007543590772316951,
+      "loss": 1.6897,
+      "theoretical_loss": 4.33393837729076,
+      "tokens_seen": 219414528
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007543430290872619,
+      "loss": 1.5929,
+      "theoretical_loss": 4.333779793070871,
+      "tokens_seen": 219480064
+    },
+    {
+      "epoch": 0.07,
+      "objective/train/docs_used": 79188,
+      "objective/train/instantaneous_batch_size": 32,
+      "objective/train/instantaneous_microbatch_size": 32768,
+      "objective/train/original_loss": 1.5970346927642822,
+      "objective/train/theoretical_loss": 4.33362126945087,
+      "objective/train/tokens_used": 240005600,
+      "theoretical_loss": 4.33362126945087,
+      "tokens_seen": 219545600
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007543269809428285,
+      "loss": 1.6301,
+      "theoretical_loss": 4.33362126945087,
+      "tokens_seen": 219545600
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007543109327983953,
+      "loss": 1.6278,
+      "theoretical_loss": 4.333462806389517,
+      "tokens_seen": 219611136
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007542948846539619,
+      "loss": 1.6423,
+      "theoretical_loss": 4.33330440384562,
+      "tokens_seen": 219676672
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007542788365095287,
+      "loss": 1.6139,
+      "theoretical_loss": 4.333146061778021,
+      "tokens_seen": 219742208
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007542627883650953,
+      "loss": 1.6051,
+      "theoretical_loss": 4.332987780145606,
+      "tokens_seen": 219807744
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007542467402206621,
+      "loss": 1.608,
+      "theoretical_loss": 4.3328295589073,
+      "tokens_seen": 219873280
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007542306920762287,
+      "loss": 1.5542,
+      "theoretical_loss": 4.332671398022067,
+      "tokens_seen": 219938816
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007542146439317955,
+      "loss": 1.6946,
+      "theoretical_loss": 4.332513297448912,
+      "tokens_seen": 220004352
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007541985957873621,
+      "loss": 1.5519,
+      "theoretical_loss": 4.332355257146881,
+      "tokens_seen": 220069888
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007541825476429289,
+      "loss": 1.6058,
+      "theoretical_loss": 4.3321972770750605,
+      "tokens_seen": 220135424
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007541664994984955,
+      "loss": 1.6942,
+      "theoretical_loss": 4.332039357192572,
+      "tokens_seen": 220200960
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007541504513540623,
+      "loss": 1.7263,
+      "theoretical_loss": 4.331881497458584,
+      "tokens_seen": 220266496
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007541344032096289,
+      "loss": 1.737,
+      "theoretical_loss": 4.3317236978323,
+      "tokens_seen": 220332032
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007541183550651957,
+      "loss": 1.6201,
+      "theoretical_loss": 4.3315659582729635,
+      "tokens_seen": 220397568
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007541023069207623,
+      "loss": 1.8591,
+      "theoretical_loss": 4.331408278739861,
+      "tokens_seen": 220463104
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007540862587763291,
+      "loss": 1.6871,
+      "theoretical_loss": 4.331250659192316,
+      "tokens_seen": 220528640
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007540702106318957,
+      "loss": 1.7414,
+      "theoretical_loss": 4.33109309958969,
+      "tokens_seen": 220594176
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007540541624874625,
+      "loss": 1.6949,
+      "theoretical_loss": 4.330935599891389,
+      "tokens_seen": 220659712
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007540381143430291,
+      "loss": 1.5956,
+      "theoretical_loss": 4.330778160056855,
+      "tokens_seen": 220725248
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007540220661985959,
+      "loss": 1.6693,
+      "theoretical_loss": 4.330620780045571,
+      "tokens_seen": 220790784
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007540060180541625,
+      "loss": 1.7746,
+      "theoretical_loss": 4.330463459817057,
+      "tokens_seen": 220856320
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007539899699097293,
+      "loss": 1.739,
+      "theoretical_loss": 4.3303061993308765,
+      "tokens_seen": 220921856
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007539739217652959,
+      "loss": 1.679,
+      "theoretical_loss": 4.330148998546628,
+      "tokens_seen": 220987392
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007539578736208627,
+      "loss": 1.6877,
+      "theoretical_loss": 4.329991857423953,
+      "tokens_seen": 221052928
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007539418254764293,
+      "loss": 1.677,
+      "theoretical_loss": 4.329834775922529,
+      "tokens_seen": 221118464
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007539257773319961,
+      "loss": 1.6982,
+      "theoretical_loss": 4.329677754002074,
+      "tokens_seen": 221184000
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007539097291875627,
+      "loss": 1.6958,
+      "theoretical_loss": 4.329520791622348,
+      "tokens_seen": 221249536
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007538936810431295,
+      "loss": 1.5988,
+      "theoretical_loss": 4.329363888743145,
+      "tokens_seen": 221315072
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007538776328986961,
+      "loss": 1.7823,
+      "theoretical_loss": 4.329207045324301,
+      "tokens_seen": 221380608
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007538615847542629,
+      "loss": 1.556,
+      "theoretical_loss": 4.3290502613256905,
+      "tokens_seen": 221446144
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007538455366098295,
+      "loss": 1.6921,
+      "theoretical_loss": 4.328893536707228,
+      "tokens_seen": 221511680
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007538294884653963,
+      "loss": 1.6912,
+      "theoretical_loss": 4.328736871428864,
+      "tokens_seen": 221577216
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.000753813440320963,
+      "loss": 1.6371,
+      "theoretical_loss": 4.328580265450592,
+      "tokens_seen": 221642752
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007537973921765297,
+      "loss": 1.7237,
+      "theoretical_loss": 4.32842371873244,
+      "tokens_seen": 221708288
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007537813440320963,
+      "loss": 1.7639,
+      "theoretical_loss": 4.328267231234476,
+      "tokens_seen": 221773824
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.000753765295887663,
+      "loss": 1.7211,
+      "theoretical_loss": 4.328110802916811,
+      "tokens_seen": 221839360
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007537492477432297,
+      "loss": 1.6728,
+      "theoretical_loss": 4.327954433739588,
+      "tokens_seen": 221904896
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007537331995987964,
+      "loss": 1.6379,
+      "theoretical_loss": 4.327798123662992,
+      "tokens_seen": 221970432
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007537171514543632,
+      "loss": 1.6861,
+      "theoretical_loss": 4.327641872647248,
+      "tokens_seen": 222035968
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007537011033099298,
+      "loss": 1.7967,
+      "theoretical_loss": 4.327485680652615,
+      "tokens_seen": 222101504
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007536850551654966,
+      "loss": 1.6718,
+      "theoretical_loss": 4.327329547639396,
+      "tokens_seen": 222167040
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007536690070210632,
+      "loss": 1.6342,
+      "theoretical_loss": 4.327173473567927,
+      "tokens_seen": 222232576
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00075365295887663,
+      "loss": 1.6883,
+      "theoretical_loss": 4.327017458398587,
+      "tokens_seen": 222298112
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007536369107321966,
+      "loss": 1.5868,
+      "theoretical_loss": 4.32686150209179,
+      "tokens_seen": 222363648
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007536208625877632,
+      "loss": 1.8894,
+      "theoretical_loss": 4.32670560460799,
+      "tokens_seen": 222429184
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00075360481444333,
+      "loss": 1.607,
+      "theoretical_loss": 4.326549765907679,
+      "tokens_seen": 222494720
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007535887662988966,
+      "loss": 1.8075,
+      "theoretical_loss": 4.326393985951386,
+      "tokens_seen": 222560256
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007535727181544634,
+      "loss": 1.5828,
+      "theoretical_loss": 4.326238264699679,
+      "tokens_seen": 222625792
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00075355667001003,
+      "loss": 1.5931,
+      "theoretical_loss": 4.326082602113164,
+      "tokens_seen": 222691328
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007535406218655968,
+      "loss": 1.6439,
+      "theoretical_loss": 4.325926998152487,
+      "tokens_seen": 222756864
+    },
+    {
+      "epoch": 0.07,
+      "objective/train/docs_used": 80307,
+      "objective/train/instantaneous_batch_size": 32,
+      "objective/train/instantaneous_microbatch_size": 32768,
+      "objective/train/original_loss": 1.6677907705307007,
+      "objective/train/theoretical_loss": 4.325771452778328,
+      "objective/train/tokens_used": 243282400,
+      "theoretical_loss": 4.325771452778328,
+      "tokens_seen": 222822400
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007535245737211634,
+      "loss": 1.5579,
+      "theoretical_loss": 4.325771452778328,
+      "tokens_seen": 222822400
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007535085255767302,
+      "loss": 1.726,
+      "theoretical_loss": 4.325615965951408,
+      "tokens_seen": 222887936
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007534924774322969,
+      "loss": 1.6604,
+      "theoretical_loss": 4.325460537632483,
+      "tokens_seen": 222953472
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007534764292878636,
+      "loss": 1.6836,
+      "theoretical_loss": 4.325305167782352,
+      "tokens_seen": 223019008
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007534603811434303,
+      "loss": 1.6851,
+      "theoretical_loss": 4.325149856361845,
+      "tokens_seen": 223084544
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.000753444332998997,
+      "loss": 1.6246,
+      "theoretical_loss": 4.3249946033318345,
+      "tokens_seen": 223150080
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007534282848545637,
+      "loss": 1.5557,
+      "theoretical_loss": 4.32483940865323,
+      "tokens_seen": 223215616
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007534122367101304,
+      "loss": 1.6875,
+      "theoretical_loss": 4.324684272286978,
+      "tokens_seen": 223281152
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007533961885656971,
+      "loss": 1.6429,
+      "theoretical_loss": 4.324529194194062,
+      "tokens_seen": 223346688
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007533801404212638,
+      "loss": 1.7568,
+      "theoretical_loss": 4.324374174335503,
+      "tokens_seen": 223412224
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007533640922768305,
+      "loss": 1.5455,
+      "theoretical_loss": 4.324219212672363,
+      "tokens_seen": 223477760
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007533480441323972,
+      "loss": 1.6705,
+      "theoretical_loss": 4.324064309165734,
+      "tokens_seen": 223543296
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007533319959879639,
+      "loss": 1.6054,
+      "theoretical_loss": 4.3239094637767534,
+      "tokens_seen": 223608832
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007533159478435306,
+      "loss": 1.6056,
+      "theoretical_loss": 4.323754676466592,
+      "tokens_seen": 223674368
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007532998996990973,
+      "loss": 1.6672,
+      "theoretical_loss": 4.323599947196458,
+      "tokens_seen": 223739904
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.000753283851554664,
+      "loss": 1.6007,
+      "theoretical_loss": 4.323445275927597,
+      "tokens_seen": 223805440
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007532678034102307,
+      "loss": 1.7509,
+      "theoretical_loss": 4.323290662621292,
+      "tokens_seen": 223870976
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007532517552657974,
+      "loss": 1.6039,
+      "theoretical_loss": 4.323136107238865,
+      "tokens_seen": 223936512
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007532357071213641,
+      "loss": 1.5947,
+      "theoretical_loss": 4.322981609741671,
+      "tokens_seen": 224002048
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007532196589769308,
+      "loss": 1.7144,
+      "theoretical_loss": 4.322827170091106,
+      "tokens_seen": 224067584
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007532036108324975,
+      "loss": 1.6347,
+      "theoretical_loss": 4.322672788248601,
+      "tokens_seen": 224133120
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007531875626880642,
+      "loss": 1.6187,
+      "theoretical_loss": 4.3225184641756265,
+      "tokens_seen": 224198656
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007531715145436309,
+      "loss": 1.5413,
+      "theoretical_loss": 4.322364197833684,
+      "tokens_seen": 224264192
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007531554663991976,
+      "loss": 1.7619,
+      "theoretical_loss": 4.322209989184319,
+      "tokens_seen": 224329728
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007531394182547644,
+      "loss": 1.5852,
+      "theoretical_loss": 4.32205583818911,
+      "tokens_seen": 224395264
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.000753123370110331,
+      "loss": 1.6715,
+      "theoretical_loss": 4.321901744809672,
+      "tokens_seen": 224460800
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007531073219658978,
+      "loss": 1.6177,
+      "theoretical_loss": 4.321747709007658,
+      "tokens_seen": 224526336
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007530912738214644,
+      "loss": 1.6659,
+      "theoretical_loss": 4.321593730744759,
+      "tokens_seen": 224591872
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007530752256770312,
+      "loss": 1.6454,
+      "theoretical_loss": 4.3214398099826985,
+      "tokens_seen": 224657408
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007530591775325978,
+      "loss": 1.6124,
+      "theoretical_loss": 4.321285946683242,
+      "tokens_seen": 224722944
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007530431293881646,
+      "loss": 1.4957,
+      "theoretical_loss": 4.321132140808186,
+      "tokens_seen": 224788480
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007530270812437312,
+      "loss": 1.7706,
+      "theoretical_loss": 4.320978392319368,
+      "tokens_seen": 224854016
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.000753011033099298,
+      "loss": 1.6766,
+      "theoretical_loss": 4.320824701178661,
+      "tokens_seen": 224919552
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007529949849548646,
+      "loss": 1.7005,
+      "theoretical_loss": 4.320671067347972,
+      "tokens_seen": 224985088
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007529789368104314,
+      "loss": 1.6329,
+      "theoretical_loss": 4.320517490789246,
+      "tokens_seen": 225050624
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.000752962888665998,
+      "loss": 1.6954,
+      "theoretical_loss": 4.320363971464466,
+      "tokens_seen": 225116160
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007529468405215648,
+      "loss": 1.5983,
+      "theoretical_loss": 4.320210509335649,
+      "tokens_seen": 225181696
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007529307923771314,
+      "loss": 1.6613,
+      "theoretical_loss": 4.320057104364848,
+      "tokens_seen": 225247232
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007529147442326982,
+      "loss": 1.7164,
+      "theoretical_loss": 4.3199037565141545,
+      "tokens_seen": 225312768
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007528986960882648,
+      "loss": 1.5651,
+      "theoretical_loss": 4.319750465745694,
+      "tokens_seen": 225378304
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007528826479438316,
+      "loss": 1.6675,
+      "theoretical_loss": 4.31959723202163,
+      "tokens_seen": 225443840
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007528665997993982,
+      "loss": 1.6431,
+      "theoretical_loss": 4.319444055304161,
+      "tokens_seen": 225509376
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.000752850551654965,
+      "loss": 1.6055,
+      "theoretical_loss": 4.319290935555521,
+      "tokens_seen": 225574912
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007528345035105316,
+      "loss": 1.5909,
+      "theoretical_loss": 4.319137872737979,
+      "tokens_seen": 225640448
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007528184553660984,
+      "loss": 1.6031,
+      "theoretical_loss": 4.318984866813844,
+      "tokens_seen": 225705984
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.000752802407221665,
+      "loss": 1.6253,
+      "theoretical_loss": 4.318831917745458,
+      "tokens_seen": 225771520
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007527863590772318,
+      "loss": 1.6518,
+      "theoretical_loss": 4.318679025495197,
+      "tokens_seen": 225837056
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007527703109327984,
+      "loss": 1.6471,
+      "theoretical_loss": 4.318526190025478,
+      "tokens_seen": 225902592
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007527542627883652,
+      "loss": 1.5365,
+      "theoretical_loss": 4.318373411298749,
+      "tokens_seen": 225968128
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007527382146439318,
+      "loss": 1.6307,
+      "theoretical_loss": 4.318220689277496,
+      "tokens_seen": 226033664
+    },
+    {
+      "epoch": 0.07,
+      "objective/train/docs_used": 81440,
+      "objective/train/instantaneous_batch_size": 32,
+      "objective/train/instantaneous_microbatch_size": 32768,
+      "objective/train/original_loss": 1.5563585758209229,
+      "objective/train/theoretical_loss": 4.318068023924241,
+      "objective/train/tokens_used": 246559200,
+      "theoretical_loss": 4.318068023924241,
+      "tokens_seen": 226099200
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007527221664994986,
+      "loss": 1.5078,
+      "theoretical_loss": 4.318068023924241,
+      "tokens_seen": 226099200
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007527061183550652,
+      "loss": 1.5606,
+      "theoretical_loss": 4.317915415201538,
+      "tokens_seen": 226164736
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.000752690070210632,
+      "loss": 1.6603,
+      "theoretical_loss": 4.317762863071982,
+      "tokens_seen": 226230272
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007526740220661986,
+      "loss": 1.7236,
+      "theoretical_loss": 4.317610367498201,
+      "tokens_seen": 226295808
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007526579739217654,
+      "loss": 1.7993,
+      "theoretical_loss": 4.317457928442856,
+      "tokens_seen": 226361344
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.000752641925777332,
+      "loss": 1.6412,
+      "theoretical_loss": 4.317305545868648,
+      "tokens_seen": 226426880
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007526258776328988,
+      "loss": 1.503,
+      "theoretical_loss": 4.317153219738311,
+      "tokens_seen": 226492416
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007526098294884654,
+      "loss": 1.7393,
+      "theoretical_loss": 4.317000950014615,
+      "tokens_seen": 226557952
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007525937813440322,
+      "loss": 1.5782,
+      "theoretical_loss": 4.316848736660363,
+      "tokens_seen": 226623488
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007525777331995988,
+      "loss": 1.6813,
+      "theoretical_loss": 4.316696579638396,
+      "tokens_seen": 226689024
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007525616850551656,
+      "loss": 1.5585,
+      "theoretical_loss": 4.316544478911592,
+      "tokens_seen": 226754560
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007525456369107322,
+      "loss": 1.6094,
+      "theoretical_loss": 4.316392434442858,
+      "tokens_seen": 226820096
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.000752529588766299,
+      "loss": 1.6799,
+      "theoretical_loss": 4.316240446195142,
+      "tokens_seen": 226885632
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007525135406218656,
+      "loss": 1.512,
+      "theoretical_loss": 4.3160885141314225,
+      "tokens_seen": 226951168
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007524974924774324,
+      "loss": 1.5751,
+      "theoretical_loss": 4.315936638214718,
+      "tokens_seen": 227016704
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.000752481444332999,
+      "loss": 1.6076,
+      "theoretical_loss": 4.315784818408079,
+      "tokens_seen": 227082240
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007524653961885658,
+      "loss": 1.7199,
+      "theoretical_loss": 4.31563305467459,
+      "tokens_seen": 227147776
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007524493480441324,
+      "loss": 1.7079,
+      "theoretical_loss": 4.315481346977374,
+      "tokens_seen": 227213312
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007524332998996992,
+      "loss": 1.6606,
+      "theoretical_loss": 4.315329695279584,
+      "tokens_seen": 227278848
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007524172517552658,
+      "loss": 1.6311,
+      "theoretical_loss": 4.315178099544413,
+      "tokens_seen": 227344384
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007524012036108326,
+      "loss": 1.6426,
+      "theoretical_loss": 4.315026559735085,
+      "tokens_seen": 227409920
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007523851554663992,
+      "loss": 1.6613,
+      "theoretical_loss": 4.314875075814861,
+      "tokens_seen": 227475456
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007523691073219659,
+      "loss": 1.6062,
+      "theoretical_loss": 4.314723647747035,
+      "tokens_seen": 227540992
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007523530591775327,
+      "loss": 1.5894,
+      "theoretical_loss": 4.314572275494937,
+      "tokens_seen": 227606528
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007523370110330993,
+      "loss": 1.6215,
+      "theoretical_loss": 4.3144209590219305,
+      "tokens_seen": 227672064
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.000752320962888666,
+      "loss": 1.7094,
+      "theoretical_loss": 4.314269698291415,
+      "tokens_seen": 227737600
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007523049147442327,
+      "loss": 1.6197,
+      "theoretical_loss": 4.3141184932668235,
+      "tokens_seen": 227803136
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007522888665997995,
+      "loss": 1.5909,
+      "theoretical_loss": 4.3139673439116235,
+      "tokens_seen": 227868672
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007522728184553661,
+      "loss": 1.6152,
+      "theoretical_loss": 4.313816250189317,
+      "tokens_seen": 227934208
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007522567703109329,
+      "loss": 1.6349,
+      "theoretical_loss": 4.313665212063441,
+      "tokens_seen": 227999744
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007522407221664995,
+      "loss": 1.5664,
+      "theoretical_loss": 4.313514229497566,
+      "tokens_seen": 228065280
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007522246740220661,
+      "loss": 1.6162,
+      "theoretical_loss": 4.313363302455299,
+      "tokens_seen": 228130816
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007522086258776329,
+      "loss": 1.6323,
+      "theoretical_loss": 4.313212430900277,
+      "tokens_seen": 228196352
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007521925777331995,
+      "loss": 1.5688,
+      "theoretical_loss": 4.313061614796176,
+      "tokens_seen": 228261888
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007521765295887663,
+      "loss": 1.6629,
+      "theoretical_loss": 4.312910854106702,
+      "tokens_seen": 228327424
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.000752160481444333,
+      "loss": 1.7265,
+      "theoretical_loss": 4.3127601487956,
+      "tokens_seen": 228392960
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007521444332998997,
+      "loss": 1.6531,
+      "theoretical_loss": 4.312609498826644,
+      "tokens_seen": 228458496
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007521283851554664,
+      "loss": 1.709,
+      "theoretical_loss": 4.312458904163645,
+      "tokens_seen": 228524032
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007521123370110331,
+      "loss": 1.5928,
+      "theoretical_loss": 4.312308364770448,
+      "tokens_seen": 228589568
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007520962888665998,
+      "loss": 1.6832,
+      "theoretical_loss": 4.312157880610931,
+      "tokens_seen": 228655104
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007520802407221665,
+      "loss": 1.611,
+      "theoretical_loss": 4.312007451649006,
+      "tokens_seen": 228720640
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007520641925777332,
+      "loss": 1.6157,
+      "theoretical_loss": 4.31185707784862,
+      "tokens_seen": 228786176
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007520481444332999,
+      "loss": 1.6948,
+      "theoretical_loss": 4.311706759173753,
+      "tokens_seen": 228851712
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007520320962888666,
+      "loss": 1.7721,
+      "theoretical_loss": 4.311556495588419,
+      "tokens_seen": 228917248
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007520160481444333,
+      "loss": 1.7458,
+      "theoretical_loss": 4.3114062870566645,
+      "tokens_seen": 228982784
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.000752,
+      "loss": 1.5913,
+      "theoretical_loss": 4.311256133542573,
+      "tokens_seen": 229048320
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007519839518555667,
+      "loss": 1.574,
+      "theoretical_loss": 4.311106035010259,
+      "tokens_seen": 229113856
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007519679037111334,
+      "loss": 1.608,
+      "theoretical_loss": 4.310955991423871,
+      "tokens_seen": 229179392
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007519518555667001,
+      "loss": 1.6548,
+      "theoretical_loss": 4.310806002747592,
+      "tokens_seen": 229244928
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007519358074222669,
+      "loss": 1.683,
+      "theoretical_loss": 4.310656068945637,
+      "tokens_seen": 229310464
+    },
+    {
+      "epoch": 0.07,
+      "objective/train/docs_used": 82339,
+      "objective/train/instantaneous_batch_size": 32,
+      "objective/train/instantaneous_microbatch_size": 32768,
+      "objective/train/original_loss": 1.5768122673034668,
+      "objective/train/theoretical_loss": 4.310506189982257,
+      "objective/train/tokens_used": 249836000,
+      "theoretical_loss": 4.310506189982257,
+      "tokens_seen": 229376000
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007519197592778335,
+      "loss": 1.4792,
+      "theoretical_loss": 4.310506189982257,
+      "tokens_seen": 229376000
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007519037111334003,
+      "loss": 1.6147,
+      "theoretical_loss": 4.310356365821734,
+      "tokens_seen": 229441536
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007518876629889669,
+      "loss": 1.5856,
+      "theoretical_loss": 4.310206596428387,
+      "tokens_seen": 229507072
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007518716148445337,
+      "loss": 1.6637,
+      "theoretical_loss": 4.310056881766561,
+      "tokens_seen": 229572608
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007518555667001003,
+      "loss": 1.6138,
+      "theoretical_loss": 4.309907221800645,
+      "tokens_seen": 229638144
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007518395185556671,
+      "loss": 1.5819,
+      "theoretical_loss": 4.309757616495053,
+      "tokens_seen": 229703680
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007518234704112337,
+      "loss": 1.6515,
+      "theoretical_loss": 4.309608065814234,
+      "tokens_seen": 229769216
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007518074222668005,
+      "loss": 1.5929,
+      "theoretical_loss": 4.309458569722673,
+      "tokens_seen": 229834752
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007517913741223671,
+      "loss": 1.6339,
+      "theoretical_loss": 4.309309128184886,
+      "tokens_seen": 229900288
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007517753259779339,
+      "loss": 1.642,
+      "theoretical_loss": 4.309159741165422,
+      "tokens_seen": 229965824
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007517592778335005,
+      "loss": 1.6325,
+      "theoretical_loss": 4.309010408628865,
+      "tokens_seen": 230031360
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007517432296890673,
+      "loss": 1.5926,
+      "theoretical_loss": 4.308861130539829,
+      "tokens_seen": 230096896
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007517271815446339,
+      "loss": 1.5324,
+      "theoretical_loss": 4.308711906862965,
+      "tokens_seen": 230162432
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007517111334002007,
+      "loss": 1.736,
+      "theoretical_loss": 4.308562737562953,
+      "tokens_seen": 230227968
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007516950852557673,
+      "loss": 1.6326,
+      "theoretical_loss": 4.30841362260451,
+      "tokens_seen": 230293504
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007516790371113341,
+      "loss": 1.5781,
+      "theoretical_loss": 4.308264561952381,
+      "tokens_seen": 230359040
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007516629889669007,
+      "loss": 1.6477,
+      "theoretical_loss": 4.30811555557135,
+      "tokens_seen": 230424576
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007516469408224675,
+      "loss": 1.5454,
+      "theoretical_loss": 4.307966603426227,
+      "tokens_seen": 230490112
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007516308926780341,
+      "loss": 1.5479,
+      "theoretical_loss": 4.307817705481861,
+      "tokens_seen": 230555648
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007516148445336009,
+      "loss": 1.588,
+      "theoretical_loss": 4.307668861703131,
+      "tokens_seen": 230621184
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007515987963891675,
+      "loss": 1.6417,
+      "theoretical_loss": 4.307520072054947,
+      "tokens_seen": 230686720
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007515827482447343,
+      "loss": 1.7475,
+      "theoretical_loss": 4.307371336502254,
+      "tokens_seen": 230752256
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007515667001003009,
+      "loss": 1.5841,
+      "theoretical_loss": 4.307222655010031,
+      "tokens_seen": 230817792
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007515506519558677,
+      "loss": 1.6067,
+      "theoretical_loss": 4.307074027543287,
+      "tokens_seen": 230883328
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007515346038114343,
+      "loss": 1.6151,
+      "theoretical_loss": 4.3069254540670645,
+      "tokens_seen": 230948864
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007515185556670011,
+      "loss": 1.6335,
+      "theoretical_loss": 4.306776934546436,
+      "tokens_seen": 231014400
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007515025075225677,
+      "loss": 1.5974,
+      "theoretical_loss": 4.306628468946512,
+      "tokens_seen": 231079936
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007514864593781345,
+      "loss": 1.6839,
+      "theoretical_loss": 4.306480057232431,
+      "tokens_seen": 231145472
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007514704112337011,
+      "loss": 1.6051,
+      "theoretical_loss": 4.306331699369366,
+      "tokens_seen": 231211008
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007514543630892679,
+      "loss": 1.6544,
+      "theoretical_loss": 4.30618339532252,
+      "tokens_seen": 231276544
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007514383149448345,
+      "loss": 1.6898,
+      "theoretical_loss": 4.306035145057132,
+      "tokens_seen": 231342080
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007514222668004013,
+      "loss": 1.6831,
+      "theoretical_loss": 4.30588694853847,
+      "tokens_seen": 231407616
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007514062186559679,
+      "loss": 1.5407,
+      "theoretical_loss": 4.305738805731836,
+      "tokens_seen": 231473152
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007513901705115347,
+      "loss": 1.7381,
+      "theoretical_loss": 4.305590716602563,
+      "tokens_seen": 231538688
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007513741223671013,
+      "loss": 1.6949,
+      "theoretical_loss": 4.305442681116018,
+      "tokens_seen": 231604224
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007513580742226681,
+      "loss": 1.5807,
+      "theoretical_loss": 4.305294699237598,
+      "tokens_seen": 231669760
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007513420260782347,
+      "loss": 1.6686,
+      "theoretical_loss": 4.305146770932734,
+      "tokens_seen": 231735296
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007513259779338015,
+      "loss": 1.599,
+      "theoretical_loss": 4.304998896166888,
+      "tokens_seen": 231800832
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007513099297893681,
+      "loss": 1.7105,
+      "theoretical_loss": 4.304851074905554,
+      "tokens_seen": 231866368
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007512938816449349,
+      "loss": 1.6424,
+      "theoretical_loss": 4.304703307114258,
+      "tokens_seen": 231931904
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007512778335005015,
+      "loss": 1.7342,
+      "theoretical_loss": 4.304555592758559,
+      "tokens_seen": 231997440
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007512617853560683,
+      "loss": 1.6645,
+      "theoretical_loss": 4.304407931804047,
+      "tokens_seen": 232062976
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007512457372116349,
+      "loss": 1.6135,
+      "theoretical_loss": 4.304260324216344,
+      "tokens_seen": 232128512
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007512296890672017,
+      "loss": 1.6263,
+      "theoretical_loss": 4.304112769961102,
+      "tokens_seen": 232194048
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007512136409227683,
+      "loss": 1.7394,
+      "theoretical_loss": 4.303965269004008,
+      "tokens_seen": 232259584
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007511975927783351,
+      "loss": 1.704,
+      "theoretical_loss": 4.30381782131078,
+      "tokens_seen": 232325120
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007511815446339017,
+      "loss": 1.6265,
+      "theoretical_loss": 4.303670426847166,
+      "tokens_seen": 232390656
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007511654964894685,
+      "loss": 1.6788,
+      "theoretical_loss": 4.303523085578946,
+      "tokens_seen": 232456192
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007511494483450351,
+      "loss": 1.5481,
+      "theoretical_loss": 4.303375797471935,
+      "tokens_seen": 232521728
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007511334002006019,
+      "loss": 1.6114,
+      "theoretical_loss": 4.303228562491974,
+      "tokens_seen": 232587264
+    },
+    {
+      "epoch": 0.07,
+      "objective/train/docs_used": 83567,
+      "objective/train/instantaneous_batch_size": 32,
+      "objective/train/instantaneous_microbatch_size": 32768,
+      "objective/train/original_loss": 1.4511950016021729,
+      "objective/train/theoretical_loss": 4.303081380604939,
+      "objective/train/tokens_used": 253112800,
+      "theoretical_loss": 4.303081380604939,
+      "tokens_seen": 232652800
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007511173520561685,
+      "loss": 1.5394,
+      "theoretical_loss": 4.303081380604939,
+      "tokens_seen": 232652800
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007511013039117353,
+      "loss": 1.5252,
+      "theoretical_loss": 4.302934251776739,
+      "tokens_seen": 232718336
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.000751085255767302,
+      "loss": 1.6582,
+      "theoretical_loss": 4.302787175973311,
+      "tokens_seen": 232783872
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007510692076228687,
+      "loss": 1.5324,
+      "theoretical_loss": 4.302640153160625,
+      "tokens_seen": 232849408
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007510531594784353,
+      "loss": 1.6077,
+      "theoretical_loss": 4.302493183304683,
+      "tokens_seen": 232914944
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007510371113340021,
+      "loss": 1.5084,
+      "theoretical_loss": 4.302346266371517,
+      "tokens_seen": 232980480
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007510210631895687,
+      "loss": 1.645,
+      "theoretical_loss": 4.302199402327191,
+      "tokens_seen": 233046016
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007510050150451355,
+      "loss": 1.6992,
+      "theoretical_loss": 4.302052591137802,
+      "tokens_seen": 233111552
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007509889669007021,
+      "loss": 1.5375,
+      "theoretical_loss": 4.301905832769474,
+      "tokens_seen": 233177088
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007509729187562688,
+      "loss": 1.5687,
+      "theoretical_loss": 4.301759127188367,
+      "tokens_seen": 233242624
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007509568706118356,
+      "loss": 1.6591,
+      "theoretical_loss": 4.301612474360669,
+      "tokens_seen": 233308160
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007509408224674022,
+      "loss": 1.6299,
+      "theoretical_loss": 4.301465874252601,
+      "tokens_seen": 233373696
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.000750924774322969,
+      "loss": 1.5226,
+      "theoretical_loss": 4.301319326830413,
+      "tokens_seen": 233439232
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007509087261785356,
+      "loss": 1.6108,
+      "theoretical_loss": 4.301172832060389,
+      "tokens_seen": 233504768
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007508926780341024,
+      "loss": 1.7491,
+      "theoretical_loss": 4.3010263899088415,
+      "tokens_seen": 233570304
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.000750876629889669,
+      "loss": 1.64,
+      "theoretical_loss": 4.300880000342114,
+      "tokens_seen": 233635840
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007508605817452358,
+      "loss": 1.6839,
+      "theoretical_loss": 4.300733663326583,
+      "tokens_seen": 233701376
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007508445336008024,
+      "loss": 1.6464,
+      "theoretical_loss": 4.300587378828655,
+      "tokens_seen": 233766912
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.000750828485456369,
+      "loss": 1.7037,
+      "theoretical_loss": 4.300441146814766,
+      "tokens_seen": 233832448
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007508124373119358,
+      "loss": 1.7115,
+      "theoretical_loss": 4.300294967251386,
+      "tokens_seen": 233897984
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007507963891675024,
+      "loss": 1.5955,
+      "theoretical_loss": 4.300148840105011,
+      "tokens_seen": 233963520
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007507803410230692,
+      "loss": 1.6468,
+      "theoretical_loss": 4.300002765342173,
+      "tokens_seen": 234029056
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007507642928786359,
+      "loss": 1.5222,
+      "theoretical_loss": 4.299856742929431,
+      "tokens_seen": 234094592
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007507482447342026,
+      "loss": 1.5984,
+      "theoretical_loss": 4.299710772833377,
+      "tokens_seen": 234160128
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007507321965897694,
+      "loss": 1.6461,
+      "theoretical_loss": 4.299564855020631,
+      "tokens_seen": 234225664
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.000750716148445336,
+      "loss": 1.6699,
+      "theoretical_loss": 4.299418989457848,
+      "tokens_seen": 234291200
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007507001003009028,
+      "loss": 1.6128,
+      "theoretical_loss": 4.299273176111709,
+      "tokens_seen": 234356736
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007506840521564694,
+      "loss": 1.5536,
+      "theoretical_loss": 4.299127414948927,
+      "tokens_seen": 234422272
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007506680040120362,
+      "loss": 1.6108,
+      "theoretical_loss": 4.298981705936248,
+      "tokens_seen": 234487808
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007506519558676028,
+      "loss": 1.7277,
+      "theoretical_loss": 4.298836049040444,
+      "tokens_seen": 234553344
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007506359077231696,
+      "loss": 1.5894,
+      "theoretical_loss": 4.298690444228322,
+      "tokens_seen": 234618880
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007506198595787362,
+      "loss": 1.6374,
+      "theoretical_loss": 4.298544891466717,
+      "tokens_seen": 234684416
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.000750603811434303,
+      "loss": 1.6616,
+      "theoretical_loss": 4.298399390722493,
+      "tokens_seen": 234749952
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007505877632898696,
+      "loss": 1.6846,
+      "theoretical_loss": 4.298253941962548,
+      "tokens_seen": 234815488
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007505717151454364,
+      "loss": 1.7153,
+      "theoretical_loss": 4.2981085451538075,
+      "tokens_seen": 234881024
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.000750555667001003,
+      "loss": 1.6791,
+      "theoretical_loss": 4.297963200263228,
+      "tokens_seen": 234946560
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007505396188565698,
+      "loss": 1.5973,
+      "theoretical_loss": 4.297817907257797,
+      "tokens_seen": 235012096
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007505235707121364,
+      "loss": 1.6408,
+      "theoretical_loss": 4.29767266610453,
+      "tokens_seen": 235077632
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007505075225677032,
+      "loss": 1.6739,
+      "theoretical_loss": 4.297527476770476,
+      "tokens_seen": 235143168
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007504914744232698,
+      "loss": 1.7313,
+      "theoretical_loss": 4.297382339222711,
+      "tokens_seen": 235208704
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007504754262788366,
+      "loss": 1.512,
+      "theoretical_loss": 4.297237253428342,
+      "tokens_seen": 235274240
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007504593781344032,
+      "loss": 1.6907,
+      "theoretical_loss": 4.297092219354509,
+      "tokens_seen": 235339776
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00075044332998997,
+      "loss": 1.7227,
+      "theoretical_loss": 4.296947236968376,
+      "tokens_seen": 235405312
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007504272818455366,
+      "loss": 1.5305,
+      "theoretical_loss": 4.296802306237143,
+      "tokens_seen": 235470848
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007504112337011034,
+      "loss": 1.6608,
+      "theoretical_loss": 4.296657427128036,
+      "tokens_seen": 235536384
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00075039518555667,
+      "loss": 1.7321,
+      "theoretical_loss": 4.296512599608311,
+      "tokens_seen": 235601920
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007503791374122368,
+      "loss": 1.6408,
+      "theoretical_loss": 4.296367823645259,
+      "tokens_seen": 235667456
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007503630892678034,
+      "loss": 1.6847,
+      "theoretical_loss": 4.296223099206193,
+      "tokens_seen": 235732992
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007503470411233702,
+      "loss": 1.6661,
+      "theoretical_loss": 4.296078426258462,
+      "tokens_seen": 235798528
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007503309929789368,
+      "loss": 1.6597,
+      "theoretical_loss": 4.295933804769442,
+      "tokens_seen": 235864064
+    },
+    {
+      "epoch": 0.07,
+      "objective/train/docs_used": 84479,
+      "objective/train/instantaneous_batch_size": 32,
+      "objective/train/instantaneous_microbatch_size": 32768,
+      "objective/train/original_loss": 1.6005480289459229,
+      "objective/train/theoretical_loss": 4.295789234706538,
+      "objective/train/tokens_used": 256389600,
+      "theoretical_loss": 4.295789234706538,
+      "tokens_seen": 235929600
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007503149448345036,
+      "loss": 1.6153,
+      "theoretical_loss": 4.295789234706538,
+      "tokens_seen": 235929600
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007502988966900702,
+      "loss": 1.6297,
+      "theoretical_loss": 4.295644716037188,
+      "tokens_seen": 235995136
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.000750282848545637,
+      "loss": 1.6977,
+      "theoretical_loss": 4.295500248728856,
+      "tokens_seen": 236060672
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007502668004012036,
+      "loss": 1.5492,
+      "theoretical_loss": 4.295355832749038,
+      "tokens_seen": 236126208
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007502507522567704,
+      "loss": 1.6194,
+      "theoretical_loss": 4.295211468065258,
+      "tokens_seen": 236191744
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.000750234704112337,
+      "loss": 1.4883,
+      "theoretical_loss": 4.295067154645071,
+      "tokens_seen": 236257280
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007502186559679038,
+      "loss": 1.6352,
+      "theoretical_loss": 4.294922892456061,
+      "tokens_seen": 236322816
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007502026078234704,
+      "loss": 1.55,
+      "theoretical_loss": 4.294778681465841,
+      "tokens_seen": 236388352
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007501865596790372,
+      "loss": 1.6804,
+      "theoretical_loss": 4.294634521642053,
+      "tokens_seen": 236453888
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007501705115346038,
+      "loss": 1.6414,
+      "theoretical_loss": 4.294490412952371,
+      "tokens_seen": 236519424
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007501544633901706,
+      "loss": 1.6467,
+      "theoretical_loss": 4.294346355364494,
+      "tokens_seen": 236584960
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007501384152457372,
+      "loss": 1.6515,
+      "theoretical_loss": 4.294202348846156,
+      "tokens_seen": 236650496
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.000750122367101304,
+      "loss": 1.6922,
+      "theoretical_loss": 4.294058393365114,
+      "tokens_seen": 236716032
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007501063189568706,
+      "loss": 1.5542,
+      "theoretical_loss": 4.293914488889161,
+      "tokens_seen": 236781568
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007500902708124374,
+      "loss": 1.6803,
+      "theoretical_loss": 4.293770635386112,
+      "tokens_seen": 236847104
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.000750074222668004,
+      "loss": 1.727,
+      "theoretical_loss": 4.293626832823818,
+      "tokens_seen": 236912640
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007500581745235708,
+      "loss": 1.678,
+      "theoretical_loss": 4.293483081170153,
+      "tokens_seen": 236978176
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007500421263791374,
+      "loss": 1.6602,
+      "theoretical_loss": 4.293339380393027,
+      "tokens_seen": 237043712
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007500260782347042,
+      "loss": 1.5453,
+      "theoretical_loss": 4.293195730460372,
+      "tokens_seen": 237109248
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007500100300902708,
+      "loss": 1.5695,
+      "theoretical_loss": 4.293052131340154,
+      "tokens_seen": 237174784
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007499939819458376,
+      "loss": 1.636,
+      "theoretical_loss": 4.292908583000365,
+      "tokens_seen": 237240320
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007499779338014042,
+      "loss": 1.6634,
+      "theoretical_loss": 4.29276508540903,
+      "tokens_seen": 237305856
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.000749961885656971,
+      "loss": 1.5336,
+      "theoretical_loss": 4.292621638534198,
+      "tokens_seen": 237371392
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007499458375125376,
+      "loss": 1.6682,
+      "theoretical_loss": 4.29247824234395,
+      "tokens_seen": 237436928
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007499297893681044,
+      "loss": 1.6177,
+      "theoretical_loss": 4.292334896806397,
+      "tokens_seen": 237502464
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.000749913741223671,
+      "loss": 1.6931,
+      "theoretical_loss": 4.292191601889673,
+      "tokens_seen": 237568000
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007498976930792378,
+      "loss": 1.6678,
+      "theoretical_loss": 4.2920483575619475,
+      "tokens_seen": 237633536
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007498816449348044,
+      "loss": 1.7043,
+      "theoretical_loss": 4.291905163791417,
+      "tokens_seen": 237699072
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007498655967903712,
+      "loss": 1.6763,
+      "theoretical_loss": 4.291762020546304,
+      "tokens_seen": 237764608
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007498495486459378,
+      "loss": 1.6054,
+      "theoretical_loss": 4.291618927794862,
+      "tokens_seen": 237830144
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007498335005015046,
+      "loss": 1.6275,
+      "theoretical_loss": 4.291475885505374,
+      "tokens_seen": 237895680
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007498174523570712,
+      "loss": 1.6571,
+      "theoretical_loss": 4.291332893646149,
+      "tokens_seen": 237961216
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.000749801404212638,
+      "loss": 1.6808,
+      "theoretical_loss": 4.291189952185527,
+      "tokens_seen": 238026752
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007497853560682046,
+      "loss": 1.6213,
+      "theoretical_loss": 4.291047061091875,
+      "tokens_seen": 238092288
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007497693079237714,
+      "loss": 1.6086,
+      "theoretical_loss": 4.290904220333591,
+      "tokens_seen": 238157824
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.000749753259779338,
+      "loss": 1.4797,
+      "theoretical_loss": 4.2907614298790975,
+      "tokens_seen": 238223360
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007497372116349048,
+      "loss": 1.5973,
+      "theoretical_loss": 4.290618689696848,
+      "tokens_seen": 238288896
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007497211634904714,
+      "loss": 1.6309,
+      "theoretical_loss": 4.290475999755326,
+      "tokens_seen": 238354432
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007497051153460382,
+      "loss": 1.5985,
+      "theoretical_loss": 4.29033336002304,
+      "tokens_seen": 238419968
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007496890672016048,
+      "loss": 1.6572,
+      "theoretical_loss": 4.290190770468529,
+      "tokens_seen": 238485504
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007496730190571716,
+      "loss": 1.6657,
+      "theoretical_loss": 4.290048231060361,
+      "tokens_seen": 238551040
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007496569709127382,
+      "loss": 1.6352,
+      "theoretical_loss": 4.289905741767129,
+      "tokens_seen": 238616576
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.000749640922768305,
+      "loss": 1.6211,
+      "theoretical_loss": 4.289763302557457,
+      "tokens_seen": 238682112
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007496248746238716,
+      "loss": 1.5304,
+      "theoretical_loss": 4.289620913399998,
+      "tokens_seen": 238747648
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007496088264794384,
+      "loss": 1.6546,
+      "theoretical_loss": 4.2894785742634305,
+      "tokens_seen": 238813184
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.000749592778335005,
+      "loss": 1.5993,
+      "theoretical_loss": 4.289336285116463,
+      "tokens_seen": 238878720
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007495767301905718,
+      "loss": 1.6602,
+      "theoretical_loss": 4.289194045927832,
+      "tokens_seen": 238944256
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007495606820461385,
+      "loss": 1.6194,
+      "theoretical_loss": 4.289051856666303,
+      "tokens_seen": 239009792
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007495446339017051,
+      "loss": 1.7735,
+      "theoretical_loss": 4.288909717300667,
+      "tokens_seen": 239075328
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007495285857572719,
+      "loss": 1.7223,
+      "theoretical_loss": 4.288767627799745,
+      "tokens_seen": 239140864
+    },
+    {
+      "epoch": 0.07,
+      "objective/train/docs_used": 85591,
+      "objective/train/instantaneous_batch_size": 32,
+      "objective/train/instantaneous_microbatch_size": 32768,
+      "objective/train/original_loss": 1.5911835432052612,
+      "objective/train/theoretical_loss": 4.288625588132385,
+      "objective/train/tokens_used": 259666400,
+      "theoretical_loss": 4.288625588132385,
+      "tokens_seen": 239206400
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007495125376128385,
+      "loss": 1.6132,
+      "theoretical_loss": 4.288625588132385,
+      "tokens_seen": 239206400
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007494964894684053,
+      "loss": 1.5748,
+      "theoretical_loss": 4.288483598267465,
+      "tokens_seen": 239271936
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007494804413239719,
+      "loss": 1.5424,
+      "theoretical_loss": 4.288341658173888,
+      "tokens_seen": 239337472
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007494643931795387,
+      "loss": 1.5214,
+      "theoretical_loss": 4.288199767820587,
+      "tokens_seen": 239403008
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007494483450351053,
+      "loss": 1.6185,
+      "theoretical_loss": 4.2880579271765225,
+      "tokens_seen": 239468544
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007494322968906721,
+      "loss": 1.6753,
+      "theoretical_loss": 4.287916136210682,
+      "tokens_seen": 239534080
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007494162487462387,
+      "loss": 1.6175,
+      "theoretical_loss": 4.287774394892082,
+      "tokens_seen": 239599616
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007494002006018055,
+      "loss": 1.6588,
+      "theoretical_loss": 4.287632703189766,
+      "tokens_seen": 239665152
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007493841524573721,
+      "loss": 1.6161,
+      "theoretical_loss": 4.2874910610728065,
+      "tokens_seen": 239730688
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007493681043129389,
+      "loss": 1.4604,
+      "theoretical_loss": 4.287349468510302,
+      "tokens_seen": 239796224
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007493520561685055,
+      "loss": 1.7263,
+      "theoretical_loss": 4.28720792547138,
+      "tokens_seen": 239861760
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007493360080240723,
+      "loss": 1.604,
+      "theoretical_loss": 4.287066431925194,
+      "tokens_seen": 239927296
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007493199598796389,
+      "loss": 1.5719,
+      "theoretical_loss": 4.286924987840926,
+      "tokens_seen": 239992832
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007493039117352057,
+      "loss": 1.5682,
+      "theoretical_loss": 4.2867835931877885,
+      "tokens_seen": 240058368
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007492878635907723,
+      "loss": 1.7062,
+      "theoretical_loss": 4.286642247935017,
+      "tokens_seen": 240123904
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007492718154463391,
+      "loss": 1.5984,
+      "theoretical_loss": 4.286500952051876,
+      "tokens_seen": 240189440
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007492557673019057,
+      "loss": 1.5649,
+      "theoretical_loss": 4.286359705507659,
+      "tokens_seen": 240254976
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007492397191574725,
+      "loss": 1.617,
+      "theoretical_loss": 4.286218508271686,
+      "tokens_seen": 240320512
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007492236710130391,
+      "loss": 1.6426,
+      "theoretical_loss": 4.286077360313303,
+      "tokens_seen": 240386048
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007492076228686059,
+      "loss": 1.6051,
+      "theoretical_loss": 4.285936261601887,
+      "tokens_seen": 240451584
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007491915747241725,
+      "loss": 1.528,
+      "theoretical_loss": 4.285795212106839,
+      "tokens_seen": 240517120
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007491755265797393,
+      "loss": 1.6518,
+      "theoretical_loss": 4.2856542117975875,
+      "tokens_seen": 240582656
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007491594784353059,
+      "loss": 1.5736,
+      "theoretical_loss": 4.285513260643591,
+      "tokens_seen": 240648192
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007491434302908727,
+      "loss": 1.5997,
+      "theoretical_loss": 4.285372358614332,
+      "tokens_seen": 240713728
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007491273821464393,
+      "loss": 1.5507,
+      "theoretical_loss": 4.285231505679324,
+      "tokens_seen": 240779264
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007491113340020061,
+      "loss": 1.5485,
+      "theoretical_loss": 4.285090701808103,
+      "tokens_seen": 240844800
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007490952858575727,
+      "loss": 1.5818,
+      "theoretical_loss": 4.284949946970236,
+      "tokens_seen": 240910336
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007490792377131395,
+      "loss": 1.6634,
+      "theoretical_loss": 4.284809241135315,
+      "tokens_seen": 240975872
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007490631895687061,
+      "loss": 1.7095,
+      "theoretical_loss": 4.284668584272962,
+      "tokens_seen": 241041408
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007490471414242729,
+      "loss": 1.5739,
+      "theoretical_loss": 4.284527976352823,
+      "tokens_seen": 241106944
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007490310932798395,
+      "loss": 1.5536,
+      "theoretical_loss": 4.284387417344572,
+      "tokens_seen": 241172480
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007490150451354063,
+      "loss": 1.6238,
+      "theoretical_loss": 4.28424690721791,
+      "tokens_seen": 241238016
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007489989969909729,
+      "loss": 1.6059,
+      "theoretical_loss": 4.284106445942565,
+      "tokens_seen": 241303552
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007489829488465397,
+      "loss": 1.6157,
+      "theoretical_loss": 4.2839660334882925,
+      "tokens_seen": 241369088
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007489669007021063,
+      "loss": 1.5834,
+      "theoretical_loss": 4.283825669824875,
+      "tokens_seen": 241434624
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007489508525576731,
+      "loss": 1.5091,
+      "theoretical_loss": 4.28368535492212,
+      "tokens_seen": 241500160
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007489348044132397,
+      "loss": 1.5947,
+      "theoretical_loss": 4.283545088749865,
+      "tokens_seen": 241565696
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007489187562688065,
+      "loss": 1.7086,
+      "theoretical_loss": 4.2834048712779715,
+      "tokens_seen": 241631232
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007489027081243731,
+      "loss": 1.6833,
+      "theoretical_loss": 4.28326470247633,
+      "tokens_seen": 241696768
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007488866599799399,
+      "loss": 1.5479,
+      "theoretical_loss": 4.283124582314856,
+      "tokens_seen": 241762304
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007488706118355065,
+      "loss": 1.654,
+      "theoretical_loss": 4.282984510763493,
+      "tokens_seen": 241827840
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007488545636910733,
+      "loss": 1.5723,
+      "theoretical_loss": 4.282844487792209,
+      "tokens_seen": 241893376
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007488385155466399,
+      "loss": 1.5084,
+      "theoretical_loss": 4.282704513371002,
+      "tokens_seen": 241958912
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007488224674022067,
+      "loss": 1.5897,
+      "theoretical_loss": 4.282564587469896,
+      "tokens_seen": 242024448
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007488064192577733,
+      "loss": 1.6446,
+      "theoretical_loss": 4.282424710058938,
+      "tokens_seen": 242089984
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007487903711133401,
+      "loss": 1.6017,
+      "theoretical_loss": 4.282284881108206,
+      "tokens_seen": 242155520
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007487743229689067,
+      "loss": 1.73,
+      "theoretical_loss": 4.282145100587802,
+      "tokens_seen": 242221056
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007487582748244735,
+      "loss": 1.5746,
+      "theoretical_loss": 4.282005368467856,
+      "tokens_seen": 242286592
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007487422266800401,
+      "loss": 1.5327,
+      "theoretical_loss": 4.2818656847185235,
+      "tokens_seen": 242352128
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007487261785356069,
+      "loss": 1.5259,
+      "theoretical_loss": 4.281726049309987,
+      "tokens_seen": 242417664
+    },
+    {
+      "epoch": 0.07,
+      "objective/train/docs_used": 86546,
+      "objective/train/instantaneous_batch_size": 32,
+      "objective/train/instantaneous_microbatch_size": 32768,
+      "objective/train/original_loss": 1.6260560750961304,
+      "objective/train/theoretical_loss": 4.281586462212455,
+      "objective/train/tokens_used": 262943200,
+      "theoretical_loss": 4.281586462212455,
+      "tokens_seen": 242483200
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007487101303911735,
+      "loss": 1.5872,
+      "theoretical_loss": 4.281586462212455,
+      "tokens_seen": 242483200
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007486940822467403,
+      "loss": 1.609,
+      "theoretical_loss": 4.281446923396162,
+      "tokens_seen": 242548736
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007486780341023069,
+      "loss": 1.7132,
+      "theoretical_loss": 4.281307432831371,
+      "tokens_seen": 242614272
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007486619859578737,
+      "loss": 1.6274,
+      "theoretical_loss": 4.281167990488369,
+      "tokens_seen": 242679808
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007486459378134403,
+      "loss": 1.7038,
+      "theoretical_loss": 4.281028596337469,
+      "tokens_seen": 242745344
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007486298896690071,
+      "loss": 1.5809,
+      "theoretical_loss": 4.280889250349014,
+      "tokens_seen": 242810880
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007486138415245737,
+      "loss": 1.6707,
+      "theoretical_loss": 4.280749952493368,
+      "tokens_seen": 242876416
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007485977933801405,
+      "loss": 1.5787,
+      "theoretical_loss": 4.280610702740926,
+      "tokens_seen": 242941952
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007485817452357071,
+      "loss": 1.7007,
+      "theoretical_loss": 4.280471501062106,
+      "tokens_seen": 243007488
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007485656970912739,
+      "loss": 1.6596,
+      "theoretical_loss": 4.280332347427353,
+      "tokens_seen": 243073024
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007485496489468405,
+      "loss": 1.6144,
+      "theoretical_loss": 4.280193241807139,
+      "tokens_seen": 243138560
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007485336008024073,
+      "loss": 1.6062,
+      "theoretical_loss": 4.280054184171963,
+      "tokens_seen": 243204096
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007485175526579739,
+      "loss": 1.6145,
+      "theoretical_loss": 4.279915174492345,
+      "tokens_seen": 243269632
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007485015045135407,
+      "loss": 1.5874,
+      "theoretical_loss": 4.279776212738838,
+      "tokens_seen": 243335168
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007484854563691073,
+      "loss": 1.6082,
+      "theoretical_loss": 4.279637298882016,
+      "tokens_seen": 243400704
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007484694082246741,
+      "loss": 1.5853,
+      "theoretical_loss": 4.27949843289248,
+      "tokens_seen": 243466240
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007484533600802407,
+      "loss": 1.6289,
+      "theoretical_loss": 4.2793596147408595,
+      "tokens_seen": 243531776
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007484373119358075,
+      "loss": 1.5293,
+      "theoretical_loss": 4.279220844397806,
+      "tokens_seen": 243597312
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007484212637913741,
+      "loss": 1.7669,
+      "theoretical_loss": 4.279082121834001,
+      "tokens_seen": 243662848
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007484052156469409,
+      "loss": 1.5515,
+      "theoretical_loss": 4.278943447020148,
+      "tokens_seen": 243728384
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007483891675025075,
+      "loss": 1.4456,
+      "theoretical_loss": 4.278804819926978,
+      "tokens_seen": 243793920
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007483731193580743,
+      "loss": 1.7349,
+      "theoretical_loss": 4.278666240525249,
+      "tokens_seen": 243859456
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007483570712136409,
+      "loss": 1.6376,
+      "theoretical_loss": 4.278527708785743,
+      "tokens_seen": 243924992
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007483410230692077,
+      "loss": 1.5392,
+      "theoretical_loss": 4.278389224679268,
+      "tokens_seen": 243990528
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007483249749247745,
+      "loss": 1.5989,
+      "theoretical_loss": 4.27825078817666,
+      "tokens_seen": 244056064
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007483089267803411,
+      "loss": 1.6789,
+      "theoretical_loss": 4.2781123992487755,
+      "tokens_seen": 244121600
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007482928786359077,
+      "loss": 1.6446,
+      "theoretical_loss": 4.277974057866503,
+      "tokens_seen": 244187136
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007482768304914745,
+      "loss": 1.6487,
+      "theoretical_loss": 4.277835764000752,
+      "tokens_seen": 244252672
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007482607823470411,
+      "loss": 1.6469,
+      "theoretical_loss": 4.277697517622459,
+      "tokens_seen": 244318208
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007482447342026079,
+      "loss": 1.6675,
+      "theoretical_loss": 4.277559318702588,
+      "tokens_seen": 244383744
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007482286860581746,
+      "loss": 1.6186,
+      "theoretical_loss": 4.277421167212125,
+      "tokens_seen": 244449280
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007482126379137413,
+      "loss": 1.6237,
+      "theoretical_loss": 4.277283063122083,
+      "tokens_seen": 244514816
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.000748196589769308,
+      "loss": 1.5999,
+      "theoretical_loss": 4.277145006403503,
+      "tokens_seen": 244580352
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007481805416248747,
+      "loss": 1.5871,
+      "theoretical_loss": 4.2770069970274465,
+      "tokens_seen": 244645888
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007481644934804414,
+      "loss": 1.5793,
+      "theoretical_loss": 4.276869034965005,
+      "tokens_seen": 244711424
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.000748148445336008,
+      "loss": 1.5585,
+      "theoretical_loss": 4.276731120187293,
+      "tokens_seen": 244776960
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007481323971915748,
+      "loss": 1.7133,
+      "theoretical_loss": 4.2765932526654495,
+      "tokens_seen": 244842496
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007481163490471414,
+      "loss": 1.5657,
+      "theoretical_loss": 4.276455432370643,
+      "tokens_seen": 244908032
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007481003009027082,
+      "loss": 1.6833,
+      "theoretical_loss": 4.276317659274062,
+      "tokens_seen": 244973568
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007480842527582748,
+      "loss": 1.6325,
+      "theoretical_loss": 4.276179933346924,
+      "tokens_seen": 245039104
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007480682046138416,
+      "loss": 1.706,
+      "theoretical_loss": 4.27604225456047,
+      "tokens_seen": 245104640
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007480521564694082,
+      "loss": 1.565,
+      "theoretical_loss": 4.275904622885967,
+      "tokens_seen": 245170176
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.000748036108324975,
+      "loss": 1.6839,
+      "theoretical_loss": 4.275767038294708,
+      "tokens_seen": 245235712
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007480200601805416,
+      "loss": 1.6496,
+      "theoretical_loss": 4.275629500758008,
+      "tokens_seen": 245301248
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007480040120361084,
+      "loss": 1.6672,
+      "theoretical_loss": 4.27549201024721,
+      "tokens_seen": 245366784
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.000747987963891675,
+      "loss": 1.5975,
+      "theoretical_loss": 4.275354566733682,
+      "tokens_seen": 245432320
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007479719157472418,
+      "loss": 1.7334,
+      "theoretical_loss": 4.2752171701888155,
+      "tokens_seen": 245497856
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007479558676028084,
+      "loss": 1.5798,
+      "theoretical_loss": 4.275079820584029,
+      "tokens_seen": 245563392
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007479398194583752,
+      "loss": 1.6385,
+      "theoretical_loss": 4.274942517890764,
+      "tokens_seen": 245628928
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007479237713139418,
+      "loss": 1.8012,
+      "theoretical_loss": 4.2748052620804895,
+      "tokens_seen": 245694464
+    },
+    {
+      "epoch": 0.07,
+      "objective/train/docs_used": 87764,
+      "objective/train/instantaneous_batch_size": 32,
+      "objective/train/instantaneous_microbatch_size": 32768,
+      "objective/train/original_loss": 1.7244846820831299,
+      "objective/train/theoretical_loss": 4.274668053124696,
+      "objective/train/tokens_used": 266220000,
+      "theoretical_loss": 4.274668053124696,
+      "tokens_seen": 245760000
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007479077231695086,
+      "loss": 1.67,
+      "theoretical_loss": 4.274668053124696,
+      "tokens_seen": 245760000
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007478916750250752,
+      "loss": 1.6337,
+      "theoretical_loss": 4.2745308909949005,
+      "tokens_seen": 245825536
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.000747875626880642,
+      "loss": 1.6494,
+      "theoretical_loss": 4.274393775662648,
+      "tokens_seen": 245891072
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007478595787362086,
+      "loss": 1.6913,
+      "theoretical_loss": 4.274256707099502,
+      "tokens_seen": 245956608
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007478435305917754,
+      "loss": 1.5855,
+      "theoretical_loss": 4.274119685277058,
+      "tokens_seen": 246022144
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.000747827482447342,
+      "loss": 1.4986,
+      "theoretical_loss": 4.273982710166931,
+      "tokens_seen": 246087680
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007478114343029088,
+      "loss": 1.508,
+      "theoretical_loss": 4.273845781740762,
+      "tokens_seen": 246153216
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007477953861584754,
+      "loss": 1.5012,
+      "theoretical_loss": 4.273708899970218,
+      "tokens_seen": 246218752
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007477793380140422,
+      "loss": 1.5291,
+      "theoretical_loss": 4.273572064826991,
+      "tokens_seen": 246284288
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007477632898696088,
+      "loss": 1.6974,
+      "theoretical_loss": 4.273435276282795,
+      "tokens_seen": 246349824
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007477472417251756,
+      "loss": 1.7677,
+      "theoretical_loss": 4.2732985343093715,
+      "tokens_seen": 246415360
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007477311935807422,
+      "loss": 1.5992,
+      "theoretical_loss": 4.273161838878485,
+      "tokens_seen": 246480896
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.000747715145436309,
+      "loss": 1.6791,
+      "theoretical_loss": 4.273025189961926,
+      "tokens_seen": 246546432
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007476990972918756,
+      "loss": 1.594,
+      "theoretical_loss": 4.272888587531508,
+      "tokens_seen": 246611968
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007476830491474424,
+      "loss": 1.6518,
+      "theoretical_loss": 4.272752031559071,
+      "tokens_seen": 246677504
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.000747667001003009,
+      "loss": 1.7312,
+      "theoretical_loss": 4.272615522016476,
+      "tokens_seen": 246743040
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007476509528585758,
+      "loss": 1.7256,
+      "theoretical_loss": 4.272479058875614,
+      "tokens_seen": 246808576
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007476349047141424,
+      "loss": 1.5071,
+      "theoretical_loss": 4.272342642108396,
+      "tokens_seen": 246874112
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007476188565697092,
+      "loss": 1.6228,
+      "theoretical_loss": 4.272206271686758,
+      "tokens_seen": 246939648
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007476028084252758,
+      "loss": 1.5508,
+      "theoretical_loss": 4.272069947582662,
+      "tokens_seen": 247005184
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007475867602808426,
+      "loss": 1.6406,
+      "theoretical_loss": 4.271933669768094,
+      "tokens_seen": 247070720
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007475707121364092,
+      "loss": 1.6354,
+      "theoretical_loss": 4.271797438215064,
+      "tokens_seen": 247136256
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.000747554663991976,
+      "loss": 1.6948,
+      "theoretical_loss": 4.271661252895605,
+      "tokens_seen": 247201792
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007475386158475426,
+      "loss": 1.5651,
+      "theoretical_loss": 4.271525113781777,
+      "tokens_seen": 247267328
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007475225677031094,
+      "loss": 1.6706,
+      "theoretical_loss": 4.271389020845662,
+      "tokens_seen": 247332864
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.000747506519558676,
+      "loss": 1.6613,
+      "theoretical_loss": 4.27125297405937,
+      "tokens_seen": 247398400
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0007474904714142428,
+      "loss": 1.5238,
+      "theoretical_loss": 4.271116973395028,
+      "tokens_seen": 247463936
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007474744232698094,
+      "loss": 1.6983,
+      "theoretical_loss": 4.270981018824795,
+      "tokens_seen": 247529472
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007474583751253762,
+      "loss": 1.5716,
+      "theoretical_loss": 4.270845110320851,
+      "tokens_seen": 247595008
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007474423269809428,
+      "loss": 1.4968,
+      "theoretical_loss": 4.270709247855397,
+      "tokens_seen": 247660544
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007474262788365096,
+      "loss": 1.5398,
+      "theoretical_loss": 4.270573431400663,
+      "tokens_seen": 247726080
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007474102306920762,
+      "loss": 1.574,
+      "theoretical_loss": 4.270437660928902,
+      "tokens_seen": 247791616
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.000747394182547643,
+      "loss": 1.6017,
+      "theoretical_loss": 4.270301936412389,
+      "tokens_seen": 247857152
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007473781344032096,
+      "loss": 1.6093,
+      "theoretical_loss": 4.2701662578234245,
+      "tokens_seen": 247922688
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007473620862587764,
+      "loss": 1.6365,
+      "theoretical_loss": 4.270030625134334,
+      "tokens_seen": 247988224
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.000747346038114343,
+      "loss": 1.5936,
+      "theoretical_loss": 4.269895038317464,
+      "tokens_seen": 248053760
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007473299899699098,
+      "loss": 1.6421,
+      "theoretical_loss": 4.269759497345188,
+      "tokens_seen": 248119296
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007473139418254764,
+      "loss": 1.6416,
+      "theoretical_loss": 4.269624002189901,
+      "tokens_seen": 248184832
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007472978936810432,
+      "loss": 1.6085,
+      "theoretical_loss": 4.269488552824024,
+      "tokens_seen": 248250368
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007472818455366098,
+      "loss": 1.4617,
+      "theoretical_loss": 4.269353149220001,
+      "tokens_seen": 248315904
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007472657973921766,
+      "loss": 1.6036,
+      "theoretical_loss": 4.2692177913503,
+      "tokens_seen": 248381440
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007472497492477432,
+      "loss": 1.7317,
+      "theoretical_loss": 4.2690824791874125,
+      "tokens_seen": 248446976
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00074723370110331,
+      "loss": 1.5024,
+      "theoretical_loss": 4.2689472127038535,
+      "tokens_seen": 248512512
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007472176529588766,
+      "loss": 1.5484,
+      "theoretical_loss": 4.2688119918721625,
+      "tokens_seen": 248578048
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007472016048144434,
+      "loss": 1.4788,
+      "theoretical_loss": 4.268676816664902,
+      "tokens_seen": 248643584
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00074718555667001,
+      "loss": 1.6269,
+      "theoretical_loss": 4.26854168705466,
+      "tokens_seen": 248709120
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007471695085255768,
+      "loss": 1.6235,
+      "theoretical_loss": 4.2684066030140455,
+      "tokens_seen": 248774656
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007471534603811434,
+      "loss": 1.5541,
+      "theoretical_loss": 4.268271564515693,
+      "tokens_seen": 248840192
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007471374122367102,
+      "loss": 1.5264,
+      "theoretical_loss": 4.26813657153226,
+      "tokens_seen": 248905728
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.000747121364092277,
+      "loss": 1.5801,
+      "theoretical_loss": 4.268001624036429,
+      "tokens_seen": 248971264
+    },
+    {
+      "epoch": 0.08,
+      "objective/train/docs_used": 88717,
+      "objective/train/instantaneous_batch_size": 32,
+      "objective/train/instantaneous_microbatch_size": 32768,
+      "objective/train/original_loss": 1.622373342514038,
+      "objective/train/theoretical_loss": 4.267866722000902,
+      "objective/train/tokens_used": 269496800,
+      "theoretical_loss": 4.267866722000902,
+      "tokens_seen": 249036800
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007471053159478436,
+      "loss": 1.6075,
+      "theoretical_loss": 4.267866722000902,
+      "tokens_seen": 249036800
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007470892678034103,
+      "loss": 1.6464,
+      "theoretical_loss": 4.26773186539841,
+      "tokens_seen": 249102336
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.000747073219658977,
+      "loss": 1.6316,
+      "theoretical_loss": 4.267597054201705,
+      "tokens_seen": 249167872
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007470571715145438,
+      "loss": 1.6576,
+      "theoretical_loss": 4.267462288383561,
+      "tokens_seen": 249233408
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007470411233701104,
+      "loss": 1.6169,
+      "theoretical_loss": 4.267327567916777,
+      "tokens_seen": 249298944
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007470250752256772,
+      "loss": 1.6454,
+      "theoretical_loss": 4.267192892774176,
+      "tokens_seen": 249364480
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007470090270812438,
+      "loss": 1.6379,
+      "theoretical_loss": 4.267058262928604,
+      "tokens_seen": 249430016
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007469929789368106,
+      "loss": 1.5889,
+      "theoretical_loss": 4.26692367835293,
+      "tokens_seen": 249495552
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007469769307923772,
+      "loss": 1.5403,
+      "theoretical_loss": 4.266789139020045,
+      "tokens_seen": 249561088
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.000746960882647944,
+      "loss": 1.6017,
+      "theoretical_loss": 4.266654644902868,
+      "tokens_seen": 249626624
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007469448345035106,
+      "loss": 1.484,
+      "theoretical_loss": 4.266520195974336,
+      "tokens_seen": 249692160
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007469287863590774,
+      "loss": 1.6305,
+      "theoretical_loss": 4.266385792207412,
+      "tokens_seen": 249757696
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.000746912738214644,
+      "loss": 1.6925,
+      "theoretical_loss": 4.266251433575082,
+      "tokens_seen": 249823232
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007468966900702106,
+      "loss": 1.6038,
+      "theoretical_loss": 4.266117120050355,
+      "tokens_seen": 249888768
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007468806419257774,
+      "loss": 1.6286,
+      "theoretical_loss": 4.265982851606263,
+      "tokens_seen": 249954304
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.000746864593781344,
+      "loss": 1.6135,
+      "theoretical_loss": 4.265848628215862,
+      "tokens_seen": 250019840
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007468485456369108,
+      "loss": 1.6459,
+      "theoretical_loss": 4.265714449852229,
+      "tokens_seen": 250085376
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007468324974924775,
+      "loss": 1.698,
+      "theoretical_loss": 4.265580316488467,
+      "tokens_seen": 250150912
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007468164493480442,
+      "loss": 1.6662,
+      "theoretical_loss": 4.2654462280977015,
+      "tokens_seen": 250216448
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007468004012036109,
+      "loss": 1.7066,
+      "theoretical_loss": 4.265312184653078,
+      "tokens_seen": 250281984
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007467843530591776,
+      "loss": 1.5266,
+      "theoretical_loss": 4.2651781861277716,
+      "tokens_seen": 250347520
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007467683049147443,
+      "loss": 1.5015,
+      "theoretical_loss": 4.265044232494972,
+      "tokens_seen": 250413056
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007467522567703109,
+      "loss": 1.6865,
+      "theoretical_loss": 4.264910323727898,
+      "tokens_seen": 250478592
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007467362086258777,
+      "loss": 1.5655,
+      "theoretical_loss": 4.264776459799791,
+      "tokens_seen": 250544128
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007467201604814443,
+      "loss": 1.6014,
+      "theoretical_loss": 4.264642640683911,
+      "tokens_seen": 250609664
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007467041123370111,
+      "loss": 1.4893,
+      "theoretical_loss": 4.264508866353546,
+      "tokens_seen": 250675200
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007466880641925777,
+      "loss": 1.5473,
+      "theoretical_loss": 4.264375136782004,
+      "tokens_seen": 250740736
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007466720160481445,
+      "loss": 1.6749,
+      "theoretical_loss": 4.264241451942617,
+      "tokens_seen": 250806272
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007466559679037111,
+      "loss": 1.6507,
+      "theoretical_loss": 4.264107811808739,
+      "tokens_seen": 250871808
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007466399197592779,
+      "loss": 1.6085,
+      "theoretical_loss": 4.263974216353749,
+      "tokens_seen": 250937344
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007466238716148445,
+      "loss": 1.6564,
+      "theoretical_loss": 4.2638406655510455,
+      "tokens_seen": 251002880
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007466078234704113,
+      "loss": 1.6496,
+      "theoretical_loss": 4.263707159374051,
+      "tokens_seen": 251068416
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007465917753259779,
+      "loss": 1.6642,
+      "theoretical_loss": 4.263573697796213,
+      "tokens_seen": 251133952
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007465757271815447,
+      "loss": 1.5706,
+      "theoretical_loss": 4.263440280790999,
+      "tokens_seen": 251199488
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007465596790371113,
+      "loss": 1.6217,
+      "theoretical_loss": 4.263306908331899,
+      "tokens_seen": 251265024
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007465436308926781,
+      "loss": 1.5774,
+      "theoretical_loss": 4.2631735803924276,
+      "tokens_seen": 251330560
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007465275827482447,
+      "loss": 1.6148,
+      "theoretical_loss": 4.263040296946122,
+      "tokens_seen": 251396096
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007465115346038115,
+      "loss": 1.5488,
+      "theoretical_loss": 4.26290705796654,
+      "tokens_seen": 251461632
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007464954864593781,
+      "loss": 1.6859,
+      "theoretical_loss": 4.262773863427265,
+      "tokens_seen": 251527168
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007464794383149449,
+      "loss": 1.6363,
+      "theoretical_loss": 4.262640713301899,
+      "tokens_seen": 251592704
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007464633901705115,
+      "loss": 1.5869,
+      "theoretical_loss": 4.26250760756407,
+      "tokens_seen": 251658240
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007464473420260783,
+      "loss": 1.6862,
+      "theoretical_loss": 4.262374546187428,
+      "tokens_seen": 251723776
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007464312938816449,
+      "loss": 1.6634,
+      "theoretical_loss": 4.262241529145643,
+      "tokens_seen": 251789312
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007464152457372117,
+      "loss": 1.6101,
+      "theoretical_loss": 4.262108556412411,
+      "tokens_seen": 251854848
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007463991975927783,
+      "loss": 1.6915,
+      "theoretical_loss": 4.261975627961448,
+      "tokens_seen": 251920384
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007463831494483451,
+      "loss": 1.6473,
+      "theoretical_loss": 4.2618427437664925,
+      "tokens_seen": 251985920
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007463671013039117,
+      "loss": 1.7384,
+      "theoretical_loss": 4.261709903801307,
+      "tokens_seen": 252051456
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007463510531594785,
+      "loss": 1.5923,
+      "theoretical_loss": 4.261577108039674,
+      "tokens_seen": 252116992
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007463350050150451,
+      "loss": 1.6084,
+      "theoretical_loss": 4.261444356455402,
+      "tokens_seen": 252182528
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007463189568706119,
+      "loss": 1.617,
+      "theoretical_loss": 4.261311649022318,
+      "tokens_seen": 252248064
+    },
+    {
+      "epoch": 0.08,
+      "objective/train/docs_used": 89977,
+      "objective/train/instantaneous_batch_size": 32,
+      "objective/train/instantaneous_microbatch_size": 32768,
+      "objective/train/original_loss": 1.5997533798217773,
+      "objective/train/theoretical_loss": 4.261178985714273,
+      "objective/train/tokens_used": 272773600,
+      "theoretical_loss": 4.261178985714273,
+      "tokens_seen": 252313600
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007463029087261785,
+      "loss": 1.5685,
+      "theoretical_loss": 4.261178985714273,
+      "tokens_seen": 252313600
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007462868605817453,
+      "loss": 1.6051,
+      "theoretical_loss": 4.26104636650514,
+      "tokens_seen": 252379136
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007462708124373119,
+      "loss": 1.6011,
+      "theoretical_loss": 4.260913791368815,
+      "tokens_seen": 252444672
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007462547642928787,
+      "loss": 1.633,
+      "theoretical_loss": 4.260781260279216,
+      "tokens_seen": 252510208
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007462387161484453,
+      "loss": 1.536,
+      "theoretical_loss": 4.260648773210281,
+      "tokens_seen": 252575744
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007462226680040121,
+      "loss": 1.5701,
+      "theoretical_loss": 4.2605163301359745,
+      "tokens_seen": 252641280
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007462066198595787,
+      "loss": 1.6052,
+      "theoretical_loss": 4.260383931030279,
+      "tokens_seen": 252706816
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007461905717151455,
+      "loss": 1.4953,
+      "theoretical_loss": 4.2602515758672,
+      "tokens_seen": 252772352
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007461745235707121,
+      "loss": 1.5352,
+      "theoretical_loss": 4.260119264620768,
+      "tokens_seen": 252837888
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007461584754262789,
+      "loss": 1.6613,
+      "theoretical_loss": 4.259986997265033,
+      "tokens_seen": 252903424
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007461424272818455,
+      "loss": 1.7471,
+      "theoretical_loss": 4.259854773774066,
+      "tokens_seen": 252968960
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007461263791374123,
+      "loss": 1.7108,
+      "theoretical_loss": 4.259722594121963,
+      "tokens_seen": 253034496
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007461103309929789,
+      "loss": 1.538,
+      "theoretical_loss": 4.259590458282839,
+      "tokens_seen": 253100032
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007460942828485457,
+      "loss": 1.5588,
+      "theoretical_loss": 4.259458366230835,
+      "tokens_seen": 253165568
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007460782347041123,
+      "loss": 1.5723,
+      "theoretical_loss": 4.2593263179401095,
+      "tokens_seen": 253231104
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007460621865596791,
+      "loss": 1.595,
+      "theoretical_loss": 4.259194313384846,
+      "tokens_seen": 253296640
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007460461384152457,
+      "loss": 1.6265,
+      "theoretical_loss": 4.259062352539247,
+      "tokens_seen": 253362176
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007460300902708125,
+      "loss": 1.6009,
+      "theoretical_loss": 4.258930435377541,
+      "tokens_seen": 253427712
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007460140421263791,
+      "loss": 1.577,
+      "theoretical_loss": 4.258798561873974,
+      "tokens_seen": 253493248
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007459979939819459,
+      "loss": 1.5744,
+      "theoretical_loss": 4.258666732002817,
+      "tokens_seen": 253558784
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007459819458375125,
+      "loss": 1.5501,
+      "theoretical_loss": 4.258534945738362,
+      "tokens_seen": 253624320
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007459658976930793,
+      "loss": 1.7433,
+      "theoretical_loss": 4.258403203054923,
+      "tokens_seen": 253689856
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007459498495486459,
+      "loss": 1.6291,
+      "theoretical_loss": 4.258271503926833,
+      "tokens_seen": 253755392
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007459338014042127,
+      "loss": 1.5855,
+      "theoretical_loss": 4.258139848328451,
+      "tokens_seen": 253820928
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007459177532597794,
+      "loss": 1.5645,
+      "theoretical_loss": 4.258008236234154,
+      "tokens_seen": 253886464
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007459017051153461,
+      "loss": 1.5791,
+      "theoretical_loss": 4.257876667618344,
+      "tokens_seen": 253952000
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007458856569709128,
+      "loss": 1.6261,
+      "theoretical_loss": 4.257745142455442,
+      "tokens_seen": 254017536
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007458696088264795,
+      "loss": 1.6497,
+      "theoretical_loss": 4.257613660719892,
+      "tokens_seen": 254083072
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007458535606820462,
+      "loss": 1.6368,
+      "theoretical_loss": 4.25748222238616,
+      "tokens_seen": 254148608
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007458375125376129,
+      "loss": 1.7427,
+      "theoretical_loss": 4.2573508274287315,
+      "tokens_seen": 254214144
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007458214643931796,
+      "loss": 1.5152,
+      "theoretical_loss": 4.257219475822116,
+      "tokens_seen": 254279680
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007458054162487463,
+      "loss": 1.6003,
+      "theoretical_loss": 4.257088167540843,
+      "tokens_seen": 254345216
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.000745789368104313,
+      "loss": 1.6177,
+      "theoretical_loss": 4.256956902559465,
+      "tokens_seen": 254410752
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007457733199598797,
+      "loss": 1.5215,
+      "theoretical_loss": 4.256825680852554,
+      "tokens_seen": 254476288
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007457572718154464,
+      "loss": 1.6095,
+      "theoretical_loss": 4.256694502394704,
+      "tokens_seen": 254541824
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007457412236710131,
+      "loss": 1.5408,
+      "theoretical_loss": 4.256563367160533,
+      "tokens_seen": 254607360
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007457251755265798,
+      "loss": 1.5503,
+      "theoretical_loss": 4.256432275124676,
+      "tokens_seen": 254672896
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007457091273821465,
+      "loss": 1.5718,
+      "theoretical_loss": 4.256301226261794,
+      "tokens_seen": 254738432
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007456930792377133,
+      "loss": 1.4958,
+      "theoretical_loss": 4.256170220546565,
+      "tokens_seen": 254803968
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007456770310932799,
+      "loss": 1.5251,
+      "theoretical_loss": 4.2560392579536925,
+      "tokens_seen": 254869504
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007456609829488467,
+      "loss": 1.6568,
+      "theoretical_loss": 4.255908338457898,
+      "tokens_seen": 254935040
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007456449348044133,
+      "loss": 1.6467,
+      "theoretical_loss": 4.255777462033926,
+      "tokens_seen": 255000576
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00074562888665998,
+      "loss": 1.5805,
+      "theoretical_loss": 4.255646628656542,
+      "tokens_seen": 255066112
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007456128385155467,
+      "loss": 1.5746,
+      "theoretical_loss": 4.255515838300534,
+      "tokens_seen": 255131648
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007455967903711135,
+      "loss": 1.64,
+      "theoretical_loss": 4.255385090940708,
+      "tokens_seen": 255197184
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007455807422266801,
+      "loss": 1.5045,
+      "theoretical_loss": 4.255254386551894,
+      "tokens_seen": 255262720
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007455646940822469,
+      "loss": 1.5491,
+      "theoretical_loss": 4.255123725108943,
+      "tokens_seen": 255328256
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007455486459378135,
+      "loss": 1.6597,
+      "theoretical_loss": 4.254993106586725,
+      "tokens_seen": 255393792
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007455325977933803,
+      "loss": 1.5919,
+      "theoretical_loss": 4.254862530960134,
+      "tokens_seen": 255459328
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007455165496489469,
+      "loss": 1.7126,
+      "theoretical_loss": 4.254731998204083,
+      "tokens_seen": 255524864
+    },
+    {
+      "epoch": 0.08,
+      "objective/train/docs_used": 90841,
+      "objective/train/instantaneous_batch_size": 32,
+      "objective/train/instantaneous_microbatch_size": 32768,
+      "objective/train/original_loss": 1.5629866123199463,
+      "objective/train/theoretical_loss": 4.254601508293508,
+      "objective/train/tokens_used": 276050400,
+      "theoretical_loss": 4.254601508293508,
+      "tokens_seen": 255590400
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007455005015045136,
+      "loss": 1.6265,
+      "theoretical_loss": 4.254601508293508,
+      "tokens_seen": 255590400
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007454844533600803,
+      "loss": 1.7083,
+      "theoretical_loss": 4.254471061203364,
+      "tokens_seen": 255655936
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.000745468405215647,
+      "loss": 1.48,
+      "theoretical_loss": 4.254340656908628,
+      "tokens_seen": 255721472
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007454523570712137,
+      "loss": 1.5372,
+      "theoretical_loss": 4.254210295384298,
+      "tokens_seen": 255787008
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007454363089267804,
+      "loss": 1.5572,
+      "theoretical_loss": 4.254079976605394,
+      "tokens_seen": 255852544
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007454202607823471,
+      "loss": 1.567,
+      "theoretical_loss": 4.253949700546956,
+      "tokens_seen": 255918080
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007454042126379138,
+      "loss": 1.4579,
+      "theoretical_loss": 4.2538194671840435,
+      "tokens_seen": 255983616
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007453881644934805,
+      "loss": 1.5505,
+      "theoretical_loss": 4.25368927649174,
+      "tokens_seen": 256049152
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007453721163490472,
+      "loss": 1.4673,
+      "theoretical_loss": 4.253559128445148,
+      "tokens_seen": 256114688
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007453560682046138,
+      "loss": 1.6469,
+      "theoretical_loss": 4.253429023019391,
+      "tokens_seen": 256180224
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007453400200601806,
+      "loss": 1.7815,
+      "theoretical_loss": 4.253298960189614,
+      "tokens_seen": 256245760
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007453239719157472,
+      "loss": 1.5467,
+      "theoretical_loss": 4.253168939930982,
+      "tokens_seen": 256311296
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.000745307923771314,
+      "loss": 1.7182,
+      "theoretical_loss": 4.253038962218682,
+      "tokens_seen": 256376832
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007452918756268806,
+      "loss": 1.5841,
+      "theoretical_loss": 4.25290902702792,
+      "tokens_seen": 256442368
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007452758274824474,
+      "loss": 1.6418,
+      "theoretical_loss": 4.252779134333926,
+      "tokens_seen": 256507904
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.000745259779338014,
+      "loss": 1.5929,
+      "theoretical_loss": 4.252649284111946,
+      "tokens_seen": 256573440
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007452437311935808,
+      "loss": 1.6688,
+      "theoretical_loss": 4.252519476337251,
+      "tokens_seen": 256638976
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007452276830491474,
+      "loss": 1.5167,
+      "theoretical_loss": 4.252389710985131,
+      "tokens_seen": 256704512
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007452116349047142,
+      "loss": 1.6583,
+      "theoretical_loss": 4.252259988030896,
+      "tokens_seen": 256770048
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007451955867602808,
+      "loss": 1.5454,
+      "theoretical_loss": 4.252130307449878,
+      "tokens_seen": 256835584
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007451795386158476,
+      "loss": 1.6186,
+      "theoretical_loss": 4.252000669217429,
+      "tokens_seen": 256901120
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007451634904714142,
+      "loss": 1.6713,
+      "theoretical_loss": 4.251871073308923,
+      "tokens_seen": 256966656
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.000745147442326981,
+      "loss": 1.5235,
+      "theoretical_loss": 4.251741519699751,
+      "tokens_seen": 257032192
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007451313941825476,
+      "loss": 1.6449,
+      "theoretical_loss": 4.251612008365328,
+      "tokens_seen": 257097728
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007451153460381144,
+      "loss": 1.6395,
+      "theoretical_loss": 4.2514825392810875,
+      "tokens_seen": 257163264
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.000745099297893681,
+      "loss": 1.6222,
+      "theoretical_loss": 4.251353112422486,
+      "tokens_seen": 257228800
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007450832497492478,
+      "loss": 1.558,
+      "theoretical_loss": 4.251223727764999,
+      "tokens_seen": 257294336
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007450672016048144,
+      "loss": 1.5734,
+      "theoretical_loss": 4.251094385284121,
+      "tokens_seen": 257359872
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007450511534603812,
+      "loss": 1.6303,
+      "theoretical_loss": 4.25096508495537,
+      "tokens_seen": 257425408
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007450351053159478,
+      "loss": 1.6462,
+      "theoretical_loss": 4.250835826754281,
+      "tokens_seen": 257490944
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007450190571715146,
+      "loss": 1.5893,
+      "theoretical_loss": 4.2507066106564135,
+      "tokens_seen": 257556480
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007450030090270812,
+      "loss": 1.71,
+      "theoretical_loss": 4.250577436637344,
+      "tokens_seen": 257622016
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.000744986960882648,
+      "loss": 1.5198,
+      "theoretical_loss": 4.25044830467267,
+      "tokens_seen": 257687552
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007449709127382146,
+      "loss": 1.6292,
+      "theoretical_loss": 4.250319214738011,
+      "tokens_seen": 257753088
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007449548645937814,
+      "loss": 1.6212,
+      "theoretical_loss": 4.250190166809005,
+      "tokens_seen": 257818624
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.000744938816449348,
+      "loss": 1.5907,
+      "theoretical_loss": 4.250061160861311,
+      "tokens_seen": 257884160
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007449227683049148,
+      "loss": 1.6344,
+      "theoretical_loss": 4.249932196870608,
+      "tokens_seen": 257949696
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007449067201604814,
+      "loss": 1.5848,
+      "theoretical_loss": 4.249803274812598,
+      "tokens_seen": 258015232
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007448906720160482,
+      "loss": 1.751,
+      "theoretical_loss": 4.249674394662998,
+      "tokens_seen": 258080768
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007448746238716148,
+      "loss": 1.656,
+      "theoretical_loss": 4.249545556397549,
+      "tokens_seen": 258146304
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007448585757271816,
+      "loss": 1.5503,
+      "theoretical_loss": 4.249416759992011,
+      "tokens_seen": 258211840
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007448425275827482,
+      "loss": 1.5536,
+      "theoretical_loss": 4.249288005422166,
+      "tokens_seen": 258277376
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.000744826479438315,
+      "loss": 1.584,
+      "theoretical_loss": 4.249159292663812,
+      "tokens_seen": 258342912
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007448104312938816,
+      "loss": 1.6758,
+      "theoretical_loss": 4.249030621692772,
+      "tokens_seen": 258408448
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007447943831494484,
+      "loss": 1.4921,
+      "theoretical_loss": 4.248901992484885,
+      "tokens_seen": 258473984
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.000744778335005015,
+      "loss": 1.6655,
+      "theoretical_loss": 4.248773405016014,
+      "tokens_seen": 258539520
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007447622868605818,
+      "loss": 1.5606,
+      "theoretical_loss": 4.248644859262039,
+      "tokens_seen": 258605056
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007447462387161484,
+      "loss": 1.6209,
+      "theoretical_loss": 4.248516355198861,
+      "tokens_seen": 258670592
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007447301905717152,
+      "loss": 1.6271,
+      "theoretical_loss": 4.2483878928024,
+      "tokens_seen": 258736128
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007447141424272819,
+      "loss": 1.7272,
+      "theoretical_loss": 4.248259472048598,
+      "tokens_seen": 258801664
+    },
+    {
+      "epoch": 0.08,
+      "objective/train/docs_used": 92089,
+      "objective/train/instantaneous_batch_size": 32,
+      "objective/train/instantaneous_microbatch_size": 32768,
+      "objective/train/original_loss": 1.6121582984924316,
+      "objective/train/theoretical_loss": 4.248131092913416,
+      "objective/train/tokens_used": 279327200,
+      "theoretical_loss": 4.248131092913416,
+      "tokens_seen": 258867200
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007446980942828486,
+      "loss": 1.5968,
+      "theoretical_loss": 4.248131092913416,
+      "tokens_seen": 258867200
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007446820461384153,
+      "loss": 1.6051,
+      "theoretical_loss": 4.248002755372836,
+      "tokens_seen": 258932736
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.000744665997993982,
+      "loss": 1.556,
+      "theoretical_loss": 4.247874459402857,
+      "tokens_seen": 258998272
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007446499498495487,
+      "loss": 1.5964,
+      "theoretical_loss": 4.2477462049795,
+      "tokens_seen": 259063808
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007446339017051154,
+      "loss": 1.6101,
+      "theoretical_loss": 4.2476179920788075,
+      "tokens_seen": 259129344
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007446178535606821,
+      "loss": 1.639,
+      "theoretical_loss": 4.247489820676839,
+      "tokens_seen": 259194880
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007446018054162488,
+      "loss": 1.6686,
+      "theoretical_loss": 4.247361690749676,
+      "tokens_seen": 259260416
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007445857572718155,
+      "loss": 1.6614,
+      "theoretical_loss": 4.2472336022734165,
+      "tokens_seen": 259325952
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007445697091273822,
+      "loss": 1.5972,
+      "theoretical_loss": 4.2471055552241825,
+      "tokens_seen": 259391488
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007445536609829489,
+      "loss": 1.59,
+      "theoretical_loss": 4.246977549578113,
+      "tokens_seen": 259457024
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007445376128385156,
+      "loss": 1.6897,
+      "theoretical_loss": 4.246849585311368,
+      "tokens_seen": 259522560
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007445215646940823,
+      "loss": 1.5626,
+      "theoretical_loss": 4.246721662400128,
+      "tokens_seen": 259588096
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.000744505516549649,
+      "loss": 1.6635,
+      "theoretical_loss": 4.246593780820589,
+      "tokens_seen": 259653632
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007444894684052157,
+      "loss": 1.5579,
+      "theoretical_loss": 4.2464659405489735,
+      "tokens_seen": 259719168
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007444734202607824,
+      "loss": 1.588,
+      "theoretical_loss": 4.246338141561518,
+      "tokens_seen": 259784704
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007444573721163491,
+      "loss": 1.5711,
+      "theoretical_loss": 4.24621038383448,
+      "tokens_seen": 259850240
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007444413239719158,
+      "loss": 1.5605,
+      "theoretical_loss": 4.246082667344139,
+      "tokens_seen": 259915776
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007444252758274825,
+      "loss": 1.5478,
+      "theoretical_loss": 4.245954992066792,
+      "tokens_seen": 259981312
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007444092276830492,
+      "loss": 1.6295,
+      "theoretical_loss": 4.245827357978754,
+      "tokens_seen": 260046848
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.000744393179538616,
+      "loss": 1.5641,
+      "theoretical_loss": 4.245699765056365,
+      "tokens_seen": 260112384
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007443771313941826,
+      "loss": 1.5023,
+      "theoretical_loss": 4.245572213275978,
+      "tokens_seen": 260177920
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007443610832497493,
+      "loss": 1.6892,
+      "theoretical_loss": 4.245444702613971,
+      "tokens_seen": 260243456
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.000744345035105316,
+      "loss": 1.5438,
+      "theoretical_loss": 4.245317233046738,
+      "tokens_seen": 260308992
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007443289869608828,
+      "loss": 1.604,
+      "theoretical_loss": 4.2451898045506935,
+      "tokens_seen": 260374528
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007443129388164494,
+      "loss": 1.6275,
+      "theoretical_loss": 4.245062417102272,
+      "tokens_seen": 260440064
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007442968906720162,
+      "loss": 1.5233,
+      "theoretical_loss": 4.244935070677927,
+      "tokens_seen": 260505600
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007442808425275828,
+      "loss": 1.5382,
+      "theoretical_loss": 4.244807765254132,
+      "tokens_seen": 260571136
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007442647943831496,
+      "loss": 1.4302,
+      "theoretical_loss": 4.244680500807378,
+      "tokens_seen": 260636672
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007442487462387162,
+      "loss": 1.6044,
+      "theoretical_loss": 4.244553277314178,
+      "tokens_seen": 260702208
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.000744232698094283,
+      "loss": 1.5054,
+      "theoretical_loss": 4.244426094751063,
+      "tokens_seen": 260767744
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007442166499498496,
+      "loss": 1.608,
+      "theoretical_loss": 4.244298953094583,
+      "tokens_seen": 260833280
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007442006018054164,
+      "loss": 1.5169,
+      "theoretical_loss": 4.244171852321308,
+      "tokens_seen": 260898816
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.000744184553660983,
+      "loss": 1.6222,
+      "theoretical_loss": 4.244044792407828,
+      "tokens_seen": 260964352
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007441685055165498,
+      "loss": 1.5103,
+      "theoretical_loss": 4.24391777333075,
+      "tokens_seen": 261029888
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007441524573721164,
+      "loss": 1.6287,
+      "theoretical_loss": 4.243790795066703,
+      "tokens_seen": 261095424
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007441364092276832,
+      "loss": 1.7086,
+      "theoretical_loss": 4.243663857592332,
+      "tokens_seen": 261160960
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007441203610832498,
+      "loss": 1.7049,
+      "theoretical_loss": 4.243536960884305,
+      "tokens_seen": 261226496
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007441043129388166,
+      "loss": 1.6148,
+      "theoretical_loss": 4.243410104919307,
+      "tokens_seen": 261292032
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007440882647943832,
+      "loss": 1.5788,
+      "theoretical_loss": 4.243283289674042,
+      "tokens_seen": 261357568
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007440722166499499,
+      "loss": 1.5389,
+      "theoretical_loss": 4.243156515125234,
+      "tokens_seen": 261423104
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007440561685055166,
+      "loss": 1.5889,
+      "theoretical_loss": 4.243029781249625,
+      "tokens_seen": 261488640
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007440401203610833,
+      "loss": 1.5932,
+      "theoretical_loss": 4.2429030880239775,
+      "tokens_seen": 261554176
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00074402407221665,
+      "loss": 1.5755,
+      "theoretical_loss": 4.242776435425072,
+      "tokens_seen": 261619712
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007440080240722167,
+      "loss": 1.5043,
+      "theoretical_loss": 4.24264982342971,
+      "tokens_seen": 261685248
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007439919759277834,
+      "loss": 1.6232,
+      "theoretical_loss": 4.242523252014709,
+      "tokens_seen": 261750784
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007439759277833501,
+      "loss": 1.434,
+      "theoretical_loss": 4.242396721156908,
+      "tokens_seen": 261816320
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007439598796389168,
+      "loss": 1.5584,
+      "theoretical_loss": 4.242270230833164,
+      "tokens_seen": 261881856
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007439438314944835,
+      "loss": 1.6205,
+      "theoretical_loss": 4.242143781020353,
+      "tokens_seen": 261947392
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007439277833500501,
+      "loss": 1.6467,
+      "theoretical_loss": 4.24201737169537,
+      "tokens_seen": 262012928
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007439117352056169,
+      "loss": 1.6356,
+      "theoretical_loss": 4.24189100283513,
+      "tokens_seen": 262078464
+    },
+    {
+      "epoch": 0.08,
+      "objective/train/docs_used": 93300,
+      "objective/train/instantaneous_batch_size": 32,
+      "objective/train/instantaneous_microbatch_size": 32768,
+      "objective/train/original_loss": 1.5407991409301758,
+      "objective/train/theoretical_loss": 4.241764674416565,
+      "objective/train/tokens_used": 282604000,
+      "theoretical_loss": 4.241764674416565,
+      "tokens_seen": 262144000
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007438956870611835,
+      "loss": 1.5449,
+      "theoretical_loss": 4.241764674416565,
+      "tokens_seen": 262144000
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007438796389167503,
+      "loss": 1.5976,
+      "theoretical_loss": 4.241638386416628,
+      "tokens_seen": 262209536
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007438635907723169,
+      "loss": 1.6151,
+      "theoretical_loss": 4.241512138812288,
+      "tokens_seen": 262275072
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007438475426278837,
+      "loss": 1.5972,
+      "theoretical_loss": 4.2413859315805365,
+      "tokens_seen": 262340608
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007438314944834503,
+      "loss": 1.5658,
+      "theoretical_loss": 4.241259764698381,
+      "tokens_seen": 262406144
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007438154463390171,
+      "loss": 1.6079,
+      "theoretical_loss": 4.24113363814285,
+      "tokens_seen": 262471680
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007437993981945837,
+      "loss": 1.5915,
+      "theoretical_loss": 4.241007551890988,
+      "tokens_seen": 262537216
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007437833500501505,
+      "loss": 1.6023,
+      "theoretical_loss": 4.2408815059198615,
+      "tokens_seen": 262602752
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007437673019057171,
+      "loss": 1.7017,
+      "theoretical_loss": 4.240755500206554,
+      "tokens_seen": 262668288
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007437512537612839,
+      "loss": 1.4861,
+      "theoretical_loss": 4.240629534728167,
+      "tokens_seen": 262733824
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007437352056168505,
+      "loss": 1.5669,
+      "theoretical_loss": 4.240503609461822,
+      "tokens_seen": 262799360
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007437191574724173,
+      "loss": 1.5445,
+      "theoretical_loss": 4.24037772438466,
+      "tokens_seen": 262864896
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007437031093279839,
+      "loss": 1.5621,
+      "theoretical_loss": 4.240251879473839,
+      "tokens_seen": 262930432
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007436870611835507,
+      "loss": 1.5906,
+      "theoretical_loss": 4.240126074706536,
+      "tokens_seen": 262995968
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007436710130391173,
+      "loss": 1.5684,
+      "theoretical_loss": 4.240000310059948,
+      "tokens_seen": 263061504
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007436549648946841,
+      "loss": 1.6725,
+      "theoretical_loss": 4.239874585511288,
+      "tokens_seen": 263127040
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007436389167502507,
+      "loss": 1.6853,
+      "theoretical_loss": 4.239748901037791,
+      "tokens_seen": 263192576
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007436228686058175,
+      "loss": 1.5701,
+      "theoretical_loss": 4.2396232566167065,
+      "tokens_seen": 263258112
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007436068204613841,
+      "loss": 1.6427,
+      "theoretical_loss": 4.239497652225307,
+      "tokens_seen": 263323648
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007435907723169509,
+      "loss": 1.5741,
+      "theoretical_loss": 4.23937208784088,
+      "tokens_seen": 263389184
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007435747241725175,
+      "loss": 1.5592,
+      "theoretical_loss": 4.239246563440734,
+      "tokens_seen": 263454720
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007435586760280843,
+      "loss": 1.6337,
+      "theoretical_loss": 4.239121079002194,
+      "tokens_seen": 263520256
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.000743542627883651,
+      "loss": 1.6201,
+      "theoretical_loss": 4.238995634502606,
+      "tokens_seen": 263585792
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007435265797392177,
+      "loss": 1.5932,
+      "theoretical_loss": 4.238870229919331,
+      "tokens_seen": 263651328
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007435105315947844,
+      "loss": 1.5494,
+      "theoretical_loss": 4.2387448652297515,
+      "tokens_seen": 263716864
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007434944834503511,
+      "loss": 1.5638,
+      "theoretical_loss": 4.238619540411268,
+      "tokens_seen": 263782400
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007434784353059178,
+      "loss": 1.5212,
+      "theoretical_loss": 4.238494255441296,
+      "tokens_seen": 263847936
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007434623871614845,
+      "loss": 1.5204,
+      "theoretical_loss": 4.238369010297275,
+      "tokens_seen": 263913472
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007434463390170512,
+      "loss": 1.6772,
+      "theoretical_loss": 4.238243804956658,
+      "tokens_seen": 263979008
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007434302908726179,
+      "loss": 1.5227,
+      "theoretical_loss": 4.23811863939692,
+      "tokens_seen": 264044544
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007434142427281846,
+      "loss": 1.5964,
+      "theoretical_loss": 4.237993513595552,
+      "tokens_seen": 264110080
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007433981945837513,
+      "loss": 1.4756,
+      "theoretical_loss": 4.2378684275300635,
+      "tokens_seen": 264175616
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.000743382146439318,
+      "loss": 1.6068,
+      "theoretical_loss": 4.237743381177983,
+      "tokens_seen": 264241152
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007433660982948847,
+      "loss": 1.6288,
+      "theoretical_loss": 4.237618374516858,
+      "tokens_seen": 264306688
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007433500501504514,
+      "loss": 1.7239,
+      "theoretical_loss": 4.237493407524253,
+      "tokens_seen": 264372224
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007433340020060181,
+      "loss": 1.6119,
+      "theoretical_loss": 4.23736848017775,
+      "tokens_seen": 264437760
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007433179538615848,
+      "loss": 1.5926,
+      "theoretical_loss": 4.237243592454951,
+      "tokens_seen": 264503296
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007433019057171515,
+      "loss": 1.516,
+      "theoretical_loss": 4.2371187443334755,
+      "tokens_seen": 264568832
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007432858575727182,
+      "loss": 1.6483,
+      "theoretical_loss": 4.236993935790962,
+      "tokens_seen": 264634368
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007432698094282849,
+      "loss": 1.6067,
+      "theoretical_loss": 4.236869166805064,
+      "tokens_seen": 264699904
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007432537612838516,
+      "loss": 1.7355,
+      "theoretical_loss": 4.236744437353458,
+      "tokens_seen": 264765440
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007432377131394183,
+      "loss": 1.5652,
+      "theoretical_loss": 4.236619747413836,
+      "tokens_seen": 264830976
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.000743221664994985,
+      "loss": 1.4592,
+      "theoretical_loss": 4.236495096963906,
+      "tokens_seen": 264896512
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007432056168505517,
+      "loss": 1.62,
+      "theoretical_loss": 4.236370485981398,
+      "tokens_seen": 264962048
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007431895687061184,
+      "loss": 1.5871,
+      "theoretical_loss": 4.236245914444058,
+      "tokens_seen": 265027584
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007431735205616851,
+      "loss": 1.5937,
+      "theoretical_loss": 4.2361213823296495,
+      "tokens_seen": 265093120
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007431574724172518,
+      "loss": 1.37,
+      "theoretical_loss": 4.235996889615957,
+      "tokens_seen": 265158656
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007431414242728185,
+      "loss": 1.5311,
+      "theoretical_loss": 4.235872436280779,
+      "tokens_seen": 265224192
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007431253761283852,
+      "loss": 1.5256,
+      "theoretical_loss": 4.235748022301935,
+      "tokens_seen": 265289728
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007431093279839519,
+      "loss": 1.3591,
+      "theoretical_loss": 4.235623647657261,
+      "tokens_seen": 265355264
+    },
+    {
+      "epoch": 0.08,
+      "objective/train/docs_used": 94246,
+      "objective/train/instantaneous_batch_size": 32,
+      "objective/train/instantaneous_microbatch_size": 32768,
+      "objective/train/original_loss": 1.5605759620666504,
+      "objective/train/theoretical_loss": 4.235499312324611,
+      "objective/train/tokens_used": 285880800,
+      "theoretical_loss": 4.235499312324611,
+      "tokens_seen": 265420800
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007430932798395186,
+      "loss": 1.5757,
+      "theoretical_loss": 4.235499312324611,
+      "tokens_seen": 265420800
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007430772316950853,
+      "loss": 1.5962,
+      "theoretical_loss": 4.235375016281858,
+      "tokens_seen": 265486336
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.000743061183550652,
+      "loss": 1.6213,
+      "theoretical_loss": 4.235250759506892,
+      "tokens_seen": 265551872
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007430451354062187,
+      "loss": 1.6001,
+      "theoretical_loss": 4.235126541977621,
+      "tokens_seen": 265617408
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007430290872617854,
+      "loss": 1.6254,
+      "theoretical_loss": 4.2350023636719705,
+      "tokens_seen": 265682944
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007430130391173521,
+      "loss": 1.5425,
+      "theoretical_loss": 4.234878224567885,
+      "tokens_seen": 265748480
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007429969909729188,
+      "loss": 1.5673,
+      "theoretical_loss": 4.2347541246433265,
+      "tokens_seen": 265814016
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007429809428284855,
+      "loss": 1.7503,
+      "theoretical_loss": 4.234630063876273,
+      "tokens_seen": 265879552
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007429648946840523,
+      "loss": 1.7047,
+      "theoretical_loss": 4.234506042244723,
+      "tokens_seen": 265945088
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007429488465396189,
+      "loss": 1.6124,
+      "theoretical_loss": 4.234382059726691,
+      "tokens_seen": 266010624
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007429327983951857,
+      "loss": 1.534,
+      "theoretical_loss": 4.234258116300211,
+      "tokens_seen": 266076160
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007429167502507523,
+      "loss": 1.5566,
+      "theoretical_loss": 4.234134211943332,
+      "tokens_seen": 266141696
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.000742900702106319,
+      "loss": 1.5992,
+      "theoretical_loss": 4.234010346634123,
+      "tokens_seen": 266207232
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007428846539618857,
+      "loss": 1.7117,
+      "theoretical_loss": 4.23388652035067,
+      "tokens_seen": 266272768
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007428686058174525,
+      "loss": 1.6602,
+      "theoretical_loss": 4.233762733071076,
+      "tokens_seen": 266338304
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007428525576730191,
+      "loss": 1.5712,
+      "theoretical_loss": 4.233638984773464,
+      "tokens_seen": 266403840
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007428365095285859,
+      "loss": 1.5781,
+      "theoretical_loss": 4.2335152754359715,
+      "tokens_seen": 266469376
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007428204613841525,
+      "loss": 1.6273,
+      "theoretical_loss": 4.233391605036756,
+      "tokens_seen": 266534912
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007428044132397193,
+      "loss": 1.5873,
+      "theoretical_loss": 4.233267973553991,
+      "tokens_seen": 266600448
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007427883650952859,
+      "loss": 1.6577,
+      "theoretical_loss": 4.233144380965869,
+      "tokens_seen": 266665984
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007427723169508527,
+      "loss": 1.6224,
+      "theoretical_loss": 4.2330208272505985,
+      "tokens_seen": 266731520
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007427562688064193,
+      "loss": 1.609,
+      "theoretical_loss": 4.232897312386408,
+      "tokens_seen": 266797056
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007427402206619861,
+      "loss": 1.5896,
+      "theoretical_loss": 4.232773836351541,
+      "tokens_seen": 266862592
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007427241725175527,
+      "loss": 1.5432,
+      "theoretical_loss": 4.2326503991242586,
+      "tokens_seen": 266928128
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007427081243731195,
+      "loss": 1.6541,
+      "theoretical_loss": 4.232527000682843,
+      "tokens_seen": 266993664
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007426920762286861,
+      "loss": 1.5926,
+      "theoretical_loss": 4.232403641005589,
+      "tokens_seen": 267059200
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007426760280842528,
+      "loss": 1.5599,
+      "theoretical_loss": 4.232280320070812,
+      "tokens_seen": 267124736
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007426599799398195,
+      "loss": 1.5699,
+      "theoretical_loss": 4.232157037856844,
+      "tokens_seen": 267190272
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007426439317953862,
+      "loss": 1.5946,
+      "theoretical_loss": 4.232033794342033,
+      "tokens_seen": 267255808
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007426278836509529,
+      "loss": 1.526,
+      "theoretical_loss": 4.231910589504748,
+      "tokens_seen": 267321344
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007426118355065196,
+      "loss": 1.5774,
+      "theoretical_loss": 4.2317874233233725,
+      "tokens_seen": 267386880
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007425957873620863,
+      "loss": 1.6703,
+      "theoretical_loss": 4.231664295776307,
+      "tokens_seen": 267452416
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.000742579739217653,
+      "loss": 1.7569,
+      "theoretical_loss": 4.231541206841971,
+      "tokens_seen": 267517952
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007425636910732197,
+      "loss": 1.5637,
+      "theoretical_loss": 4.2314181564988,
+      "tokens_seen": 267583488
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007425476429287864,
+      "loss": 1.6282,
+      "theoretical_loss": 4.23129514472525,
+      "tokens_seen": 267649024
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.000742531594784353,
+      "loss": 1.5033,
+      "theoretical_loss": 4.231172171499789,
+      "tokens_seen": 267714560
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007425155466399198,
+      "loss": 1.5924,
+      "theoretical_loss": 4.231049236800907,
+      "tokens_seen": 267780096
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007424994984954864,
+      "loss": 1.5738,
+      "theoretical_loss": 4.230926340607109,
+      "tokens_seen": 267845632
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007424834503510532,
+      "loss": 1.6115,
+      "theoretical_loss": 4.230803482896918,
+      "tokens_seen": 267911168
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007424674022066198,
+      "loss": 1.6685,
+      "theoretical_loss": 4.230680663648872,
+      "tokens_seen": 267976704
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007424513540621866,
+      "loss": 1.5675,
+      "theoretical_loss": 4.230557882841531,
+      "tokens_seen": 268042240
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007424353059177532,
+      "loss": 1.6846,
+      "theoretical_loss": 4.2304351404534675,
+      "tokens_seen": 268107776
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00074241925777332,
+      "loss": 1.6865,
+      "theoretical_loss": 4.230312436463272,
+      "tokens_seen": 268173312
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007424032096288866,
+      "loss": 1.6701,
+      "theoretical_loss": 4.230189770849556,
+      "tokens_seen": 268238848
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007423871614844534,
+      "loss": 1.6526,
+      "theoretical_loss": 4.230067143590943,
+      "tokens_seen": 268304384
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00074237111334002,
+      "loss": 1.6089,
+      "theoretical_loss": 4.229944554666076,
+      "tokens_seen": 268369920
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007423550651955868,
+      "loss": 1.5078,
+      "theoretical_loss": 4.229822004053617,
+      "tokens_seen": 268435456
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007423390170511535,
+      "loss": 1.6715,
+      "theoretical_loss": 4.229699491732242,
+      "tokens_seen": 268500992
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007423229689067202,
+      "loss": 1.5602,
+      "theoretical_loss": 4.2295770176806435,
+      "tokens_seen": 268566528
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007423069207622869,
+      "loss": 1.5834,
+      "theoretical_loss": 4.229454581877535,
+      "tokens_seen": 268632064
+    },
+    {
+      "epoch": 0.08,
+      "objective/train/docs_used": 95521,
+      "objective/train/instantaneous_batch_size": 32,
+      "objective/train/instantaneous_microbatch_size": 32768,
+      "objective/train/original_loss": 1.6754567623138428,
+      "objective/train/theoretical_loss": 4.229332184301644,
+      "objective/train/tokens_used": 289157600,
+      "theoretical_loss": 4.229332184301644,
+      "tokens_seen": 268697600
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007422908726178536,
+      "loss": 1.6305,
+      "theoretical_loss": 4.229332184301644,
+      "tokens_seen": 268697600
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007422748244734203,
+      "loss": 1.5105,
+      "theoretical_loss": 4.229209824931714,
+      "tokens_seen": 268763136
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.000742258776328987,
+      "loss": 1.5247,
+      "theoretical_loss": 4.22908750374651,
+      "tokens_seen": 268828672
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007422427281845537,
+      "loss": 1.6184,
+      "theoretical_loss": 4.22896522072481,
+      "tokens_seen": 268894208
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007422266800401204,
+      "loss": 1.5818,
+      "theoretical_loss": 4.22884297584541,
+      "tokens_seen": 268959744
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007422106318956871,
+      "loss": 1.5809,
+      "theoretical_loss": 4.228720769087124,
+      "tokens_seen": 269025280
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007421945837512538,
+      "loss": 1.558,
+      "theoretical_loss": 4.22859860042878,
+      "tokens_seen": 269090816
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007421785356068205,
+      "loss": 1.6743,
+      "theoretical_loss": 4.2284764698492285,
+      "tokens_seen": 269156352
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007421624874623872,
+      "loss": 1.5943,
+      "theoretical_loss": 4.22835437732733,
+      "tokens_seen": 269221888
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007421464393179539,
+      "loss": 1.6661,
+      "theoretical_loss": 4.228232322841967,
+      "tokens_seen": 269287424
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007421303911735206,
+      "loss": 1.5661,
+      "theoretical_loss": 4.228110306372036,
+      "tokens_seen": 269352960
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007421143430290873,
+      "loss": 1.7021,
+      "theoretical_loss": 4.227988327896453,
+      "tokens_seen": 269418496
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.000742098294884654,
+      "loss": 1.5548,
+      "theoretical_loss": 4.2278663873941476,
+      "tokens_seen": 269484032
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007420822467402207,
+      "loss": 1.5474,
+      "theoretical_loss": 4.227744484844068,
+      "tokens_seen": 269549568
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007420661985957874,
+      "loss": 1.5843,
+      "theoretical_loss": 4.227622620225182,
+      "tokens_seen": 269615104
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007420501504513541,
+      "loss": 1.6371,
+      "theoretical_loss": 4.227500793516466,
+      "tokens_seen": 269680640
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007420341023069208,
+      "loss": 1.645,
+      "theoretical_loss": 4.227379004696923,
+      "tokens_seen": 269746176
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007420180541624875,
+      "loss": 1.5229,
+      "theoretical_loss": 4.227257253745566,
+      "tokens_seen": 269811712
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007420020060180542,
+      "loss": 1.5522,
+      "theoretical_loss": 4.227135540641426,
+      "tokens_seen": 269877248
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007419859578736209,
+      "loss": 1.6947,
+      "theoretical_loss": 4.227013865363553,
+      "tokens_seen": 269942784
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007419699097291876,
+      "loss": 1.6148,
+      "theoretical_loss": 4.226892227891012,
+      "tokens_seen": 270008320
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007419538615847543,
+      "loss": 1.6242,
+      "theoretical_loss": 4.2267706282028845,
+      "tokens_seen": 270073856
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.000741937813440321,
+      "loss": 1.585,
+      "theoretical_loss": 4.226649066278269,
+      "tokens_seen": 270139392
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007419217652958877,
+      "loss": 1.5902,
+      "theoretical_loss": 4.226527542096282,
+      "tokens_seen": 270204928
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007419057171514544,
+      "loss": 1.4568,
+      "theoretical_loss": 4.226406055636053,
+      "tokens_seen": 270270464
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007418896690070211,
+      "loss": 1.4898,
+      "theoretical_loss": 4.226284606876731,
+      "tokens_seen": 270336000
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007418736208625878,
+      "loss": 1.6609,
+      "theoretical_loss": 4.226163195797481,
+      "tokens_seen": 270401536
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007418575727181545,
+      "loss": 1.5374,
+      "theoretical_loss": 4.226041822377486,
+      "tokens_seen": 270467072
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007418415245737212,
+      "loss": 1.5465,
+      "theoretical_loss": 4.225920486595941,
+      "tokens_seen": 270532608
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007418254764292879,
+      "loss": 1.6082,
+      "theoretical_loss": 4.225799188432063,
+      "tokens_seen": 270598144
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007418094282848546,
+      "loss": 1.5496,
+      "theoretical_loss": 4.225677927865083,
+      "tokens_seen": 270663680
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007417933801404213,
+      "loss": 1.6383,
+      "theoretical_loss": 4.225556704874247,
+      "tokens_seen": 270729216
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.000741777331995988,
+      "loss": 1.5524,
+      "theoretical_loss": 4.22543551943882,
+      "tokens_seen": 270794752
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007417612838515547,
+      "loss": 1.5021,
+      "theoretical_loss": 4.225314371538083,
+      "tokens_seen": 270860288
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007417452357071214,
+      "loss": 1.6118,
+      "theoretical_loss": 4.225193261151332,
+      "tokens_seen": 270925824
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007417291875626881,
+      "loss": 1.6192,
+      "theoretical_loss": 4.22507218825788,
+      "tokens_seen": 270991360
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007417131394182548,
+      "loss": 1.4778,
+      "theoretical_loss": 4.224951152837058,
+      "tokens_seen": 271056896
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007416970912738215,
+      "loss": 1.5281,
+      "theoretical_loss": 4.2248301548682115,
+      "tokens_seen": 271122432
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007416810431293882,
+      "loss": 1.6422,
+      "theoretical_loss": 4.2247091943307025,
+      "tokens_seen": 271187968
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.000741664994984955,
+      "loss": 1.5512,
+      "theoretical_loss": 4.224588271203911,
+      "tokens_seen": 271253504
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007416489468405216,
+      "loss": 1.6742,
+      "theoretical_loss": 4.22446738546723,
+      "tokens_seen": 271319040
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007416328986960883,
+      "loss": 1.6599,
+      "theoretical_loss": 4.224346537100074,
+      "tokens_seen": 271384576
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.000741616850551655,
+      "loss": 1.634,
+      "theoretical_loss": 4.22422572608187,
+      "tokens_seen": 271450112
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007416008024072218,
+      "loss": 1.6747,
+      "theoretical_loss": 4.224104952392061,
+      "tokens_seen": 271515648
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007415847542627884,
+      "loss": 1.5533,
+      "theoretical_loss": 4.223984216010107,
+      "tokens_seen": 271581184
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007415687061183552,
+      "loss": 1.4202,
+      "theoretical_loss": 4.223863516915486,
+      "tokens_seen": 271646720
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007415526579739218,
+      "loss": 1.4197,
+      "theoretical_loss": 4.223742855087691,
+      "tokens_seen": 271712256
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007415366098294886,
+      "loss": 1.5972,
+      "theoretical_loss": 4.223622230506231,
+      "tokens_seen": 271777792
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007415205616850552,
+      "loss": 1.5212,
+      "theoretical_loss": 4.22350164315063,
+      "tokens_seen": 271843328
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.000741504513540622,
+      "loss": 1.57,
+      "theoretical_loss": 4.22338109300043,
+      "tokens_seen": 271908864
+    },
+    {
+      "epoch": 0.08,
+      "objective/train/docs_used": 96402,
+      "objective/train/instantaneous_batch_size": 32,
+      "objective/train/instantaneous_microbatch_size": 32768,
+      "objective/train/original_loss": 1.7149293422698975,
+      "objective/train/theoretical_loss": 4.22326058003519,
+      "objective/train/tokens_used": 292434400,
+      "theoretical_loss": 4.22326058003519,
+      "tokens_seen": 271974400
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007414884653961886,
+      "loss": 1.6574,
+      "theoretical_loss": 4.22326058003519,
+      "tokens_seen": 271974400
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007414724172517554,
+      "loss": 1.6241,
+      "theoretical_loss": 4.223140104234482,
+      "tokens_seen": 272039936
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.000741456369107322,
+      "loss": 1.6037,
+      "theoretical_loss": 4.2230196655778975,
+      "tokens_seen": 272105472
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007414403209628888,
+      "loss": 1.6278,
+      "theoretical_loss": 4.222899264045042,
+      "tokens_seen": 272171008
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007414242728184554,
+      "loss": 1.6132,
+      "theoretical_loss": 4.2227788996155375,
+      "tokens_seen": 272236544
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007414082246740222,
+      "loss": 1.5246,
+      "theoretical_loss": 4.222658572269022,
+      "tokens_seen": 272302080
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007413921765295888,
+      "loss": 1.648,
+      "theoretical_loss": 4.222538281985151,
+      "tokens_seen": 272367616
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007413761283851556,
+      "loss": 1.6319,
+      "theoretical_loss": 4.2224180287435935,
+      "tokens_seen": 272433152
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007413600802407222,
+      "loss": 1.6037,
+      "theoretical_loss": 4.222297812524037,
+      "tokens_seen": 272498688
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.000741344032096289,
+      "loss": 1.4991,
+      "theoretical_loss": 4.222177633306183,
+      "tokens_seen": 272564224
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007413279839518556,
+      "loss": 1.6058,
+      "theoretical_loss": 4.222057491069751,
+      "tokens_seen": 272629760
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007413119358074224,
+      "loss": 1.5762,
+      "theoretical_loss": 4.221937385794474,
+      "tokens_seen": 272695296
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.000741295887662989,
+      "loss": 1.5151,
+      "theoretical_loss": 4.221817317460104,
+      "tokens_seen": 272760832
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007412798395185557,
+      "loss": 1.5332,
+      "theoretical_loss": 4.221697286046407,
+      "tokens_seen": 272826368
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007412637913741224,
+      "loss": 1.556,
+      "theoretical_loss": 4.221577291533165,
+      "tokens_seen": 272891904
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007412477432296891,
+      "loss": 1.5723,
+      "theoretical_loss": 4.221457333900176,
+      "tokens_seen": 272957440
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007412316950852558,
+      "loss": 1.5868,
+      "theoretical_loss": 4.221337413127254,
+      "tokens_seen": 273022976
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007412156469408225,
+      "loss": 1.5757,
+      "theoretical_loss": 4.221217529194231,
+      "tokens_seen": 273088512
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007411995987963892,
+      "loss": 1.5992,
+      "theoretical_loss": 4.22109768208095,
+      "tokens_seen": 273154048
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007411835506519559,
+      "loss": 1.6333,
+      "theoretical_loss": 4.220977871767275,
+      "tokens_seen": 273219584
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007411675025075226,
+      "loss": 1.6518,
+      "theoretical_loss": 4.220858098233082,
+      "tokens_seen": 273285120
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007411514543630893,
+      "loss": 1.5201,
+      "theoretical_loss": 4.220738361458266,
+      "tokens_seen": 273350656
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.000741135406218656,
+      "loss": 1.5227,
+      "theoretical_loss": 4.220618661422735,
+      "tokens_seen": 273416192
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007411193580742227,
+      "loss": 1.5981,
+      "theoretical_loss": 4.2204989981064145,
+      "tokens_seen": 273481728
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007411033099297894,
+      "loss": 1.5229,
+      "theoretical_loss": 4.220379371489246,
+      "tokens_seen": 273547264
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007410872617853561,
+      "loss": 1.4596,
+      "theoretical_loss": 4.220259781551185,
+      "tokens_seen": 273612800
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007410712136409228,
+      "loss": 1.53,
+      "theoretical_loss": 4.2201402282722045,
+      "tokens_seen": 273678336
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007410551654964895,
+      "loss": 1.4904,
+      "theoretical_loss": 4.2200207116322925,
+      "tokens_seen": 273743872
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007410391173520562,
+      "loss": 1.7368,
+      "theoretical_loss": 4.219901231611452,
+      "tokens_seen": 273809408
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007410230692076229,
+      "loss": 1.5563,
+      "theoretical_loss": 4.219781788189703,
+      "tokens_seen": 273874944
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007410070210631896,
+      "loss": 1.5882,
+      "theoretical_loss": 4.2196623813470815,
+      "tokens_seen": 273940480
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007409909729187563,
+      "loss": 1.6127,
+      "theoretical_loss": 4.219543011063637,
+      "tokens_seen": 274006016
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.000740974924774323,
+      "loss": 1.5707,
+      "theoretical_loss": 4.219423677319437,
+      "tokens_seen": 274071552
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007409588766298897,
+      "loss": 1.5745,
+      "theoretical_loss": 4.219304380094562,
+      "tokens_seen": 274137088
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007409428284854564,
+      "loss": 1.5717,
+      "theoretical_loss": 4.2191851193691114,
+      "tokens_seen": 274202624
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007409267803410231,
+      "loss": 1.6123,
+      "theoretical_loss": 4.219065895123197,
+      "tokens_seen": 274268160
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007409107321965898,
+      "loss": 1.6337,
+      "theoretical_loss": 4.218946707336949,
+      "tokens_seen": 274333696
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007408946840521565,
+      "loss": 1.5212,
+      "theoretical_loss": 4.2188275559905115,
+      "tokens_seen": 274399232
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007408786359077232,
+      "loss": 1.5648,
+      "theoretical_loss": 4.218708441064044,
+      "tokens_seen": 274464768
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007408625877632899,
+      "loss": 1.6287,
+      "theoretical_loss": 4.218589362537723,
+      "tokens_seen": 274530304
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007408465396188566,
+      "loss": 1.5189,
+      "theoretical_loss": 4.218470320391739,
+      "tokens_seen": 274595840
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007408304914744233,
+      "loss": 1.7048,
+      "theoretical_loss": 4.218351314606298,
+      "tokens_seen": 274661376
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00074081444332999,
+      "loss": 1.5479,
+      "theoretical_loss": 4.218232345161622,
+      "tokens_seen": 274726912
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007407983951855567,
+      "loss": 1.5818,
+      "theoretical_loss": 4.21811341203795,
+      "tokens_seen": 274792448
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007407823470411234,
+      "loss": 1.6653,
+      "theoretical_loss": 4.217994515215534,
+      "tokens_seen": 274857984
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007407662988966901,
+      "loss": 1.5963,
+      "theoretical_loss": 4.217875654674642,
+      "tokens_seen": 274923520
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007407502507522568,
+      "loss": 1.6549,
+      "theoretical_loss": 4.217756830395559,
+      "tokens_seen": 274989056
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007407342026078235,
+      "loss": 1.483,
+      "theoretical_loss": 4.217638042358583,
+      "tokens_seen": 275054592
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007407181544633902,
+      "loss": 1.5156,
+      "theoretical_loss": 4.21751929054403,
+      "tokens_seen": 275120128
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007407021063189569,
+      "loss": 1.6518,
+      "theoretical_loss": 4.2174005749322285,
+      "tokens_seen": 275185664
+    },
+    {
+      "epoch": 0.08,
+      "objective/train/docs_used": 97580,
+      "objective/train/instantaneous_batch_size": 32,
+      "objective/train/instantaneous_microbatch_size": 32768,
+      "objective/train/original_loss": 1.5536854267120361,
+      "objective/train/theoretical_loss": 4.2172818955035245,
+      "objective/train/tokens_used": 295711200,
+      "theoretical_loss": 4.2172818955035245,
+      "tokens_seen": 275251200
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007406860581745236,
+      "loss": 1.6128,
+      "theoretical_loss": 4.2172818955035245,
+      "tokens_seen": 275251200
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007406700100300903,
+      "loss": 1.5872,
+      "theoretical_loss": 4.217163252238279,
+      "tokens_seen": 275316736
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.000740653961885657,
+      "loss": 1.6388,
+      "theoretical_loss": 4.217044645116869,
+      "tokens_seen": 275382272
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007406379137412237,
+      "loss": 1.5348,
+      "theoretical_loss": 4.216926074119684,
+      "tokens_seen": 275447808
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007406218655967904,
+      "loss": 1.5398,
+      "theoretical_loss": 4.216807539227132,
+      "tokens_seen": 275513344
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007406058174523571,
+      "loss": 1.5788,
+      "theoretical_loss": 4.2166890404196335,
+      "tokens_seen": 275578880
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007405897693079238,
+      "loss": 1.5592,
+      "theoretical_loss": 4.216570577677627,
+      "tokens_seen": 275644416
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007405737211634905,
+      "loss": 1.4793,
+      "theoretical_loss": 4.216452150981566,
+      "tokens_seen": 275709952
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007405576730190572,
+      "loss": 1.587,
+      "theoretical_loss": 4.216333760311916,
+      "tokens_seen": 275775488
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007405416248746239,
+      "loss": 1.6332,
+      "theoretical_loss": 4.216215405649161,
+      "tokens_seen": 275841024
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007405255767301906,
+      "loss": 1.5481,
+      "theoretical_loss": 4.2160970869738,
+      "tokens_seen": 275906560
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007405095285857573,
+      "loss": 1.4537,
+      "theoretical_loss": 4.215978804266345,
+      "tokens_seen": 275972096
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.000740493480441324,
+      "loss": 1.6983,
+      "theoretical_loss": 4.215860557507324,
+      "tokens_seen": 276037632
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007404774322968907,
+      "loss": 1.6022,
+      "theoretical_loss": 4.215742346677283,
+      "tokens_seen": 276103168
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007404613841524574,
+      "loss": 1.6303,
+      "theoretical_loss": 4.215624171756779,
+      "tokens_seen": 276168704
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007404453360080241,
+      "loss": 1.5424,
+      "theoretical_loss": 4.215506032726386,
+      "tokens_seen": 276234240
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007404292878635908,
+      "loss": 1.4775,
+      "theoretical_loss": 4.215387929566694,
+      "tokens_seen": 276299776
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007404132397191575,
+      "loss": 1.6119,
+      "theoretical_loss": 4.215269862258307,
+      "tokens_seen": 276365312
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007403971915747242,
+      "loss": 1.5651,
+      "theoretical_loss": 4.215151830781843,
+      "tokens_seen": 276430848
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007403811434302909,
+      "loss": 1.6032,
+      "theoretical_loss": 4.2150338351179375,
+      "tokens_seen": 276496384
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007403650952858576,
+      "loss": 1.6364,
+      "theoretical_loss": 4.21491587524724,
+      "tokens_seen": 276561920
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007403490471414243,
+      "loss": 1.7387,
+      "theoretical_loss": 4.214797951150415,
+      "tokens_seen": 276627456
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.000740332998996991,
+      "loss": 1.6563,
+      "theoretical_loss": 4.214680062808142,
+      "tokens_seen": 276692992
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007403169508525577,
+      "loss": 1.5149,
+      "theoretical_loss": 4.214562210201114,
+      "tokens_seen": 276758528
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007403009027081244,
+      "loss": 1.596,
+      "theoretical_loss": 4.214444393310042,
+      "tokens_seen": 276824064
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007402848545636911,
+      "loss": 1.5568,
+      "theoretical_loss": 4.2143266121156495,
+      "tokens_seen": 276889600
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007402688064192578,
+      "loss": 1.646,
+      "theoretical_loss": 4.2142088665986766,
+      "tokens_seen": 276955136
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007402527582748245,
+      "loss": 1.6845,
+      "theoretical_loss": 4.214091156739878,
+      "tokens_seen": 277020672
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007402367101303912,
+      "loss": 1.5336,
+      "theoretical_loss": 4.213973482520021,
+      "tokens_seen": 277086208
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007402206619859579,
+      "loss": 1.6861,
+      "theoretical_loss": 4.213855843919891,
+      "tokens_seen": 277151744
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007402046138415247,
+      "loss": 1.5853,
+      "theoretical_loss": 4.2137382409202875,
+      "tokens_seen": 277217280
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007401885656970913,
+      "loss": 1.5205,
+      "theoretical_loss": 4.2136206735020245,
+      "tokens_seen": 277282816
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.000740172517552658,
+      "loss": 1.7472,
+      "theoretical_loss": 4.21350314164593,
+      "tokens_seen": 277348352
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007401564694082247,
+      "loss": 1.5482,
+      "theoretical_loss": 4.213385645332848,
+      "tokens_seen": 277413888
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007401404212637915,
+      "loss": 1.5338,
+      "theoretical_loss": 4.213268184543638,
+      "tokens_seen": 277479424
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007401243731193581,
+      "loss": 1.6345,
+      "theoretical_loss": 4.213150759259172,
+      "tokens_seen": 277544960
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007401083249749249,
+      "loss": 1.3764,
+      "theoretical_loss": 4.213033369460339,
+      "tokens_seen": 277610496
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007400922768304915,
+      "loss": 1.5424,
+      "theoretical_loss": 4.212916015128041,
+      "tokens_seen": 277676032
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007400762286860583,
+      "loss": 1.6079,
+      "theoretical_loss": 4.212798696243197,
+      "tokens_seen": 277741568
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007400601805416249,
+      "loss": 1.5246,
+      "theoretical_loss": 4.212681412786738,
+      "tokens_seen": 277807104
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007400441323971917,
+      "loss": 1.7072,
+      "theoretical_loss": 4.212564164739613,
+      "tokens_seen": 277872640
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007400280842527583,
+      "loss": 1.5011,
+      "theoretical_loss": 4.212446952082783,
+      "tokens_seen": 277938176
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007400120361083251,
+      "loss": 1.698,
+      "theoretical_loss": 4.212329774797225,
+      "tokens_seen": 278003712
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007399959879638917,
+      "loss": 1.6597,
+      "theoretical_loss": 4.212212632863931,
+      "tokens_seen": 278069248
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007399799398194585,
+      "loss": 1.4884,
+      "theoretical_loss": 4.212095526263905,
+      "tokens_seen": 278134784
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007399638916750251,
+      "loss": 1.5212,
+      "theoretical_loss": 4.211978454978171,
+      "tokens_seen": 278200320
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007399478435305919,
+      "loss": 1.6093,
+      "theoretical_loss": 4.211861418987762,
+      "tokens_seen": 278265856
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007399317953861585,
+      "loss": 1.6447,
+      "theoretical_loss": 4.211744418273728,
+      "tokens_seen": 278331392
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007399157472417253,
+      "loss": 1.5559,
+      "theoretical_loss": 4.211627452817136,
+      "tokens_seen": 278396928
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007398996990972919,
+      "loss": 1.501,
+      "theoretical_loss": 4.211510522599063,
+      "tokens_seen": 278462464
+    },
+    {
+      "epoch": 0.08,
+      "objective/train/docs_used": 98403,
+      "objective/train/instantaneous_batch_size": 32,
+      "objective/train/instantaneous_microbatch_size": 32768,
+      "objective/train/original_loss": 1.496962070465088,
+      "objective/train/theoretical_loss": 4.2113936276006045,
+      "objective/train/tokens_used": 298988000,
+      "theoretical_loss": 4.2113936276006045,
+      "tokens_seen": 278528000
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007398836509528586,
+      "loss": 1.5531,
+      "theoretical_loss": 4.2113936276006045,
+      "tokens_seen": 278528000
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007398676028084253,
+      "loss": 1.4953,
+      "theoretical_loss": 4.211276767802868,
+      "tokens_seen": 278593536
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.000739851554663992,
+      "loss": 1.6817,
+      "theoretical_loss": 4.211159943186978,
+      "tokens_seen": 278659072
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007398355065195587,
+      "loss": 1.5426,
+      "theoretical_loss": 4.211043153734071,
+      "tokens_seen": 278724608
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007398194583751254,
+      "loss": 1.504,
+      "theoretical_loss": 4.2109263994253,
+      "tokens_seen": 278790144
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007398034102306921,
+      "loss": 1.5324,
+      "theoretical_loss": 4.21080968024183,
+      "tokens_seen": 278855680
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007397873620862588,
+      "loss": 1.5969,
+      "theoretical_loss": 4.210692996164845,
+      "tokens_seen": 278921216
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007397713139418255,
+      "loss": 1.5886,
+      "theoretical_loss": 4.210576347175539,
+      "tokens_seen": 278986752
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007397552657973922,
+      "loss": 1.5332,
+      "theoretical_loss": 4.210459733255123,
+      "tokens_seen": 279052288
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007397392176529589,
+      "loss": 1.7381,
+      "theoretical_loss": 4.2103431543848195,
+      "tokens_seen": 279117824
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007397231695085256,
+      "loss": 1.5301,
+      "theoretical_loss": 4.210226610545872,
+      "tokens_seen": 279183360
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007397071213640923,
+      "loss": 1.4012,
+      "theoretical_loss": 4.21011010171953,
+      "tokens_seen": 279248896
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.000739691073219659,
+      "loss": 1.5633,
+      "theoretical_loss": 4.209993627887064,
+      "tokens_seen": 279314432
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007396750250752257,
+      "loss": 1.4734,
+      "theoretical_loss": 4.209877189029756,
+      "tokens_seen": 279379968
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007396589769307924,
+      "loss": 1.4957,
+      "theoretical_loss": 4.209760785128903,
+      "tokens_seen": 279445504
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007396429287863591,
+      "loss": 1.5449,
+      "theoretical_loss": 4.2096444161658155,
+      "tokens_seen": 279511040
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007396268806419258,
+      "loss": 1.4702,
+      "theoretical_loss": 4.209528082121819,
+      "tokens_seen": 279576576
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007396108324974925,
+      "loss": 1.4972,
+      "theoretical_loss": 4.209411782978256,
+      "tokens_seen": 279642112
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007395947843530592,
+      "loss": 1.6041,
+      "theoretical_loss": 4.209295518716477,
+      "tokens_seen": 279707648
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007395787362086259,
+      "loss": 1.6686,
+      "theoretical_loss": 4.209179289317854,
+      "tokens_seen": 279773184
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007395626880641926,
+      "loss": 1.4353,
+      "theoretical_loss": 4.209063094763767,
+      "tokens_seen": 279838720
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007395466399197593,
+      "loss": 1.4785,
+      "theoretical_loss": 4.208946935035616,
+      "tokens_seen": 279904256
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.000739530591775326,
+      "loss": 1.6187,
+      "theoretical_loss": 4.2088308101148115,
+      "tokens_seen": 279969792
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007395145436308927,
+      "loss": 1.5968,
+      "theoretical_loss": 4.208714719982778,
+      "tokens_seen": 280035328
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007394984954864594,
+      "loss": 1.6957,
+      "theoretical_loss": 4.208598664620958,
+      "tokens_seen": 280100864
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007394824473420261,
+      "loss": 1.7062,
+      "theoretical_loss": 4.208482644010804,
+      "tokens_seen": 280166400
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007394663991975928,
+      "loss": 1.4989,
+      "theoretical_loss": 4.2083666581337855,
+      "tokens_seen": 280231936
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007394503510531595,
+      "loss": 1.4967,
+      "theoretical_loss": 4.208250706971385,
+      "tokens_seen": 280297472
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007394343029087262,
+      "loss": 1.6891,
+      "theoretical_loss": 4.208134790505099,
+      "tokens_seen": 280363008
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007394182547642929,
+      "loss": 1.6591,
+      "theoretical_loss": 4.20801890871644,
+      "tokens_seen": 280428544
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0007394022066198596,
+      "loss": 1.7174,
+      "theoretical_loss": 4.207903061586931,
+      "tokens_seen": 280494080
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007393861584754263,
+      "loss": 1.3745,
+      "theoretical_loss": 4.207787249098113,
+      "tokens_seen": 280559616
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.000739370110330993,
+      "loss": 1.5797,
+      "theoretical_loss": 4.20767147123154,
+      "tokens_seen": 280625152
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007393540621865597,
+      "loss": 1.6767,
+      "theoretical_loss": 4.20755572796878,
+      "tokens_seen": 280690688
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007393380140421264,
+      "loss": 1.5703,
+      "theoretical_loss": 4.207440019291413,
+      "tokens_seen": 280756224
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007393219658976931,
+      "loss": 1.543,
+      "theoretical_loss": 4.2073243451810365,
+      "tokens_seen": 280821760
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007393059177532598,
+      "loss": 1.583,
+      "theoretical_loss": 4.20720870561926,
+      "tokens_seen": 280887296
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007392898696088265,
+      "loss": 1.4603,
+      "theoretical_loss": 4.2070931005877075,
+      "tokens_seen": 280952832
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007392738214643932,
+      "loss": 1.6722,
+      "theoretical_loss": 4.206977530068017,
+      "tokens_seen": 281018368
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007392577733199599,
+      "loss": 1.5182,
+      "theoretical_loss": 4.206861994041842,
+      "tokens_seen": 281083904
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007392417251755266,
+      "loss": 1.5638,
+      "theoretical_loss": 4.206746492490847,
+      "tokens_seen": 281149440
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007392256770310933,
+      "loss": 1.6387,
+      "theoretical_loss": 4.206631025396714,
+      "tokens_seen": 281214976
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.00073920962888666,
+      "loss": 1.5261,
+      "theoretical_loss": 4.206515592741135,
+      "tokens_seen": 281280512
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007391935807422267,
+      "loss": 1.597,
+      "theoretical_loss": 4.20640019450582,
+      "tokens_seen": 281346048
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007391775325977934,
+      "loss": 1.6273,
+      "theoretical_loss": 4.206284830672491,
+      "tokens_seen": 281411584
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007391614844533601,
+      "loss": 1.6355,
+      "theoretical_loss": 4.206169501222885,
+      "tokens_seen": 281477120
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007391454363089268,
+      "loss": 1.5453,
+      "theoretical_loss": 4.20605420613875,
+      "tokens_seen": 281542656
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007391293881644935,
+      "loss": 1.661,
+      "theoretical_loss": 4.205938945401851,
+      "tokens_seen": 281608192
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007391133400200602,
+      "loss": 1.5877,
+      "theoretical_loss": 4.205823718993966,
+      "tokens_seen": 281673728
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007390972918756269,
+      "loss": 1.6022,
+      "theoretical_loss": 4.2057085268968875,
+      "tokens_seen": 281739264
+    },
+    {
+      "epoch": 0.09,
+      "objective/train/docs_used": 99654,
+      "objective/train/instantaneous_batch_size": 32,
+      "objective/train/instantaneous_microbatch_size": 32768,
+      "objective/train/original_loss": 1.5881729125976562,
+      "objective/train/theoretical_loss": 4.205593369092421,
+      "objective/train/tokens_used": 302264800,
+      "theoretical_loss": 4.205593369092421,
+      "tokens_seen": 281804800
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007390812437311936,
+      "loss": 1.5664,
+      "theoretical_loss": 4.205593369092421,
+      "tokens_seen": 281804800
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007390651955867603,
+      "loss": 1.5111,
+      "theoretical_loss": 4.205478245562386,
+      "tokens_seen": 281870336
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.000739049147442327,
+      "loss": 1.3946,
+      "theoretical_loss": 4.205363156288615,
+      "tokens_seen": 281935872
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007390330992978937,
+      "loss": 1.6569,
+      "theoretical_loss": 4.205248101252957,
+      "tokens_seen": 282001408
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007390170511534604,
+      "loss": 1.5346,
+      "theoretical_loss": 4.205133080437273,
+      "tokens_seen": 282066944
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007390010030090271,
+      "loss": 1.6258,
+      "theoretical_loss": 4.205018093823437,
+      "tokens_seen": 282132480
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007389849548645938,
+      "loss": 1.5873,
+      "theoretical_loss": 4.204903141393338,
+      "tokens_seen": 282198016
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007389689067201605,
+      "loss": 1.5391,
+      "theoretical_loss": 4.204788223128879,
+      "tokens_seen": 282263552
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007389528585757272,
+      "loss": 1.6691,
+      "theoretical_loss": 4.204673339011976,
+      "tokens_seen": 282329088
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.000738936810431294,
+      "loss": 1.6469,
+      "theoretical_loss": 4.204558489024561,
+      "tokens_seen": 282394624
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007389207622868606,
+      "loss": 1.5518,
+      "theoretical_loss": 4.204443673148575,
+      "tokens_seen": 282460160
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007389047141424273,
+      "loss": 1.6271,
+      "theoretical_loss": 4.204328891365977,
+      "tokens_seen": 282525696
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.000738888665997994,
+      "loss": 1.5519,
+      "theoretical_loss": 4.2042141436587395,
+      "tokens_seen": 282591232
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007388726178535607,
+      "loss": 1.6809,
+      "theoretical_loss": 4.204099430008846,
+      "tokens_seen": 282656768
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007388565697091274,
+      "loss": 1.561,
+      "theoretical_loss": 4.203984750398296,
+      "tokens_seen": 282722304
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007388405215646942,
+      "loss": 1.4828,
+      "theoretical_loss": 4.203870104809101,
+      "tokens_seen": 282787840
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007388244734202608,
+      "loss": 1.574,
+      "theoretical_loss": 4.203755493223289,
+      "tokens_seen": 282853376
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007388084252758276,
+      "loss": 1.5327,
+      "theoretical_loss": 4.203640915622898,
+      "tokens_seen": 282918912
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007387923771313942,
+      "loss": 1.5892,
+      "theoretical_loss": 4.203526371989983,
+      "tokens_seen": 282984448
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.000738776328986961,
+      "loss": 1.6255,
+      "theoretical_loss": 4.203411862306609,
+      "tokens_seen": 283049984
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007387602808425276,
+      "loss": 1.6154,
+      "theoretical_loss": 4.203297386554859,
+      "tokens_seen": 283115520
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007387442326980944,
+      "loss": 1.6293,
+      "theoretical_loss": 4.203182944716826,
+      "tokens_seen": 283181056
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007387281845536611,
+      "loss": 1.6613,
+      "theoretical_loss": 4.203068536774619,
+      "tokens_seen": 283246592
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007387121364092278,
+      "loss": 1.5847,
+      "theoretical_loss": 4.202954162710357,
+      "tokens_seen": 283312128
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007386960882647945,
+      "loss": 1.6116,
+      "theoretical_loss": 4.202839822506178,
+      "tokens_seen": 283377664
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007386800401203612,
+      "loss": 1.542,
+      "theoretical_loss": 4.202725516144228,
+      "tokens_seen": 283443200
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007386639919759279,
+      "loss": 1.6279,
+      "theoretical_loss": 4.202611243606671,
+      "tokens_seen": 283508736
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007386479438314946,
+      "loss": 1.5786,
+      "theoretical_loss": 4.202497004875681,
+      "tokens_seen": 283574272
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007386318956870613,
+      "loss": 1.6656,
+      "theoretical_loss": 4.202382799933449,
+      "tokens_seen": 283639808
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.000738615847542628,
+      "loss": 1.6199,
+      "theoretical_loss": 4.2022686287621775,
+      "tokens_seen": 283705344
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007385997993981946,
+      "loss": 1.472,
+      "theoretical_loss": 4.20215449134408,
+      "tokens_seen": 283770880
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007385837512537614,
+      "loss": 1.5804,
+      "theoretical_loss": 4.202040387661389,
+      "tokens_seen": 283836416
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.000738567703109328,
+      "loss": 1.5252,
+      "theoretical_loss": 4.201926317696347,
+      "tokens_seen": 283901952
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007385516549648948,
+      "loss": 1.5568,
+      "theoretical_loss": 4.201812281431209,
+      "tokens_seen": 283967488
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007385356068204614,
+      "loss": 1.6331,
+      "theoretical_loss": 4.201698278848246,
+      "tokens_seen": 284033024
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007385195586760282,
+      "loss": 1.5696,
+      "theoretical_loss": 4.201584309929743,
+      "tokens_seen": 284098560
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007385035105315948,
+      "loss": 1.6039,
+      "theoretical_loss": 4.201470374657993,
+      "tokens_seen": 284164096
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007384874623871615,
+      "loss": 1.6283,
+      "theoretical_loss": 4.201356473015308,
+      "tokens_seen": 284229632
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007384714142427282,
+      "loss": 1.5742,
+      "theoretical_loss": 4.201242604984013,
+      "tokens_seen": 284295168
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007384553660982949,
+      "loss": 1.6418,
+      "theoretical_loss": 4.201128770546442,
+      "tokens_seen": 284360704
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007384393179538616,
+      "loss": 1.6433,
+      "theoretical_loss": 4.201014969684948,
+      "tokens_seen": 284426240
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007384232698094283,
+      "loss": 1.5238,
+      "theoretical_loss": 4.200901202381893,
+      "tokens_seen": 284491776
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.000738407221664995,
+      "loss": 1.6351,
+      "theoretical_loss": 4.200787468619653,
+      "tokens_seen": 284557312
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007383911735205617,
+      "loss": 1.7295,
+      "theoretical_loss": 4.200673768380621,
+      "tokens_seen": 284622848
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007383751253761284,
+      "loss": 1.6412,
+      "theoretical_loss": 4.200560101647198,
+      "tokens_seen": 284688384
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007383590772316951,
+      "loss": 1.6357,
+      "theoretical_loss": 4.2004464684018,
+      "tokens_seen": 284753920
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007383430290872618,
+      "loss": 1.5205,
+      "theoretical_loss": 4.200332868626861,
+      "tokens_seen": 284819456
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007383269809428285,
+      "loss": 1.5761,
+      "theoretical_loss": 4.200219302304821,
+      "tokens_seen": 284884992
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007383109327983952,
+      "loss": 1.6343,
+      "theoretical_loss": 4.200105769418138,
+      "tokens_seen": 284950528
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007382948846539619,
+      "loss": 1.6184,
+      "theoretical_loss": 4.199992269949281,
+      "tokens_seen": 285016064
+    },
+    {
+      "epoch": 0.09,
+      "objective/train/docs_used": 100958,
+      "objective/train/instantaneous_batch_size": 32,
+      "objective/train/instantaneous_microbatch_size": 32768,
+      "objective/train/original_loss": 1.4548814296722412,
+      "objective/train/theoretical_loss": 4.199878803880733,
+      "objective/train/tokens_used": 305541600,
+      "theoretical_loss": 4.199878803880733,
+      "tokens_seen": 285081600
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007382788365095286,
+      "loss": 1.4891,
+      "theoretical_loss": 4.199878803880733,
+      "tokens_seen": 285081600
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007382627883650953,
+      "loss": 1.6365,
+      "theoretical_loss": 4.1997653711949905,
+      "tokens_seen": 285147136
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.000738246740220662,
+      "loss": 1.571,
+      "theoretical_loss": 4.199651971874562,
+      "tokens_seen": 285212672
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007382306920762287,
+      "loss": 1.5463,
+      "theoretical_loss": 4.199538605901973,
+      "tokens_seen": 285278208
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007382146439317954,
+      "loss": 1.5488,
+      "theoretical_loss": 4.199425273259756,
+      "tokens_seen": 285343744
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007381985957873621,
+      "loss": 1.6161,
+      "theoretical_loss": 4.199311973930461,
+      "tokens_seen": 285409280
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007381825476429288,
+      "loss": 1.6467,
+      "theoretical_loss": 4.199198707896651,
+      "tokens_seen": 285474816
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007381664994984955,
+      "loss": 1.5671,
+      "theoretical_loss": 4.199085475140899,
+      "tokens_seen": 285540352
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007381504513540622,
+      "loss": 1.4941,
+      "theoretical_loss": 4.1989722756457954,
+      "tokens_seen": 285605888
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007381344032096289,
+      "loss": 1.5462,
+      "theoretical_loss": 4.198859109393941,
+      "tokens_seen": 285671424
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007381183550651956,
+      "loss": 1.4696,
+      "theoretical_loss": 4.198745976367949,
+      "tokens_seen": 285736960
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007381023069207623,
+      "loss": 1.5923,
+      "theoretical_loss": 4.198632876550449,
+      "tokens_seen": 285802496
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.000738086258776329,
+      "loss": 1.6282,
+      "theoretical_loss": 4.19851980992408,
+      "tokens_seen": 285868032
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007380702106318957,
+      "loss": 1.5349,
+      "theoretical_loss": 4.198406776471497,
+      "tokens_seen": 285933568
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007380541624874624,
+      "loss": 1.5021,
+      "theoretical_loss": 4.198293776175365,
+      "tokens_seen": 285999104
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007380381143430291,
+      "loss": 1.6033,
+      "theoretical_loss": 4.198180809018366,
+      "tokens_seen": 286064640
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007380220661985958,
+      "loss": 1.6567,
+      "theoretical_loss": 4.198067874983192,
+      "tokens_seen": 286130176
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007380060180541625,
+      "loss": 1.6084,
+      "theoretical_loss": 4.197954974052546,
+      "tokens_seen": 286195712
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007379899699097292,
+      "loss": 1.5787,
+      "theoretical_loss": 4.19784210620915,
+      "tokens_seen": 286261248
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007379739217652959,
+      "loss": 1.5521,
+      "theoretical_loss": 4.197729271435736,
+      "tokens_seen": 286326784
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007379578736208626,
+      "loss": 1.5467,
+      "theoretical_loss": 4.197616469715046,
+      "tokens_seen": 286392320
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007379418254764293,
+      "loss": 1.579,
+      "theoretical_loss": 4.19750370102984,
+      "tokens_seen": 286457856
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.000737925777331996,
+      "loss": 1.5574,
+      "theoretical_loss": 4.197390965362887,
+      "tokens_seen": 286523392
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007379097291875627,
+      "loss": 1.6028,
+      "theoretical_loss": 4.197278262696971,
+      "tokens_seen": 286588928
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007378936810431294,
+      "loss": 1.6448,
+      "theoretical_loss": 4.197165593014889,
+      "tokens_seen": 286654464
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007378776328986961,
+      "loss": 1.6405,
+      "theoretical_loss": 4.197052956299449,
+      "tokens_seen": 286720000
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007378615847542628,
+      "loss": 1.5308,
+      "theoretical_loss": 4.196940352533474,
+      "tokens_seen": 286785536
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007378455366098295,
+      "loss": 1.5081,
+      "theoretical_loss": 4.196827781699799,
+      "tokens_seen": 286851072
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007378294884653962,
+      "loss": 1.5871,
+      "theoretical_loss": 4.196715243781273,
+      "tokens_seen": 286916608
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007378134403209629,
+      "loss": 1.5453,
+      "theoretical_loss": 4.196602738760754,
+      "tokens_seen": 286982144
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007377973921765296,
+      "loss": 1.5823,
+      "theoretical_loss": 4.196490266621119,
+      "tokens_seen": 287047680
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007377813440320963,
+      "loss": 1.5715,
+      "theoretical_loss": 4.196377827345252,
+      "tokens_seen": 287113216
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.000737765295887663,
+      "loss": 1.5327,
+      "theoretical_loss": 4.196265420916053,
+      "tokens_seen": 287178752
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007377492477432297,
+      "loss": 1.598,
+      "theoretical_loss": 4.196153047316433,
+      "tokens_seen": 287244288
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007377331995987964,
+      "loss": 1.5752,
+      "theoretical_loss": 4.196040706529319,
+      "tokens_seen": 287309824
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007377171514543631,
+      "loss": 1.5872,
+      "theoretical_loss": 4.195928398537647,
+      "tokens_seen": 287375360
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007377011033099298,
+      "loss": 1.762,
+      "theoretical_loss": 4.195816123324367,
+      "tokens_seen": 287440896
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007376850551654965,
+      "loss": 1.6078,
+      "theoretical_loss": 4.195703880872443,
+      "tokens_seen": 287506432
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007376690070210632,
+      "loss": 1.5071,
+      "theoretical_loss": 4.195591671164852,
+      "tokens_seen": 287571968
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007376529588766299,
+      "loss": 1.5424,
+      "theoretical_loss": 4.195479494184581,
+      "tokens_seen": 287637504
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007376369107321966,
+      "loss": 1.4977,
+      "theoretical_loss": 4.195367349914631,
+      "tokens_seen": 287703040
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007376208625877633,
+      "loss": 1.5777,
+      "theoretical_loss": 4.195255238338016,
+      "tokens_seen": 287768576
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.00073760481444333,
+      "loss": 1.6417,
+      "theoretical_loss": 4.195143159437764,
+      "tokens_seen": 287834112
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007375887662988967,
+      "loss": 1.6226,
+      "theoretical_loss": 4.195031113196913,
+      "tokens_seen": 287899648
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007375727181544634,
+      "loss": 1.6726,
+      "theoretical_loss": 4.194919099598516,
+      "tokens_seen": 287965184
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007375566700100301,
+      "loss": 1.4275,
+      "theoretical_loss": 4.194807118625636,
+      "tokens_seen": 288030720
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007375406218655968,
+      "loss": 1.521,
+      "theoretical_loss": 4.194695170261353,
+      "tokens_seen": 288096256
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007375245737211636,
+      "loss": 1.4216,
+      "theoretical_loss": 4.194583254488754,
+      "tokens_seen": 288161792
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007375085255767302,
+      "loss": 1.5601,
+      "theoretical_loss": 4.194471371290944,
+      "tokens_seen": 288227328
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.000737492477432297,
+      "loss": 1.6122,
+      "theoretical_loss": 4.194359520651036,
+      "tokens_seen": 288292864
+    },
+    {
+      "epoch": 0.09,
+      "objective/train/docs_used": 101843,
+      "objective/train/instantaneous_batch_size": 32,
+      "objective/train/instantaneous_microbatch_size": 32768,
+      "objective/train/original_loss": 1.7517683506011963,
+      "objective/train/theoretical_loss": 4.1942477025521585,
+      "objective/train/tokens_used": 308818400,
+      "theoretical_loss": 4.1942477025521585,
+      "tokens_seen": 288358400
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007374764292878637,
+      "loss": 1.6522,
+      "theoretical_loss": 4.1942477025521585,
+      "tokens_seen": 288358400
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007374603811434304,
+      "loss": 1.6579,
+      "theoretical_loss": 4.194135916977452,
+      "tokens_seen": 288423936
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.000737444332998997,
+      "loss": 1.6069,
+      "theoretical_loss": 4.19402416391007,
+      "tokens_seen": 288489472
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007374282848545638,
+      "loss": 1.4984,
+      "theoretical_loss": 4.193912443333177,
+      "tokens_seen": 288555008
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007374122367101305,
+      "loss": 1.5211,
+      "theoretical_loss": 4.193800755229951,
+      "tokens_seen": 288620544
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007373961885656972,
+      "loss": 1.6569,
+      "theoretical_loss": 4.193689099583582,
+      "tokens_seen": 288686080
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007373801404212639,
+      "loss": 1.5611,
+      "theoretical_loss": 4.193577476377275,
+      "tokens_seen": 288751616
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007373640922768306,
+      "loss": 1.5658,
+      "theoretical_loss": 4.193465885594242,
+      "tokens_seen": 288817152
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007373480441323973,
+      "loss": 1.5874,
+      "theoretical_loss": 4.1933543272177145,
+      "tokens_seen": 288882688
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.000737331995987964,
+      "loss": 1.5188,
+      "theoretical_loss": 4.193242801230931,
+      "tokens_seen": 288948224
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007373159478435307,
+      "loss": 1.6054,
+      "theoretical_loss": 4.193131307617145,
+      "tokens_seen": 289013760
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007372998996990974,
+      "loss": 1.6708,
+      "theoretical_loss": 4.193019846359622,
+      "tokens_seen": 289079296
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007372838515546641,
+      "loss": 1.6052,
+      "theoretical_loss": 4.192908417441639,
+      "tokens_seen": 289144832
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007372678034102308,
+      "loss": 1.6682,
+      "theoretical_loss": 4.192797020846487,
+      "tokens_seen": 289210368
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007372517552657975,
+      "loss": 1.6583,
+      "theoretical_loss": 4.192685656557468,
+      "tokens_seen": 289275904
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007372357071213642,
+      "loss": 1.6163,
+      "theoretical_loss": 4.1925743245578975,
+      "tokens_seen": 289341440
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007372196589769309,
+      "loss": 1.4386,
+      "theoretical_loss": 4.1924630248311034,
+      "tokens_seen": 289406976
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007372036108324975,
+      "loss": 1.5358,
+      "theoretical_loss": 4.192351757360425,
+      "tokens_seen": 289472512
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007371875626880643,
+      "loss": 1.6138,
+      "theoretical_loss": 4.1922405221292145,
+      "tokens_seen": 289538048
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007371715145436309,
+      "loss": 1.4933,
+      "theoretical_loss": 4.192129319120838,
+      "tokens_seen": 289603584
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007371554663991977,
+      "loss": 1.5791,
+      "theoretical_loss": 4.1920181483186685,
+      "tokens_seen": 289669120
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007371394182547643,
+      "loss": 1.6491,
+      "theoretical_loss": 4.1919070097060995,
+      "tokens_seen": 289734656
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007371233701103311,
+      "loss": 1.5349,
+      "theoretical_loss": 4.19179590326653,
+      "tokens_seen": 289800192
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007371073219658977,
+      "loss": 1.5031,
+      "theoretical_loss": 4.191684828983375,
+      "tokens_seen": 289865728
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007370912738214645,
+      "loss": 1.6173,
+      "theoretical_loss": 4.191573786840061,
+      "tokens_seen": 289931264
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007370752256770311,
+      "loss": 1.5673,
+      "theoretical_loss": 4.191462776820025,
+      "tokens_seen": 289996800
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007370591775325978,
+      "loss": 1.5083,
+      "theoretical_loss": 4.191351798906719,
+      "tokens_seen": 290062336
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007370431293881645,
+      "loss": 1.4875,
+      "theoretical_loss": 4.191240853083604,
+      "tokens_seen": 290127872
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007370270812437312,
+      "loss": 1.6398,
+      "theoretical_loss": 4.191129939334159,
+      "tokens_seen": 290193408
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007370110330992979,
+      "loss": 1.6251,
+      "theoretical_loss": 4.191019057641868,
+      "tokens_seen": 290258944
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007369949849548646,
+      "loss": 1.5814,
+      "theoretical_loss": 4.1909082079902324,
+      "tokens_seen": 290324480
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007369789368104313,
+      "loss": 1.5124,
+      "theoretical_loss": 4.190797390362764,
+      "tokens_seen": 290390016
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.000736962888665998,
+      "loss": 1.5789,
+      "theoretical_loss": 4.190686604742986,
+      "tokens_seen": 290455552
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007369468405215647,
+      "loss": 1.6906,
+      "theoretical_loss": 4.190575851114437,
+      "tokens_seen": 290521088
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007369307923771314,
+      "loss": 1.6629,
+      "theoretical_loss": 4.190465129460662,
+      "tokens_seen": 290586624
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007369147442326981,
+      "loss": 1.6839,
+      "theoretical_loss": 4.190354439765224,
+      "tokens_seen": 290652160
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007368986960882648,
+      "loss": 1.5767,
+      "theoretical_loss": 4.190243782011696,
+      "tokens_seen": 290717696
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007368826479438315,
+      "loss": 1.4519,
+      "theoretical_loss": 4.190133156183663,
+      "tokens_seen": 290783232
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007368665997993982,
+      "loss": 1.526,
+      "theoretical_loss": 4.190022562264721,
+      "tokens_seen": 290848768
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007368505516549649,
+      "loss": 1.6035,
+      "theoretical_loss": 4.189912000238479,
+      "tokens_seen": 290914304
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007368345035105316,
+      "loss": 1.5362,
+      "theoretical_loss": 4.189801470088559,
+      "tokens_seen": 290979840
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007368184553660983,
+      "loss": 1.4881,
+      "theoretical_loss": 4.189690971798596,
+      "tokens_seen": 291045376
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.000736802407221665,
+      "loss": 1.5363,
+      "theoretical_loss": 4.189580505352232,
+      "tokens_seen": 291110912
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007367863590772317,
+      "loss": 1.5927,
+      "theoretical_loss": 4.189470070733129,
+      "tokens_seen": 291176448
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007367703109327984,
+      "loss": 1.5632,
+      "theoretical_loss": 4.189359667924952,
+      "tokens_seen": 291241984
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007367542627883651,
+      "loss": 1.6798,
+      "theoretical_loss": 4.189249296911386,
+      "tokens_seen": 291307520
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007367382146439318,
+      "loss": 1.6025,
+      "theoretical_loss": 4.189138957676125,
+      "tokens_seen": 291373056
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007367221664994985,
+      "loss": 1.6616,
+      "theoretical_loss": 4.1890286502028715,
+      "tokens_seen": 291438592
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007367061183550652,
+      "loss": 1.4996,
+      "theoretical_loss": 4.188918374475347,
+      "tokens_seen": 291504128
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007366900702106319,
+      "loss": 1.664,
+      "theoretical_loss": 4.188808130477279,
+      "tokens_seen": 291569664
+    },
+    {
+      "epoch": 0.09,
+      "objective/train/docs_used": 102968,
+      "objective/train/instantaneous_batch_size": 32,
+      "objective/train/instantaneous_microbatch_size": 32768,
+      "objective/train/original_loss": 1.6024798154830933,
+      "objective/train/theoretical_loss": 4.1886979181924096,
+      "objective/train/tokens_used": 312095200,
+      "theoretical_loss": 4.1886979181924096,
+      "tokens_seen": 291635200
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007366740220661986,
+      "loss": 1.5594,
+      "theoretical_loss": 4.1886979181924096,
+      "tokens_seen": 291635200
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007366579739217653,
+      "loss": 1.4848,
+      "theoretical_loss": 4.188587737604493,
+      "tokens_seen": 291700736
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.000736641925777332,
+      "loss": 1.5697,
+      "theoretical_loss": 4.188477588697295,
+      "tokens_seen": 291766272
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007366258776328987,
+      "loss": 1.625,
+      "theoretical_loss": 4.188367471454593,
+      "tokens_seen": 291831808
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007366098294884654,
+      "loss": 1.6012,
+      "theoretical_loss": 4.188257385860177,
+      "tokens_seen": 291897344
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007365937813440321,
+      "loss": 1.5497,
+      "theoretical_loss": 4.188147331897849,
+      "tokens_seen": 291962880
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007365777331995988,
+      "loss": 1.5374,
+      "theoretical_loss": 4.1880373095514205,
+      "tokens_seen": 292028416
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007365616850551655,
+      "loss": 1.6771,
+      "theoretical_loss": 4.18792731880472,
+      "tokens_seen": 292093952
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007365456369107322,
+      "loss": 1.5743,
+      "theoretical_loss": 4.187817359641581,
+      "tokens_seen": 292159488
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007365295887662989,
+      "loss": 1.6635,
+      "theoretical_loss": 4.1877074320458565,
+      "tokens_seen": 292225024
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007365135406218656,
+      "loss": 1.5803,
+      "theoretical_loss": 4.187597536001406,
+      "tokens_seen": 292290560
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007364974924774323,
+      "loss": 1.6695,
+      "theoretical_loss": 4.187487671492102,
+      "tokens_seen": 292356096
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.000736481444332999,
+      "loss": 1.5485,
+      "theoretical_loss": 4.18737783850183,
+      "tokens_seen": 292421632
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007364653961885657,
+      "loss": 1.6169,
+      "theoretical_loss": 4.187268037014486,
+      "tokens_seen": 292487168
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007364493480441324,
+      "loss": 1.6168,
+      "theoretical_loss": 4.187158267013979,
+      "tokens_seen": 292552704
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007364332998996991,
+      "loss": 1.5953,
+      "theoretical_loss": 4.18704852848423,
+      "tokens_seen": 292618240
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007364172517552658,
+      "loss": 1.6573,
+      "theoretical_loss": 4.18693882140917,
+      "tokens_seen": 292683776
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007364012036108325,
+      "loss": 1.5318,
+      "theoretical_loss": 4.186829145772743,
+      "tokens_seen": 292749312
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007363851554663992,
+      "loss": 1.5916,
+      "theoretical_loss": 4.186719501558905,
+      "tokens_seen": 292814848
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007363691073219659,
+      "loss": 1.5141,
+      "theoretical_loss": 4.186609888751623,
+      "tokens_seen": 292880384
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007363530591775326,
+      "loss": 1.6541,
+      "theoretical_loss": 4.186500307334878,
+      "tokens_seen": 292945920
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007363370110330993,
+      "loss": 1.6081,
+      "theoretical_loss": 4.186390757292659,
+      "tokens_seen": 293011456
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007363209628886661,
+      "loss": 1.6245,
+      "theoretical_loss": 4.18628123860897,
+      "tokens_seen": 293076992
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007363049147442327,
+      "loss": 1.4708,
+      "theoretical_loss": 4.186171751267825,
+      "tokens_seen": 293142528
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007362888665997995,
+      "loss": 1.5364,
+      "theoretical_loss": 4.186062295253249,
+      "tokens_seen": 293208064
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007362728184553661,
+      "loss": 1.6285,
+      "theoretical_loss": 4.185952870549283,
+      "tokens_seen": 293273600
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007362567703109329,
+      "loss": 1.5606,
+      "theoretical_loss": 4.185843477139974,
+      "tokens_seen": 293339136
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007362407221664995,
+      "loss": 1.4876,
+      "theoretical_loss": 4.185734115009383,
+      "tokens_seen": 293404672
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007362246740220663,
+      "loss": 1.6881,
+      "theoretical_loss": 4.185624784141585,
+      "tokens_seen": 293470208
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.000736208625877633,
+      "loss": 1.5761,
+      "theoretical_loss": 4.185515484520664,
+      "tokens_seen": 293535744
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007361925777331997,
+      "loss": 1.6294,
+      "theoretical_loss": 4.1854062161307155,
+      "tokens_seen": 293601280
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007361765295887663,
+      "loss": 1.5766,
+      "theoretical_loss": 4.185296978955848,
+      "tokens_seen": 293666816
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007361604814443331,
+      "loss": 1.6546,
+      "theoretical_loss": 4.185187772980181,
+      "tokens_seen": 293732352
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007361444332998997,
+      "loss": 1.5794,
+      "theoretical_loss": 4.185078598187846,
+      "tokens_seen": 293797888
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007361283851554665,
+      "loss": 1.5979,
+      "theoretical_loss": 4.184969454562985,
+      "tokens_seen": 293863424
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007361123370110332,
+      "loss": 1.7826,
+      "theoretical_loss": 4.1848603420897525,
+      "tokens_seen": 293928960
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007360962888665999,
+      "loss": 1.5757,
+      "theoretical_loss": 4.184751260752316,
+      "tokens_seen": 293994496
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007360802407221666,
+      "loss": 1.5794,
+      "theoretical_loss": 4.18464221053485,
+      "tokens_seen": 294060032
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007360641925777333,
+      "loss": 1.5862,
+      "theoretical_loss": 4.184533191421547,
+      "tokens_seen": 294125568
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007360481444333,
+      "loss": 1.6176,
+      "theoretical_loss": 4.184424203396606,
+      "tokens_seen": 294191104
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007360320962888667,
+      "loss": 1.5429,
+      "theoretical_loss": 4.184315246444239,
+      "tokens_seen": 294256640
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007360160481444334,
+      "loss": 1.6508,
+      "theoretical_loss": 4.18420632054867,
+      "tokens_seen": 294322176
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007360000000000001,
+      "loss": 1.5789,
+      "theoretical_loss": 4.1840974256941355,
+      "tokens_seen": 294387712
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007359839518555668,
+      "loss": 1.5864,
+      "theoretical_loss": 4.183988561864879,
+      "tokens_seen": 294453248
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007359679037111335,
+      "loss": 1.6938,
+      "theoretical_loss": 4.183879729045163,
+      "tokens_seen": 294518784
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007359518555667002,
+      "loss": 1.495,
+      "theoretical_loss": 4.183770927219255,
+      "tokens_seen": 294584320
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007359358074222669,
+      "loss": 1.5571,
+      "theoretical_loss": 4.183662156371436,
+      "tokens_seen": 294649856
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007359197592778336,
+      "loss": 1.725,
+      "theoretical_loss": 4.183553416485999,
+      "tokens_seen": 294715392
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007359037111334003,
+      "loss": 1.6613,
+      "theoretical_loss": 4.183444707547249,
+      "tokens_seen": 294780928
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.000735887662988967,
+      "loss": 1.641,
+      "theoretical_loss": 4.1833360295395,
+      "tokens_seen": 294846464
+    },
+    {
+      "epoch": 0.09,
+      "objective/train/docs_used": 103866,
+      "objective/train/instantaneous_batch_size": 32,
+      "objective/train/instantaneous_microbatch_size": 32768,
+      "objective/train/original_loss": 1.6054697036743164,
+      "objective/train/theoretical_loss": 4.183227382447081,
+      "objective/train/tokens_used": 315372000,
+      "theoretical_loss": 4.183227382447081,
+      "tokens_seen": 294912000
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007358716148445337,
+      "loss": 1.6279,
+      "theoretical_loss": 4.183227382447081,
+      "tokens_seen": 294912000
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007358555667001004,
+      "loss": 1.6136,
+      "theoretical_loss": 4.183118766254328,
+      "tokens_seen": 294977536
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007358395185556671,
+      "loss": 1.5475,
+      "theoretical_loss": 4.183010180945593,
+      "tokens_seen": 295043072
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007358234704112338,
+      "loss": 1.5088,
+      "theoretical_loss": 4.182901626505237,
+      "tokens_seen": 295108608
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007358074222668004,
+      "loss": 1.6321,
+      "theoretical_loss": 4.182793102917632,
+      "tokens_seen": 295174144
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007357913741223672,
+      "loss": 1.6214,
+      "theoretical_loss": 4.182684610167162,
+      "tokens_seen": 295239680
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007357753259779338,
+      "loss": 1.5931,
+      "theoretical_loss": 4.1825761482382235,
+      "tokens_seen": 295305216
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007357592778335006,
+      "loss": 1.5504,
+      "theoretical_loss": 4.182467717115221,
+      "tokens_seen": 295370752
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007357432296890672,
+      "loss": 1.5749,
+      "theoretical_loss": 4.182359316782576,
+      "tokens_seen": 295436288
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.000735727181544634,
+      "loss": 1.5405,
+      "theoretical_loss": 4.1822509472247145,
+      "tokens_seen": 295501824
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007357111334002006,
+      "loss": 1.5905,
+      "theoretical_loss": 4.18214260842608,
+      "tokens_seen": 295567360
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007356950852557674,
+      "loss": 1.6108,
+      "theoretical_loss": 4.182034300371122,
+      "tokens_seen": 295632896
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.000735679037111334,
+      "loss": 1.5572,
+      "theoretical_loss": 4.181926023044306,
+      "tokens_seen": 295698432
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007356629889669007,
+      "loss": 1.5761,
+      "theoretical_loss": 4.181817776430106,
+      "tokens_seen": 295763968
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007356469408224674,
+      "loss": 1.7044,
+      "theoretical_loss": 4.181709560513008,
+      "tokens_seen": 295829504
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007356308926780341,
+      "loss": 1.6393,
+      "theoretical_loss": 4.181601375277509,
+      "tokens_seen": 295895040
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007356148445336008,
+      "loss": 1.6263,
+      "theoretical_loss": 4.181493220708117,
+      "tokens_seen": 295960576
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007355987963891675,
+      "loss": 1.555,
+      "theoretical_loss": 4.181385096789353,
+      "tokens_seen": 296026112
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007355827482447342,
+      "loss": 1.6019,
+      "theoretical_loss": 4.181277003505747,
+      "tokens_seen": 296091648
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007355667001003009,
+      "loss": 1.4783,
+      "theoretical_loss": 4.181168940841843,
+      "tokens_seen": 296157184
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007355506519558676,
+      "loss": 1.5435,
+      "theoretical_loss": 4.181060908782191,
+      "tokens_seen": 296222720
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007355346038114343,
+      "loss": 1.4945,
+      "theoretical_loss": 4.18095290731136,
+      "tokens_seen": 296288256
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.000735518555667001,
+      "loss": 1.6571,
+      "theoretical_loss": 4.180844936413922,
+      "tokens_seen": 296353792
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007355025075225677,
+      "loss": 1.5707,
+      "theoretical_loss": 4.180736996074465,
+      "tokens_seen": 296419328
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007354864593781344,
+      "loss": 1.6334,
+      "theoretical_loss": 4.180629086277589,
+      "tokens_seen": 296484864
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007354704112337011,
+      "loss": 1.5318,
+      "theoretical_loss": 4.180521207007902,
+      "tokens_seen": 296550400
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007354543630892678,
+      "loss": 1.6292,
+      "theoretical_loss": 4.180413358250024,
+      "tokens_seen": 296615936
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007354383149448345,
+      "loss": 1.5584,
+      "theoretical_loss": 4.180305539988588,
+      "tokens_seen": 296681472
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007354222668004012,
+      "loss": 1.6523,
+      "theoretical_loss": 4.180197752208235,
+      "tokens_seen": 296747008
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007354062186559679,
+      "loss": 1.541,
+      "theoretical_loss": 4.1800899948936205,
+      "tokens_seen": 296812544
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007353901705115346,
+      "loss": 1.5914,
+      "theoretical_loss": 4.179982268029409,
+      "tokens_seen": 296878080
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007353741223671013,
+      "loss": 1.463,
+      "theoretical_loss": 4.179874571600277,
+      "tokens_seen": 296943616
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.000735358074222668,
+      "loss": 1.5426,
+      "theoretical_loss": 4.1797669055909115,
+      "tokens_seen": 297009152
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007353420260782347,
+      "loss": 1.6375,
+      "theoretical_loss": 4.1796592699860105,
+      "tokens_seen": 297074688
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007353259779338014,
+      "loss": 1.6646,
+      "theoretical_loss": 4.179551664770283,
+      "tokens_seen": 297140224
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007353099297893681,
+      "loss": 1.5421,
+      "theoretical_loss": 4.1794440899284515,
+      "tokens_seen": 297205760
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007352938816449348,
+      "loss": 1.4364,
+      "theoretical_loss": 4.179336545445245,
+      "tokens_seen": 297271296
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007352778335005015,
+      "loss": 1.5667,
+      "theoretical_loss": 4.1792290313054075,
+      "tokens_seen": 297336832
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007352617853560682,
+      "loss": 1.5237,
+      "theoretical_loss": 4.179121547493692,
+      "tokens_seen": 297402368
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007352457372116349,
+      "loss": 1.5959,
+      "theoretical_loss": 4.179014093994862,
+      "tokens_seen": 297467904
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007352296890672016,
+      "loss": 1.6273,
+      "theoretical_loss": 4.178906670793695,
+      "tokens_seen": 297533440
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007352136409227683,
+      "loss": 1.6422,
+      "theoretical_loss": 4.178799277874977,
+      "tokens_seen": 297598976
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.000735197592778335,
+      "loss": 1.5832,
+      "theoretical_loss": 4.178691915223505,
+      "tokens_seen": 297664512
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007351815446339017,
+      "loss": 1.6341,
+      "theoretical_loss": 4.178584582824088,
+      "tokens_seen": 297730048
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007351654964894684,
+      "loss": 1.5198,
+      "theoretical_loss": 4.1784772806615464,
+      "tokens_seen": 297795584
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007351494483450351,
+      "loss": 1.5324,
+      "theoretical_loss": 4.178370008720709,
+      "tokens_seen": 297861120
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007351334002006018,
+      "loss": 1.5899,
+      "theoretical_loss": 4.178262766986418,
+      "tokens_seen": 297926656
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007351173520561686,
+      "loss": 1.6902,
+      "theoretical_loss": 4.178155555443525,
+      "tokens_seen": 297992192
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007351013039117352,
+      "loss": 1.5224,
+      "theoretical_loss": 4.178048374076894,
+      "tokens_seen": 298057728
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.000735085255767302,
+      "loss": 1.5546,
+      "theoretical_loss": 4.1779412228714,
+      "tokens_seen": 298123264
+    },
+    {
+      "epoch": 0.09,
+      "objective/train/docs_used": 105192,
+      "objective/train/instantaneous_batch_size": 32,
+      "objective/train/instantaneous_microbatch_size": 32768,
+      "objective/train/original_loss": 1.5836049318313599,
+      "objective/train/theoretical_loss": 4.177834101811927,
+      "objective/train/tokens_used": 318648800,
+      "theoretical_loss": 4.177834101811927,
+      "tokens_seen": 298188800
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007350692076228686,
+      "loss": 1.5477,
+      "theoretical_loss": 4.177834101811927,
+      "tokens_seen": 298188800
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007350531594784354,
+      "loss": 1.5409,
+      "theoretical_loss": 4.1777270108833715,
+      "tokens_seen": 298254336
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.000735037111334002,
+      "loss": 1.5651,
+      "theoretical_loss": 4.177619950070639,
+      "tokens_seen": 298319872
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007350210631895688,
+      "loss": 1.5792,
+      "theoretical_loss": 4.177512919358649,
+      "tokens_seen": 298385408
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007350050150451354,
+      "loss": 1.6706,
+      "theoretical_loss": 4.17740591873233,
+      "tokens_seen": 298450944
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007349889669007022,
+      "loss": 1.5278,
+      "theoretical_loss": 4.177298948176619,
+      "tokens_seen": 298516480
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007349729187562688,
+      "loss": 1.4575,
+      "theoretical_loss": 4.17719200767647,
+      "tokens_seen": 298582016
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007349568706118356,
+      "loss": 1.5257,
+      "theoretical_loss": 4.177085097216842,
+      "tokens_seen": 298647552
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007349408224674022,
+      "loss": 1.5551,
+      "theoretical_loss": 4.1769782167827065,
+      "tokens_seen": 298713088
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.000734924774322969,
+      "loss": 1.5615,
+      "theoretical_loss": 4.176871366359047,
+      "tokens_seen": 298778624
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007349087261785356,
+      "loss": 1.6923,
+      "theoretical_loss": 4.176764545930858,
+      "tokens_seen": 298844160
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007348926780341024,
+      "loss": 1.6758,
+      "theoretical_loss": 4.176657755483143,
+      "tokens_seen": 298909696
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.000734876629889669,
+      "loss": 1.5424,
+      "theoretical_loss": 4.176550995000917,
+      "tokens_seen": 298975232
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007348605817452358,
+      "loss": 1.5976,
+      "theoretical_loss": 4.176444264469206,
+      "tokens_seen": 299040768
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007348445336008024,
+      "loss": 1.6364,
+      "theoretical_loss": 4.176337563873046,
+      "tokens_seen": 299106304
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007348284854563692,
+      "loss": 1.5765,
+      "theoretical_loss": 4.176230893197486,
+      "tokens_seen": 299171840
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007348124373119358,
+      "loss": 1.5696,
+      "theoretical_loss": 4.1761242524275834,
+      "tokens_seen": 299237376
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007347963891675026,
+      "loss": 1.4667,
+      "theoretical_loss": 4.176017641548407,
+      "tokens_seen": 299302912
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007347803410230692,
+      "loss": 1.5479,
+      "theoretical_loss": 4.175911060545037,
+      "tokens_seen": 299368448
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.000734764292878636,
+      "loss": 1.5657,
+      "theoretical_loss": 4.175804509402562,
+      "tokens_seen": 299433984
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007347482447342027,
+      "loss": 1.641,
+      "theoretical_loss": 4.175697988106085,
+      "tokens_seen": 299499520
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007347321965897694,
+      "loss": 1.5938,
+      "theoretical_loss": 4.175591496640718,
+      "tokens_seen": 299565056
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.000734716148445336,
+      "loss": 1.6269,
+      "theoretical_loss": 4.175485034991581,
+      "tokens_seen": 299630592
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007347001003009028,
+      "loss": 1.5371,
+      "theoretical_loss": 4.17537860314381,
+      "tokens_seen": 299696128
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007346840521564695,
+      "loss": 1.536,
+      "theoretical_loss": 4.175272201082547,
+      "tokens_seen": 299761664
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007346680040120362,
+      "loss": 1.5328,
+      "theoretical_loss": 4.175165828792946,
+      "tokens_seen": 299827200
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007346519558676029,
+      "loss": 1.5621,
+      "theoretical_loss": 4.1750594862601735,
+      "tokens_seen": 299892736
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007346359077231696,
+      "loss": 1.585,
+      "theoretical_loss": 4.1749531734694045,
+      "tokens_seen": 299958272
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007346198595787363,
+      "loss": 1.6496,
+      "theoretical_loss": 4.174846890405825,
+      "tokens_seen": 300023808
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.000734603811434303,
+      "loss": 1.4407,
+      "theoretical_loss": 4.174740637054632,
+      "tokens_seen": 300089344
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007345877632898697,
+      "loss": 1.4657,
+      "theoretical_loss": 4.174634413401034,
+      "tokens_seen": 300154880
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007345717151454364,
+      "loss": 1.6096,
+      "theoretical_loss": 4.174528219430247,
+      "tokens_seen": 300220416
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007345556670010031,
+      "loss": 1.6406,
+      "theoretical_loss": 4.174422055127502,
+      "tokens_seen": 300285952
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007345396188565698,
+      "loss": 1.5824,
+      "theoretical_loss": 4.174315920478037,
+      "tokens_seen": 300351488
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007345235707121365,
+      "loss": 1.5873,
+      "theoretical_loss": 4.174209815467102,
+      "tokens_seen": 300417024
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007345075225677032,
+      "loss": 1.6349,
+      "theoretical_loss": 4.174103740079958,
+      "tokens_seen": 300482560
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007344914744232699,
+      "loss": 1.4045,
+      "theoretical_loss": 4.1739976943018755,
+      "tokens_seen": 300548096
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007344754262788366,
+      "loss": 1.6427,
+      "theoretical_loss": 4.173891678118135,
+      "tokens_seen": 300613632
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007344593781344033,
+      "loss": 1.6638,
+      "theoretical_loss": 4.173785691514031,
+      "tokens_seen": 300679168
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.00073444332998997,
+      "loss": 1.6682,
+      "theoretical_loss": 4.173679734474862,
+      "tokens_seen": 300744704
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007344272818455367,
+      "loss": 1.6007,
+      "theoretical_loss": 4.173573806985945,
+      "tokens_seen": 300810240
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007344112337011033,
+      "loss": 1.63,
+      "theoretical_loss": 4.173467909032602,
+      "tokens_seen": 300875776
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007343951855566701,
+      "loss": 1.6031,
+      "theoretical_loss": 4.173362040600166,
+      "tokens_seen": 300941312
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007343791374122367,
+      "loss": 1.5375,
+      "theoretical_loss": 4.173256201673983,
+      "tokens_seen": 301006848
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007343630892678035,
+      "loss": 1.6008,
+      "theoretical_loss": 4.173150392239406,
+      "tokens_seen": 301072384
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007343470411233701,
+      "loss": 1.6719,
+      "theoretical_loss": 4.173044612281802,
+      "tokens_seen": 301137920
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007343309929789369,
+      "loss": 1.5542,
+      "theoretical_loss": 4.172938861786546,
+      "tokens_seen": 301203456
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007343149448345035,
+      "loss": 1.5171,
+      "theoretical_loss": 4.1728331407390264,
+      "tokens_seen": 301268992
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007342988966900703,
+      "loss": 1.5447,
+      "theoretical_loss": 4.172727449124636,
+      "tokens_seen": 301334528
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007342828485456369,
+      "loss": 1.5496,
+      "theoretical_loss": 4.172621786928786,
+      "tokens_seen": 301400064
+    },
+    {
+      "epoch": 0.09,
+      "objective/train/docs_used": 106059,
+      "objective/train/instantaneous_batch_size": 32,
+      "objective/train/instantaneous_microbatch_size": 32768,
+      "objective/train/original_loss": 1.758809208869934,
+      "objective/train/theoretical_loss": 4.17251615413689,
+      "objective/train/tokens_used": 321925600,
+      "theoretical_loss": 4.17251615413689,
+      "tokens_seen": 301465600
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007342668004012036,
+      "loss": 1.6491,
+      "theoretical_loss": 4.17251615413689,
+      "tokens_seen": 301465600
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007342507522567703,
+      "loss": 1.536,
+      "theoretical_loss": 4.172410550734378,
+      "tokens_seen": 301531136
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.000734234704112337,
+      "loss": 1.575,
+      "theoretical_loss": 4.172304976706689,
+      "tokens_seen": 301596672
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007342186559679037,
+      "loss": 1.5799,
+      "theoretical_loss": 4.17219943203927,
+      "tokens_seen": 301662208
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007342026078234704,
+      "loss": 1.516,
+      "theoretical_loss": 4.172093916717581,
+      "tokens_seen": 301727744
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007341865596790371,
+      "loss": 1.639,
+      "theoretical_loss": 4.171988430727091,
+      "tokens_seen": 301793280
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007341705115346038,
+      "loss": 1.7349,
+      "theoretical_loss": 4.17188297405328,
+      "tokens_seen": 301858816
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007341544633901705,
+      "loss": 1.6165,
+      "theoretical_loss": 4.171777546681638,
+      "tokens_seen": 301924352
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007341384152457372,
+      "loss": 1.6024,
+      "theoretical_loss": 4.171672148597665,
+      "tokens_seen": 301989888
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007341223671013039,
+      "loss": 1.5757,
+      "theoretical_loss": 4.171566779786872,
+      "tokens_seen": 302055424
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007341063189568706,
+      "loss": 1.5407,
+      "theoretical_loss": 4.17146144023478,
+      "tokens_seen": 302120960
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007340902708124373,
+      "loss": 1.5774,
+      "theoretical_loss": 4.171356129926921,
+      "tokens_seen": 302186496
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.000734074222668004,
+      "loss": 1.6298,
+      "theoretical_loss": 4.171250848848835,
+      "tokens_seen": 302252032
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007340581745235707,
+      "loss": 1.5455,
+      "theoretical_loss": 4.171145596986076,
+      "tokens_seen": 302317568
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007340421263791374,
+      "loss": 1.5813,
+      "theoretical_loss": 4.171040374324204,
+      "tokens_seen": 302383104
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007340260782347041,
+      "loss": 1.5171,
+      "theoretical_loss": 4.170935180848793,
+      "tokens_seen": 302448640
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007340100300902708,
+      "loss": 1.5307,
+      "theoretical_loss": 4.170830016545425,
+      "tokens_seen": 302514176
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007339939819458375,
+      "loss": 1.5617,
+      "theoretical_loss": 4.1707248813996936,
+      "tokens_seen": 302579712
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007339779338014042,
+      "loss": 1.6199,
+      "theoretical_loss": 4.170619775397201,
+      "tokens_seen": 302645248
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007339618856569709,
+      "loss": 1.5442,
+      "theoretical_loss": 4.1705146985235615,
+      "tokens_seen": 302710784
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007339458375125376,
+      "loss": 1.5901,
+      "theoretical_loss": 4.170409650764397,
+      "tokens_seen": 302776320
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007339297893681043,
+      "loss": 1.6718,
+      "theoretical_loss": 4.170304632105344,
+      "tokens_seen": 302841856
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007339137412236711,
+      "loss": 1.6428,
+      "theoretical_loss": 4.170199642532045,
+      "tokens_seen": 302907392
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007338976930792377,
+      "loss": 1.568,
+      "theoretical_loss": 4.170094682030154,
+      "tokens_seen": 302972928
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007338816449348045,
+      "loss": 1.6485,
+      "theoretical_loss": 4.169989750585335,
+      "tokens_seen": 303038464
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007338655967903711,
+      "loss": 1.4992,
+      "theoretical_loss": 4.169884848183264,
+      "tokens_seen": 303104000
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007338495486459379,
+      "loss": 1.3649,
+      "theoretical_loss": 4.169779974809624,
+      "tokens_seen": 303169536
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007338335005015045,
+      "loss": 1.5366,
+      "theoretical_loss": 4.169675130450111,
+      "tokens_seen": 303235072
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007338174523570713,
+      "loss": 1.6338,
+      "theoretical_loss": 4.16957031509043,
+      "tokens_seen": 303300608
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007338014042126379,
+      "loss": 1.5639,
+      "theoretical_loss": 4.169465528716295,
+      "tokens_seen": 303366144
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007337853560682047,
+      "loss": 1.4963,
+      "theoretical_loss": 4.169360771313434,
+      "tokens_seen": 303431680
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007337693079237713,
+      "loss": 1.6657,
+      "theoretical_loss": 4.169256042867579,
+      "tokens_seen": 303497216
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007337532597793381,
+      "loss": 1.6183,
+      "theoretical_loss": 4.169151343364476,
+      "tokens_seen": 303562752
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007337372116349047,
+      "loss": 1.6496,
+      "theoretical_loss": 4.169046672789882,
+      "tokens_seen": 303628288
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007337211634904715,
+      "loss": 1.6686,
+      "theoretical_loss": 4.168942031129562,
+      "tokens_seen": 303693824
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007337051153460381,
+      "loss": 1.5519,
+      "theoretical_loss": 4.168837418369292,
+      "tokens_seen": 303759360
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007336890672016049,
+      "loss": 1.6705,
+      "theoretical_loss": 4.168732834494857,
+      "tokens_seen": 303824896
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007336730190571715,
+      "loss": 1.5031,
+      "theoretical_loss": 4.1686282794920535,
+      "tokens_seen": 303890432
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007336569709127383,
+      "loss": 1.5267,
+      "theoretical_loss": 4.168523753346687,
+      "tokens_seen": 303955968
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007336409227683049,
+      "loss": 1.7368,
+      "theoretical_loss": 4.168419256044573,
+      "tokens_seen": 304021504
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007336248746238717,
+      "loss": 1.565,
+      "theoretical_loss": 4.168314787571538,
+      "tokens_seen": 304087040
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007336088264794383,
+      "loss": 1.5134,
+      "theoretical_loss": 4.168210347913418,
+      "tokens_seen": 304152576
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007335927783350051,
+      "loss": 1.5636,
+      "theoretical_loss": 4.168105937056059,
+      "tokens_seen": 304218112
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007335767301905717,
+      "loss": 1.5359,
+      "theoretical_loss": 4.168001554985317,
+      "tokens_seen": 304283648
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007335606820461385,
+      "loss": 1.5894,
+      "theoretical_loss": 4.167897201687056,
+      "tokens_seen": 304349184
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007335446339017051,
+      "loss": 1.5073,
+      "theoretical_loss": 4.167792877147155,
+      "tokens_seen": 304414720
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007335285857572719,
+      "loss": 1.5895,
+      "theoretical_loss": 4.1676885813514986,
+      "tokens_seen": 304480256
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007335125376128385,
+      "loss": 1.5639,
+      "theoretical_loss": 4.167584314285982,
+      "tokens_seen": 304545792
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007334964894684053,
+      "loss": 1.5099,
+      "theoretical_loss": 4.167480075936512,
+      "tokens_seen": 304611328
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.000733480441323972,
+      "loss": 1.6157,
+      "theoretical_loss": 4.167375866289004,
+      "tokens_seen": 304676864
+    },
+    {
+      "epoch": 0.09,
+      "objective/train/docs_used": 107321,
+      "objective/train/instantaneous_batch_size": 32,
+      "objective/train/instantaneous_microbatch_size": 32768,
+      "objective/train/original_loss": 1.6312581300735474,
+      "objective/train/theoretical_loss": 4.167271685329383,
+      "objective/train/tokens_used": 325202400,
+      "theoretical_loss": 4.167271685329383,
+      "tokens_seen": 304742400
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007334643931795387,
+      "loss": 1.5691,
+      "theoretical_loss": 4.167271685329383,
+      "tokens_seen": 304742400
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007334483450351053,
+      "loss": 1.5939,
+      "theoretical_loss": 4.167167533043585,
+      "tokens_seen": 304807936
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007334322968906721,
+      "loss": 1.5378,
+      "theoretical_loss": 4.1670634094175565,
+      "tokens_seen": 304873472
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007334162487462387,
+      "loss": 1.6314,
+      "theoretical_loss": 4.166959314437252,
+      "tokens_seen": 304939008
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007334002006018055,
+      "loss": 1.5821,
+      "theoretical_loss": 4.166855248088637,
+      "tokens_seen": 305004544
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007333841524573721,
+      "loss": 1.5085,
+      "theoretical_loss": 4.1667512103576865,
+      "tokens_seen": 305070080
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007333681043129389,
+      "loss": 1.7104,
+      "theoretical_loss": 4.166647201230386,
+      "tokens_seen": 305135616
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007333520561685056,
+      "loss": 1.5186,
+      "theoretical_loss": 4.166543220692731,
+      "tokens_seen": 305201152
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007333360080240723,
+      "loss": 1.6194,
+      "theoretical_loss": 4.166439268730724,
+      "tokens_seen": 305266688
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.000733319959879639,
+      "loss": 1.6088,
+      "theoretical_loss": 4.166335345330383,
+      "tokens_seen": 305332224
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007333039117352057,
+      "loss": 1.658,
+      "theoretical_loss": 4.16623145047773,
+      "tokens_seen": 305397760
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007332878635907724,
+      "loss": 1.6187,
+      "theoretical_loss": 4.166127584158801,
+      "tokens_seen": 305463296
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007332718154463391,
+      "loss": 1.7476,
+      "theoretical_loss": 4.166023746359639,
+      "tokens_seen": 305528832
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007332557673019058,
+      "loss": 1.6207,
+      "theoretical_loss": 4.165919937066299,
+      "tokens_seen": 305594368
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007332397191574725,
+      "loss": 1.6563,
+      "theoretical_loss": 4.165816156264844,
+      "tokens_seen": 305659904
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007332236710130392,
+      "loss": 1.6873,
+      "theoretical_loss": 4.165712403941348,
+      "tokens_seen": 305725440
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007332076228686059,
+      "loss": 1.6141,
+      "theoretical_loss": 4.1656086800818946,
+      "tokens_seen": 305790976
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007331915747241726,
+      "loss": 1.8297,
+      "theoretical_loss": 4.165504984672577,
+      "tokens_seen": 305856512
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007331755265797393,
+      "loss": 1.4954,
+      "theoretical_loss": 4.165401317699498,
+      "tokens_seen": 305922048
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.000733159478435306,
+      "loss": 1.6272,
+      "theoretical_loss": 4.16529767914877,
+      "tokens_seen": 305987584
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007331434302908727,
+      "loss": 1.5968,
+      "theoretical_loss": 4.165194069006516,
+      "tokens_seen": 306053120
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007331273821464394,
+      "loss": 1.5917,
+      "theoretical_loss": 4.165090487258867,
+      "tokens_seen": 306118656
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007331113340020061,
+      "loss": 1.6375,
+      "theoretical_loss": 4.164986933891968,
+      "tokens_seen": 306184192
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007330952858575728,
+      "loss": 1.5956,
+      "theoretical_loss": 4.164883408891968,
+      "tokens_seen": 306249728
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007330792377131395,
+      "loss": 1.5718,
+      "theoretical_loss": 4.164779912245029,
+      "tokens_seen": 306315264
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007330631895687062,
+      "loss": 1.6585,
+      "theoretical_loss": 4.1646764439373225,
+      "tokens_seen": 306380800
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007330471414242729,
+      "loss": 1.5583,
+      "theoretical_loss": 4.164573003955029,
+      "tokens_seen": 306446336
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007330310932798396,
+      "loss": 1.5588,
+      "theoretical_loss": 4.164469592284338,
+      "tokens_seen": 306511872
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007330150451354062,
+      "loss": 1.6941,
+      "theoretical_loss": 4.164366208911453,
+      "tokens_seen": 306577408
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.000732998996990973,
+      "loss": 1.6402,
+      "theoretical_loss": 4.1642628538225805,
+      "tokens_seen": 306642944
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007329829488465396,
+      "loss": 1.654,
+      "theoretical_loss": 4.1641595270039415,
+      "tokens_seen": 306708480
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007329669007021064,
+      "loss": 1.6779,
+      "theoretical_loss": 4.1640562284417655,
+      "tokens_seen": 306774016
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.000732950852557673,
+      "loss": 1.6115,
+      "theoretical_loss": 4.16395295812229,
+      "tokens_seen": 306839552
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007329348044132398,
+      "loss": 1.5958,
+      "theoretical_loss": 4.1638497160317645,
+      "tokens_seen": 306905088
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007329187562688064,
+      "loss": 1.6348,
+      "theoretical_loss": 4.163746502156448,
+      "tokens_seen": 306970624
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007329027081243732,
+      "loss": 1.5914,
+      "theoretical_loss": 4.163643316482606,
+      "tokens_seen": 307036160
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007328866599799398,
+      "loss": 1.6536,
+      "theoretical_loss": 4.1635401589965175,
+      "tokens_seen": 307101696
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007328706118355065,
+      "loss": 1.7747,
+      "theoretical_loss": 4.163437029684469,
+      "tokens_seen": 307167232
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007328545636910732,
+      "loss": 1.6501,
+      "theoretical_loss": 4.163333928532758,
+      "tokens_seen": 307232768
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007328385155466399,
+      "loss": 1.6067,
+      "theoretical_loss": 4.163230855527688,
+      "tokens_seen": 307298304
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007328224674022066,
+      "loss": 1.5235,
+      "theoretical_loss": 4.163127810655578,
+      "tokens_seen": 307363840
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007328064192577733,
+      "loss": 1.5547,
+      "theoretical_loss": 4.163024793902752,
+      "tokens_seen": 307429376
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.00073279037111334,
+      "loss": 1.6222,
+      "theoretical_loss": 4.162921805255543,
+      "tokens_seen": 307494912
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007327743229689067,
+      "loss": 1.6553,
+      "theoretical_loss": 4.162818844700298,
+      "tokens_seen": 307560448
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007327582748244734,
+      "loss": 1.577,
+      "theoretical_loss": 4.162715912223369,
+      "tokens_seen": 307625984
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007327422266800401,
+      "loss": 1.5632,
+      "theoretical_loss": 4.162613007811122,
+      "tokens_seen": 307691520
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007327261785356068,
+      "loss": 1.6673,
+      "theoretical_loss": 4.1625101314499275,
+      "tokens_seen": 307757056
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007327101303911736,
+      "loss": 1.7133,
+      "theoretical_loss": 4.162407283126169,
+      "tokens_seen": 307822592
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007326940822467402,
+      "loss": 1.6526,
+      "theoretical_loss": 4.1623044628262384,
+      "tokens_seen": 307888128
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.000732678034102307,
+      "loss": 1.6482,
+      "theoretical_loss": 4.162201670536537,
+      "tokens_seen": 307953664
+    },
+    {
+      "epoch": 0.09,
+      "objective/train/docs_used": 108601,
+      "objective/train/instantaneous_batch_size": 32,
+      "objective/train/instantaneous_microbatch_size": 32768,
+      "objective/train/original_loss": 1.5907751321792603,
+      "objective/train/theoretical_loss": 4.162098906243477,
+      "objective/train/tokens_used": 328479200,
+      "theoretical_loss": 4.162098906243477,
+      "tokens_seen": 308019200
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007326619859578736,
+      "loss": 1.6646,
+      "theoretical_loss": 4.162098906243477,
+      "tokens_seen": 308019200
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007326459378134404,
+      "loss": 1.5776,
+      "theoretical_loss": 4.161996169933477,
+      "tokens_seen": 308084736
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.000732629889669007,
+      "loss": 1.6395,
+      "theoretical_loss": 4.161893461592968,
+      "tokens_seen": 308150272
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007326138415245738,
+      "loss": 1.6916,
+      "theoretical_loss": 4.16179078120839,
+      "tokens_seen": 308215808
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007325977933801404,
+      "loss": 1.657,
+      "theoretical_loss": 4.161688128766191,
+      "tokens_seen": 308281344
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007325817452357072,
+      "loss": 1.595,
+      "theoretical_loss": 4.16158550425283,
+      "tokens_seen": 308346880
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007325656970912738,
+      "loss": 1.7356,
+      "theoretical_loss": 4.161482907654775,
+      "tokens_seen": 308412416
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007325496489468406,
+      "loss": 1.5232,
+      "theoretical_loss": 4.161380338958502,
+      "tokens_seen": 308477952
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007325336008024072,
+      "loss": 1.7097,
+      "theoretical_loss": 4.161277798150498,
+      "tokens_seen": 308543488
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.000732517552657974,
+      "loss": 1.5382,
+      "theoretical_loss": 4.16117528521726,
+      "tokens_seen": 308609024
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007325015045135406,
+      "loss": 1.6999,
+      "theoretical_loss": 4.161072800145292,
+      "tokens_seen": 308674560
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007324854563691074,
+      "loss": 1.6401,
+      "theoretical_loss": 4.16097034292111,
+      "tokens_seen": 308740096
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.000732469408224674,
+      "loss": 1.604,
+      "theoretical_loss": 4.160867913531238,
+      "tokens_seen": 308805632
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007324533600802408,
+      "loss": 1.6734,
+      "theoretical_loss": 4.160765511962209,
+      "tokens_seen": 308871168
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007324373119358074,
+      "loss": 1.5299,
+      "theoretical_loss": 4.160663138200567,
+      "tokens_seen": 308936704
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007324212637913742,
+      "loss": 1.5198,
+      "theoretical_loss": 4.1605607922328645,
+      "tokens_seen": 309002240
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007324052156469408,
+      "loss": 1.5991,
+      "theoretical_loss": 4.160458474045662,
+      "tokens_seen": 309067776
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007323891675025076,
+      "loss": 1.6336,
+      "theoretical_loss": 4.16035618362553,
+      "tokens_seen": 309133312
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007323731193580742,
+      "loss": 1.5597,
+      "theoretical_loss": 4.16025392095905,
+      "tokens_seen": 309198848
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.000732357071213641,
+      "loss": 1.6787,
+      "theoretical_loss": 4.160151686032814,
+      "tokens_seen": 309264384
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007323410230692076,
+      "loss": 1.5706,
+      "theoretical_loss": 4.160049478833416,
+      "tokens_seen": 309329920
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007323249749247744,
+      "loss": 1.6222,
+      "theoretical_loss": 4.159947299347468,
+      "tokens_seen": 309395456
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.000732308926780341,
+      "loss": 1.6116,
+      "theoretical_loss": 4.159845147561587,
+      "tokens_seen": 309460992
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007322928786359078,
+      "loss": 1.5138,
+      "theoretical_loss": 4.1597430234624,
+      "tokens_seen": 309526528
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007322768304914744,
+      "loss": 1.5193,
+      "theoretical_loss": 4.159640927036541,
+      "tokens_seen": 309592064
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007322607823470412,
+      "loss": 1.7338,
+      "theoretical_loss": 4.159538858270659,
+      "tokens_seen": 309657600
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007322447342026078,
+      "loss": 1.5676,
+      "theoretical_loss": 4.159436817151407,
+      "tokens_seen": 309723136
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007322286860581746,
+      "loss": 1.4932,
+      "theoretical_loss": 4.159334803665448,
+      "tokens_seen": 309788672
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007322126379137412,
+      "loss": 1.6678,
+      "theoretical_loss": 4.159232817799458,
+      "tokens_seen": 309854208
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.000732196589769308,
+      "loss": 1.6643,
+      "theoretical_loss": 4.1591308595401175,
+      "tokens_seen": 309919744
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007321805416248746,
+      "loss": 1.6542,
+      "theoretical_loss": 4.159028928874118,
+      "tokens_seen": 309985280
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007321644934804414,
+      "loss": 1.693,
+      "theoretical_loss": 4.158927025788162,
+      "tokens_seen": 310050816
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.000732148445336008,
+      "loss": 1.6669,
+      "theoretical_loss": 4.158825150268958,
+      "tokens_seen": 310116352
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007321323971915748,
+      "loss": 1.6518,
+      "theoretical_loss": 4.158723302303227,
+      "tokens_seen": 310181888
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007321163490471414,
+      "loss": 1.6339,
+      "theoretical_loss": 4.158621481877697,
+      "tokens_seen": 310247424
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007321003009027082,
+      "loss": 1.6746,
+      "theoretical_loss": 4.158519688979106,
+      "tokens_seen": 310312960
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007320842527582748,
+      "loss": 1.5232,
+      "theoretical_loss": 4.1584179235942,
+      "tokens_seen": 310378496
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007320682046138416,
+      "loss": 1.6539,
+      "theoretical_loss": 4.158316185709737,
+      "tokens_seen": 310444032
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007320521564694082,
+      "loss": 1.6137,
+      "theoretical_loss": 4.158214475312481,
+      "tokens_seen": 310509568
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.000732036108324975,
+      "loss": 1.6964,
+      "theoretical_loss": 4.158112792389206,
+      "tokens_seen": 310575104
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007320200601805416,
+      "loss": 1.5929,
+      "theoretical_loss": 4.158011136926698,
+      "tokens_seen": 310640640
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007320040120361084,
+      "loss": 1.604,
+      "theoretical_loss": 4.157909508911747,
+      "tokens_seen": 310706176
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.000731987963891675,
+      "loss": 1.6161,
+      "theoretical_loss": 4.157807908331157,
+      "tokens_seen": 310771712
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007319719157472418,
+      "loss": 1.6194,
+      "theoretical_loss": 4.157706335171738,
+      "tokens_seen": 310837248
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007319558676028085,
+      "loss": 1.6468,
+      "theoretical_loss": 4.15760478942031,
+      "tokens_seen": 310902784
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007319398194583752,
+      "loss": 1.5963,
+      "theoretical_loss": 4.157503271063703,
+      "tokens_seen": 310968320
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007319237713139419,
+      "loss": 1.6221,
+      "theoretical_loss": 4.157401780088756,
+      "tokens_seen": 311033856
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007319077231695086,
+      "loss": 1.6468,
+      "theoretical_loss": 4.157300316482315,
+      "tokens_seen": 311099392
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007318916750250753,
+      "loss": 1.7145,
+      "theoretical_loss": 4.157198880231238,
+      "tokens_seen": 311164928
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.000731875626880642,
+      "loss": 1.5167,
+      "theoretical_loss": 4.157097471322389,
+      "tokens_seen": 311230464
+    },
+    {
+      "epoch": 0.09,
+      "objective/train/docs_used": 109622,
+      "objective/train/instantaneous_batch_size": 32,
+      "objective/train/instantaneous_microbatch_size": 32768,
+      "objective/train/original_loss": 1.8168914318084717,
+      "objective/train/theoretical_loss": 4.156996089742645,
+      "objective/train/tokens_used": 331756000,
+      "theoretical_loss": 4.156996089742645,
+      "tokens_seen": 311296000
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007318595787362087,
+      "loss": 1.7238,
+      "theoretical_loss": 4.156996089742645,
+      "tokens_seen": 311296000
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007318435305917754,
+      "loss": 1.6184,
+      "theoretical_loss": 4.156894735478888,
+      "tokens_seen": 311361536
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007318274824473421,
+      "loss": 1.52,
+      "theoretical_loss": 4.156793408518011,
+      "tokens_seen": 311427072
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007318114343029088,
+      "loss": 1.5543,
+      "theoretical_loss": 4.156692108846916,
+      "tokens_seen": 311492608
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007317953861584755,
+      "loss": 1.4828,
+      "theoretical_loss": 4.156590836452514,
+      "tokens_seen": 311558144
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007317793380140422,
+      "loss": 1.6621,
+      "theoretical_loss": 4.156489591321726,
+      "tokens_seen": 311623680
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007317632898696089,
+      "loss": 1.7598,
+      "theoretical_loss": 4.15638837344148,
+      "tokens_seen": 311689216
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007317472417251756,
+      "loss": 1.4587,
+      "theoretical_loss": 4.156287182798714,
+      "tokens_seen": 311754752
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007317311935807423,
+      "loss": 1.6337,
+      "theoretical_loss": 4.156186019380375,
+      "tokens_seen": 311820288
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.000731715145436309,
+      "loss": 1.6263,
+      "theoretical_loss": 4.156084883173419,
+      "tokens_seen": 311885824
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007316990972918757,
+      "loss": 1.6484,
+      "theoretical_loss": 4.155983774164811,
+      "tokens_seen": 311951360
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007316830491474424,
+      "loss": 1.6904,
+      "theoretical_loss": 4.1558826923415255,
+      "tokens_seen": 312016896
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007316670010030091,
+      "loss": 1.6563,
+      "theoretical_loss": 4.155781637690545,
+      "tokens_seen": 312082432
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007316509528585758,
+      "loss": 1.6017,
+      "theoretical_loss": 4.1556806101988615,
+      "tokens_seen": 312147968
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007316349047141425,
+      "loss": 1.4874,
+      "theoretical_loss": 4.155579609853476,
+      "tokens_seen": 312213504
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007316188565697092,
+      "loss": 1.6783,
+      "theoretical_loss": 4.155478636641398,
+      "tokens_seen": 312279040
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007316028084252759,
+      "loss": 1.5442,
+      "theoretical_loss": 4.155377690549646,
+      "tokens_seen": 312344576
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007315867602808425,
+      "loss": 1.6007,
+      "theoretical_loss": 4.155276771565248,
+      "tokens_seen": 312410112
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007315707121364093,
+      "loss": 1.5979,
+      "theoretical_loss": 4.155175879675241,
+      "tokens_seen": 312475648
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007315546639919759,
+      "loss": 1.5809,
+      "theoretical_loss": 4.155075014866671,
+      "tokens_seen": 312541184
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007315386158475427,
+      "loss": 1.7356,
+      "theoretical_loss": 4.15497417712659,
+      "tokens_seen": 312606720
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007315225677031093,
+      "loss": 1.5843,
+      "theoretical_loss": 4.1548733664420645,
+      "tokens_seen": 312672256
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007315065195586761,
+      "loss": 1.5196,
+      "theoretical_loss": 4.154772582800165,
+      "tokens_seen": 312737792
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007314904714142427,
+      "loss": 1.6866,
+      "theoretical_loss": 4.154671826187972,
+      "tokens_seen": 312803328
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007314744232698095,
+      "loss": 1.5664,
+      "theoretical_loss": 4.154571096592576,
+      "tokens_seen": 312868864
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007314583751253761,
+      "loss": 1.6195,
+      "theoretical_loss": 4.154470394001077,
+      "tokens_seen": 312934400
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007314423269809429,
+      "loss": 1.6015,
+      "theoretical_loss": 4.154369718400583,
+      "tokens_seen": 312999936
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007314262788365095,
+      "loss": 1.576,
+      "theoretical_loss": 4.154269069778207,
+      "tokens_seen": 313065472
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007314102306920763,
+      "loss": 1.5558,
+      "theoretical_loss": 4.154168448121078,
+      "tokens_seen": 313131008
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007313941825476429,
+      "loss": 1.6493,
+      "theoretical_loss": 4.15406785341633,
+      "tokens_seen": 313196544
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007313781344032097,
+      "loss": 1.5579,
+      "theoretical_loss": 4.153967285651105,
+      "tokens_seen": 313262080
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007313620862587763,
+      "loss": 1.5085,
+      "theoretical_loss": 4.153866744812555,
+      "tokens_seen": 313327616
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007313460381143431,
+      "loss": 1.7063,
+      "theoretical_loss": 4.1537662308878405,
+      "tokens_seen": 313393152
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0007313299899699097,
+      "loss": 1.5337,
+      "theoretical_loss": 4.153665743864131,
+      "tokens_seen": 313458688
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0007313139418254765,
+      "loss": 1.7316,
+      "theoretical_loss": 4.153565283728606,
+      "tokens_seen": 313524224
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0007312978936810431,
+      "loss": 1.5111,
+      "theoretical_loss": 4.153464850468452,
+      "tokens_seen": 313589760
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0007312818455366099,
+      "loss": 1.6305,
+      "theoretical_loss": 4.1533644440708635,
+      "tokens_seen": 313655296
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0007312657973921765,
+      "loss": 1.619,
+      "theoretical_loss": 4.153264064523047,
+      "tokens_seen": 313720832
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0007312497492477433,
+      "loss": 1.5846,
+      "theoretical_loss": 4.1531637118122156,
+      "tokens_seen": 313786368
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0007312337011033099,
+      "loss": 1.749,
+      "theoretical_loss": 4.153063385925591,
+      "tokens_seen": 313851904
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0007312176529588767,
+      "loss": 1.6386,
+      "theoretical_loss": 4.152963086850405,
+      "tokens_seen": 313917440
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0007312016048144433,
+      "loss": 1.601,
+      "theoretical_loss": 4.152862814573895,
+      "tokens_seen": 313982976
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0007311855566700101,
+      "loss": 1.6169,
+      "theoretical_loss": 4.152762569083313,
+      "tokens_seen": 314048512
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0007311695085255767,
+      "loss": 1.6395,
+      "theoretical_loss": 4.1526623503659135,
+      "tokens_seen": 314114048
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0007311534603811435,
+      "loss": 1.5309,
+      "theoretical_loss": 4.152562158408962,
+      "tokens_seen": 314179584
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0007311374122367101,
+      "loss": 1.617,
+      "theoretical_loss": 4.1524619931997355,
+      "tokens_seen": 314245120
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0007311213640922769,
+      "loss": 1.5898,
+      "theoretical_loss": 4.152361854725515,
+      "tokens_seen": 314310656
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0007311053159478435,
+      "loss": 1.6312,
+      "theoretical_loss": 4.152261742973594,
+      "tokens_seen": 314376192
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0007310892678034103,
+      "loss": 1.6135,
+      "theoretical_loss": 4.152161657931273,
+      "tokens_seen": 314441728
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0007310732196589769,
+      "loss": 1.5081,
+      "theoretical_loss": 4.15206159958586,
+      "tokens_seen": 314507264
+    },
+    {
+      "epoch": 0.1,
+      "objective/train/docs_used": 110811,
+      "objective/train/instantaneous_batch_size": 32,
+      "objective/train/instantaneous_microbatch_size": 32768,
+      "objective/train/original_loss": 1.65950608253479,
+      "objective/train/theoretical_loss": 4.1519615679246735,
+      "objective/train/tokens_used": 335032800,
+      "theoretical_loss": 4.1519615679246735,
+      "tokens_seen": 314572800
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0007310571715145437,
+      "loss": 1.5273,
+      "theoretical_loss": 4.1519615679246735,
+      "tokens_seen": 314572800
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0007310411233701103,
+      "loss": 1.6243,
+      "theoretical_loss": 4.151861562935041,
+      "tokens_seen": 314638336
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0007310250752256771,
+      "loss": 1.6176,
+      "theoretical_loss": 4.151761584604298,
+      "tokens_seen": 314703872
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0007310090270812437,
+      "loss": 1.5744,
+      "theoretical_loss": 4.1516616329197875,
+      "tokens_seen": 314769408
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0007309929789368105,
+      "loss": 1.5553,
+      "theoretical_loss": 4.151561707868863,
+      "tokens_seen": 314834944
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0007309769307923771,
+      "loss": 1.6404,
+      "theoretical_loss": 4.151461809438885,
+      "tokens_seen": 314900480
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0007309608826479439,
+      "loss": 1.6536,
+      "theoretical_loss": 4.1513619376172235,
+      "tokens_seen": 314966016
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0007309448345035105,
+      "loss": 1.5826,
+      "theoretical_loss": 4.151262092391257,
+      "tokens_seen": 315031552
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0007309287863590773,
+      "loss": 1.5484,
+      "theoretical_loss": 4.1511622737483735,
+      "tokens_seen": 315097088
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0007309127382146439,
+      "loss": 1.5388,
+      "theoretical_loss": 4.151062481675968,
+      "tokens_seen": 315162624
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0007308966900702107,
+      "loss": 1.672,
+      "theoretical_loss": 4.150962716161445,
+      "tokens_seen": 315228160
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0007308806419257773,
+      "loss": 1.5627,
+      "theoretical_loss": 4.150862977192217,
+      "tokens_seen": 315293696
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0007308645937813441,
+      "loss": 1.5826,
+      "theoretical_loss": 4.150763264755706,
+      "tokens_seen": 315359232
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0007308485456369107,
+      "loss": 1.5799,
+      "theoretical_loss": 4.150663578839342,
+      "tokens_seen": 315424768
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0007308324974924775,
+      "loss": 1.5342,
+      "theoretical_loss": 4.150563919430562,
+      "tokens_seen": 315490304
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0007308164493480441,
+      "loss": 1.5803,
+      "theoretical_loss": 4.150464286516816,
+      "tokens_seen": 315555840
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0007308004012036109,
+      "loss": 1.5861,
+      "theoretical_loss": 4.150364680085558,
+      "tokens_seen": 315621376
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0007307843530591775,
+      "loss": 1.519,
+      "theoretical_loss": 4.150265100124253,
+      "tokens_seen": 315686912
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0007307683049147443,
+      "loss": 1.5708,
+      "theoretical_loss": 4.150165546620373,
+      "tokens_seen": 315752448
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0007307522567703109,
+      "loss": 1.6274,
+      "theoretical_loss": 4.1500660195614,
+      "tokens_seen": 315817984
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0007307362086258777,
+      "loss": 1.6334,
+      "theoretical_loss": 4.149966518934824,
+      "tokens_seen": 315883520
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0007307201604814443,
+      "loss": 1.6195,
+      "theoretical_loss": 4.149867044728143,
+      "tokens_seen": 315949056
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0007307041123370111,
+      "loss": 1.7246,
+      "theoretical_loss": 4.149767596928863,
+      "tokens_seen": 316014592
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0007306880641925777,
+      "loss": 1.6548,
+      "theoretical_loss": 4.149668175524502,
+      "tokens_seen": 316080128
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0007306720160481445,
+      "loss": 1.6405,
+      "theoretical_loss": 4.14956878050258,
+      "tokens_seen": 316145664
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0007306559679037111,
+      "loss": 1.8254,
+      "theoretical_loss": 4.1494694118506334,
+      "tokens_seen": 316211200
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0007306399197592779,
+      "loss": 1.6047,
+      "theoretical_loss": 4.1493700695562,
+      "tokens_seen": 316276736
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0007306238716148446,
+      "loss": 1.5901,
+      "theoretical_loss": 4.14927075360683,
+      "tokens_seen": 316342272
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0007306078234704113,
+      "loss": 1.6434,
+      "theoretical_loss": 4.149171463990082,
+      "tokens_seen": 316407808
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.000730591775325978,
+      "loss": 1.5657,
+      "theoretical_loss": 4.149072200693521,
+      "tokens_seen": 316473344
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0007305757271815447,
+      "loss": 1.6584,
+      "theoretical_loss": 4.148972963704722,
+      "tokens_seen": 316538880
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0007305596790371114,
+      "loss": 1.652,
+      "theoretical_loss": 4.148873753011269,
+      "tokens_seen": 316604416
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0007305436308926781,
+      "loss": 1.6189,
+      "theoretical_loss": 4.148774568600752,
+      "tokens_seen": 316669952
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0007305275827482448,
+      "loss": 1.6358,
+      "theoretical_loss": 4.148675410460772,
+      "tokens_seen": 316735488
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0007305115346038115,
+      "loss": 1.4701,
+      "theoretical_loss": 4.148576278578937,
+      "tokens_seen": 316801024
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0007304954864593782,
+      "loss": 1.5516,
+      "theoretical_loss": 4.148477172942863,
+      "tokens_seen": 316866560
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0007304794383149449,
+      "loss": 1.6256,
+      "theoretical_loss": 4.1483780935401775,
+      "tokens_seen": 316932096
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0007304633901705116,
+      "loss": 1.5445,
+      "theoretical_loss": 4.148279040358511,
+      "tokens_seen": 316997632
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0007304473420260783,
+      "loss": 1.625,
+      "theoretical_loss": 4.148180013385507,
+      "tokens_seen": 317063168
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.000730431293881645,
+      "loss": 1.7008,
+      "theoretical_loss": 4.148081012608817,
+      "tokens_seen": 317128704
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0007304152457372117,
+      "loss": 1.6663,
+      "theoretical_loss": 4.147982038016096,
+      "tokens_seen": 317194240
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0007303991975927784,
+      "loss": 1.6141,
+      "theoretical_loss": 4.147883089595015,
+      "tokens_seen": 317259776
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0007303831494483451,
+      "loss": 1.6443,
+      "theoretical_loss": 4.147784167333247,
+      "tokens_seen": 317325312
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0007303671013039118,
+      "loss": 1.4877,
+      "theoretical_loss": 4.147685271218477,
+      "tokens_seen": 317390848
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0007303510531594785,
+      "loss": 1.5337,
+      "theoretical_loss": 4.147586401238396,
+      "tokens_seen": 317456384
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0007303350050150452,
+      "loss": 1.6511,
+      "theoretical_loss": 4.147487557380705,
+      "tokens_seen": 317521920
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0007303189568706119,
+      "loss": 1.5331,
+      "theoretical_loss": 4.147388739633113,
+      "tokens_seen": 317587456
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0007303029087261786,
+      "loss": 1.5758,
+      "theoretical_loss": 4.147289947983337,
+      "tokens_seen": 317652992
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0007302868605817453,
+      "loss": 1.7897,
+      "theoretical_loss": 4.1471911824191015,
+      "tokens_seen": 317718528
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.000730270812437312,
+      "loss": 1.6711,
+      "theoretical_loss": 4.147092442928141,
+      "tokens_seen": 317784064
+    },
+    {
+      "epoch": 0.1,
+      "objective/train/docs_used": 111685,
+      "objective/train/instantaneous_batch_size": 32,
+      "objective/train/instantaneous_microbatch_size": 32768,
+      "objective/train/original_loss": 1.6283586025238037,
+      "objective/train/theoretical_loss": 4.146993729498197,
+      "objective/train/tokens_used": 338309600,
+      "theoretical_loss": 4.146993729498197,
+      "tokens_seen": 317849600
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0007302547642928787,
+      "loss": 1.6262,
+      "theoretical_loss": 4.146993729498197,
+      "tokens_seen": 317849600
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0007302387161484454,
+      "loss": 1.6615,
+      "theoretical_loss": 4.146895042117021,
+      "tokens_seen": 317915136
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0007302226680040121,
+      "loss": 1.6932,
+      "theoretical_loss": 4.146796380772369,
+      "tokens_seen": 317980672
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0007302066198595788,
+      "loss": 1.5095,
+      "theoretical_loss": 4.146697745452011,
+      "tokens_seen": 318046208
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0007301905717151454,
+      "loss": 1.5231,
+      "theoretical_loss": 4.146599136143719,
+      "tokens_seen": 318111744
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0007301745235707122,
+      "loss": 1.6505,
+      "theoretical_loss": 4.146500552835278,
+      "tokens_seen": 318177280
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0007301584754262788,
+      "loss": 1.6085,
+      "theoretical_loss": 4.146401995514479,
+      "tokens_seen": 318242816
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0007301424272818456,
+      "loss": 1.6205,
+      "theoretical_loss": 4.146303464169123,
+      "tokens_seen": 318308352
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0007301263791374122,
+      "loss": 1.46,
+      "theoretical_loss": 4.146204958787017,
+      "tokens_seen": 318373888
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.000730110330992979,
+      "loss": 1.4363,
+      "theoretical_loss": 4.146106479355978,
+      "tokens_seen": 318439424
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0007300942828485456,
+      "loss": 1.4894,
+      "theoretical_loss": 4.1460080258638285,
+      "tokens_seen": 318504960
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0007300782347041124,
+      "loss": 1.6025,
+      "theoretical_loss": 4.145909598298404,
+      "tokens_seen": 318570496
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.000730062186559679,
+      "loss": 1.6009,
+      "theoretical_loss": 4.145811196647544,
+      "tokens_seen": 318636032
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0007300461384152458,
+      "loss": 1.453,
+      "theoretical_loss": 4.145712820899098,
+      "tokens_seen": 318701568
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0007300300902708124,
+      "loss": 1.5069,
+      "theoretical_loss": 4.145614471040923,
+      "tokens_seen": 318767104
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0007300140421263792,
+      "loss": 1.5945,
+      "theoretical_loss": 4.145516147060884,
+      "tokens_seen": 318832640
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0007299979939819458,
+      "loss": 1.7832,
+      "theoretical_loss": 4.145417848946857,
+      "tokens_seen": 318898176
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0007299819458375126,
+      "loss": 1.4607,
+      "theoretical_loss": 4.145319576686721,
+      "tokens_seen": 318963712
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0007299658976930792,
+      "loss": 1.5956,
+      "theoretical_loss": 4.1452213302683685,
+      "tokens_seen": 319029248
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.000729949849548646,
+      "loss": 1.685,
+      "theoretical_loss": 4.145123109679696,
+      "tokens_seen": 319094784
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0007299338014042126,
+      "loss": 1.738,
+      "theoretical_loss": 4.1450249149086105,
+      "tokens_seen": 319160320
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0007299177532597794,
+      "loss": 1.7129,
+      "theoretical_loss": 4.144926745943026,
+      "tokens_seen": 319225856
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.000729901705115346,
+      "loss": 1.6884,
+      "theoretical_loss": 4.1448286027708665,
+      "tokens_seen": 319291392
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0007298856569709128,
+      "loss": 1.6157,
+      "theoretical_loss": 4.144730485380062,
+      "tokens_seen": 319356928
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0007298696088264794,
+      "loss": 1.6194,
+      "theoretical_loss": 4.144632393758551,
+      "tokens_seen": 319422464
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0007298535606820462,
+      "loss": 1.6735,
+      "theoretical_loss": 4.144534327894281,
+      "tokens_seen": 319488000
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0007298375125376128,
+      "loss": 1.4938,
+      "theoretical_loss": 4.144436287775206,
+      "tokens_seen": 319553536
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0007298214643931796,
+      "loss": 1.5247,
+      "theoretical_loss": 4.144338273389291,
+      "tokens_seen": 319619072
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0007298054162487462,
+      "loss": 1.698,
+      "theoretical_loss": 4.144240284724505,
+      "tokens_seen": 319684608
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.000729789368104313,
+      "loss": 1.6583,
+      "theoretical_loss": 4.144142321768831,
+      "tokens_seen": 319750144
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0007297733199598796,
+      "loss": 1.567,
+      "theoretical_loss": 4.144044384510253,
+      "tokens_seen": 319815680
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0007297572718154464,
+      "loss": 1.5985,
+      "theoretical_loss": 4.1439464729367685,
+      "tokens_seen": 319881216
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.000729741223671013,
+      "loss": 1.6317,
+      "theoretical_loss": 4.14384858703638,
+      "tokens_seen": 319946752
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0007297251755265798,
+      "loss": 1.7177,
+      "theoretical_loss": 4.1437507267971,
+      "tokens_seen": 320012288
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0007297091273821464,
+      "loss": 1.6724,
+      "theoretical_loss": 4.143652892206948,
+      "tokens_seen": 320077824
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0007296930792377132,
+      "loss": 1.6509,
+      "theoretical_loss": 4.1435550832539505,
+      "tokens_seen": 320143360
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0007296770310932798,
+      "loss": 1.6096,
+      "theoretical_loss": 4.143457299926146,
+      "tokens_seen": 320208896
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0007296609829488466,
+      "loss": 1.6542,
+      "theoretical_loss": 4.143359542211576,
+      "tokens_seen": 320274432
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0007296449348044132,
+      "loss": 1.4936,
+      "theoretical_loss": 4.143261810098293,
+      "tokens_seen": 320339968
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.00072962888665998,
+      "loss": 1.5946,
+      "theoretical_loss": 4.1431641035743585,
+      "tokens_seen": 320405504
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0007296128385155466,
+      "loss": 1.8647,
+      "theoretical_loss": 4.143066422627838,
+      "tokens_seen": 320471040
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0007295967903711134,
+      "loss": 1.7195,
+      "theoretical_loss": 4.142968767246808,
+      "tokens_seen": 320536576
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.00072958074222668,
+      "loss": 1.5384,
+      "theoretical_loss": 4.142871137419354,
+      "tokens_seen": 320602112
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0007295646940822468,
+      "loss": 1.557,
+      "theoretical_loss": 4.142773533133565,
+      "tokens_seen": 320667648
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0007295486459378134,
+      "loss": 1.5675,
+      "theoretical_loss": 4.142675954377543,
+      "tokens_seen": 320733184
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0007295325977933802,
+      "loss": 1.5436,
+      "theoretical_loss": 4.142578401139396,
+      "tokens_seen": 320798720
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0007295165496489468,
+      "loss": 1.6971,
+      "theoretical_loss": 4.142480873407239,
+      "tokens_seen": 320864256
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0007295005015045136,
+      "loss": 1.6449,
+      "theoretical_loss": 4.1423833711691955,
+      "tokens_seen": 320929792
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0007294844533600802,
+      "loss": 1.6426,
+      "theoretical_loss": 4.142285894413398,
+      "tokens_seen": 320995328
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.000729468405215647,
+      "loss": 1.6954,
+      "theoretical_loss": 4.142188443127985,
+      "tokens_seen": 321060864
+    },
+    {
+      "epoch": 0.1,
+      "objective/train/docs_used": 112942,
+      "objective/train/instantaneous_batch_size": 32,
+      "objective/train/instantaneous_microbatch_size": 32768,
+      "objective/train/original_loss": 1.700814962387085,
+      "objective/train/theoretical_loss": 4.142091017301105,
+      "objective/train/tokens_used": 341586400,
+      "theoretical_loss": 4.142091017301105,
+      "tokens_seen": 321126400
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0007294523570712136,
+      "loss": 1.7337,
+      "theoretical_loss": 4.142091017301105,
+      "tokens_seen": 321126400
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0007294363089267804,
+      "loss": 1.6901,
+      "theoretical_loss": 4.141993616920914,
+      "tokens_seen": 321191936
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.000729420260782347,
+      "loss": 1.6759,
+      "theoretical_loss": 4.141896241975575,
+      "tokens_seen": 321257472
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0007294042126379138,
+      "loss": 1.4865,
+      "theoretical_loss": 4.141798892453259,
+      "tokens_seen": 321323008
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0007293881644934804,
+      "loss": 1.6282,
+      "theoretical_loss": 4.141701568342145,
+      "tokens_seen": 321388544
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0007293721163490472,
+      "loss": 1.5739,
+      "theoretical_loss": 4.141604269630422,
+      "tokens_seen": 321454080
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0007293560682046138,
+      "loss": 1.5523,
+      "theoretical_loss": 4.1415069963062825,
+      "tokens_seen": 321519616
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0007293400200601806,
+      "loss": 1.6926,
+      "theoretical_loss": 4.1414097483579315,
+      "tokens_seen": 321585152
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0007293239719157472,
+      "loss": 1.6747,
+      "theoretical_loss": 4.1413125257735794,
+      "tokens_seen": 321650688
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.000729307923771314,
+      "loss": 1.6173,
+      "theoretical_loss": 4.141215328541445,
+      "tokens_seen": 321716224
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0007292918756268806,
+      "loss": 1.6028,
+      "theoretical_loss": 4.141118156649753,
+      "tokens_seen": 321781760
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0007292758274824474,
+      "loss": 1.5543,
+      "theoretical_loss": 4.1410210100867415,
+      "tokens_seen": 321847296
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.000729259779338014,
+      "loss": 1.5896,
+      "theoretical_loss": 4.14092388884065,
+      "tokens_seen": 321912832
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0007292437311935808,
+      "loss": 1.5221,
+      "theoretical_loss": 4.1408267928997295,
+      "tokens_seen": 321978368
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0007292276830491475,
+      "loss": 1.6314,
+      "theoretical_loss": 4.140729722252239,
+      "tokens_seen": 322043904
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0007292116349047142,
+      "loss": 1.7016,
+      "theoretical_loss": 4.140632676886442,
+      "tokens_seen": 322109440
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0007291955867602809,
+      "loss": 1.6146,
+      "theoretical_loss": 4.140535656790615,
+      "tokens_seen": 322174976
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0007291795386158476,
+      "loss": 1.6725,
+      "theoretical_loss": 4.140438661953038,
+      "tokens_seen": 322240512
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0007291634904714143,
+      "loss": 1.6458,
+      "theoretical_loss": 4.140341692362,
+      "tokens_seen": 322306048
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.000729147442326981,
+      "loss": 1.555,
+      "theoretical_loss": 4.140244748005799,
+      "tokens_seen": 322371584
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0007291313941825477,
+      "loss": 1.6082,
+      "theoretical_loss": 4.14014782887274,
+      "tokens_seen": 322437120
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0007291153460381144,
+      "loss": 1.6463,
+      "theoretical_loss": 4.140050934951135,
+      "tokens_seen": 322502656
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0007290992978936812,
+      "loss": 1.6962,
+      "theoretical_loss": 4.139954066229304,
+      "tokens_seen": 322568192
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0007290832497492478,
+      "loss": 1.6344,
+      "theoretical_loss": 4.139857222695578,
+      "tokens_seen": 322633728
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0007290672016048146,
+      "loss": 1.7656,
+      "theoretical_loss": 4.139760404338291,
+      "tokens_seen": 322699264
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0007290511534603812,
+      "loss": 1.628,
+      "theoretical_loss": 4.139663611145786,
+      "tokens_seen": 322764800
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.000729035105315948,
+      "loss": 1.5753,
+      "theoretical_loss": 4.139566843106417,
+      "tokens_seen": 322830336
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0007290190571715146,
+      "loss": 1.7131,
+      "theoretical_loss": 4.139470100208542,
+      "tokens_seen": 322895872
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0007290030090270814,
+      "loss": 1.806,
+      "theoretical_loss": 4.139373382440528,
+      "tokens_seen": 322961408
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.000728986960882648,
+      "loss": 1.6066,
+      "theoretical_loss": 4.1392766897907505,
+      "tokens_seen": 323026944
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0007289709127382148,
+      "loss": 1.6261,
+      "theoretical_loss": 4.139180022247592,
+      "tokens_seen": 323092480
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0007289548645937814,
+      "loss": 1.6827,
+      "theoretical_loss": 4.139083379799443,
+      "tokens_seen": 323158016
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0007289388164493481,
+      "loss": 1.66,
+      "theoretical_loss": 4.1389867624347,
+      "tokens_seen": 323223552
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0007289227683049148,
+      "loss": 1.7963,
+      "theoretical_loss": 4.138890170141771,
+      "tokens_seen": 323289088
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0007289067201604815,
+      "loss": 1.7597,
+      "theoretical_loss": 4.138793602909068,
+      "tokens_seen": 323354624
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0007288906720160482,
+      "loss": 1.8073,
+      "theoretical_loss": 4.138697060725013,
+      "tokens_seen": 323420160
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0007288746238716149,
+      "loss": 1.6887,
+      "theoretical_loss": 4.138600543578034,
+      "tokens_seen": 323485696
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0007288585757271816,
+      "loss": 1.7487,
+      "theoretical_loss": 4.138504051456568,
+      "tokens_seen": 323551232
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0007288425275827483,
+      "loss": 1.6694,
+      "theoretical_loss": 4.13840758434906,
+      "tokens_seen": 323616768
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.000728826479438315,
+      "loss": 1.6603,
+      "theoretical_loss": 4.1383111422439605,
+      "tokens_seen": 323682304
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0007288104312938817,
+      "loss": 1.6991,
+      "theoretical_loss": 4.13821472512973,
+      "tokens_seen": 323747840
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0007287943831494483,
+      "loss": 1.6797,
+      "theoretical_loss": 4.138118332994837,
+      "tokens_seen": 323813376
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0007287783350050151,
+      "loss": 1.7166,
+      "theoretical_loss": 4.138021965827753,
+      "tokens_seen": 323878912
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0007287622868605817,
+      "loss": 1.6586,
+      "theoretical_loss": 4.1379256236169635,
+      "tokens_seen": 323944448
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0007287462387161485,
+      "loss": 1.6492,
+      "theoretical_loss": 4.137829306350957,
+      "tokens_seen": 324009984
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0007287301905717151,
+      "loss": 1.757,
+      "theoretical_loss": 4.137733014018233,
+      "tokens_seen": 324075520
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0007287141424272819,
+      "loss": 1.6995,
+      "theoretical_loss": 4.137636746607295,
+      "tokens_seen": 324141056
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0007286980942828485,
+      "loss": 1.5696,
+      "theoretical_loss": 4.137540504106657,
+      "tokens_seen": 324206592
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0007286820461384153,
+      "loss": 1.7136,
+      "theoretical_loss": 4.137444286504841,
+      "tokens_seen": 324272128
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0007286659979939819,
+      "loss": 1.5873,
+      "theoretical_loss": 4.137348093790372,
+      "tokens_seen": 324337664
+    },
+    {
+      "epoch": 0.1,
+      "objective/train/docs_used": 113857,
+      "objective/train/instantaneous_batch_size": 32,
+      "objective/train/instantaneous_microbatch_size": 32768,
+      "objective/train/original_loss": 1.7934999465942383,
+      "objective/train/theoretical_loss": 4.13725192595179,
+      "objective/train/tokens_used": 344863200,
+      "theoretical_loss": 4.13725192595179,
+      "tokens_seen": 324403200
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0007286499498495487,
+      "loss": 1.767,
+      "theoretical_loss": 4.13725192595179,
+      "tokens_seen": 324403200
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0007286339017051153,
+      "loss": 1.7679,
+      "theoretical_loss": 4.1371557829776355,
+      "tokens_seen": 324468736
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0007286178535606821,
+      "loss": 1.6285,
+      "theoretical_loss": 4.13705966485646,
+      "tokens_seen": 324534272
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0007286018054162487,
+      "loss": 1.6915,
+      "theoretical_loss": 4.136963571576825,
+      "tokens_seen": 324599808
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0007285857572718155,
+      "loss": 1.5742,
+      "theoretical_loss": 4.136867503127292,
+      "tokens_seen": 324665344
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0007285697091273821,
+      "loss": 1.6896,
+      "theoretical_loss": 4.136771459496439,
+      "tokens_seen": 324730880
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0007285536609829489,
+      "loss": 1.6767,
+      "theoretical_loss": 4.136675440672844,
+      "tokens_seen": 324796416
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0007285376128385155,
+      "loss": 1.7684,
+      "theoretical_loss": 4.136579446645098,
+      "tokens_seen": 324861952
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0007285215646940823,
+      "loss": 1.8015,
+      "theoretical_loss": 4.136483477401798,
+      "tokens_seen": 324927488
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0007285055165496489,
+      "loss": 1.5625,
+      "theoretical_loss": 4.136387532931546,
+      "tokens_seen": 324993024
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0007284894684052157,
+      "loss": 1.6688,
+      "theoretical_loss": 4.136291613222955,
+      "tokens_seen": 325058560
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0007284734202607823,
+      "loss": 1.5664,
+      "theoretical_loss": 4.136195718264644,
+      "tokens_seen": 325124096
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0007284573721163491,
+      "loss": 1.6453,
+      "theoretical_loss": 4.136099848045239,
+      "tokens_seen": 325189632
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0007284413239719157,
+      "loss": 1.5347,
+      "theoretical_loss": 4.136004002553375,
+      "tokens_seen": 325255168
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0007284252758274825,
+      "loss": 1.6747,
+      "theoretical_loss": 4.135908181777693,
+      "tokens_seen": 325320704
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0007284092276830491,
+      "loss": 1.5615,
+      "theoretical_loss": 4.135812385706842,
+      "tokens_seen": 325386240
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0007283931795386159,
+      "loss": 1.6777,
+      "theoretical_loss": 4.135716614329479,
+      "tokens_seen": 325451776
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0007283771313941825,
+      "loss": 1.6178,
+      "theoretical_loss": 4.135620867634269,
+      "tokens_seen": 325517312
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0007283610832497493,
+      "loss": 1.5208,
+      "theoretical_loss": 4.135525145609881,
+      "tokens_seen": 325582848
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0007283450351053159,
+      "loss": 1.6594,
+      "theoretical_loss": 4.135429448244997,
+      "tokens_seen": 325648384
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0007283289869608827,
+      "loss": 1.6468,
+      "theoretical_loss": 4.135333775528302,
+      "tokens_seen": 325713920
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0007283129388164493,
+      "loss": 1.6528,
+      "theoretical_loss": 4.13523812744849,
+      "tokens_seen": 325779456
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0007282968906720161,
+      "loss": 1.6653,
+      "theoretical_loss": 4.135142503994263,
+      "tokens_seen": 325844992
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0007282808425275827,
+      "loss": 1.5799,
+      "theoretical_loss": 4.13504690515433,
+      "tokens_seen": 325910528
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0007282647943831495,
+      "loss": 1.7105,
+      "theoretical_loss": 4.134951330917408,
+      "tokens_seen": 325976064
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0007282487462387161,
+      "loss": 1.6631,
+      "theoretical_loss": 4.134855781272218,
+      "tokens_seen": 326041600
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0007282326980942829,
+      "loss": 1.6487,
+      "theoretical_loss": 4.134760256207495,
+      "tokens_seen": 326107136
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0007282166499498495,
+      "loss": 1.63,
+      "theoretical_loss": 4.1346647557119764,
+      "tokens_seen": 326172672
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0007282006018054163,
+      "loss": 1.5818,
+      "theoretical_loss": 4.134569279774407,
+      "tokens_seen": 326238208
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0007281845536609829,
+      "loss": 1.7232,
+      "theoretical_loss": 4.134473828383541,
+      "tokens_seen": 326303744
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0007281685055165497,
+      "loss": 1.6898,
+      "theoretical_loss": 4.13437840152814,
+      "tokens_seen": 326369280
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0007281524573721163,
+      "loss": 1.6883,
+      "theoretical_loss": 4.134282999196973,
+      "tokens_seen": 326434816
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0007281364092276831,
+      "loss": 1.5962,
+      "theoretical_loss": 4.134187621378813,
+      "tokens_seen": 326500352
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0007281203610832497,
+      "loss": 1.6067,
+      "theoretical_loss": 4.134092268062446,
+      "tokens_seen": 326565888
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0007281043129388165,
+      "loss": 1.5378,
+      "theoretical_loss": 4.133996939236661,
+      "tokens_seen": 326631424
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0007280882647943831,
+      "loss": 1.6856,
+      "theoretical_loss": 4.133901634890256,
+      "tokens_seen": 326696960
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0007280722166499499,
+      "loss": 1.6527,
+      "theoretical_loss": 4.1338063550120365,
+      "tokens_seen": 326762496
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0007280561685055165,
+      "loss": 1.6668,
+      "theoretical_loss": 4.133711099590815,
+      "tokens_seen": 326828032
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0007280401203610833,
+      "loss": 1.7179,
+      "theoretical_loss": 4.133615868615411,
+      "tokens_seen": 326893568
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0007280240722166499,
+      "loss": 1.6269,
+      "theoretical_loss": 4.133520662074654,
+      "tokens_seen": 326959104
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0007280080240722167,
+      "loss": 1.6931,
+      "theoretical_loss": 4.133425479957375,
+      "tokens_seen": 327024640
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0007279919759277833,
+      "loss": 1.6625,
+      "theoretical_loss": 4.133330322252419,
+      "tokens_seen": 327090176
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0007279759277833501,
+      "loss": 1.6271,
+      "theoretical_loss": 4.1332351889486345,
+      "tokens_seen": 327155712
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0007279598796389167,
+      "loss": 1.6475,
+      "theoretical_loss": 4.133140080034878,
+      "tokens_seen": 327221248
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0007279438314944835,
+      "loss": 1.5666,
+      "theoretical_loss": 4.133044995500013,
+      "tokens_seen": 327286784
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0007279277833500503,
+      "loss": 1.6712,
+      "theoretical_loss": 4.1329499353329116,
+      "tokens_seen": 327352320
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0007279117352056169,
+      "loss": 1.7416,
+      "theoretical_loss": 4.132854899522453,
+      "tokens_seen": 327417856
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0007278956870611837,
+      "loss": 1.6048,
+      "theoretical_loss": 4.132759888057521,
+      "tokens_seen": 327483392
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0007278796389167503,
+      "loss": 1.7779,
+      "theoretical_loss": 4.13266490092701,
+      "tokens_seen": 327548928
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0007278635907723171,
+      "loss": 1.6649,
+      "theoretical_loss": 4.132569938119821,
+      "tokens_seen": 327614464
+    },
+    {
+      "debugging/Compilability": 0.0,
+      "debugging/distinct-1-grams": 0.7056194307904765,
+      "debugging/entropy-1-grams": 4.971469485477989,
+      "debugging/length": 589.4444444444445,
+      "debugging/num_segments": 9,
+      "debugging/score": 0.07199942613520005,
+      "debugging/score_std": 0.01675004059337693,
+      "epoch": 0.1,
+      "objective/train/docs_used": 115127,
+      "objective/train/instantaneous_batch_size": 32,
+      "objective/train/instantaneous_microbatch_size": 32768,
+      "objective/train/original_loss": 1.6466954946517944,
+      "objective/train/theoretical_loss": 4.132474999624861,
+      "objective/train/tokens_used": 348140000,
+      "theoretical_loss": 4.132474999624861,
+      "tokens_seen": 327680000
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0007278475426278837,
+      "loss": 1.6818,
+      "theoretical_loss": 4.132474999624861,
+      "tokens_seen": 327680000
+    }
+  ],
+  "max_steps": 50354,
+  "num_train_epochs": 9223372036854775807,
+  "total_flos": 1.6722690048e+17,
+  "trial_name": null,
+  "trial_params": null
+}