diff --git "a/checkpoint-1757/trainer_state.json" "b/checkpoint-1757/trainer_state.json"
new file mode 100644--- /dev/null
+++ "b/checkpoint-1757/trainer_state.json"
@@ -0,0 +1,13864 @@
+{
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 0.743861134631668,
+  "global_step": 1757,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.0,
+      "learning_rate": 4.166666666666667e-06,
+      "loss": 3.0643,
+      "theoretical_loss": 3.321567680436603,
+      "tokens_seen": 2990538752
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 8.333333333333334e-06,
+      "loss": 3.0798,
+      "theoretical_loss": 3.3215564803546,
+      "tokens_seen": 2990669824
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 1.25e-05,
+      "loss": 2.9318,
+      "theoretical_loss": 3.321545280900887,
+      "tokens_seen": 2990800896
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 1.6666666666666667e-05,
+      "loss": 2.8098,
+      "theoretical_loss": 3.3215340820754022,
+      "tokens_seen": 2990931968
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 2.0833333333333336e-05,
+      "loss": 2.7055,
+      "theoretical_loss": 3.3215228838780817,
+      "tokens_seen": 2991063040
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 2.5e-05,
+      "loss": 2.9762,
+      "theoretical_loss": 3.3215116863088636,
+      "tokens_seen": 2991194112
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 2.916666666666667e-05,
+      "loss": 2.8724,
+      "theoretical_loss": 3.3215004893676854,
+      "tokens_seen": 2991325184
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 3.3333333333333335e-05,
+      "loss": 3.0452,
+      "theoretical_loss": 3.321489293054483,
+      "tokens_seen": 2991456256
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 3.7500000000000003e-05,
+      "loss": 2.8676,
+      "theoretical_loss": 3.321478097369195,
+      "tokens_seen": 2991587328
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 4.166666666666667e-05,
+      "loss": 2.8343,
+      "theoretical_loss": 3.321466902311758,
+      "tokens_seen": 2991718400
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 4.5833333333333334e-05,
+      "loss": 2.743,
+      "theoretical_loss": 3.3214557078821096,
+      "tokens_seen": 2991849472
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 5e-05,
+      "loss": 2.5867,
+      "theoretical_loss": 3.321444514080187,
+      "tokens_seen": 2991980544
+    },
+    {
+      "epoch": 0.01,
+      "objective/train/docs_used": 1640856,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 2.7297048568725586,
+      "objective/train/theoretical_loss": 3.321438917414603,
+      "objective/train/tokens_used": 22097376,
+      "theoretical_loss": 3.321438917414603,
+      "tokens_seen": 2992046080
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 5.4166666666666664e-05,
+      "loss": 2.7262,
+      "theoretical_loss": 3.321433320905927,
+      "tokens_seen": 2992111616
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 5.833333333333334e-05,
+      "loss": 2.6517,
+      "theoretical_loss": 3.3214221283592678,
+      "tokens_seen": 2992242688
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 6.25e-05,
+      "loss": 2.9399,
+      "theoretical_loss": 3.321410936440146,
+      "tokens_seen": 2992373760
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 6.666666666666667e-05,
+      "loss": 2.7939,
+      "theoretical_loss": 3.3213997451485,
+      "tokens_seen": 2992504832
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 7.083333333333334e-05,
+      "loss": 2.5715,
+      "theoretical_loss": 3.3213885544842654,
+      "tokens_seen": 2992635904
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 7.500000000000001e-05,
+      "loss": 2.6047,
+      "theoretical_loss": 3.321377364447381,
+      "tokens_seen": 2992766976
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 7.916666666666666e-05,
+      "loss": 2.6736,
+      "theoretical_loss": 3.3213661750377836,
+      "tokens_seen": 2992898048
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 8.333333333333334e-05,
+      "loss": 2.6853,
+      "theoretical_loss": 3.3213549862554106,
+      "tokens_seen": 2993029120
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 8.75e-05,
+      "loss": 2.3665,
+      "theoretical_loss": 3.3213437981001994,
+      "tokens_seen": 2993160192
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 9.166666666666667e-05,
+      "loss": 2.618,
+      "theoretical_loss": 3.3213326105720875,
+      "tokens_seen": 2993291264
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 9.583333333333334e-05,
+      "loss": 2.6614,
+      "theoretical_loss": 3.3213214236710122,
+      "tokens_seen": 2993422336
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0001,
+      "loss": 2.6163,
+      "theoretical_loss": 3.321310237396911,
+      "tokens_seen": 2993553408
+    },
+    {
+      "epoch": 0.01,
+      "objective/train/docs_used": 1641461,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 2.5642035007476807,
+      "objective/train/theoretical_loss": 3.3212990517497207,
+      "objective/train/tokens_used": 23735776,
+      "theoretical_loss": 3.3212990517497207,
+      "tokens_seen": 2993684480
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 9.995722840034217e-05,
+      "loss": 2.7218,
+      "theoretical_loss": 3.3212990517497207,
+      "tokens_seen": 2993684480
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 9.991445680068435e-05,
+      "loss": 2.5137,
+      "theoretical_loss": 3.3212878667293797,
+      "tokens_seen": 2993815552
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 9.987168520102653e-05,
+      "loss": 2.5283,
+      "theoretical_loss": 3.321276682335825,
+      "tokens_seen": 2993946624
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 9.98289136013687e-05,
+      "loss": 2.6367,
+      "theoretical_loss": 3.3212654985689936,
+      "tokens_seen": 2994077696
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 9.978614200171087e-05,
+      "loss": 2.5822,
+      "theoretical_loss": 3.3212543154288237,
+      "tokens_seen": 2994208768
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 9.974337040205303e-05,
+      "loss": 2.6296,
+      "theoretical_loss": 3.3212431329152525,
+      "tokens_seen": 2994339840
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 9.970059880239521e-05,
+      "loss": 2.5596,
+      "theoretical_loss": 3.321231951028217,
+      "tokens_seen": 2994470912
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 9.965782720273739e-05,
+      "loss": 2.5663,
+      "theoretical_loss": 3.3212207697676552,
+      "tokens_seen": 2994601984
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 9.961505560307956e-05,
+      "loss": 2.5138,
+      "theoretical_loss": 3.3212095891335043,
+      "tokens_seen": 2994733056
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 9.957228400342173e-05,
+      "loss": 2.5938,
+      "theoretical_loss": 3.321198409125702,
+      "tokens_seen": 2994864128
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 9.95295124037639e-05,
+      "loss": 2.4583,
+      "theoretical_loss": 3.321187229744186,
+      "tokens_seen": 2994995200
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 9.948674080410608e-05,
+      "loss": 2.5265,
+      "theoretical_loss": 3.321176050988893,
+      "tokens_seen": 2995126272
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 9.944396920444825e-05,
+      "loss": 2.7376,
+      "theoretical_loss": 3.3211648728597614,
+      "tokens_seen": 2995257344
+    },
+    {
+      "epoch": 0.02,
+      "objective/train/docs_used": 1642666,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 2.7178521156311035,
+      "objective/train/theoretical_loss": 3.3211592840299864,
+      "objective/train/tokens_used": 25374176,
+      "theoretical_loss": 3.3211592840299864,
+      "tokens_seen": 2995322880
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 9.940119760479042e-05,
+      "loss": 2.4464,
+      "theoretical_loss": 3.3211536953567284,
+      "tokens_seen": 2995388416
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 9.93584260051326e-05,
+      "loss": 2.657,
+      "theoretical_loss": 3.321142518479731,
+      "tokens_seen": 2995519488
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 9.931565440547476e-05,
+      "loss": 2.5525,
+      "theoretical_loss": 3.321131342228708,
+      "tokens_seen": 2995650560
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 9.927288280581694e-05,
+      "loss": 2.6099,
+      "theoretical_loss": 3.321120166603596,
+      "tokens_seen": 2995781632
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 9.923011120615912e-05,
+      "loss": 2.4712,
+      "theoretical_loss": 3.3211089916043326,
+      "tokens_seen": 2995912704
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 9.918733960650128e-05,
+      "loss": 2.514,
+      "theoretical_loss": 3.3210978172308554,
+      "tokens_seen": 2996043776
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 9.914456800684346e-05,
+      "loss": 2.4037,
+      "theoretical_loss": 3.3210866434831026,
+      "tokens_seen": 2996174848
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 9.910179640718563e-05,
+      "loss": 2.4715,
+      "theoretical_loss": 3.3210754703610106,
+      "tokens_seen": 2996305920
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 9.90590248075278e-05,
+      "loss": 2.4805,
+      "theoretical_loss": 3.321064297864518,
+      "tokens_seen": 2996436992
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 9.901625320786998e-05,
+      "loss": 2.4755,
+      "theoretical_loss": 3.3210531259935627,
+      "tokens_seen": 2996568064
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 9.897348160821215e-05,
+      "loss": 2.5174,
+      "theoretical_loss": 3.321041954748081,
+      "tokens_seen": 2996699136
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 9.893071000855433e-05,
+      "loss": 2.5996,
+      "theoretical_loss": 3.321030784128012,
+      "tokens_seen": 2996830208
+    },
+    {
+      "epoch": 0.02,
+      "objective/train/docs_used": 1643300,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 2.879695415496826,
+      "objective/train/theoretical_loss": 3.321019614133292,
+      "objective/train/tokens_used": 27012576,
+      "theoretical_loss": 3.321019614133292,
+      "tokens_seen": 2996961280
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 9.888793840889649e-05,
+      "loss": 2.5696,
+      "theoretical_loss": 3.321019614133292,
+      "tokens_seen": 2996961280
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 9.884516680923867e-05,
+      "loss": 2.607,
+      "theoretical_loss": 3.3210084447638595,
+      "tokens_seen": 2997092352
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 9.880239520958085e-05,
+      "loss": 2.4604,
+      "theoretical_loss": 3.320997276019652,
+      "tokens_seen": 2997223424
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 9.875962360992301e-05,
+      "loss": 2.4603,
+      "theoretical_loss": 3.3209861079006067,
+      "tokens_seen": 2997354496
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 9.871685201026519e-05,
+      "loss": 2.4374,
+      "theoretical_loss": 3.320974940406662,
+      "tokens_seen": 2997485568
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 9.867408041060736e-05,
+      "loss": 2.5285,
+      "theoretical_loss": 3.320963773537755,
+      "tokens_seen": 2997616640
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 9.863130881094953e-05,
+      "loss": 2.5895,
+      "theoretical_loss": 3.320952607293824,
+      "tokens_seen": 2997747712
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 9.858853721129171e-05,
+      "loss": 2.5835,
+      "theoretical_loss": 3.320941441674806,
+      "tokens_seen": 2997878784
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 9.854576561163388e-05,
+      "loss": 2.7362,
+      "theoretical_loss": 3.320930276680639,
+      "tokens_seen": 2998009856
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 9.850299401197606e-05,
+      "loss": 2.5553,
+      "theoretical_loss": 3.3209191123112607,
+      "tokens_seen": 2998140928
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 9.846022241231822e-05,
+      "loss": 2.6403,
+      "theoretical_loss": 3.320907948566609,
+      "tokens_seen": 2998272000
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 9.84174508126604e-05,
+      "loss": 2.5129,
+      "theoretical_loss": 3.3208967854466214,
+      "tokens_seen": 2998403072
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 9.837467921300258e-05,
+      "loss": 2.4238,
+      "theoretical_loss": 3.3208856229512356,
+      "tokens_seen": 2998534144
+    },
+    {
+      "epoch": 0.03,
+      "objective/train/docs_used": 1644380,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 2.4041128158569336,
+      "objective/train/theoretical_loss": 3.320880041937749,
+      "objective/train/tokens_used": 28650976,
+      "theoretical_loss": 3.320880041937749,
+      "tokens_seen": 2998599680
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 9.833190761334474e-05,
+      "loss": 2.3705,
+      "theoretical_loss": 3.3208744610803898,
+      "tokens_seen": 2998665216
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 9.828913601368692e-05,
+      "loss": 2.4602,
+      "theoretical_loss": 3.320863299834021,
+      "tokens_seen": 2998796288
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 9.824636441402908e-05,
+      "loss": 2.5492,
+      "theoretical_loss": 3.320852139212068,
+      "tokens_seen": 2998927360
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 9.820359281437126e-05,
+      "loss": 2.426,
+      "theoretical_loss": 3.3208409792144677,
+      "tokens_seen": 2999058432
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 9.816082121471344e-05,
+      "loss": 2.4079,
+      "theoretical_loss": 3.320829819841158,
+      "tokens_seen": 2999189504
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 9.81180496150556e-05,
+      "loss": 2.5189,
+      "theoretical_loss": 3.320818661092077,
+      "tokens_seen": 2999320576
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 9.807527801539777e-05,
+      "loss": 2.4085,
+      "theoretical_loss": 3.3208075029671624,
+      "tokens_seen": 2999451648
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 9.803250641573995e-05,
+      "loss": 2.4031,
+      "theoretical_loss": 3.320796345466352,
+      "tokens_seen": 2999582720
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 9.798973481608213e-05,
+      "loss": 2.5067,
+      "theoretical_loss": 3.320785188589584,
+      "tokens_seen": 2999713792
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 9.79469632164243e-05,
+      "loss": 2.4357,
+      "theoretical_loss": 3.3207740323367956,
+      "tokens_seen": 2999844864
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 9.790419161676647e-05,
+      "loss": 2.4929,
+      "theoretical_loss": 3.3207628767079242,
+      "tokens_seen": 2999975936
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 9.786142001710863e-05,
+      "loss": 2.4052,
+      "theoretical_loss": 3.3207517217029094,
+      "tokens_seen": 3000107008
+    },
+    {
+      "epoch": 0.03,
+      "objective/train/docs_used": 1645056,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 2.3271520137786865,
+      "objective/train/theoretical_loss": 3.3207405673216877,
+      "objective/train/tokens_used": 30289376,
+      "theoretical_loss": 3.3207405673216877,
+      "tokens_seen": 3000238080
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 9.781864841745081e-05,
+      "loss": 2.5201,
+      "theoretical_loss": 3.3207405673216877,
+      "tokens_seen": 3000238080
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 9.777587681779299e-05,
+      "loss": 2.5431,
+      "theoretical_loss": 3.320729413564197,
+      "tokens_seen": 3000369152
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 9.773310521813517e-05,
+      "loss": 2.5359,
+      "theoretical_loss": 3.3207182604303753,
+      "tokens_seen": 3000500224
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 9.769033361847733e-05,
+      "loss": 2.1929,
+      "theoretical_loss": 3.320707107920161,
+      "tokens_seen": 3000631296
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 9.76475620188195e-05,
+      "loss": 2.5196,
+      "theoretical_loss": 3.3206959560334917,
+      "tokens_seen": 3000762368
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 9.760479041916169e-05,
+      "loss": 2.3645,
+      "theoretical_loss": 3.320684804770305,
+      "tokens_seen": 3000893440
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 9.756201881950386e-05,
+      "loss": 2.5329,
+      "theoretical_loss": 3.3206736541305393,
+      "tokens_seen": 3001024512
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 9.751924721984602e-05,
+      "loss": 2.533,
+      "theoretical_loss": 3.3206625041141318,
+      "tokens_seen": 3001155584
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 9.74764756201882e-05,
+      "loss": 2.4702,
+      "theoretical_loss": 3.3206513547210212,
+      "tokens_seen": 3001286656
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 9.743370402053036e-05,
+      "loss": 2.6255,
+      "theoretical_loss": 3.320640205951145,
+      "tokens_seen": 3001417728
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 9.739093242087256e-05,
+      "loss": 2.5139,
+      "theoretical_loss": 3.3206290578044415,
+      "tokens_seen": 3001548800
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 9.734816082121472e-05,
+      "loss": 2.3796,
+      "theoretical_loss": 3.3206179102808484,
+      "tokens_seen": 3001679872
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 9.730538922155689e-05,
+      "loss": 2.4997,
+      "theoretical_loss": 3.3206067633803036,
+      "tokens_seen": 3001810944
+    },
+    {
+      "epoch": 0.04,
+      "objective/train/docs_used": 1646327,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 2.398160219192505,
+      "objective/train/theoretical_loss": 3.320601190163655,
+      "objective/train/tokens_used": 31927776,
+      "theoretical_loss": 3.320601190163655,
+      "tokens_seen": 3001876480
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 9.726261762189906e-05,
+      "loss": 2.3607,
+      "theoretical_loss": 3.320595617102745,
+      "tokens_seen": 3001942016
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 9.721984602224123e-05,
+      "loss": 2.4803,
+      "theoretical_loss": 3.320584471448111,
+      "tokens_seen": 3002073088
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 9.717707442258342e-05,
+      "loss": 2.348,
+      "theoretical_loss": 3.3205733264163393,
+      "tokens_seen": 3002204160
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 9.713430282292558e-05,
+      "loss": 2.2828,
+      "theoretical_loss": 3.320562182007368,
+      "tokens_seen": 3002335232
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 9.709153122326775e-05,
+      "loss": 2.4615,
+      "theoretical_loss": 3.320551038221135,
+      "tokens_seen": 3002466304
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 9.704875962360993e-05,
+      "loss": 2.3988,
+      "theoretical_loss": 3.3205398950575784,
+      "tokens_seen": 3002597376
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 9.700598802395209e-05,
+      "loss": 2.5582,
+      "theoretical_loss": 3.320528752516636,
+      "tokens_seen": 3002728448
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 9.696321642429428e-05,
+      "loss": 2.3266,
+      "theoretical_loss": 3.3205176105982463,
+      "tokens_seen": 3002859520
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 9.692044482463645e-05,
+      "loss": 2.5922,
+      "theoretical_loss": 3.320506469302347,
+      "tokens_seen": 3002990592
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 9.687767322497861e-05,
+      "loss": 2.4959,
+      "theoretical_loss": 3.3204953286288763,
+      "tokens_seen": 3003121664
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 9.683490162532079e-05,
+      "loss": 2.6068,
+      "theoretical_loss": 3.3204841885777725,
+      "tokens_seen": 3003252736
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 9.679213002566297e-05,
+      "loss": 2.3996,
+      "theoretical_loss": 3.3204730491489727,
+      "tokens_seen": 3003383808
+    },
+    {
+      "epoch": 0.04,
+      "objective/train/docs_used": 1647543,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 2.3885486125946045,
+      "objective/train/theoretical_loss": 3.3204619103424164,
+      "objective/train/tokens_used": 33566176,
+      "theoretical_loss": 3.3204619103424164,
+      "tokens_seen": 3003514880
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 9.674935842600514e-05,
+      "loss": 2.4692,
+      "theoretical_loss": 3.3204619103424164,
+      "tokens_seen": 3003514880
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 9.670658682634731e-05,
+      "loss": 2.35,
+      "theoretical_loss": 3.3204507721580403,
+      "tokens_seen": 3003645952
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 9.666381522668948e-05,
+      "loss": 2.3168,
+      "theoretical_loss": 3.3204396345957834,
+      "tokens_seen": 3003777024
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 9.662104362703166e-05,
+      "loss": 2.4671,
+      "theoretical_loss": 3.320428497655584,
+      "tokens_seen": 3003908096
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 9.657827202737383e-05,
+      "loss": 2.552,
+      "theoretical_loss": 3.320417361337379,
+      "tokens_seen": 3004039168
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 9.6535500427716e-05,
+      "loss": 2.3902,
+      "theoretical_loss": 3.3204062256411078,
+      "tokens_seen": 3004170240
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 9.649272882805818e-05,
+      "loss": 2.491,
+      "theoretical_loss": 3.320395090566708,
+      "tokens_seen": 3004301312
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 9.644995722840034e-05,
+      "loss": 2.4628,
+      "theoretical_loss": 3.3203839561141173,
+      "tokens_seen": 3004432384
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 9.640718562874252e-05,
+      "loss": 2.4858,
+      "theoretical_loss": 3.320372822283275,
+      "tokens_seen": 3004563456
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 9.63644140290847e-05,
+      "loss": 2.5038,
+      "theoretical_loss": 3.3203616890741183,
+      "tokens_seen": 3004694528
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 9.632164242942686e-05,
+      "loss": 2.2651,
+      "theoretical_loss": 3.3203505564865856,
+      "tokens_seen": 3004825600
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 9.627887082976904e-05,
+      "loss": 2.4591,
+      "theoretical_loss": 3.3203394245206153,
+      "tokens_seen": 3004956672
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 9.623609923011121e-05,
+      "loss": 2.4003,
+      "theoretical_loss": 3.320328293176145,
+      "tokens_seen": 3005087744
+    },
+    {
+      "epoch": 0.05,
+      "objective/train/docs_used": 1648109,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 2.3964030742645264,
+      "objective/train/theoretical_loss": 3.3203227277369534,
+      "objective/train/tokens_used": 35204576,
+      "theoretical_loss": 3.3203227277369534,
+      "tokens_seen": 3005153280
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 9.619332763045337e-05,
+      "loss": 2.3952,
+      "theoretical_loss": 3.320317162453114,
+      "tokens_seen": 3005218816
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 9.615055603079556e-05,
+      "loss": 2.4367,
+      "theoretical_loss": 3.3203060323514593,
+      "tokens_seen": 3005349888
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 9.610778443113773e-05,
+      "loss": 2.4468,
+      "theoretical_loss": 3.3202949028711197,
+      "tokens_seen": 3005480960
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 9.60650128314799e-05,
+      "loss": 2.3551,
+      "theoretical_loss": 3.3202837740120335,
+      "tokens_seen": 3005612032
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 9.602224123182207e-05,
+      "loss": 2.3886,
+      "theoretical_loss": 3.3202726457741387,
+      "tokens_seen": 3005743104
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 9.597946963216424e-05,
+      "loss": 2.4953,
+      "theoretical_loss": 3.320261518157374,
+      "tokens_seen": 3005874176
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 9.593669803250643e-05,
+      "loss": 2.3074,
+      "theoretical_loss": 3.3202503911616765,
+      "tokens_seen": 3006005248
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 9.589392643284859e-05,
+      "loss": 2.4135,
+      "theoretical_loss": 3.320239264786986,
+      "tokens_seen": 3006136320
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 9.585115483319077e-05,
+      "loss": 2.431,
+      "theoretical_loss": 3.3202281390332393,
+      "tokens_seen": 3006267392
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 9.580838323353294e-05,
+      "loss": 2.3277,
+      "theoretical_loss": 3.320217013900376,
+      "tokens_seen": 3006398464
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 9.576561163387511e-05,
+      "loss": 2.5083,
+      "theoretical_loss": 3.3202058893883333,
+      "tokens_seen": 3006529536
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 9.572284003421729e-05,
+      "loss": 2.4582,
+      "theoretical_loss": 3.3201947654970505,
+      "tokens_seen": 3006660608
+    },
+    {
+      "epoch": 0.05,
+      "objective/train/docs_used": 1649212,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 2.5212416648864746,
+      "objective/train/theoretical_loss": 3.320183642226465,
+      "objective/train/tokens_used": 36842976,
+      "theoretical_loss": 3.320183642226465,
+      "tokens_seen": 3006791680
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 9.568006843455946e-05,
+      "loss": 2.3596,
+      "theoretical_loss": 3.320183642226465,
+      "tokens_seen": 3006791680
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 9.563729683490164e-05,
+      "loss": 2.3923,
+      "theoretical_loss": 3.3201725195765155,
+      "tokens_seen": 3006922752
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 9.55945252352438e-05,
+      "loss": 2.3563,
+      "theoretical_loss": 3.3201613975471402,
+      "tokens_seen": 3007053824
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 9.555175363558598e-05,
+      "loss": 2.4285,
+      "theoretical_loss": 3.3201502761382775,
+      "tokens_seen": 3007184896
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 9.550898203592816e-05,
+      "loss": 2.2928,
+      "theoretical_loss": 3.320139155349866,
+      "tokens_seen": 3007315968
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 9.546621043627032e-05,
+      "loss": 2.4617,
+      "theoretical_loss": 3.3201280351818436,
+      "tokens_seen": 3007447040
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 9.542343883661249e-05,
+      "loss": 2.4107,
+      "theoretical_loss": 3.320116915634149,
+      "tokens_seen": 3007578112
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 9.538066723695466e-05,
+      "loss": 2.5715,
+      "theoretical_loss": 3.3201057967067205,
+      "tokens_seen": 3007709184
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 9.533789563729684e-05,
+      "loss": 2.4217,
+      "theoretical_loss": 3.3200946783994962,
+      "tokens_seen": 3007840256
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 9.529512403763902e-05,
+      "loss": 2.4315,
+      "theoretical_loss": 3.3200835607124146,
+      "tokens_seen": 3007971328
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 9.525235243798119e-05,
+      "loss": 2.4307,
+      "theoretical_loss": 3.3200724436454143,
+      "tokens_seen": 3008102400
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 9.520958083832335e-05,
+      "loss": 2.5032,
+      "theoretical_loss": 3.3200613271984336,
+      "tokens_seen": 3008233472
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 9.516680923866553e-05,
+      "loss": 2.4379,
+      "theoretical_loss": 3.3200502113714108,
+      "tokens_seen": 3008364544
+    },
+    {
+      "epoch": 0.06,
+      "objective/train/docs_used": 1649940,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 2.399945020675659,
+      "objective/train/theoretical_loss": 3.3200446536903643,
+      "objective/train/tokens_used": 38481376,
+      "theoretical_loss": 3.3200446536903643,
+      "tokens_seen": 3008430080
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 9.512403763900771e-05,
+      "loss": 2.5107,
+      "theoretical_loss": 3.3200390961642845,
+      "tokens_seen": 3008495616
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 9.508126603934989e-05,
+      "loss": 2.4659,
+      "theoretical_loss": 3.3200279815769926,
+      "tokens_seen": 3008626688
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 9.503849443969205e-05,
+      "loss": 2.4327,
+      "theoretical_loss": 3.3200168676094743,
+      "tokens_seen": 3008757760
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 9.499572284003421e-05,
+      "loss": 2.2681,
+      "theoretical_loss": 3.320005754261668,
+      "tokens_seen": 3008888832
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 9.49529512403764e-05,
+      "loss": 2.3802,
+      "theoretical_loss": 3.319994641533511,
+      "tokens_seen": 3009019904
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 9.491017964071857e-05,
+      "loss": 2.5461,
+      "theoretical_loss": 3.319983529424943,
+      "tokens_seen": 3009150976
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 9.486740804106075e-05,
+      "loss": 2.4021,
+      "theoretical_loss": 3.3199724179359027,
+      "tokens_seen": 3009282048
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 9.482463644140291e-05,
+      "loss": 2.4038,
+      "theoretical_loss": 3.319961307066327,
+      "tokens_seen": 3009413120
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 9.478186484174508e-05,
+      "loss": 2.4687,
+      "theoretical_loss": 3.3199501968161558,
+      "tokens_seen": 3009544192
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 9.473909324208726e-05,
+      "loss": 2.4401,
+      "theoretical_loss": 3.319939087185327,
+      "tokens_seen": 3009675264
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 9.469632164242944e-05,
+      "loss": 2.4778,
+      "theoretical_loss": 3.3199279781737796,
+      "tokens_seen": 3009806336
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 9.46535500427716e-05,
+      "loss": 2.4166,
+      "theoretical_loss": 3.3199168697814514,
+      "tokens_seen": 3009937408
+    },
+    {
+      "epoch": 0.06,
+      "objective/train/docs_used": 1651249,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 2.412529230117798,
+      "objective/train/theoretical_loss": 3.3199057620082812,
+      "objective/train/tokens_used": 40119776,
+      "theoretical_loss": 3.3199057620082812,
+      "tokens_seen": 3010068480
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 9.461077844311378e-05,
+      "loss": 2.3839,
+      "theoretical_loss": 3.3199057620082812,
+      "tokens_seen": 3010068480
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 9.456800684345594e-05,
+      "loss": 2.5879,
+      "theoretical_loss": 3.319894654854208,
+      "tokens_seen": 3010199552
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 9.452523524379812e-05,
+      "loss": 2.444,
+      "theoretical_loss": 3.3198835483191695,
+      "tokens_seen": 3010330624
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 9.44824636441403e-05,
+      "loss": 2.2849,
+      "theoretical_loss": 3.319872442403105,
+      "tokens_seen": 3010461696
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 9.443969204448247e-05,
+      "loss": 2.568,
+      "theoretical_loss": 3.3198613371059524,
+      "tokens_seen": 3010592768
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 9.439692044482464e-05,
+      "loss": 2.4131,
+      "theoretical_loss": 3.319850232427651,
+      "tokens_seen": 3010723840
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 9.435414884516681e-05,
+      "loss": 2.4222,
+      "theoretical_loss": 3.3198391283681383,
+      "tokens_seen": 3010854912
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 9.431137724550899e-05,
+      "loss": 2.3993,
+      "theoretical_loss": 3.3198280249273546,
+      "tokens_seen": 3010985984
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 9.426860564585116e-05,
+      "loss": 2.5294,
+      "theoretical_loss": 3.319816922105237,
+      "tokens_seen": 3011117056
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 9.422583404619333e-05,
+      "loss": 2.4529,
+      "theoretical_loss": 3.319805819901724,
+      "tokens_seen": 3011248128
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 9.418306244653551e-05,
+      "loss": 2.4924,
+      "theoretical_loss": 3.3197947183167553,
+      "tokens_seen": 3011379200
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 9.414029084687767e-05,
+      "loss": 2.6054,
+      "theoretical_loss": 3.319783617350269,
+      "tokens_seen": 3011510272
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 9.409751924721985e-05,
+      "loss": 2.388,
+      "theoretical_loss": 3.319772517002204,
+      "tokens_seen": 3011641344
+    },
+    {
+      "epoch": 0.07,
+      "objective/train/docs_used": 1651905,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 2.8164331912994385,
+      "objective/train/theoretical_loss": 3.31976696706006,
+      "objective/train/tokens_used": 41758176,
+      "theoretical_loss": 3.31976696706006,
+      "tokens_seen": 3011706880
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 9.405474764756203e-05,
+      "loss": 2.4623,
+      "theoretical_loss": 3.319761417272498,
+      "tokens_seen": 3011772416
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 9.40119760479042e-05,
+      "loss": 2.3955,
+      "theoretical_loss": 3.319750318161091,
+      "tokens_seen": 3011903488
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 9.396920444824637e-05,
+      "loss": 2.2424,
+      "theoretical_loss": 3.3197392196679205,
+      "tokens_seen": 3012034560
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 9.392643284858854e-05,
+      "loss": 2.3568,
+      "theoretical_loss": 3.3197281217929255,
+      "tokens_seen": 3012165632
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 9.388366124893072e-05,
+      "loss": 2.3788,
+      "theoretical_loss": 3.319717024536045,
+      "tokens_seen": 3012296704
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 9.38408896492729e-05,
+      "loss": 2.3081,
+      "theoretical_loss": 3.3197059278972176,
+      "tokens_seen": 3012427776
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 9.379811804961506e-05,
+      "loss": 2.4277,
+      "theoretical_loss": 3.3196948318763817,
+      "tokens_seen": 3012558848
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 9.375534644995724e-05,
+      "loss": 2.4135,
+      "theoretical_loss": 3.319683736473476,
+      "tokens_seen": 3012689920
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 9.37125748502994e-05,
+      "loss": 2.3474,
+      "theoretical_loss": 3.3196726416884395,
+      "tokens_seen": 3012820992
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 9.366980325064158e-05,
+      "loss": 2.5489,
+      "theoretical_loss": 3.3196615475212106,
+      "tokens_seen": 3012952064
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 9.362703165098376e-05,
+      "loss": 2.4998,
+      "theoretical_loss": 3.3196504539717284,
+      "tokens_seen": 3013083136
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 9.358426005132592e-05,
+      "loss": 2.5438,
+      "theoretical_loss": 3.3196393610399317,
+      "tokens_seen": 3013214208
+    },
+    {
+      "epoch": 0.07,
+      "objective/train/docs_used": 1652881,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 2.3386740684509277,
+      "objective/train/theoretical_loss": 3.3196282687257583,
+      "objective/train/tokens_used": 43396576,
+      "theoretical_loss": 3.3196282687257583,
+      "tokens_seen": 3013345280
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 9.35414884516681e-05,
+      "loss": 2.5351,
+      "theoretical_loss": 3.3196282687257583,
+      "tokens_seen": 3013345280
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 9.349871685201027e-05,
+      "loss": 2.3127,
+      "theoretical_loss": 3.3196171770291483,
+      "tokens_seen": 3013476352
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 9.345594525235244e-05,
+      "loss": 2.435,
+      "theoretical_loss": 3.3196060859500394,
+      "tokens_seen": 3013607424
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 9.341317365269462e-05,
+      "loss": 2.3466,
+      "theoretical_loss": 3.319594995488371,
+      "tokens_seen": 3013738496
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 9.337040205303679e-05,
+      "loss": 2.5683,
+      "theoretical_loss": 3.3195839056440812,
+      "tokens_seen": 3013869568
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 9.332763045337895e-05,
+      "loss": 2.4838,
+      "theoretical_loss": 3.3195728164171094,
+      "tokens_seen": 3014000640
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 9.328485885372113e-05,
+      "loss": 2.5372,
+      "theoretical_loss": 3.319561727807394,
+      "tokens_seen": 3014131712
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 9.324208725406331e-05,
+      "loss": 2.4055,
+      "theoretical_loss": 3.3195506398148744,
+      "tokens_seen": 3014262784
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 9.319931565440549e-05,
+      "loss": 2.5566,
+      "theoretical_loss": 3.319539552439489,
+      "tokens_seen": 3014393856
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 9.315654405474765e-05,
+      "loss": 2.4051,
+      "theoretical_loss": 3.3195284656811763,
+      "tokens_seen": 3014524928
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 9.311377245508982e-05,
+      "loss": 2.5032,
+      "theoretical_loss": 3.319517379539876,
+      "tokens_seen": 3014656000
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 9.3071000855432e-05,
+      "loss": 2.427,
+      "theoretical_loss": 3.3195062940155258,
+      "tokens_seen": 3014787072
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 9.302822925577417e-05,
+      "loss": 2.4468,
+      "theoretical_loss": 3.3194952091080654,
+      "tokens_seen": 3014918144
+    },
+    {
+      "epoch": 0.08,
+      "objective/train/docs_used": 1653310,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 2.2789435386657715,
+      "objective/train/theoretical_loss": 3.3194896668856497,
+      "objective/train/tokens_used": 45034976,
+      "theoretical_loss": 3.3194896668856497,
+      "tokens_seen": 3014983680
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 9.298545765611635e-05,
+      "loss": 2.3447,
+      "theoretical_loss": 3.3194841248174334,
+      "tokens_seen": 3015049216
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 9.294268605645852e-05,
+      "loss": 2.3587,
+      "theoretical_loss": 3.3194730411435684,
+      "tokens_seen": 3015180288
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 9.289991445680068e-05,
+      "loss": 2.6264,
+      "theoretical_loss": 3.3194619580864098,
+      "tokens_seen": 3015311360
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 9.285714285714286e-05,
+      "loss": 2.57,
+      "theoretical_loss": 3.3194508756458965,
+      "tokens_seen": 3015442432
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 9.281437125748504e-05,
+      "loss": 2.3972,
+      "theoretical_loss": 3.319439793821967,
+      "tokens_seen": 3015573504
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 9.27715996578272e-05,
+      "loss": 2.4522,
+      "theoretical_loss": 3.3194287126145596,
+      "tokens_seen": 3015704576
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 9.272882805816938e-05,
+      "loss": 2.4546,
+      "theoretical_loss": 3.3194176320236144,
+      "tokens_seen": 3015835648
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 9.268605645851154e-05,
+      "loss": 2.6088,
+      "theoretical_loss": 3.31940655204907,
+      "tokens_seen": 3015966720
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 9.264328485885372e-05,
+      "loss": 2.4454,
+      "theoretical_loss": 3.319395472690865,
+      "tokens_seen": 3016097792
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 9.26005132591959e-05,
+      "loss": 2.3876,
+      "theoretical_loss": 3.3193843939489382,
+      "tokens_seen": 3016228864
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 9.255774165953807e-05,
+      "loss": 2.4971,
+      "theoretical_loss": 3.319373315823229,
+      "tokens_seen": 3016359936
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 9.251497005988024e-05,
+      "loss": 2.5668,
+      "theoretical_loss": 3.3193622383136763,
+      "tokens_seen": 3016491008
+    },
+    {
+      "epoch": 0.08,
+      "objective/train/docs_used": 1654644,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 2.148563861846924,
+      "objective/train/theoretical_loss": 3.3193511614202187,
+      "objective/train/tokens_used": 46673376,
+      "theoretical_loss": 3.3193511614202187,
+      "tokens_seen": 3016622080
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 9.247219846022241e-05,
+      "loss": 2.3435,
+      "theoretical_loss": 3.3193511614202187,
+      "tokens_seen": 3016622080
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 9.242942686056459e-05,
+      "loss": 2.5671,
+      "theoretical_loss": 3.319340085142796,
+      "tokens_seen": 3016753152
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 9.238665526090677e-05,
+      "loss": 2.455,
+      "theoretical_loss": 3.319329009481346,
+      "tokens_seen": 3016884224
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 9.234388366124893e-05,
+      "loss": 2.5706,
+      "theoretical_loss": 3.3193179344358086,
+      "tokens_seen": 3017015296
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 9.230111206159111e-05,
+      "loss": 2.3164,
+      "theoretical_loss": 3.319306860006122,
+      "tokens_seen": 3017146368
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 9.225834046193327e-05,
+      "loss": 2.317,
+      "theoretical_loss": 3.319295786192226,
+      "tokens_seen": 3017277440
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 9.221556886227547e-05,
+      "loss": 2.3955,
+      "theoretical_loss": 3.319284712994059,
+      "tokens_seen": 3017408512
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 9.217279726261763e-05,
+      "loss": 2.4648,
+      "theoretical_loss": 3.3192736404115606,
+      "tokens_seen": 3017539584
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 9.21300256629598e-05,
+      "loss": 2.3474,
+      "theoretical_loss": 3.3192625684446693,
+      "tokens_seen": 3017670656
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 9.208725406330197e-05,
+      "loss": 2.2225,
+      "theoretical_loss": 3.3192514970933242,
+      "tokens_seen": 3017801728
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 9.204448246364414e-05,
+      "loss": 2.4497,
+      "theoretical_loss": 3.319240426357465,
+      "tokens_seen": 3017932800
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 9.200171086398632e-05,
+      "loss": 2.4471,
+      "theoretical_loss": 3.31922935623703,
+      "tokens_seen": 3018063872
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 9.19589392643285e-05,
+      "loss": 2.3816,
+      "theoretical_loss": 3.3192182867319584,
+      "tokens_seen": 3018194944
+    },
+    {
+      "epoch": 0.09,
+      "objective/train/docs_used": 1655335,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 2.1034493446350098,
+      "objective/train/theoretical_loss": 3.319212752210165,
+      "objective/train/tokens_used": 48311776,
+      "theoretical_loss": 3.319212752210165,
+      "tokens_seen": 3018260480
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 9.191616766467066e-05,
+      "loss": 2.3561,
+      "theoretical_loss": 3.3192072178421896,
+      "tokens_seen": 3018326016
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 9.187339606501284e-05,
+      "loss": 2.4573,
+      "theoretical_loss": 3.319196149567662,
+      "tokens_seen": 3018457088
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 9.1830624465355e-05,
+      "loss": 2.3837,
+      "theoretical_loss": 3.3191850819083157,
+      "tokens_seen": 3018588160
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 9.178785286569718e-05,
+      "loss": 2.4923,
+      "theoretical_loss": 3.319174014864089,
+      "tokens_seen": 3018719232
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 9.174508126603936e-05,
+      "loss": 2.6121,
+      "theoretical_loss": 3.319162948434921,
+      "tokens_seen": 3018850304
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 9.170230966638152e-05,
+      "loss": 2.5103,
+      "theoretical_loss": 3.319151882620752,
+      "tokens_seen": 3018981376
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 9.16595380667237e-05,
+      "loss": 2.3644,
+      "theoretical_loss": 3.3191408174215193,
+      "tokens_seen": 3019112448
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 9.161676646706587e-05,
+      "loss": 2.5582,
+      "theoretical_loss": 3.3191297528371635,
+      "tokens_seen": 3019243520
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 9.157399486740804e-05,
+      "loss": 2.3977,
+      "theoretical_loss": 3.319118688867623,
+      "tokens_seen": 3019374592
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 9.153122326775022e-05,
+      "loss": 2.3253,
+      "theoretical_loss": 3.319107625512837,
+      "tokens_seen": 3019505664
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 9.148845166809239e-05,
+      "loss": 2.5582,
+      "theoretical_loss": 3.3190965627727445,
+      "tokens_seen": 3019636736
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 9.144568006843457e-05,
+      "loss": 2.402,
+      "theoretical_loss": 3.3190855006472857,
+      "tokens_seen": 3019767808
+    },
+    {
+      "epoch": 0.09,
+      "objective/train/docs_used": 1656670,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 2.027528762817383,
+      "objective/train/theoretical_loss": 3.3190744391363984,
+      "objective/train/tokens_used": 49950176,
+      "theoretical_loss": 3.3190744391363984,
+      "tokens_seen": 3019898880
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 9.140290846877674e-05,
+      "loss": 2.4387,
+      "theoretical_loss": 3.3190744391363984,
+      "tokens_seen": 3019898880
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 9.136013686911891e-05,
+      "loss": 2.5059,
+      "theoretical_loss": 3.3190633782400223,
+      "tokens_seen": 3020029952
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 9.131736526946109e-05,
+      "loss": 2.3856,
+      "theoretical_loss": 3.3190523179580973,
+      "tokens_seen": 3020161024
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 9.127459366980325e-05,
+      "loss": 2.5568,
+      "theoretical_loss": 3.3190412582905617,
+      "tokens_seen": 3020292096
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 9.123182207014542e-05,
+      "loss": 2.5093,
+      "theoretical_loss": 3.319030199237355,
+      "tokens_seen": 3020423168
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 9.118905047048761e-05,
+      "loss": 2.3476,
+      "theoretical_loss": 3.3190191407984164,
+      "tokens_seen": 3020554240
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 9.114627887082977e-05,
+      "loss": 2.4233,
+      "theoretical_loss": 3.3190080829736854,
+      "tokens_seen": 3020685312
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 9.110350727117195e-05,
+      "loss": 2.4302,
+      "theoretical_loss": 3.318997025763101,
+      "tokens_seen": 3020816384
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 9.106073567151412e-05,
+      "loss": 2.5779,
+      "theoretical_loss": 3.318985969166602,
+      "tokens_seen": 3020947456
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 9.101796407185628e-05,
+      "loss": 2.4611,
+      "theoretical_loss": 3.3189749131841286,
+      "tokens_seen": 3021078528
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 9.097519247219847e-05,
+      "loss": 2.3387,
+      "theoretical_loss": 3.3189638578156195,
+      "tokens_seen": 3021209600
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 9.093242087254064e-05,
+      "loss": 2.6853,
+      "theoretical_loss": 3.3189528030610136,
+      "tokens_seen": 3021340672
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 9.088964927288282e-05,
+      "loss": 2.4887,
+      "theoretical_loss": 3.318941748920251,
+      "tokens_seen": 3021471744
+    },
+    {
+      "epoch": 0.1,
+      "objective/train/docs_used": 1657192,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 2.7123944759368896,
+      "objective/train/theoretical_loss": 3.318936222080042,
+      "objective/train/tokens_used": 51588576,
+      "theoretical_loss": 3.318936222080042,
+      "tokens_seen": 3021537280
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 9.084687767322498e-05,
+      "loss": 2.4619,
+      "theoretical_loss": 3.318930695393271,
+      "tokens_seen": 3021602816
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 9.080410607356715e-05,
+      "loss": 2.4821,
+      "theoretical_loss": 3.3189196424800116,
+      "tokens_seen": 3021733888
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 9.076133447390934e-05,
+      "loss": 2.4471,
+      "theoretical_loss": 3.3189085901804134,
+      "tokens_seen": 3021864960
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 9.07185628742515e-05,
+      "loss": 2.3988,
+      "theoretical_loss": 3.3188975384944155,
+      "tokens_seen": 3021996032
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 9.067579127459367e-05,
+      "loss": 2.469,
+      "theoretical_loss": 3.318886487421957,
+      "tokens_seen": 3022127104
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 9.063301967493585e-05,
+      "loss": 2.5398,
+      "theoretical_loss": 3.318875436962977,
+      "tokens_seen": 3022258176
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 9.059024807527801e-05,
+      "loss": 2.3125,
+      "theoretical_loss": 3.3188643871174155,
+      "tokens_seen": 3022389248
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 9.05474764756202e-05,
+      "loss": 2.3502,
+      "theoretical_loss": 3.318853337885211,
+      "tokens_seen": 3022520320
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 9.050470487596237e-05,
+      "loss": 2.4073,
+      "theoretical_loss": 3.318842289266304,
+      "tokens_seen": 3022651392
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 9.046193327630453e-05,
+      "loss": 2.3899,
+      "theoretical_loss": 3.3188312412606327,
+      "tokens_seen": 3022782464
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 9.041916167664671e-05,
+      "loss": 2.3516,
+      "theoretical_loss": 3.3188201938681368,
+      "tokens_seen": 3022913536
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 9.037639007698889e-05,
+      "loss": 2.3968,
+      "theoretical_loss": 3.318809147088756,
+      "tokens_seen": 3023044608
+    },
+    {
+      "epoch": 0.11,
+      "objective/train/docs_used": 1658380,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 2.6505117416381836,
+      "objective/train/theoretical_loss": 3.3187981009224297,
+      "objective/train/tokens_used": 53226976,
+      "theoretical_loss": 3.3187981009224297,
+      "tokens_seen": 3023175680
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 9.033361847733107e-05,
+      "loss": 2.4709,
+      "theoretical_loss": 3.3187981009224297,
+      "tokens_seen": 3023175680
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 9.029084687767323e-05,
+      "loss": 2.3951,
+      "theoretical_loss": 3.3187870553690972,
+      "tokens_seen": 3023306752
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 9.02480752780154e-05,
+      "loss": 2.5401,
+      "theoretical_loss": 3.3187760104286976,
+      "tokens_seen": 3023437824
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 9.020530367835757e-05,
+      "loss": 2.4571,
+      "theoretical_loss": 3.3187649661011704,
+      "tokens_seen": 3023568896
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 9.016253207869975e-05,
+      "loss": 2.362,
+      "theoretical_loss": 3.3187539223864557,
+      "tokens_seen": 3023699968
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 9.011976047904193e-05,
+      "loss": 2.4072,
+      "theoretical_loss": 3.318742879284492,
+      "tokens_seen": 3023831040
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 9.00769888793841e-05,
+      "loss": 2.3097,
+      "theoretical_loss": 3.3187318367952194,
+      "tokens_seen": 3023962112
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 9.003421727972626e-05,
+      "loss": 2.3313,
+      "theoretical_loss": 3.318720794918577,
+      "tokens_seen": 3024093184
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 8.999144568006844e-05,
+      "loss": 2.3635,
+      "theoretical_loss": 3.3187097536545047,
+      "tokens_seen": 3024224256
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 8.994867408041062e-05,
+      "loss": 2.4897,
+      "theoretical_loss": 3.3186987130029415,
+      "tokens_seen": 3024355328
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 8.990590248075278e-05,
+      "loss": 2.5518,
+      "theoretical_loss": 3.3186876729638266,
+      "tokens_seen": 3024486400
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 8.986313088109496e-05,
+      "loss": 2.4191,
+      "theoretical_loss": 3.3186766335371005,
+      "tokens_seen": 3024617472
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 8.982035928143712e-05,
+      "loss": 2.4865,
+      "theoretical_loss": 3.318665594722702,
+      "tokens_seen": 3024748544
+    },
+    {
+      "epoch": 0.11,
+      "objective/train/docs_used": 1658975,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 2.6697089672088623,
+      "objective/train/theoretical_loss": 3.3186600755451066,
+      "objective/train/tokens_used": 54865376,
+      "theoretical_loss": 3.3186600755451066,
+      "tokens_seen": 3024814080
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 8.97775876817793e-05,
+      "loss": 2.4613,
+      "theoretical_loss": 3.3186545565205705,
+      "tokens_seen": 3024879616
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 8.973481608212148e-05,
+      "loss": 2.4463,
+      "theoretical_loss": 3.318643518930646,
+      "tokens_seen": 3025010688
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 8.969204448246365e-05,
+      "loss": 2.3875,
+      "theoretical_loss": 3.3186324819528674,
+      "tokens_seen": 3025141760
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 8.964927288280582e-05,
+      "loss": 2.4101,
+      "theoretical_loss": 3.318621445587175,
+      "tokens_seen": 3025272832
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 8.960650128314799e-05,
+      "loss": 2.3602,
+      "theoretical_loss": 3.3186104098335076,
+      "tokens_seen": 3025403904
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 8.956372968349017e-05,
+      "loss": 2.41,
+      "theoretical_loss": 3.318599374691805,
+      "tokens_seen": 3025534976
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 8.952095808383235e-05,
+      "loss": 2.4345,
+      "theoretical_loss": 3.318588340162007,
+      "tokens_seen": 3025666048
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 8.947818648417451e-05,
+      "loss": 2.5437,
+      "theoretical_loss": 3.3185773062440527,
+      "tokens_seen": 3025797120
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 8.943541488451669e-05,
+      "loss": 2.3191,
+      "theoretical_loss": 3.3185662729378826,
+      "tokens_seen": 3025928192
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 8.939264328485885e-05,
+      "loss": 2.3322,
+      "theoretical_loss": 3.318555240243435,
+      "tokens_seen": 3026059264
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 8.934987168520103e-05,
+      "loss": 2.3374,
+      "theoretical_loss": 3.3185442081606507,
+      "tokens_seen": 3026190336
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 8.930710008554321e-05,
+      "loss": 2.4206,
+      "theoretical_loss": 3.3185331766894683,
+      "tokens_seen": 3026321408
+    },
+    {
+      "epoch": 0.12,
+      "objective/train/docs_used": 1660130,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 2.2490313053131104,
+      "objective/train/theoretical_loss": 3.318522145829828,
+      "objective/train/tokens_used": 56503776,
+      "theoretical_loss": 3.318522145829828,
+      "tokens_seen": 3026452480
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 8.926432848588537e-05,
+      "loss": 2.2883,
+      "theoretical_loss": 3.318522145829828,
+      "tokens_seen": 3026452480
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 8.922155688622755e-05,
+      "loss": 2.4436,
+      "theoretical_loss": 3.318511115581669,
+      "tokens_seen": 3026583552
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 8.917878528656972e-05,
+      "loss": 2.4165,
+      "theoretical_loss": 3.3185000859449314,
+      "tokens_seen": 3026714624
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 8.91360136869119e-05,
+      "loss": 2.3836,
+      "theoretical_loss": 3.3184890569195544,
+      "tokens_seen": 3026845696
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 8.909324208725407e-05,
+      "loss": 2.3455,
+      "theoretical_loss": 3.3184780285054782,
+      "tokens_seen": 3026976768
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 8.905047048759624e-05,
+      "loss": 2.4005,
+      "theoretical_loss": 3.318467000702642,
+      "tokens_seen": 3027107840
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 8.900769888793842e-05,
+      "loss": 2.425,
+      "theoretical_loss": 3.3184559735109853,
+      "tokens_seen": 3027238912
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 8.896492728828058e-05,
+      "loss": 2.4123,
+      "theoretical_loss": 3.3184449469304482,
+      "tokens_seen": 3027369984
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 8.892215568862276e-05,
+      "loss": 2.4926,
+      "theoretical_loss": 3.3184339209609703,
+      "tokens_seen": 3027501056
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 8.887938408896494e-05,
+      "loss": 2.2946,
+      "theoretical_loss": 3.318422895602491,
+      "tokens_seen": 3027632128
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 8.88366124893071e-05,
+      "loss": 2.2669,
+      "theoretical_loss": 3.31841187085495,
+      "tokens_seen": 3027763200
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 8.879384088964928e-05,
+      "loss": 2.2708,
+      "theoretical_loss": 3.318400846718288,
+      "tokens_seen": 3027894272
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 8.875106928999145e-05,
+      "loss": 2.3823,
+      "theoretical_loss": 3.318389823192443,
+      "tokens_seen": 3028025344
+    },
+    {
+      "epoch": 0.12,
+      "objective/train/docs_used": 1661412,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 2.202181100845337,
+      "objective/train/theoretical_loss": 3.3183843116585585,
+      "objective/train/tokens_used": 58142176,
+      "theoretical_loss": 3.3183843116585585,
+      "tokens_seen": 3028090880
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 8.870829769033362e-05,
+      "loss": 2.4362,
+      "theoretical_loss": 3.318378800277356,
+      "tokens_seen": 3028156416
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 8.86655260906758e-05,
+      "loss": 2.3161,
+      "theoretical_loss": 3.3183677779729663,
+      "tokens_seen": 3028287488
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 8.862275449101797e-05,
+      "loss": 2.4133,
+      "theoretical_loss": 3.3183567562792136,
+      "tokens_seen": 3028418560
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 8.857998289136013e-05,
+      "loss": 2.464,
+      "theoretical_loss": 3.3183457351960377,
+      "tokens_seen": 3028549632
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 8.853721129170231e-05,
+      "loss": 2.4854,
+      "theoretical_loss": 3.3183347147233784,
+      "tokens_seen": 3028680704
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 8.849443969204449e-05,
+      "loss": 2.2685,
+      "theoretical_loss": 3.3183236948611756,
+      "tokens_seen": 3028811776
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 8.845166809238667e-05,
+      "loss": 2.1281,
+      "theoretical_loss": 3.3183126756093686,
+      "tokens_seen": 3028942848
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 8.840889649272883e-05,
+      "loss": 2.3911,
+      "theoretical_loss": 3.318301656967898,
+      "tokens_seen": 3029073920
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 8.8366124893071e-05,
+      "loss": 2.2723,
+      "theoretical_loss": 3.3182906389367024,
+      "tokens_seen": 3029204992
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 8.832335329341318e-05,
+      "loss": 2.2601,
+      "theoretical_loss": 3.3182796215157224,
+      "tokens_seen": 3029336064
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 8.828058169375535e-05,
+      "loss": 2.3874,
+      "theoretical_loss": 3.318268604704898,
+      "tokens_seen": 3029467136
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 8.823781009409753e-05,
+      "loss": 2.3443,
+      "theoretical_loss": 3.318257588504168,
+      "tokens_seen": 3029598208
+    },
+    {
+      "epoch": 0.13,
+      "objective/train/docs_used": 1662079,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 2.219090700149536,
+      "objective/train/theoretical_loss": 3.318246572913474,
+      "objective/train/tokens_used": 59780576,
+      "theoretical_loss": 3.318246572913474,
+      "tokens_seen": 3029729280
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 8.81950384944397e-05,
+      "loss": 2.4424,
+      "theoretical_loss": 3.318246572913474,
+      "tokens_seen": 3029729280
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 8.815226689478186e-05,
+      "loss": 2.4172,
+      "theoretical_loss": 3.318235557932754,
+      "tokens_seen": 3029860352
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 8.810949529512404e-05,
+      "loss": 2.2985,
+      "theoretical_loss": 3.318224543561948,
+      "tokens_seen": 3029991424
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 8.806672369546622e-05,
+      "loss": 2.5111,
+      "theoretical_loss": 3.3182135298009974,
+      "tokens_seen": 3030122496
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 8.80239520958084e-05,
+      "loss": 2.481,
+      "theoretical_loss": 3.3182025166498406,
+      "tokens_seen": 3030253568
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 8.798118049615056e-05,
+      "loss": 2.2891,
+      "theoretical_loss": 3.3181915041084182,
+      "tokens_seen": 3030384640
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 8.793840889649273e-05,
+      "loss": 2.2625,
+      "theoretical_loss": 3.3181804921766695,
+      "tokens_seen": 3030515712
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 8.78956372968349e-05,
+      "loss": 2.6118,
+      "theoretical_loss": 3.318169480854535,
+      "tokens_seen": 3030646784
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 8.785286569717708e-05,
+      "loss": 2.3715,
+      "theoretical_loss": 3.318158470141954,
+      "tokens_seen": 3030777856
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 8.781009409751925e-05,
+      "loss": 2.4212,
+      "theoretical_loss": 3.3181474600388667,
+      "tokens_seen": 3030908928
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 8.776732249786143e-05,
+      "loss": 2.1904,
+      "theoretical_loss": 3.318136450545213,
+      "tokens_seen": 3031040000
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 8.772455089820359e-05,
+      "loss": 2.3964,
+      "theoretical_loss": 3.318125441660933,
+      "tokens_seen": 3031171072
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 8.768177929854577e-05,
+      "loss": 2.2051,
+      "theoretical_loss": 3.318114433385966,
+      "tokens_seen": 3031302144
+    },
+    {
+      "epoch": 0.13,
+      "objective/train/docs_used": 1662642,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 2.347285270690918,
+      "objective/train/theoretical_loss": 3.3181089294769563,
+      "objective/train/tokens_used": 61418976,
+      "theoretical_loss": 3.3181089294769563,
+      "tokens_seen": 3031367680
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 8.763900769888795e-05,
+      "loss": 2.4334,
+      "theoretical_loss": 3.3181034257202526,
+      "tokens_seen": 3031433216
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 8.759623609923011e-05,
+      "loss": 2.3486,
+      "theoretical_loss": 3.318092418663732,
+      "tokens_seen": 3031564288
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 8.755346449957229e-05,
+      "loss": 2.4626,
+      "theoretical_loss": 3.3180814122163453,
+      "tokens_seen": 3031695360
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 8.751069289991445e-05,
+      "loss": 2.4253,
+      "theoretical_loss": 3.3180704063780313,
+      "tokens_seen": 3031826432
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 8.746792130025663e-05,
+      "loss": 2.4214,
+      "theoretical_loss": 3.318059401148731,
+      "tokens_seen": 3031957504
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 8.742514970059881e-05,
+      "loss": 2.536,
+      "theoretical_loss": 3.3180483965283836,
+      "tokens_seen": 3032088576
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 8.738237810094098e-05,
+      "loss": 2.3038,
+      "theoretical_loss": 3.318037392516929,
+      "tokens_seen": 3032219648
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 8.733960650128315e-05,
+      "loss": 2.3862,
+      "theoretical_loss": 3.318026389114308,
+      "tokens_seen": 3032350720
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 8.729683490162532e-05,
+      "loss": 2.2624,
+      "theoretical_loss": 3.3180153863204596,
+      "tokens_seen": 3032481792
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 8.72540633019675e-05,
+      "loss": 2.4936,
+      "theoretical_loss": 3.3180043841353246,
+      "tokens_seen": 3032612864
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 8.721129170230968e-05,
+      "loss": 2.384,
+      "theoretical_loss": 3.317993382558843,
+      "tokens_seen": 3032743936
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 8.716852010265184e-05,
+      "loss": 2.4542,
+      "theoretical_loss": 3.317982381590954,
+      "tokens_seen": 3032875008
+    },
+    {
+      "epoch": 0.14,
+      "objective/train/docs_used": 1663221,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 2.7515668869018555,
+      "objective/train/theoretical_loss": 3.3179713812315983,
+      "objective/train/tokens_used": 63057376,
+      "theoretical_loss": 3.3179713812315983,
+      "tokens_seen": 3033006080
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 8.712574850299402e-05,
+      "loss": 2.4188,
+      "theoretical_loss": 3.3179713812315983,
+      "tokens_seen": 3033006080
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 8.708297690333618e-05,
+      "loss": 2.5102,
+      "theoretical_loss": 3.317960381480716,
+      "tokens_seen": 3033137152
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 8.704020530367836e-05,
+      "loss": 2.39,
+      "theoretical_loss": 3.317949382338247,
+      "tokens_seen": 3033268224
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 8.699743370402054e-05,
+      "loss": 2.3996,
+      "theoretical_loss": 3.3179383838041314,
+      "tokens_seen": 3033399296
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 8.69546621043627e-05,
+      "loss": 2.6354,
+      "theoretical_loss": 3.317927385878309,
+      "tokens_seen": 3033530368
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 8.691189050470488e-05,
+      "loss": 2.4611,
+      "theoretical_loss": 3.31791638856072,
+      "tokens_seen": 3033661440
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 8.686911890504705e-05,
+      "loss": 2.4918,
+      "theoretical_loss": 3.317905391851305,
+      "tokens_seen": 3033792512
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 8.682634730538923e-05,
+      "loss": 2.4662,
+      "theoretical_loss": 3.3178943957500033,
+      "tokens_seen": 3033923584
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 8.67835757057314e-05,
+      "loss": 2.5224,
+      "theoretical_loss": 3.317883400256756,
+      "tokens_seen": 3034054656
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 8.674080410607357e-05,
+      "loss": 2.5573,
+      "theoretical_loss": 3.3178724053715016,
+      "tokens_seen": 3034185728
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 8.669803250641575e-05,
+      "loss": 2.4771,
+      "theoretical_loss": 3.3178614110941815,
+      "tokens_seen": 3034316800
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 8.665526090675791e-05,
+      "loss": 2.6235,
+      "theoretical_loss": 3.3178504174247356,
+      "tokens_seen": 3034447872
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 8.661248930710009e-05,
+      "loss": 2.5638,
+      "theoretical_loss": 3.317839424363104,
+      "tokens_seen": 3034578944
+    },
+    {
+      "epoch": 0.14,
+      "objective/train/docs_used": 1664363,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 2.340308666229248,
+      "objective/train/theoretical_loss": 3.3178339280602,
+      "objective/train/tokens_used": 64695776,
+      "theoretical_loss": 3.3178339280602,
+      "tokens_seen": 3034644480
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 8.656971770744227e-05,
+      "loss": 2.4057,
+      "theoretical_loss": 3.317828431909227,
+      "tokens_seen": 3034710016
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 8.652694610778443e-05,
+      "loss": 2.3668,
+      "theoretical_loss": 3.3178174400630445,
+      "tokens_seen": 3034841088
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 8.64841745081266e-05,
+      "loss": 2.616,
+      "theoretical_loss": 3.3178064488244967,
+      "tokens_seen": 3034972160
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 8.644140290846878e-05,
+      "loss": 2.402,
+      "theoretical_loss": 3.3177954581935234,
+      "tokens_seen": 3035103232
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 8.639863130881095e-05,
+      "loss": 2.3662,
+      "theoretical_loss": 3.317784468170066,
+      "tokens_seen": 3035234304
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 8.635585970915313e-05,
+      "loss": 2.3539,
+      "theoretical_loss": 3.317773478754063,
+      "tokens_seen": 3035365376
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 8.63130881094953e-05,
+      "loss": 2.4253,
+      "theoretical_loss": 3.317762489945456,
+      "tokens_seen": 3035496448
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 8.627031650983746e-05,
+      "loss": 2.3829,
+      "theoretical_loss": 3.3177515017441843,
+      "tokens_seen": 3035627520
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 8.622754491017964e-05,
+      "loss": 2.5298,
+      "theoretical_loss": 3.3177405141501883,
+      "tokens_seen": 3035758592
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 8.618477331052182e-05,
+      "loss": 2.4475,
+      "theoretical_loss": 3.317729527163409,
+      "tokens_seen": 3035889664
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 8.6142001710864e-05,
+      "loss": 2.7431,
+      "theoretical_loss": 3.3177185407837855,
+      "tokens_seen": 3036020736
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 8.609923011120616e-05,
+      "loss": 2.4028,
+      "theoretical_loss": 3.3177075550112587,
+      "tokens_seen": 3036151808
+    },
+    {
+      "epoch": 0.15,
+      "objective/train/docs_used": 1664891,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 2.417335033416748,
+      "objective/train/theoretical_loss": 3.3176965698457686,
+      "objective/train/tokens_used": 66334176,
+      "theoretical_loss": 3.3176965698457686,
+      "tokens_seen": 3036282880
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 8.605645851154833e-05,
+      "loss": 2.4356,
+      "theoretical_loss": 3.3176965698457686,
+      "tokens_seen": 3036282880
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 8.601368691189052e-05,
+      "loss": 2.4589,
+      "theoretical_loss": 3.3176855852872555,
+      "tokens_seen": 3036413952
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 8.597091531223268e-05,
+      "loss": 2.4168,
+      "theoretical_loss": 3.3176746013356597,
+      "tokens_seen": 3036545024
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 8.592814371257485e-05,
+      "loss": 2.3732,
+      "theoretical_loss": 3.317663617990922,
+      "tokens_seen": 3036676096
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 8.588537211291703e-05,
+      "loss": 2.3843,
+      "theoretical_loss": 3.3176526352529816,
+      "tokens_seen": 3036807168
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 8.584260051325919e-05,
+      "loss": 2.3937,
+      "theoretical_loss": 3.31764165312178,
+      "tokens_seen": 3036938240
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 8.579982891360138e-05,
+      "loss": 2.4782,
+      "theoretical_loss": 3.3176306715972563,
+      "tokens_seen": 3037069312
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 8.575705731394355e-05,
+      "loss": 2.5745,
+      "theoretical_loss": 3.3176196906793516,
+      "tokens_seen": 3037200384
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 8.571428571428571e-05,
+      "loss": 2.4332,
+      "theoretical_loss": 3.3176087103680056,
+      "tokens_seen": 3037331456
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 8.567151411462789e-05,
+      "loss": 2.4773,
+      "theoretical_loss": 3.3175977306631594,
+      "tokens_seen": 3037462528
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 8.562874251497006e-05,
+      "loss": 2.4218,
+      "theoretical_loss": 3.3175867515647526,
+      "tokens_seen": 3037593600
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 8.558597091531225e-05,
+      "loss": 2.3285,
+      "theoretical_loss": 3.317575773072726,
+      "tokens_seen": 3037724672
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 8.554319931565441e-05,
+      "loss": 2.2487,
+      "theoretical_loss": 3.3175647951870197,
+      "tokens_seen": 3037855744
+    },
+    {
+      "epoch": 0.15,
+      "objective/train/docs_used": 1665791,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 2.037510395050049,
+      "objective/train/theoretical_loss": 3.317559306471518,
+      "objective/train/tokens_used": 67972576,
+      "theoretical_loss": 3.317559306471518,
+      "tokens_seen": 3037921280
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 8.550042771599658e-05,
+      "loss": 2.4374,
+      "theoretical_loss": 3.3175538179075743,
+      "tokens_seen": 3037986816
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 8.545765611633876e-05,
+      "loss": 2.3843,
+      "theoretical_loss": 3.31754284123433,
+      "tokens_seen": 3038117888
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 8.541488451668092e-05,
+      "loss": 2.3408,
+      "theoretical_loss": 3.3175318651672274,
+      "tokens_seen": 3038248960
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 8.537211291702311e-05,
+      "loss": 2.45,
+      "theoretical_loss": 3.3175208897062065,
+      "tokens_seen": 3038380032
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 8.532934131736528e-05,
+      "loss": 2.5611,
+      "theoretical_loss": 3.317509914851208,
+      "tokens_seen": 3038511104
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 8.528656971770744e-05,
+      "loss": 2.2891,
+      "theoretical_loss": 3.3174989406021718,
+      "tokens_seen": 3038642176
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 8.524379811804962e-05,
+      "loss": 2.5899,
+      "theoretical_loss": 3.317487966959039,
+      "tokens_seen": 3038773248
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 8.520102651839178e-05,
+      "loss": 2.4568,
+      "theoretical_loss": 3.3174769939217494,
+      "tokens_seen": 3038904320
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 8.515825491873396e-05,
+      "loss": 2.4892,
+      "theoretical_loss": 3.317466021490244,
+      "tokens_seen": 3039035392
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 8.511548331907614e-05,
+      "loss": 2.6673,
+      "theoretical_loss": 3.3174550496644626,
+      "tokens_seen": 3039166464
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 8.50727117194183e-05,
+      "loss": 2.4839,
+      "theoretical_loss": 3.317444078444346,
+      "tokens_seen": 3039297536
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 8.502994011976048e-05,
+      "loss": 2.4649,
+      "theoretical_loss": 3.317433107829835,
+      "tokens_seen": 3039428608
+    },
+    {
+      "epoch": 0.16,
+      "objective/train/docs_used": 1666368,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 2.4885432720184326,
+      "objective/train/theoretical_loss": 3.31742213782087,
+      "objective/train/tokens_used": 69610976,
+      "theoretical_loss": 3.31742213782087,
+      "tokens_seen": 3039559680
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 8.498716852010266e-05,
+      "loss": 2.5349,
+      "theoretical_loss": 3.31742213782087,
+      "tokens_seen": 3039559680
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 8.494439692044483e-05,
+      "loss": 2.403,
+      "theoretical_loss": 3.3174111684173906,
+      "tokens_seen": 3039690752
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 8.4901625320787e-05,
+      "loss": 2.5023,
+      "theoretical_loss": 3.317400199619338,
+      "tokens_seen": 3039821824
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 8.485885372112917e-05,
+      "loss": 2.4205,
+      "theoretical_loss": 3.3173892314266524,
+      "tokens_seen": 3039952896
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 8.481608212147135e-05,
+      "loss": 2.5295,
+      "theoretical_loss": 3.3173782638392746,
+      "tokens_seen": 3040083968
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 8.477331052181353e-05,
+      "loss": 2.4369,
+      "theoretical_loss": 3.3173672968571446,
+      "tokens_seen": 3040215040
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 8.473053892215569e-05,
+      "loss": 2.5302,
+      "theoretical_loss": 3.3173563304802034,
+      "tokens_seen": 3040346112
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 8.468776732249787e-05,
+      "loss": 2.486,
+      "theoretical_loss": 3.3173453647083915,
+      "tokens_seen": 3040477184
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 8.464499572284003e-05,
+      "loss": 2.511,
+      "theoretical_loss": 3.317334399541649,
+      "tokens_seen": 3040608256
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 8.460222412318221e-05,
+      "loss": 2.6424,
+      "theoretical_loss": 3.3173234349799166,
+      "tokens_seen": 3040739328
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 8.455945252352439e-05,
+      "loss": 2.4228,
+      "theoretical_loss": 3.317312471023135,
+      "tokens_seen": 3040870400
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 8.451668092386656e-05,
+      "loss": 2.5719,
+      "theoretical_loss": 3.3173015076712447,
+      "tokens_seen": 3041001472
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 8.447390932420873e-05,
+      "loss": 2.5819,
+      "theoretical_loss": 3.3172905449241865,
+      "tokens_seen": 3041132544
+    },
+    {
+      "epoch": 0.16,
+      "objective/train/docs_used": 1667402,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 2.5028133392333984,
+      "objective/train/theoretical_loss": 3.317285063777451,
+      "objective/train/tokens_used": 71249376,
+      "theoretical_loss": 3.317285063777451,
+      "tokens_seen": 3041198080
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 8.44311377245509e-05,
+      "loss": 2.5006,
+      "theoretical_loss": 3.3172795827819,
+      "tokens_seen": 3041263616
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 8.438836612489306e-05,
+      "loss": 2.4687,
+      "theoretical_loss": 3.3172686212443274,
+      "tokens_seen": 3041394688
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 8.434559452523526e-05,
+      "loss": 2.5483,
+      "theoretical_loss": 3.317257660311408,
+      "tokens_seen": 3041525760
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 8.430282292557742e-05,
+      "loss": 2.6292,
+      "theoretical_loss": 3.3172466999830825,
+      "tokens_seen": 3041656832
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 8.42600513259196e-05,
+      "loss": 2.4604,
+      "theoretical_loss": 3.317235740259292,
+      "tokens_seen": 3041787904
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 8.421727972626176e-05,
+      "loss": 2.4137,
+      "theoretical_loss": 3.3172247811399767,
+      "tokens_seen": 3041918976
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 8.417450812660394e-05,
+      "loss": 2.5,
+      "theoretical_loss": 3.317213822625077,
+      "tokens_seen": 3042050048
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 8.413173652694612e-05,
+      "loss": 2.5009,
+      "theoretical_loss": 3.3172028647145346,
+      "tokens_seen": 3042181120
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 8.408896492728828e-05,
+      "loss": 2.4612,
+      "theoretical_loss": 3.317191907408289,
+      "tokens_seen": 3042312192
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 8.404619332763046e-05,
+      "loss": 2.4446,
+      "theoretical_loss": 3.3171809507062817,
+      "tokens_seen": 3042443264
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 8.400342172797263e-05,
+      "loss": 2.5055,
+      "theoretical_loss": 3.3171699946084523,
+      "tokens_seen": 3042574336
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 8.39606501283148e-05,
+      "loss": 2.5237,
+      "theoretical_loss": 3.3171590391147427,
+      "tokens_seen": 3042705408
+    },
+    {
+      "epoch": 0.17,
+      "objective/train/docs_used": 1668521,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 2.3456153869628906,
+      "objective/train/theoretical_loss": 3.3171480842250927,
+      "objective/train/tokens_used": 72887776,
+      "theoretical_loss": 3.3171480842250927,
+      "tokens_seen": 3042836480
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 8.391787852865698e-05,
+      "loss": 2.4685,
+      "theoretical_loss": 3.3171480842250927,
+      "tokens_seen": 3042836480
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 8.387510692899915e-05,
+      "loss": 2.4987,
+      "theoretical_loss": 3.317137129939443,
+      "tokens_seen": 3042967552
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 8.383233532934131e-05,
+      "loss": 2.4009,
+      "theoretical_loss": 3.3171261762577346,
+      "tokens_seen": 3043098624
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 8.378956372968349e-05,
+      "loss": 2.4797,
+      "theoretical_loss": 3.3171152231799086,
+      "tokens_seen": 3043229696
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 8.374679213002567e-05,
+      "loss": 2.4712,
+      "theoretical_loss": 3.317104270705905,
+      "tokens_seen": 3043360768
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 8.370402053036785e-05,
+      "loss": 2.4712,
+      "theoretical_loss": 3.3170933188356644,
+      "tokens_seen": 3043491840
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 8.366124893071001e-05,
+      "loss": 2.5555,
+      "theoretical_loss": 3.3170823675691277,
+      "tokens_seen": 3043622912
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 8.361847733105218e-05,
+      "loss": 2.5155,
+      "theoretical_loss": 3.317071416906236,
+      "tokens_seen": 3043753984
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 8.357570573139436e-05,
+      "loss": 2.4821,
+      "theoretical_loss": 3.3170604668469297,
+      "tokens_seen": 3043885056
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 8.353293413173653e-05,
+      "loss": 2.5343,
+      "theoretical_loss": 3.31704951739115,
+      "tokens_seen": 3044016128
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 8.349016253207871e-05,
+      "loss": 2.4523,
+      "theoretical_loss": 3.317038568538837,
+      "tokens_seen": 3044147200
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 8.344739093242088e-05,
+      "loss": 2.3806,
+      "theoretical_loss": 3.317027620289932,
+      "tokens_seen": 3044278272
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 8.340461933276304e-05,
+      "loss": 2.5725,
+      "theoretical_loss": 3.317016672644375,
+      "tokens_seen": 3044409344
+    },
+    {
+      "epoch": 0.17,
+      "objective/train/docs_used": 1668980,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 2.030937671661377,
+      "objective/train/theoretical_loss": 3.3170111990478337,
+      "objective/train/tokens_used": 74526176,
+      "theoretical_loss": 3.3170111990478337,
+      "tokens_seen": 3044474880
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 8.336184773310522e-05,
+      "loss": 2.3872,
+      "theoretical_loss": 3.3170057256021077,
+      "tokens_seen": 3044540416
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 8.33190761334474e-05,
+      "loss": 2.497,
+      "theoretical_loss": 3.3169947791630703,
+      "tokens_seen": 3044671488
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 8.327630453378958e-05,
+      "loss": 2.5771,
+      "theoretical_loss": 3.3169838333272037,
+      "tokens_seen": 3044802560
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 8.323353293413174e-05,
+      "loss": 2.4761,
+      "theoretical_loss": 3.316972888094449,
+      "tokens_seen": 3044933632
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 8.319076133447391e-05,
+      "loss": 2.4855,
+      "theoretical_loss": 3.3169619434647464,
+      "tokens_seen": 3045064704
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 8.314798973481609e-05,
+      "loss": 2.3585,
+      "theoretical_loss": 3.3169509994380375,
+      "tokens_seen": 3045195776
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 8.310521813515826e-05,
+      "loss": 2.5985,
+      "theoretical_loss": 3.3169400560142623,
+      "tokens_seen": 3045326848
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 8.306244653550043e-05,
+      "loss": 2.4823,
+      "theoretical_loss": 3.3169291131933623,
+      "tokens_seen": 3045457920
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 8.30196749358426e-05,
+      "loss": 2.4651,
+      "theoretical_loss": 3.316918170975278,
+      "tokens_seen": 3045588992
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 8.297690333618477e-05,
+      "loss": 2.4029,
+      "theoretical_loss": 3.31690722935995,
+      "tokens_seen": 3045720064
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 8.293413173652695e-05,
+      "loss": 2.5912,
+      "theoretical_loss": 3.3168962883473205,
+      "tokens_seen": 3045851136
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 8.289136013686913e-05,
+      "loss": 2.5166,
+      "theoretical_loss": 3.316885347937329,
+      "tokens_seen": 3045982208
+    },
+    {
+      "epoch": 0.18,
+      "objective/train/docs_used": 1670028,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 2.2350833415985107,
+      "objective/train/theoretical_loss": 3.316874408129916,
+      "objective/train/tokens_used": 76164576,
+      "theoretical_loss": 3.316874408129916,
+      "tokens_seen": 3046113280
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 8.284858853721129e-05,
+      "loss": 2.5141,
+      "theoretical_loss": 3.316874408129916,
+      "tokens_seen": 3046113280
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 8.280581693755347e-05,
+      "loss": 2.441,
+      "theoretical_loss": 3.316863468925024,
+      "tokens_seen": 3046244352
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 8.276304533789564e-05,
+      "loss": 2.5365,
+      "theoretical_loss": 3.3168525303225924,
+      "tokens_seen": 3046375424
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 8.272027373823781e-05,
+      "loss": 2.6514,
+      "theoretical_loss": 3.316841592322563,
+      "tokens_seen": 3046506496
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 8.267750213857999e-05,
+      "loss": 2.6148,
+      "theoretical_loss": 3.3168306549248765,
+      "tokens_seen": 3046637568
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 8.263473053892216e-05,
+      "loss": 2.4865,
+      "theoretical_loss": 3.316819718129474,
+      "tokens_seen": 3046768640
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 8.259195893926434e-05,
+      "loss": 2.5057,
+      "theoretical_loss": 3.3168087819362957,
+      "tokens_seen": 3046899712
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 8.25491873396065e-05,
+      "loss": 2.4055,
+      "theoretical_loss": 3.316797846345283,
+      "tokens_seen": 3047030784
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 8.250641573994868e-05,
+      "loss": 2.5902,
+      "theoretical_loss": 3.316786911356377,
+      "tokens_seen": 3047161856
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 8.246364414029086e-05,
+      "loss": 2.3611,
+      "theoretical_loss": 3.316775976969519,
+      "tokens_seen": 3047292928
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 8.242087254063302e-05,
+      "loss": 2.5959,
+      "theoretical_loss": 3.316765043184649,
+      "tokens_seen": 3047424000
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 8.23781009409752e-05,
+      "loss": 2.5646,
+      "theoretical_loss": 3.316754110001708,
+      "tokens_seen": 3047555072
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 8.233532934131736e-05,
+      "loss": 2.585,
+      "theoretical_loss": 3.3167431774206384,
+      "tokens_seen": 3047686144
+    },
+    {
+      "epoch": 0.19,
+      "objective/train/docs_used": 1670628,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 1.981849193572998,
+      "objective/train/theoretical_loss": 3.316737711355786,
+      "objective/train/tokens_used": 77802976,
+      "theoretical_loss": 3.316737711355786,
+      "tokens_seen": 3047751680
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 8.229255774165954e-05,
+      "loss": 2.3982,
+      "theoretical_loss": 3.3167322454413792,
+      "tokens_seen": 3047817216
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 8.224978614200172e-05,
+      "loss": 2.5217,
+      "theoretical_loss": 3.316721314063873,
+      "tokens_seen": 3047948288
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 8.220701454234389e-05,
+      "loss": 2.6089,
+      "theoretical_loss": 3.3167103832880604,
+      "tokens_seen": 3048079360
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 8.216424294268606e-05,
+      "loss": 2.529,
+      "theoretical_loss": 3.316699453113882,
+      "tokens_seen": 3048210432
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 8.212147134302823e-05,
+      "loss": 2.4732,
+      "theoretical_loss": 3.3166885235412784,
+      "tokens_seen": 3048341504
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 8.207869974337041e-05,
+      "loss": 2.4514,
+      "theoretical_loss": 3.316677594570192,
+      "tokens_seen": 3048472576
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 8.203592814371259e-05,
+      "loss": 2.4329,
+      "theoretical_loss": 3.316666666200563,
+      "tokens_seen": 3048603648
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 8.199315654405475e-05,
+      "loss": 2.5147,
+      "theoretical_loss": 3.316655738432332,
+      "tokens_seen": 3048734720
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 8.195038494439693e-05,
+      "loss": 2.4813,
+      "theoretical_loss": 3.3166448112654408,
+      "tokens_seen": 3048865792
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 8.190761334473909e-05,
+      "loss": 2.4503,
+      "theoretical_loss": 3.3166338846998302,
+      "tokens_seen": 3048996864
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 8.186484174508127e-05,
+      "loss": 2.5814,
+      "theoretical_loss": 3.316622958735442,
+      "tokens_seen": 3049127936
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 8.182207014542345e-05,
+      "loss": 2.5462,
+      "theoretical_loss": 3.3166120333722158,
+      "tokens_seen": 3049259008
+    },
+    {
+      "epoch": 0.19,
+      "objective/train/docs_used": 1671752,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 2.1045682430267334,
+      "objective/train/theoretical_loss": 3.3166011086100937,
+      "objective/train/tokens_used": 79441376,
+      "theoretical_loss": 3.3166011086100937,
+      "tokens_seen": 3049390080
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 8.177929854576561e-05,
+      "loss": 2.5946,
+      "theoretical_loss": 3.3166011086100937,
+      "tokens_seen": 3049390080
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 8.173652694610778e-05,
+      "loss": 2.4925,
+      "theoretical_loss": 3.3165901844490167,
+      "tokens_seen": 3049521152
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 8.169375534644996e-05,
+      "loss": 2.4937,
+      "theoretical_loss": 3.3165792608889255,
+      "tokens_seen": 3049652224
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 8.165098374679214e-05,
+      "loss": 2.5232,
+      "theoretical_loss": 3.3165683379297612,
+      "tokens_seen": 3049783296
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 8.160821214713431e-05,
+      "loss": 2.4577,
+      "theoretical_loss": 3.3165574155714657,
+      "tokens_seen": 3049914368
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 8.156544054747648e-05,
+      "loss": 2.5193,
+      "theoretical_loss": 3.3165464938139797,
+      "tokens_seen": 3050045440
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 8.152266894781864e-05,
+      "loss": 2.4776,
+      "theoretical_loss": 3.3165355726572434,
+      "tokens_seen": 3050176512
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 8.147989734816082e-05,
+      "loss": 2.4092,
+      "theoretical_loss": 3.3165246521011995,
+      "tokens_seen": 3050307584
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 8.1437125748503e-05,
+      "loss": 2.3898,
+      "theoretical_loss": 3.3165137321457885,
+      "tokens_seen": 3050438656
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 8.139435414884518e-05,
+      "loss": 2.4147,
+      "theoretical_loss": 3.3165028127909513,
+      "tokens_seen": 3050569728
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 8.135158254918734e-05,
+      "loss": 2.559,
+      "theoretical_loss": 3.3164918940366293,
+      "tokens_seen": 3050700800
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 8.130881094952951e-05,
+      "loss": 2.4779,
+      "theoretical_loss": 3.3164809758827634,
+      "tokens_seen": 3050831872
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 8.126603934987169e-05,
+      "loss": 2.5393,
+      "theoretical_loss": 3.3164700583292954,
+      "tokens_seen": 3050962944
+    },
+    {
+      "epoch": 0.2,
+      "objective/train/docs_used": 1672176,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 2.2255687713623047,
+      "objective/train/theoretical_loss": 3.316464599777692,
+      "objective/train/tokens_used": 81079776,
+      "theoretical_loss": 3.316464599777692,
+      "tokens_seen": 3051028480
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 8.122326775021386e-05,
+      "loss": 2.4892,
+      "theoretical_loss": 3.316459141376166,
+      "tokens_seen": 3051094016
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 8.118049615055604e-05,
+      "loss": 2.4738,
+      "theoretical_loss": 3.3164482250233163,
+      "tokens_seen": 3051225088
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 8.113772455089821e-05,
+      "loss": 2.5011,
+      "theoretical_loss": 3.316437309270688,
+      "tokens_seen": 3051356160
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 8.109495295124037e-05,
+      "loss": 2.4879,
+      "theoretical_loss": 3.316426394118222,
+      "tokens_seen": 3051487232
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 8.105218135158255e-05,
+      "loss": 2.441,
+      "theoretical_loss": 3.316415479565859,
+      "tokens_seen": 3051618304
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 8.100940975192473e-05,
+      "loss": 2.5491,
+      "theoretical_loss": 3.3164045656135417,
+      "tokens_seen": 3051749376
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 8.09666381522669e-05,
+      "loss": 2.502,
+      "theoretical_loss": 3.3163936522612096,
+      "tokens_seen": 3051880448
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 8.092386655260907e-05,
+      "loss": 2.3763,
+      "theoretical_loss": 3.3163827395088052,
+      "tokens_seen": 3052011520
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 8.088109495295124e-05,
+      "loss": 2.4419,
+      "theoretical_loss": 3.3163718273562695,
+      "tokens_seen": 3052142592
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 8.083832335329341e-05,
+      "loss": 2.4011,
+      "theoretical_loss": 3.3163609158035436,
+      "tokens_seen": 3052273664
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 8.07955517536356e-05,
+      "loss": 2.4094,
+      "theoretical_loss": 3.3163500048505687,
+      "tokens_seen": 3052404736
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 8.075278015397776e-05,
+      "loss": 2.4447,
+      "theoretical_loss": 3.3163390944972857,
+      "tokens_seen": 3052535808
+    },
+    {
+      "epoch": 0.2,
+      "objective/train/docs_used": 1673386,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 2.077202320098877,
+      "objective/train/theoretical_loss": 3.316328184743637,
+      "objective/train/tokens_used": 82718176,
+      "theoretical_loss": 3.316328184743637,
+      "tokens_seen": 3052666880
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 8.071000855431994e-05,
+      "loss": 2.3368,
+      "theoretical_loss": 3.316328184743637,
+      "tokens_seen": 3052666880
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 8.06672369546621e-05,
+      "loss": 2.578,
+      "theoretical_loss": 3.3163172755895634,
+      "tokens_seen": 3052797952
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 8.062446535500429e-05,
+      "loss": 2.5108,
+      "theoretical_loss": 3.3163063670350055,
+      "tokens_seen": 3052929024
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 8.058169375534646e-05,
+      "loss": 2.4558,
+      "theoretical_loss": 3.3162954590799054,
+      "tokens_seen": 3053060096
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 8.053892215568862e-05,
+      "loss": 2.5339,
+      "theoretical_loss": 3.316284551724204,
+      "tokens_seen": 3053191168
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 8.04961505560308e-05,
+      "loss": 2.3677,
+      "theoretical_loss": 3.3162736449678434,
+      "tokens_seen": 3053322240
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 8.045337895637297e-05,
+      "loss": 2.5638,
+      "theoretical_loss": 3.3162627388107637,
+      "tokens_seen": 3053453312
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 8.041060735671514e-05,
+      "loss": 2.4784,
+      "theoretical_loss": 3.316251833252908,
+      "tokens_seen": 3053584384
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 8.036783575705732e-05,
+      "loss": 2.486,
+      "theoretical_loss": 3.3162409282942154,
+      "tokens_seen": 3053715456
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 8.032506415739949e-05,
+      "loss": 2.4993,
+      "theoretical_loss": 3.316230023934629,
+      "tokens_seen": 3053846528
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 8.028229255774167e-05,
+      "loss": 2.5384,
+      "theoretical_loss": 3.3162191201740896,
+      "tokens_seen": 3053977600
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 8.023952095808383e-05,
+      "loss": 2.5113,
+      "theoretical_loss": 3.3162082170125387,
+      "tokens_seen": 3054108672
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 8.019674935842601e-05,
+      "loss": 2.2871,
+      "theoretical_loss": 3.3161973144499175,
+      "tokens_seen": 3054239744
+    },
+    {
+      "epoch": 0.21,
+      "objective/train/docs_used": 1673816,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 2.4686410427093506,
+      "objective/train/theoretical_loss": 3.316191863393187,
+      "objective/train/tokens_used": 84356576,
+      "theoretical_loss": 3.316191863393187,
+      "tokens_seen": 3054305280
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 8.015397775876819e-05,
+      "loss": 2.4141,
+      "theoretical_loss": 3.3161864124861675,
+      "tokens_seen": 3054370816
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 8.011120615911035e-05,
+      "loss": 2.5008,
+      "theoretical_loss": 3.31617551112123,
+      "tokens_seen": 3054501888
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 8.006843455945253e-05,
+      "loss": 2.5764,
+      "theoretical_loss": 3.316164610355047,
+      "tokens_seen": 3054632960
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 8.00256629597947e-05,
+      "loss": 2.4664,
+      "theoretical_loss": 3.316153710187559,
+      "tokens_seen": 3054764032
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 7.998289136013687e-05,
+      "loss": 2.5636,
+      "theoretical_loss": 3.316142810618708,
+      "tokens_seen": 3054895104
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 7.994011976047905e-05,
+      "loss": 2.54,
+      "theoretical_loss": 3.3161319116484353,
+      "tokens_seen": 3055026176
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 7.989734816082122e-05,
+      "loss": 2.6878,
+      "theoretical_loss": 3.3161210132766823,
+      "tokens_seen": 3055157248
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 7.98545765611634e-05,
+      "loss": 2.6299,
+      "theoretical_loss": 3.316110115503391,
+      "tokens_seen": 3055288320
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 7.981180496150556e-05,
+      "loss": 2.476,
+      "theoretical_loss": 3.316099218328502,
+      "tokens_seen": 3055419392
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 7.976903336184774e-05,
+      "loss": 2.5998,
+      "theoretical_loss": 3.3160883217519572,
+      "tokens_seen": 3055550464
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 7.972626176218992e-05,
+      "loss": 2.4002,
+      "theoretical_loss": 3.316077425773698,
+      "tokens_seen": 3055681536
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 7.968349016253208e-05,
+      "loss": 2.5102,
+      "theoretical_loss": 3.316066530393666,
+      "tokens_seen": 3055812608
+    },
+    {
+      "epoch": 0.21,
+      "objective/train/docs_used": 1674780,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 2.8518364429473877,
+      "objective/train/theoretical_loss": 3.3160556356118027,
+      "objective/train/tokens_used": 85994976,
+      "theoretical_loss": 3.3160556356118027,
+      "tokens_seen": 3055943680
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 7.964071856287424e-05,
+      "loss": 2.5794,
+      "theoretical_loss": 3.3160556356118027,
+      "tokens_seen": 3055943680
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 7.959794696321644e-05,
+      "loss": 2.6511,
+      "theoretical_loss": 3.3160447414280494,
+      "tokens_seen": 3056074752
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 7.95551753635586e-05,
+      "loss": 2.4842,
+      "theoretical_loss": 3.316033847842348,
+      "tokens_seen": 3056205824
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 7.951240376390078e-05,
+      "loss": 2.4747,
+      "theoretical_loss": 3.3160229548546396,
+      "tokens_seen": 3056336896
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 7.946963216424294e-05,
+      "loss": 2.4766,
+      "theoretical_loss": 3.316012062464866,
+      "tokens_seen": 3056467968
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 7.942686056458511e-05,
+      "loss": 2.611,
+      "theoretical_loss": 3.316001170672968,
+      "tokens_seen": 3056599040
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 7.93840889649273e-05,
+      "loss": 2.4415,
+      "theoretical_loss": 3.3159902794788887,
+      "tokens_seen": 3056730112
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 7.934131736526947e-05,
+      "loss": 2.4817,
+      "theoretical_loss": 3.3159793888825684,
+      "tokens_seen": 3056861184
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 7.929854576561164e-05,
+      "loss": 2.5289,
+      "theoretical_loss": 3.315968498883949,
+      "tokens_seen": 3056992256
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 7.925577416595381e-05,
+      "loss": 2.4786,
+      "theoretical_loss": 3.3159576094829726,
+      "tokens_seen": 3057123328
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 7.921300256629597e-05,
+      "loss": 2.6726,
+      "theoretical_loss": 3.3159467206795794,
+      "tokens_seen": 3057254400
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 7.917023096663817e-05,
+      "loss": 2.4774,
+      "theoretical_loss": 3.3159358324737123,
+      "tokens_seen": 3057385472
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 7.912745936698033e-05,
+      "loss": 2.3915,
+      "theoretical_loss": 3.3159249448653125,
+      "tokens_seen": 3057516544
+    },
+    {
+      "epoch": 0.22,
+      "objective/train/docs_used": 1675938,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 2.4315812587738037,
+      "objective/train/theoretical_loss": 3.3159195012851446,
+      "objective/train/tokens_used": 87633376,
+      "theoretical_loss": 3.3159195012851446,
+      "tokens_seen": 3057582080
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 7.90846877673225e-05,
+      "loss": 2.4585,
+      "theoretical_loss": 3.3159140578543216,
+      "tokens_seen": 3057647616
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 7.904191616766467e-05,
+      "loss": 2.474,
+      "theoretical_loss": 3.315903171440681,
+      "tokens_seen": 3057778688
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 7.899914456800684e-05,
+      "loss": 2.3614,
+      "theoretical_loss": 3.3158922856243325,
+      "tokens_seen": 3057909760
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 7.895637296834903e-05,
+      "loss": 2.4224,
+      "theoretical_loss": 3.3158814004052175,
+      "tokens_seen": 3058040832
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 7.89136013686912e-05,
+      "loss": 2.5027,
+      "theoretical_loss": 3.3158705157832786,
+      "tokens_seen": 3058171904
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 7.887082976903336e-05,
+      "loss": 2.4609,
+      "theoretical_loss": 3.315859631758456,
+      "tokens_seen": 3058302976
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 7.882805816937554e-05,
+      "loss": 2.5555,
+      "theoretical_loss": 3.3158487483306924,
+      "tokens_seen": 3058434048
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 7.878528656971772e-05,
+      "loss": 2.3801,
+      "theoretical_loss": 3.3158378654999288,
+      "tokens_seen": 3058565120
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 7.87425149700599e-05,
+      "loss": 2.491,
+      "theoretical_loss": 3.315826983266107,
+      "tokens_seen": 3058696192
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 7.869974337040206e-05,
+      "loss": 2.5554,
+      "theoretical_loss": 3.31581610162917,
+      "tokens_seen": 3058827264
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 7.865697177074422e-05,
+      "loss": 2.4215,
+      "theoretical_loss": 3.3158052205890574,
+      "tokens_seen": 3058958336
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 7.86142001710864e-05,
+      "loss": 2.4309,
+      "theoretical_loss": 3.315794340145712,
+      "tokens_seen": 3059089408
+    },
+    {
+      "epoch": 0.22,
+      "objective/train/docs_used": 1676633,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 2.8794491291046143,
+      "objective/train/theoretical_loss": 3.3157834602990754,
+      "objective/train/tokens_used": 89271776,
+      "theoretical_loss": 3.3157834602990754,
+      "tokens_seen": 3059220480
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 7.857142857142858e-05,
+      "loss": 2.507,
+      "theoretical_loss": 3.3157834602990754,
+      "tokens_seen": 3059220480
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 7.852865697177076e-05,
+      "loss": 2.4838,
+      "theoretical_loss": 3.3157725810490892,
+      "tokens_seen": 3059351552
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 7.848588537211292e-05,
+      "loss": 2.3672,
+      "theoretical_loss": 3.3157617023956956,
+      "tokens_seen": 3059482624
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 7.844311377245509e-05,
+      "loss": 2.5403,
+      "theoretical_loss": 3.3157508243388354,
+      "tokens_seen": 3059613696
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 7.840034217279727e-05,
+      "loss": 2.3567,
+      "theoretical_loss": 3.315739946878451,
+      "tokens_seen": 3059744768
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 7.835757057313944e-05,
+      "loss": 2.5668,
+      "theoretical_loss": 3.3157290700144837,
+      "tokens_seen": 3059875840
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 7.831479897348161e-05,
+      "loss": 2.6316,
+      "theoretical_loss": 3.315718193746876,
+      "tokens_seen": 3060006912
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 7.827202737382379e-05,
+      "loss": 2.6666,
+      "theoretical_loss": 3.315707318075569,
+      "tokens_seen": 3060137984
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 7.822925577416595e-05,
+      "loss": 2.4661,
+      "theoretical_loss": 3.3156964430005047,
+      "tokens_seen": 3060269056
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 7.818648417450813e-05,
+      "loss": 2.5179,
+      "theoretical_loss": 3.315685568521625,
+      "tokens_seen": 3060400128
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 7.814371257485031e-05,
+      "loss": 2.5853,
+      "theoretical_loss": 3.3156746946388713,
+      "tokens_seen": 3060531200
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 7.810094097519247e-05,
+      "loss": 2.5132,
+      "theoretical_loss": 3.315663821352186,
+      "tokens_seen": 3060662272
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 7.805816937553465e-05,
+      "loss": 2.6414,
+      "theoretical_loss": 3.3156529486615103,
+      "tokens_seen": 3060793344
+    },
+    {
+      "epoch": 0.23,
+      "objective/train/docs_used": 1677094,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 2.3754663467407227,
+      "objective/train/theoretical_loss": 3.315647512539658,
+      "objective/train/tokens_used": 90910176,
+      "theoretical_loss": 3.315647512539658,
+      "tokens_seen": 3060858880
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 7.801539777587682e-05,
+      "loss": 2.6056,
+      "theoretical_loss": 3.3156420765667862,
+      "tokens_seen": 3060924416
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 7.7972626176219e-05,
+      "loss": 2.5379,
+      "theoretical_loss": 3.3156312050679553,
+      "tokens_seen": 3061055488
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 7.792985457656117e-05,
+      "loss": 2.547,
+      "theoretical_loss": 3.31562033416496,
+      "tokens_seen": 3061186560
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 7.788708297690334e-05,
+      "loss": 2.533,
+      "theoretical_loss": 3.315609463857742,
+      "tokens_seen": 3061317632
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 7.784431137724552e-05,
+      "loss": 2.5029,
+      "theoretical_loss": 3.3155985941462425,
+      "tokens_seen": 3061448704
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 7.780153977758768e-05,
+      "loss": 2.478,
+      "theoretical_loss": 3.315587725030404,
+      "tokens_seen": 3061579776
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 7.775876817792986e-05,
+      "loss": 2.5078,
+      "theoretical_loss": 3.315576856510168,
+      "tokens_seen": 3061710848
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 7.771599657827204e-05,
+      "loss": 2.508,
+      "theoretical_loss": 3.315565988585477,
+      "tokens_seen": 3061841920
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 7.76732249786142e-05,
+      "loss": 2.5306,
+      "theoretical_loss": 3.3155551212562724,
+      "tokens_seen": 3061972992
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 7.763045337895638e-05,
+      "loss": 2.6,
+      "theoretical_loss": 3.3155442545224956,
+      "tokens_seen": 3062104064
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 7.758768177929855e-05,
+      "loss": 2.4638,
+      "theoretical_loss": 3.315533388384089,
+      "tokens_seen": 3062235136
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 7.754491017964072e-05,
+      "loss": 2.6104,
+      "theoretical_loss": 3.315522522840995,
+      "tokens_seen": 3062366208
+    },
+    {
+      "epoch": 0.23,
+      "objective/train/docs_used": 1678209,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 2.218628406524658,
+      "objective/train/theoretical_loss": 3.3155116578931545,
+      "objective/train/tokens_used": 92548576,
+      "theoretical_loss": 3.3155116578931545,
+      "tokens_seen": 3062497280
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 7.75021385799829e-05,
+      "loss": 2.2883,
+      "theoretical_loss": 3.3155116578931545,
+      "tokens_seen": 3062497280
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 7.745936698032507e-05,
+      "loss": 2.5558,
+      "theoretical_loss": 3.31550079354051,
+      "tokens_seen": 3062628352
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 7.741659538066724e-05,
+      "loss": 2.5281,
+      "theoretical_loss": 3.315489929783004,
+      "tokens_seen": 3062759424
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 7.737382378100941e-05,
+      "loss": 2.5825,
+      "theoretical_loss": 3.315479066620577,
+      "tokens_seen": 3062890496
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 7.733105218135159e-05,
+      "loss": 2.5783,
+      "theoretical_loss": 3.3154682040531718,
+      "tokens_seen": 3063021568
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 7.728828058169377e-05,
+      "loss": 2.4715,
+      "theoretical_loss": 3.3154573420807303,
+      "tokens_seen": 3063152640
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 7.724550898203593e-05,
+      "loss": 2.562,
+      "theoretical_loss": 3.3154464807031943,
+      "tokens_seen": 3063283712
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 7.720273738237811e-05,
+      "loss": 2.4961,
+      "theoretical_loss": 3.315435619920506,
+      "tokens_seen": 3063414784
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 7.715996578272027e-05,
+      "loss": 2.4626,
+      "theoretical_loss": 3.315424759732607,
+      "tokens_seen": 3063545856
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 7.711719418306245e-05,
+      "loss": 2.4789,
+      "theoretical_loss": 3.31541390013944,
+      "tokens_seen": 3063676928
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 7.707442258340463e-05,
+      "loss": 2.4289,
+      "theoretical_loss": 3.3154030411409465,
+      "tokens_seen": 3063808000
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 7.70316509837468e-05,
+      "loss": 2.5123,
+      "theoretical_loss": 3.3153921827370683,
+      "tokens_seen": 3063939072
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 7.698887938408896e-05,
+      "loss": 2.5002,
+      "theoretical_loss": 3.3153813249277473,
+      "tokens_seen": 3064070144
+    },
+    {
+      "epoch": 0.24,
+      "objective/train/docs_used": 1678783,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 2.781327724456787,
+      "objective/train/theoretical_loss": 3.315375896246028,
+      "objective/train/tokens_used": 94186976,
+      "theoretical_loss": 3.315375896246028,
+      "tokens_seen": 3064135680
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 7.694610778443114e-05,
+      "loss": 2.5217,
+      "theoretical_loss": 3.3153704677129263,
+      "tokens_seen": 3064201216
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 7.690333618477332e-05,
+      "loss": 2.4925,
+      "theoretical_loss": 3.3153596110925467,
+      "tokens_seen": 3064332288
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 7.68605645851155e-05,
+      "loss": 2.459,
+      "theoretical_loss": 3.315348755066551,
+      "tokens_seen": 3064463360
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 7.681779298545766e-05,
+      "loss": 2.4395,
+      "theoretical_loss": 3.31533789963488,
+      "tokens_seen": 3064594432
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 7.677502138579982e-05,
+      "loss": 2.4192,
+      "theoretical_loss": 3.3153270447974776,
+      "tokens_seen": 3064725504
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 7.6732249786142e-05,
+      "loss": 2.5295,
+      "theoretical_loss": 3.3153161905542845,
+      "tokens_seen": 3064856576
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 7.668947818648418e-05,
+      "loss": 2.4586,
+      "theoretical_loss": 3.315305336905243,
+      "tokens_seen": 3064987648
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 7.664670658682636e-05,
+      "loss": 2.4483,
+      "theoretical_loss": 3.3152944838502956,
+      "tokens_seen": 3065118720
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 7.660393498716852e-05,
+      "loss": 2.5967,
+      "theoretical_loss": 3.3152836313893843,
+      "tokens_seen": 3065249792
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 7.656116338751069e-05,
+      "loss": 2.6626,
+      "theoretical_loss": 3.3152727795224504,
+      "tokens_seen": 3065380864
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 7.651839178785287e-05,
+      "loss": 2.5101,
+      "theoretical_loss": 3.3152619282494373,
+      "tokens_seen": 3065511936
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 7.647562018819505e-05,
+      "loss": 2.3196,
+      "theoretical_loss": 3.315251077570286,
+      "tokens_seen": 3065643008
+    },
+    {
+      "epoch": 0.24,
+      "objective/train/docs_used": 1679875,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 2.6284544467926025,
+      "objective/train/theoretical_loss": 3.3152402274849395,
+      "objective/train/tokens_used": 95825376,
+      "theoretical_loss": 3.3152402274849395,
+      "tokens_seen": 3065774080
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 7.643284858853722e-05,
+      "loss": 2.5859,
+      "theoretical_loss": 3.3152402274849395,
+      "tokens_seen": 3065774080
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 7.639007698887939e-05,
+      "loss": 2.3573,
+      "theoretical_loss": 3.315229377993339,
+      "tokens_seen": 3065905152
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 7.634730538922155e-05,
+      "loss": 2.435,
+      "theoretical_loss": 3.315218529095427,
+      "tokens_seen": 3066036224
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 7.630453378956373e-05,
+      "loss": 2.4501,
+      "theoretical_loss": 3.315207680791146,
+      "tokens_seen": 3066167296
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 7.626176218990591e-05,
+      "loss": 2.484,
+      "theoretical_loss": 3.3151968330804378,
+      "tokens_seen": 3066298368
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 7.621899059024807e-05,
+      "loss": 2.4815,
+      "theoretical_loss": 3.3151859859632444,
+      "tokens_seen": 3066429440
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 7.617621899059025e-05,
+      "loss": 2.3643,
+      "theoretical_loss": 3.3151751394395084,
+      "tokens_seen": 3066560512
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 7.613344739093242e-05,
+      "loss": 2.6645,
+      "theoretical_loss": 3.3151642935091714,
+      "tokens_seen": 3066691584
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 7.60906757912746e-05,
+      "loss": 2.4228,
+      "theoretical_loss": 3.3151534481721763,
+      "tokens_seen": 3066822656
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 7.604790419161677e-05,
+      "loss": 2.5867,
+      "theoretical_loss": 3.3151426034284643,
+      "tokens_seen": 3066953728
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 7.600513259195894e-05,
+      "loss": 2.6365,
+      "theoretical_loss": 3.3151317592779788,
+      "tokens_seen": 3067084800
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 7.596236099230112e-05,
+      "loss": 2.5273,
+      "theoretical_loss": 3.315120915720661,
+      "tokens_seen": 3067215872
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 7.591958939264328e-05,
+      "loss": 2.5048,
+      "theoretical_loss": 3.315110072756454,
+      "tokens_seen": 3067346944
+    },
+    {
+      "epoch": 0.25,
+      "objective/train/docs_used": 1680552,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 2.5477449893951416,
+      "objective/train/theoretical_loss": 3.3151046514967484,
+      "objective/train/tokens_used": 97463776,
+      "theoretical_loss": 3.3151046514967484,
+      "tokens_seen": 3067412480
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 7.587681779298546e-05,
+      "loss": 2.3805,
+      "theoretical_loss": 3.315099230385299,
+      "tokens_seen": 3067478016
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 7.583404619332764e-05,
+      "loss": 2.4863,
+      "theoretical_loss": 3.3150883886071387,
+      "tokens_seen": 3067609088
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 7.57912745936698e-05,
+      "loss": 2.3692,
+      "theoretical_loss": 3.3150775474219154,
+      "tokens_seen": 3067740160
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 7.574850299401198e-05,
+      "loss": 2.5147,
+      "theoretical_loss": 3.3150667068295716,
+      "tokens_seen": 3067871232
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 7.570573139435415e-05,
+      "loss": 2.4548,
+      "theoretical_loss": 3.3150558668300487,
+      "tokens_seen": 3068002304
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 7.566295979469632e-05,
+      "loss": 2.515,
+      "theoretical_loss": 3.31504502742329,
+      "tokens_seen": 3068133376
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 7.56201881950385e-05,
+      "loss": 2.4336,
+      "theoretical_loss": 3.3150341886092374,
+      "tokens_seen": 3068264448
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 7.557741659538067e-05,
+      "loss": 2.4561,
+      "theoretical_loss": 3.3150233503878326,
+      "tokens_seen": 3068395520
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 7.553464499572285e-05,
+      "loss": 2.5634,
+      "theoretical_loss": 3.3150125127590186,
+      "tokens_seen": 3068526592
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 7.549187339606501e-05,
+      "loss": 2.5016,
+      "theoretical_loss": 3.3150016757227374,
+      "tokens_seen": 3068657664
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 7.544910179640719e-05,
+      "loss": 2.3541,
+      "theoretical_loss": 3.314990839278931,
+      "tokens_seen": 3068788736
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 7.540633019674937e-05,
+      "loss": 2.3958,
+      "theoretical_loss": 3.314980003427542,
+      "tokens_seen": 3068919808
+    },
+    {
+      "epoch": 0.25,
+      "objective/train/docs_used": 1681818,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 2.7954180240631104,
+      "objective/train/theoretical_loss": 3.3149691681685134,
+      "objective/train/tokens_used": 99102176,
+      "theoretical_loss": 3.3149691681685134,
+      "tokens_seen": 3069050880
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 7.536355859709153e-05,
+      "loss": 2.5015,
+      "theoretical_loss": 3.3149691681685134,
+      "tokens_seen": 3069050880
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 7.532078699743371e-05,
+      "loss": 2.5696,
+      "theoretical_loss": 3.314958333501786,
+      "tokens_seen": 3069181952
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 7.527801539777588e-05,
+      "loss": 2.3828,
+      "theoretical_loss": 3.3149474994273036,
+      "tokens_seen": 3069313024
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 7.523524379811805e-05,
+      "loss": 2.5782,
+      "theoretical_loss": 3.314936665945008,
+      "tokens_seen": 3069444096
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 7.519247219846023e-05,
+      "loss": 2.5156,
+      "theoretical_loss": 3.314925833054841,
+      "tokens_seen": 3069575168
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 7.51497005988024e-05,
+      "loss": 2.4908,
+      "theoretical_loss": 3.3149150007567454,
+      "tokens_seen": 3069706240
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 7.510692899914457e-05,
+      "loss": 2.5009,
+      "theoretical_loss": 3.314904169050664,
+      "tokens_seen": 3069837312
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 7.506415739948674e-05,
+      "loss": 2.4503,
+      "theoretical_loss": 3.3148933379365384,
+      "tokens_seen": 3069968384
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 7.502138579982892e-05,
+      "loss": 2.4305,
+      "theoretical_loss": 3.3148825074143113,
+      "tokens_seen": 3070099456
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 7.49786142001711e-05,
+      "loss": 2.3984,
+      "theoretical_loss": 3.314871677483925,
+      "tokens_seen": 3070230528
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 7.493584260051326e-05,
+      "loss": 2.4805,
+      "theoretical_loss": 3.3148608481453223,
+      "tokens_seen": 3070361600
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 7.489307100085543e-05,
+      "loss": 2.4796,
+      "theoretical_loss": 3.3148500193984454,
+      "tokens_seen": 3070492672
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 7.48502994011976e-05,
+      "loss": 2.5145,
+      "theoretical_loss": 3.3148391912432364,
+      "tokens_seen": 3070623744
+    },
+    {
+      "epoch": 0.26,
+      "objective/train/docs_used": 1682472,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 2.632871150970459,
+      "objective/train/theoretical_loss": 3.3148337773874896,
+      "objective/train/tokens_used": 100740576,
+      "theoretical_loss": 3.3148337773874896,
+      "tokens_seen": 3070689280
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 7.480752780153978e-05,
+      "loss": 2.4604,
+      "theoretical_loss": 3.3148283636796383,
+      "tokens_seen": 3070754816
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 7.476475620188196e-05,
+      "loss": 2.4658,
+      "theoretical_loss": 3.3148175367075927,
+      "tokens_seen": 3070885888
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 7.472198460222413e-05,
+      "loss": 2.4759,
+      "theoretical_loss": 3.314806710327043,
+      "tokens_seen": 3071016960
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 7.467921300256629e-05,
+      "loss": 2.6602,
+      "theoretical_loss": 3.3147958845379306,
+      "tokens_seen": 3071148032
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 7.463644140290847e-05,
+      "loss": 2.3896,
+      "theoretical_loss": 3.3147850593401986,
+      "tokens_seen": 3071279104
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 7.459366980325065e-05,
+      "loss": 2.587,
+      "theoretical_loss": 3.3147742347337896,
+      "tokens_seen": 3071410176
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 7.455089820359282e-05,
+      "loss": 2.719,
+      "theoretical_loss": 3.3147634107186454,
+      "tokens_seen": 3071541248
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 7.450812660393499e-05,
+      "loss": 2.5003,
+      "theoretical_loss": 3.3147525872947092,
+      "tokens_seen": 3071672320
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 7.446535500427715e-05,
+      "loss": 2.344,
+      "theoretical_loss": 3.3147417644619233,
+      "tokens_seen": 3071803392
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 7.442258340461933e-05,
+      "loss": 2.5274,
+      "theoretical_loss": 3.31473094222023,
+      "tokens_seen": 3071934464
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 7.437981180496151e-05,
+      "loss": 2.6219,
+      "theoretical_loss": 3.3147201205695715,
+      "tokens_seen": 3072065536
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 7.433704020530369e-05,
+      "loss": 2.4226,
+      "theoretical_loss": 3.3147092995098912,
+      "tokens_seen": 3072196608
+    },
+    {
+      "epoch": 0.26,
+      "objective/train/docs_used": 1683394,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 2.5833144187927246,
+      "objective/train/theoretical_loss": 3.3146984790411307,
+      "objective/train/tokens_used": 102378976,
+      "theoretical_loss": 3.3146984790411307,
+      "tokens_seen": 3072327680
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 7.429426860564585e-05,
+      "loss": 2.4399,
+      "theoretical_loss": 3.3146984790411307,
+      "tokens_seen": 3072327680
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 7.425149700598802e-05,
+      "loss": 2.6166,
+      "theoretical_loss": 3.314687659163233,
+      "tokens_seen": 3072458752
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 7.420872540633021e-05,
+      "loss": 2.2906,
+      "theoretical_loss": 3.3146768398761406,
+      "tokens_seen": 3072589824
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 7.416595380667238e-05,
+      "loss": 2.5399,
+      "theoretical_loss": 3.3146660211797956,
+      "tokens_seen": 3072720896
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 7.412318220701454e-05,
+      "loss": 2.4276,
+      "theoretical_loss": 3.3146552030741416,
+      "tokens_seen": 3072851968
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 7.408041060735672e-05,
+      "loss": 2.5037,
+      "theoretical_loss": 3.31464438555912,
+      "tokens_seen": 3072983040
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 7.403763900769888e-05,
+      "loss": 2.4357,
+      "theoretical_loss": 3.314633568634674,
+      "tokens_seen": 3073114112
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 7.399486740804107e-05,
+      "loss": 2.5825,
+      "theoretical_loss": 3.314622752300746,
+      "tokens_seen": 3073245184
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 7.395209580838324e-05,
+      "loss": 2.4643,
+      "theoretical_loss": 3.3146119365572786,
+      "tokens_seen": 3073376256
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 7.39093242087254e-05,
+      "loss": 2.3604,
+      "theoretical_loss": 3.3146011214042144,
+      "tokens_seen": 3073507328
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 7.386655260906758e-05,
+      "loss": 2.4892,
+      "theoretical_loss": 3.314590306841496,
+      "tokens_seen": 3073638400
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 7.382378100940975e-05,
+      "loss": 2.6183,
+      "theoretical_loss": 3.3145794928690657,
+      "tokens_seen": 3073769472
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 7.378100940975194e-05,
+      "loss": 2.5466,
+      "theoretical_loss": 3.3145686794868667,
+      "tokens_seen": 3073900544
+    },
+    {
+      "epoch": 0.27,
+      "objective/train/docs_used": 1684169,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 2.35479998588562,
+      "objective/train/theoretical_loss": 3.314563273017086,
+      "objective/train/tokens_used": 104017376,
+      "theoretical_loss": 3.314563273017086,
+      "tokens_seen": 3073966080
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 7.37382378100941e-05,
+      "loss": 2.5362,
+      "theoretical_loss": 3.314557866694841,
+      "tokens_seen": 3074031616
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 7.369546621043627e-05,
+      "loss": 2.647,
+      "theoretical_loss": 3.314547054492932,
+      "tokens_seen": 3074162688
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 7.365269461077845e-05,
+      "loss": 2.5716,
+      "theoretical_loss": 3.314536242881082,
+      "tokens_seen": 3074293760
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 7.360992301112061e-05,
+      "loss": 2.6096,
+      "theoretical_loss": 3.3145254318592325,
+      "tokens_seen": 3074424832
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 7.356715141146279e-05,
+      "loss": 2.4451,
+      "theoretical_loss": 3.3145146214273282,
+      "tokens_seen": 3074555904
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 7.352437981180497e-05,
+      "loss": 2.4806,
+      "theoretical_loss": 3.3145038115853103,
+      "tokens_seen": 3074686976
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 7.348160821214713e-05,
+      "loss": 2.5374,
+      "theoretical_loss": 3.314493002333122,
+      "tokens_seen": 3074818048
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 7.343883661248931e-05,
+      "loss": 2.6111,
+      "theoretical_loss": 3.3144821936707056,
+      "tokens_seen": 3074949120
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 7.339606501283149e-05,
+      "loss": 2.4581,
+      "theoretical_loss": 3.3144713855980044,
+      "tokens_seen": 3075080192
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 7.335329341317365e-05,
+      "loss": 2.4606,
+      "theoretical_loss": 3.3144605781149608,
+      "tokens_seen": 3075211264
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 7.331052181351583e-05,
+      "loss": 2.4645,
+      "theoretical_loss": 3.314449771221517,
+      "tokens_seen": 3075342336
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 7.3267750213858e-05,
+      "loss": 2.5643,
+      "theoretical_loss": 3.3144389649176165,
+      "tokens_seen": 3075473408
+    },
+    {
+      "epoch": 0.27,
+      "objective/train/docs_used": 1684867,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 2.6634860038757324,
+      "objective/train/theoretical_loss": 3.3144281592032017,
+      "objective/train/tokens_used": 105655776,
+      "theoretical_loss": 3.3144281592032017,
+      "tokens_seen": 3075604480
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 7.322497861420018e-05,
+      "loss": 2.6146,
+      "theoretical_loss": 3.3144281592032017,
+      "tokens_seen": 3075604480
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 7.318220701454235e-05,
+      "loss": 2.4308,
+      "theoretical_loss": 3.3144173540782154,
+      "tokens_seen": 3075735552
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 7.313943541488452e-05,
+      "loss": 2.4509,
+      "theoretical_loss": 3.3144065495426,
+      "tokens_seen": 3075866624
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 7.30966638152267e-05,
+      "loss": 2.4845,
+      "theoretical_loss": 3.3143957455962982,
+      "tokens_seen": 3075997696
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 7.305389221556886e-05,
+      "loss": 2.6057,
+      "theoretical_loss": 3.3143849422392533,
+      "tokens_seen": 3076128768
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 7.301112061591104e-05,
+      "loss": 2.4856,
+      "theoretical_loss": 3.3143741394714077,
+      "tokens_seen": 3076259840
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 7.296834901625322e-05,
+      "loss": 2.6096,
+      "theoretical_loss": 3.314363337292704,
+      "tokens_seen": 3076390912
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 7.292557741659538e-05,
+      "loss": 2.5767,
+      "theoretical_loss": 3.314352535703086,
+      "tokens_seen": 3076521984
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 7.288280581693756e-05,
+      "loss": 2.6051,
+      "theoretical_loss": 3.314341734702495,
+      "tokens_seen": 3076653056
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 7.284003421727973e-05,
+      "loss": 2.472,
+      "theoretical_loss": 3.3143309342908744,
+      "tokens_seen": 3076784128
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 7.279726261762189e-05,
+      "loss": 2.5424,
+      "theoretical_loss": 3.314320134468167,
+      "tokens_seen": 3076915200
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 7.275449101796408e-05,
+      "loss": 2.6118,
+      "theoretical_loss": 3.3143093352343165,
+      "tokens_seen": 3077046272
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 7.271171941830625e-05,
+      "loss": 2.4979,
+      "theoretical_loss": 3.314298536589264,
+      "tokens_seen": 3077177344
+    },
+    {
+      "epoch": 0.28,
+      "objective/train/docs_used": 1686119,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 2.6551506519317627,
+      "objective/train/theoretical_loss": 3.3142931374875197,
+      "objective/train/tokens_used": 107294176,
+      "theoretical_loss": 3.3142931374875197,
+      "tokens_seen": 3077242880
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 7.266894781864843e-05,
+      "loss": 2.5443,
+      "theoretical_loss": 3.3142877385329537,
+      "tokens_seen": 3077308416
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 7.262617621899059e-05,
+      "loss": 2.5527,
+      "theoretical_loss": 3.314276941065328,
+      "tokens_seen": 3077439488
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 7.258340461933276e-05,
+      "loss": 2.6246,
+      "theoretical_loss": 3.3142661441863295,
+      "tokens_seen": 3077570560
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 7.254063301967495e-05,
+      "loss": 2.5297,
+      "theoretical_loss": 3.3142553478959007,
+      "tokens_seen": 3077701632
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 7.249786142001711e-05,
+      "loss": 2.4888,
+      "theoretical_loss": 3.3142445521939856,
+      "tokens_seen": 3077832704
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 7.245508982035929e-05,
+      "loss": 2.5649,
+      "theoretical_loss": 3.314233757080526,
+      "tokens_seen": 3077963776
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 7.241231822070146e-05,
+      "loss": 2.5155,
+      "theoretical_loss": 3.3142229625554656,
+      "tokens_seen": 3078094848
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 7.236954662104363e-05,
+      "loss": 2.5393,
+      "theoretical_loss": 3.3142121686187465,
+      "tokens_seen": 3078225920
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 7.232677502138581e-05,
+      "loss": 2.5875,
+      "theoretical_loss": 3.3142013752703123,
+      "tokens_seen": 3078356992
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 7.228400342172798e-05,
+      "loss": 2.4805,
+      "theoretical_loss": 3.314190582510105,
+      "tokens_seen": 3078488064
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 7.224123182207014e-05,
+      "loss": 2.5378,
+      "theoretical_loss": 3.3141797903380685,
+      "tokens_seen": 3078619136
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 7.219846022241232e-05,
+      "loss": 2.5374,
+      "theoretical_loss": 3.314168998754145,
+      "tokens_seen": 3078750208
+    },
+    {
+      "epoch": 0.29,
+      "objective/train/docs_used": 1686853,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 2.4723048210144043,
+      "objective/train/theoretical_loss": 3.314158207758278,
+      "objective/train/tokens_used": 108932576,
+      "theoretical_loss": 3.314158207758278,
+      "tokens_seen": 3078881280
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 7.21556886227545e-05,
+      "loss": 2.7317,
+      "theoretical_loss": 3.314158207758278,
+      "tokens_seen": 3078881280
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 7.211291702309668e-05,
+      "loss": 2.5378,
+      "theoretical_loss": 3.3141474173504095,
+      "tokens_seen": 3079012352
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 7.207014542343884e-05,
+      "loss": 2.6327,
+      "theoretical_loss": 3.3141366275304835,
+      "tokens_seen": 3079143424
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 7.2027373823781e-05,
+      "loss": 2.5976,
+      "theoretical_loss": 3.3141258382984424,
+      "tokens_seen": 3079274496
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 7.198460222412318e-05,
+      "loss": 2.6818,
+      "theoretical_loss": 3.3141150496542293,
+      "tokens_seen": 3079405568
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 7.194183062446536e-05,
+      "loss": 2.6468,
+      "theoretical_loss": 3.3141042615977865,
+      "tokens_seen": 3079536640
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 7.189905902480754e-05,
+      "loss": 2.4813,
+      "theoretical_loss": 3.314093474129058,
+      "tokens_seen": 3079667712
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 7.18562874251497e-05,
+      "loss": 2.5804,
+      "theoretical_loss": 3.314082687247986,
+      "tokens_seen": 3079798784
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 7.181351582549187e-05,
+      "loss": 2.4319,
+      "theoretical_loss": 3.314071900954514,
+      "tokens_seen": 3079929856
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 7.177074422583405e-05,
+      "loss": 2.5641,
+      "theoretical_loss": 3.3140611152485846,
+      "tokens_seen": 3080060928
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 7.172797262617623e-05,
+      "loss": 2.609,
+      "theoretical_loss": 3.314050330130141,
+      "tokens_seen": 3080192000
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 7.16852010265184e-05,
+      "loss": 2.4717,
+      "theoretical_loss": 3.314039545599126,
+      "tokens_seen": 3080323072
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 7.164242942686057e-05,
+      "loss": 2.5039,
+      "theoretical_loss": 3.314028761655483,
+      "tokens_seen": 3080454144
+    },
+    {
+      "epoch": 0.29,
+      "objective/train/docs_used": 1688263,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 2.41924786567688,
+      "objective/train/theoretical_loss": 3.314023369903908,
+      "objective/train/tokens_used": 110570976,
+      "theoretical_loss": 3.314023369903908,
+      "tokens_seen": 3080519680
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 7.159965782720273e-05,
+      "loss": 2.4467,
+      "theoretical_loss": 3.3140179782991552,
+      "tokens_seen": 3080585216
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 7.155688622754491e-05,
+      "loss": 2.6404,
+      "theoretical_loss": 3.3140071955300847,
+      "tokens_seen": 3080716288
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 7.151411462788709e-05,
+      "loss": 2.5012,
+      "theoretical_loss": 3.3139964133482147,
+      "tokens_seen": 3080847360
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 7.147134302822926e-05,
+      "loss": 2.469,
+      "theoretical_loss": 3.3139856317534893,
+      "tokens_seen": 3080978432
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 7.142857142857143e-05,
+      "loss": 2.5803,
+      "theoretical_loss": 3.31397485074585,
+      "tokens_seen": 3081109504
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 7.13857998289136e-05,
+      "loss": 2.4955,
+      "theoretical_loss": 3.3139640703252415,
+      "tokens_seen": 3081240576
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 7.134302822925578e-05,
+      "loss": 2.5004,
+      "theoretical_loss": 3.3139532904916056,
+      "tokens_seen": 3081371648
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 7.130025662959796e-05,
+      "loss": 2.5855,
+      "theoretical_loss": 3.3139425112448864,
+      "tokens_seen": 3081502720
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 7.125748502994012e-05,
+      "loss": 2.543,
+      "theoretical_loss": 3.3139317325850257,
+      "tokens_seen": 3081633792
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 7.12147134302823e-05,
+      "loss": 2.4956,
+      "theoretical_loss": 3.3139209545119677,
+      "tokens_seen": 3081764864
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 7.117194183062446e-05,
+      "loss": 2.3803,
+      "theoretical_loss": 3.313910177025655,
+      "tokens_seen": 3081895936
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 7.112917023096664e-05,
+      "loss": 2.4136,
+      "theoretical_loss": 3.3138994001260307,
+      "tokens_seen": 3082027008
+    },
+    {
+      "epoch": 0.3,
+      "objective/train/docs_used": 1688735,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 2.326685667037964,
+      "objective/train/theoretical_loss": 3.313888623813038,
+      "objective/train/tokens_used": 112209376,
+      "theoretical_loss": 3.313888623813038,
+      "tokens_seen": 3082158080
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 7.108639863130882e-05,
+      "loss": 2.4642,
+      "theoretical_loss": 3.313888623813038,
+      "tokens_seen": 3082158080
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 7.104362703165098e-05,
+      "loss": 2.5382,
+      "theoretical_loss": 3.3138778480866202,
+      "tokens_seen": 3082289152
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 7.100085543199316e-05,
+      "loss": 2.527,
+      "theoretical_loss": 3.3138670729467203,
+      "tokens_seen": 3082420224
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 7.095808383233533e-05,
+      "loss": 2.5672,
+      "theoretical_loss": 3.3138562983932816,
+      "tokens_seen": 3082551296
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 7.09153122326775e-05,
+      "loss": 2.4753,
+      "theoretical_loss": 3.313845524426247,
+      "tokens_seen": 3082682368
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 7.087254063301968e-05,
+      "loss": 2.6285,
+      "theoretical_loss": 3.3138347510455595,
+      "tokens_seen": 3082813440
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 7.082976903336185e-05,
+      "loss": 2.4997,
+      "theoretical_loss": 3.3138239782511625,
+      "tokens_seen": 3082944512
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 7.078699743370403e-05,
+      "loss": 2.6724,
+      "theoretical_loss": 3.3138132060429992,
+      "tokens_seen": 3083075584
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 7.074422583404619e-05,
+      "loss": 2.5294,
+      "theoretical_loss": 3.313802434421013,
+      "tokens_seen": 3083206656
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 7.070145423438837e-05,
+      "loss": 2.442,
+      "theoretical_loss": 3.313791663385146,
+      "tokens_seen": 3083337728
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 7.065868263473055e-05,
+      "loss": 2.5894,
+      "theoretical_loss": 3.313780892935343,
+      "tokens_seen": 3083468800
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 7.061591103507271e-05,
+      "loss": 2.5735,
+      "theoretical_loss": 3.313770123071546,
+      "tokens_seen": 3083599872
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 7.057313943541489e-05,
+      "loss": 2.5645,
+      "theoretical_loss": 3.313759353793699,
+      "tokens_seen": 3083730944
+    },
+    {
+      "epoch": 0.3,
+      "objective/train/docs_used": 1689720,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 2.1338090896606445,
+      "objective/train/theoretical_loss": 3.313753969374489,
+      "objective/train/tokens_used": 113847776,
+      "theoretical_loss": 3.313753969374489,
+      "tokens_seen": 3083796480
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 7.053036783575706e-05,
+      "loss": 2.4857,
+      "theoretical_loss": 3.3137485851017447,
+      "tokens_seen": 3083862016
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 7.048759623609923e-05,
+      "loss": 2.6054,
+      "theoretical_loss": 3.3137378169956264,
+      "tokens_seen": 3083993088
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 7.044482463644141e-05,
+      "loss": 2.6083,
+      "theoretical_loss": 3.313727049475287,
+      "tokens_seen": 3084124160
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 7.040205303678358e-05,
+      "loss": 2.3527,
+      "theoretical_loss": 3.3137162825406707,
+      "tokens_seen": 3084255232
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 7.035928143712576e-05,
+      "loss": 2.4356,
+      "theoretical_loss": 3.31370551619172,
+      "tokens_seen": 3084386304
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 7.031650983746792e-05,
+      "loss": 2.6893,
+      "theoretical_loss": 3.313694750428378,
+      "tokens_seen": 3084517376
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 7.02737382378101e-05,
+      "loss": 2.3975,
+      "theoretical_loss": 3.313683985250589,
+      "tokens_seen": 3084648448
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 7.023096663815228e-05,
+      "loss": 2.5608,
+      "theoretical_loss": 3.313673220658295,
+      "tokens_seen": 3084779520
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 7.018819503849444e-05,
+      "loss": 2.4643,
+      "theoretical_loss": 3.31366245665144,
+      "tokens_seen": 3084910592
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 7.01454234388366e-05,
+      "loss": 2.395,
+      "theoretical_loss": 3.3136516932299673,
+      "tokens_seen": 3085041664
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 7.010265183917878e-05,
+      "loss": 2.5804,
+      "theoretical_loss": 3.3136409303938197,
+      "tokens_seen": 3085172736
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 7.005988023952096e-05,
+      "loss": 2.5382,
+      "theoretical_loss": 3.313630168142941,
+      "tokens_seen": 3085303808
+    },
+    {
+      "epoch": 0.31,
+      "objective/train/docs_used": 1690994,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 2.813086748123169,
+      "objective/train/theoretical_loss": 3.3136194064772746,
+      "objective/train/tokens_used": 115486176,
+      "theoretical_loss": 3.3136194064772746,
+      "tokens_seen": 3085434880
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 7.001710863986314e-05,
+      "loss": 2.7161,
+      "theoretical_loss": 3.3136194064772746,
+      "tokens_seen": 3085434880
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 6.99743370402053e-05,
+      "loss": 2.5303,
+      "theoretical_loss": 3.3136086453967635,
+      "tokens_seen": 3085565952
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 6.993156544054747e-05,
+      "loss": 2.5129,
+      "theoretical_loss": 3.313597884901351,
+      "tokens_seen": 3085697024
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 6.988879384088965e-05,
+      "loss": 2.4289,
+      "theoretical_loss": 3.3135871249909803,
+      "tokens_seen": 3085828096
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 6.984602224123183e-05,
+      "loss": 2.4337,
+      "theoretical_loss": 3.3135763656655954,
+      "tokens_seen": 3085959168
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 6.9803250641574e-05,
+      "loss": 2.4339,
+      "theoretical_loss": 3.313565606925139,
+      "tokens_seen": 3086090240
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 6.976047904191617e-05,
+      "loss": 2.3467,
+      "theoretical_loss": 3.313554848769555,
+      "tokens_seen": 3086221312
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 6.971770744225834e-05,
+      "loss": 2.5069,
+      "theoretical_loss": 3.313544091198786,
+      "tokens_seen": 3086352384
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 6.967493584260051e-05,
+      "loss": 2.3418,
+      "theoretical_loss": 3.313533334212776,
+      "tokens_seen": 3086483456
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 6.963216424294269e-05,
+      "loss": 2.6456,
+      "theoretical_loss": 3.3135225778114683,
+      "tokens_seen": 3086614528
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 6.958939264328487e-05,
+      "loss": 2.4547,
+      "theoretical_loss": 3.313511821994806,
+      "tokens_seen": 3086745600
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 6.954662104362704e-05,
+      "loss": 2.459,
+      "theoretical_loss": 3.313501066762733,
+      "tokens_seen": 3086876672
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 6.95038494439692e-05,
+      "loss": 2.5895,
+      "theoretical_loss": 3.3134903121151926,
+      "tokens_seen": 3087007744
+    },
+    {
+      "epoch": 0.31,
+      "objective/train/docs_used": 1691628,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 2.380619525909424,
+      "objective/train/theoretical_loss": 3.313484935010604,
+      "objective/train/tokens_used": 117124576,
+      "theoretical_loss": 3.313484935010604,
+      "tokens_seen": 3087073280
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 6.946107784431138e-05,
+      "loss": 2.5606,
+      "theoretical_loss": 3.3134795580521277,
+      "tokens_seen": 3087138816
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 6.941830624465356e-05,
+      "loss": 2.6642,
+      "theoretical_loss": 3.313468804573482,
+      "tokens_seen": 3087269888
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 6.937553464499572e-05,
+      "loss": 2.3761,
+      "theoretical_loss": 3.313458051679199,
+      "tokens_seen": 3087400960
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 6.93327630453379e-05,
+      "loss": 2.6433,
+      "theoretical_loss": 3.3134472993692223,
+      "tokens_seen": 3087532032
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 6.928999144568006e-05,
+      "loss": 2.5497,
+      "theoretical_loss": 3.3134365476434953,
+      "tokens_seen": 3087663104
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 6.924721984602224e-05,
+      "loss": 2.5376,
+      "theoretical_loss": 3.313425796501961,
+      "tokens_seen": 3087794176
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 6.920444824636442e-05,
+      "loss": 2.6459,
+      "theoretical_loss": 3.3134150459445633,
+      "tokens_seen": 3087925248
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 6.916167664670659e-05,
+      "loss": 2.7084,
+      "theoretical_loss": 3.313404295971245,
+      "tokens_seen": 3088056320
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 6.911890504704876e-05,
+      "loss": 2.5354,
+      "theoretical_loss": 3.313393546581951,
+      "tokens_seen": 3088187392
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 6.907613344739093e-05,
+      "loss": 2.4561,
+      "theoretical_loss": 3.3133827977766237,
+      "tokens_seen": 3088318464
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 6.903336184773311e-05,
+      "loss": 2.6013,
+      "theoretical_loss": 3.3133720495552064,
+      "tokens_seen": 3088449536
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 6.899059024807529e-05,
+      "loss": 2.5218,
+      "theoretical_loss": 3.3133613019176433,
+      "tokens_seen": 3088580608
+    },
+    {
+      "epoch": 0.32,
+      "objective/train/docs_used": 1692791,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 2.3386290073394775,
+      "objective/train/theoretical_loss": 3.3133505548638778,
+      "objective/train/tokens_used": 118762976,
+      "theoretical_loss": 3.3133505548638778,
+      "tokens_seen": 3088711680
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 6.894781864841745e-05,
+      "loss": 2.5461,
+      "theoretical_loss": 3.3133505548638778,
+      "tokens_seen": 3088711680
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 6.890504704875963e-05,
+      "loss": 2.6487,
+      "theoretical_loss": 3.3133398083938532,
+      "tokens_seen": 3088842752
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 6.886227544910179e-05,
+      "loss": 2.6339,
+      "theoretical_loss": 3.313329062507513,
+      "tokens_seen": 3088973824
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 6.881950384944397e-05,
+      "loss": 2.525,
+      "theoretical_loss": 3.313318317204801,
+      "tokens_seen": 3089104896
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 6.877673224978615e-05,
+      "loss": 2.4559,
+      "theoretical_loss": 3.3133075724856598,
+      "tokens_seen": 3089235968
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 6.873396065012831e-05,
+      "loss": 2.5522,
+      "theoretical_loss": 3.313296828350034,
+      "tokens_seen": 3089367040
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 6.869118905047049e-05,
+      "loss": 2.6742,
+      "theoretical_loss": 3.313286084797867,
+      "tokens_seen": 3089498112
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 6.864841745081266e-05,
+      "loss": 2.548,
+      "theoretical_loss": 3.3132753418291023,
+      "tokens_seen": 3089629184
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 6.860564585115484e-05,
+      "loss": 2.6003,
+      "theoretical_loss": 3.3132645994436833,
+      "tokens_seen": 3089760256
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 6.856287425149701e-05,
+      "loss": 2.6613,
+      "theoretical_loss": 3.313253857641554,
+      "tokens_seen": 3089891328
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 6.852010265183918e-05,
+      "loss": 2.6473,
+      "theoretical_loss": 3.3132431164226572,
+      "tokens_seen": 3090022400
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 6.847733105218136e-05,
+      "loss": 2.3945,
+      "theoretical_loss": 3.313232375786937,
+      "tokens_seen": 3090153472
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 6.843455945252352e-05,
+      "loss": 2.5502,
+      "theoretical_loss": 3.3132216357343367,
+      "tokens_seen": 3090284544
+    },
+    {
+      "epoch": 0.32,
+      "objective/train/docs_used": 1693638,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 2.6819231510162354,
+      "objective/train/theoretical_loss": 3.3132162659266893,
+      "objective/train/tokens_used": 120401376,
+      "theoretical_loss": 3.3132162659266893,
+      "tokens_seen": 3090350080
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 6.83917878528657e-05,
+      "loss": 2.4485,
+      "theoretical_loss": 3.313210896264801,
+      "tokens_seen": 3090415616
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 6.834901625320788e-05,
+      "loss": 2.6666,
+      "theoretical_loss": 3.3132001573782714,
+      "tokens_seen": 3090546688
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 6.830624465355004e-05,
+      "loss": 2.5939,
+      "theoretical_loss": 3.3131894190746936,
+      "tokens_seen": 3090677760
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 6.826347305389222e-05,
+      "loss": 2.5552,
+      "theoretical_loss": 3.3131786813540107,
+      "tokens_seen": 3090808832
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 6.822070145423439e-05,
+      "loss": 2.6067,
+      "theoretical_loss": 3.3131679442161657,
+      "tokens_seen": 3090939904
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 6.817792985457656e-05,
+      "loss": 2.6286,
+      "theoretical_loss": 3.3131572076611024,
+      "tokens_seen": 3091070976
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 6.813515825491874e-05,
+      "loss": 2.4823,
+      "theoretical_loss": 3.313146471688765,
+      "tokens_seen": 3091202048
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 6.809238665526091e-05,
+      "loss": 2.5575,
+      "theoretical_loss": 3.3131357362990967,
+      "tokens_seen": 3091333120
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 6.804961505560307e-05,
+      "loss": 2.6222,
+      "theoretical_loss": 3.3131250014920415,
+      "tokens_seen": 3091464192
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 6.800684345594526e-05,
+      "loss": 2.617,
+      "theoretical_loss": 3.313114267267543,
+      "tokens_seen": 3091595264
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 6.796407185628743e-05,
+      "loss": 2.6044,
+      "theoretical_loss": 3.3131035336255446,
+      "tokens_seen": 3091726336
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 6.792130025662961e-05,
+      "loss": 2.4566,
+      "theoretical_loss": 3.31309280056599,
+      "tokens_seen": 3091857408
+    },
+    {
+      "epoch": 0.33,
+      "objective/train/docs_used": 1694249,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 1.87570321559906,
+      "objective/train/theoretical_loss": 3.3130820680888236,
+      "objective/train/tokens_used": 122039776,
+      "theoretical_loss": 3.3130820680888236,
+      "tokens_seen": 3091988480
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 6.787852865697177e-05,
+      "loss": 2.5227,
+      "theoretical_loss": 3.3130820680888236,
+      "tokens_seen": 3091988480
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 6.783575705731394e-05,
+      "loss": 2.5406,
+      "theoretical_loss": 3.313071336193988,
+      "tokens_seen": 3092119552
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 6.779298545765613e-05,
+      "loss": 2.659,
+      "theoretical_loss": 3.313060604881428,
+      "tokens_seen": 3092250624
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 6.775021385799829e-05,
+      "loss": 2.4937,
+      "theoretical_loss": 3.313049874151087,
+      "tokens_seen": 3092381696
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 6.770744225834047e-05,
+      "loss": 2.6934,
+      "theoretical_loss": 3.313039144002908,
+      "tokens_seen": 3092512768
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 6.766467065868264e-05,
+      "loss": 2.6567,
+      "theoretical_loss": 3.313028414436836,
+      "tokens_seen": 3092643840
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 6.76218990590248e-05,
+      "loss": 2.5487,
+      "theoretical_loss": 3.313017685452814,
+      "tokens_seen": 3092774912
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 6.757912745936699e-05,
+      "loss": 2.5314,
+      "theoretical_loss": 3.3130069570507854,
+      "tokens_seen": 3092905984
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 6.753635585970916e-05,
+      "loss": 2.529,
+      "theoretical_loss": 3.312996229230695,
+      "tokens_seen": 3093037056
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 6.749358426005134e-05,
+      "loss": 2.6427,
+      "theoretical_loss": 3.312985501992485,
+      "tokens_seen": 3093168128
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 6.74508126603935e-05,
+      "loss": 2.5213,
+      "theoretical_loss": 3.312974775336101,
+      "tokens_seen": 3093299200
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 6.740804106073567e-05,
+      "loss": 2.4606,
+      "theoretical_loss": 3.312964049261486,
+      "tokens_seen": 3093430272
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 6.736526946107786e-05,
+      "loss": 2.6563,
+      "theoretical_loss": 3.3129533237685838,
+      "tokens_seen": 3093561344
+    },
+    {
+      "epoch": 0.33,
+      "objective/train/docs_used": 1695060,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 2.7868711948394775,
+      "objective/train/theoretical_loss": 3.312947961240257,
+      "objective/train/tokens_used": 123678176,
+      "theoretical_loss": 3.312947961240257,
+      "tokens_seen": 3093626880
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 6.732249786142002e-05,
+      "loss": 2.7125,
+      "theoretical_loss": 3.3129425988573376,
+      "tokens_seen": 3093692416
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 6.727972626176219e-05,
+      "loss": 2.531,
+      "theoretical_loss": 3.3129318745276923,
+      "tokens_seen": 3093823488
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 6.723695466210436e-05,
+      "loss": 2.5273,
+      "theoretical_loss": 3.3129211507795913,
+      "tokens_seen": 3093954560
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 6.719418306244653e-05,
+      "loss": 2.7121,
+      "theoretical_loss": 3.312910427612978,
+      "tokens_seen": 3094085632
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 6.715141146278872e-05,
+      "loss": 2.5665,
+      "theoretical_loss": 3.312899705027797,
+      "tokens_seen": 3094216704
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 6.710863986313089e-05,
+      "loss": 2.6584,
+      "theoretical_loss": 3.312888983023991,
+      "tokens_seen": 3094347776
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 6.706586826347305e-05,
+      "loss": 2.6025,
+      "theoretical_loss": 3.3128782616015053,
+      "tokens_seen": 3094478848
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 6.702309666381523e-05,
+      "loss": 2.6372,
+      "theoretical_loss": 3.312867540760283,
+      "tokens_seen": 3094609920
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 6.698032506415741e-05,
+      "loss": 2.5192,
+      "theoretical_loss": 3.3128568205002678,
+      "tokens_seen": 3094740992
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 6.693755346449959e-05,
+      "loss": 2.5967,
+      "theoretical_loss": 3.3128461008214036,
+      "tokens_seen": 3094872064
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 6.689478186484175e-05,
+      "loss": 2.6274,
+      "theoretical_loss": 3.3128353817236347,
+      "tokens_seen": 3095003136
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 6.685201026518392e-05,
+      "loss": 2.5711,
+      "theoretical_loss": 3.312824663206905,
+      "tokens_seen": 3095134208
+    },
+    {
+      "epoch": 0.34,
+      "objective/train/docs_used": 1695507,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 2.8500750064849854,
+      "objective/train/theoretical_loss": 3.312813945271158,
+      "objective/train/tokens_used": 125316576,
+      "theoretical_loss": 3.312813945271158,
+      "tokens_seen": 3095265280
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 6.68092386655261e-05,
+      "loss": 2.5407,
+      "theoretical_loss": 3.312813945271158,
+      "tokens_seen": 3095265280
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 6.676646706586827e-05,
+      "loss": 2.6406,
+      "theoretical_loss": 3.3128032279163375,
+      "tokens_seen": 3095396352
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 6.672369546621044e-05,
+      "loss": 2.5063,
+      "theoretical_loss": 3.312792511142388,
+      "tokens_seen": 3095527424
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 6.668092386655261e-05,
+      "loss": 2.6621,
+      "theoretical_loss": 3.3127817949492533,
+      "tokens_seen": 3095658496
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 6.663815226689478e-05,
+      "loss": 2.6269,
+      "theoretical_loss": 3.3127710793368768,
+      "tokens_seen": 3095789568
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 6.659538066723696e-05,
+      "loss": 2.6773,
+      "theoretical_loss": 3.312760364305203,
+      "tokens_seen": 3095920640
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 6.655260906757914e-05,
+      "loss": 2.6404,
+      "theoretical_loss": 3.3127496498541755,
+      "tokens_seen": 3096051712
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 6.65098374679213e-05,
+      "loss": 2.5714,
+      "theoretical_loss": 3.3127389359837385,
+      "tokens_seen": 3096182784
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 6.646706586826348e-05,
+      "loss": 2.6192,
+      "theoretical_loss": 3.3127282226938357,
+      "tokens_seen": 3096313856
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 6.642429426860564e-05,
+      "loss": 2.5385,
+      "theoretical_loss": 3.3127175099844113,
+      "tokens_seen": 3096444928
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 6.638152266894782e-05,
+      "loss": 2.632,
+      "theoretical_loss": 3.3127067978554092,
+      "tokens_seen": 3096576000
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 6.633875106929e-05,
+      "loss": 2.5296,
+      "theoretical_loss": 3.3126960863067736,
+      "tokens_seen": 3096707072
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 6.629597946963217e-05,
+      "loss": 2.5782,
+      "theoretical_loss": 3.3126853753384484,
+      "tokens_seen": 3096838144
+    },
+    {
+      "epoch": 0.34,
+      "objective/train/docs_used": 1696585,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 2.7085928916931152,
+      "objective/train/theoretical_loss": 3.3126800200718844,
+      "objective/train/tokens_used": 126954976,
+      "theoretical_loss": 3.3126800200718844,
+      "tokens_seen": 3096903680
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 6.625320786997434e-05,
+      "loss": 2.5901,
+      "theoretical_loss": 3.312674664950377,
+      "tokens_seen": 3096969216
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 6.621043627031651e-05,
+      "loss": 2.7204,
+      "theoretical_loss": 3.3126639551425043,
+      "tokens_seen": 3097100288
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 6.616766467065869e-05,
+      "loss": 2.4963,
+      "theoretical_loss": 3.3126532459147735,
+      "tokens_seen": 3097231360
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 6.612489307100087e-05,
+      "loss": 2.5793,
+      "theoretical_loss": 3.3126425372671293,
+      "tokens_seen": 3097362432
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 6.608212147134303e-05,
+      "loss": 2.6098,
+      "theoretical_loss": 3.3126318291995154,
+      "tokens_seen": 3097493504
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 6.603934987168521e-05,
+      "loss": 2.7311,
+      "theoretical_loss": 3.312621121711876,
+      "tokens_seen": 3097624576
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 6.599657827202737e-05,
+      "loss": 2.5035,
+      "theoretical_loss": 3.312610414804155,
+      "tokens_seen": 3097755648
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 6.595380667236955e-05,
+      "loss": 2.4575,
+      "theoretical_loss": 3.312599708476297,
+      "tokens_seen": 3097886720
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 6.591103507271173e-05,
+      "loss": 2.5423,
+      "theoretical_loss": 3.3125890027282447,
+      "tokens_seen": 3098017792
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 6.58682634730539e-05,
+      "loss": 2.5146,
+      "theoretical_loss": 3.3125782975599436,
+      "tokens_seen": 3098148864
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 6.582549187339607e-05,
+      "loss": 2.5976,
+      "theoretical_loss": 3.312567592971337,
+      "tokens_seen": 3098279936
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 6.578272027373824e-05,
+      "loss": 2.4906,
+      "theoretical_loss": 3.3125568889623693,
+      "tokens_seen": 3098411008
+    },
+    {
+      "epoch": 0.35,
+      "objective/train/docs_used": 1697815,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 2.453221082687378,
+      "objective/train/theoretical_loss": 3.3125461855329847,
+      "objective/train/tokens_used": 128593376,
+      "theoretical_loss": 3.3125461855329847,
+      "tokens_seen": 3098542080
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 6.573994867408042e-05,
+      "loss": 2.539,
+      "theoretical_loss": 3.3125461855329847,
+      "tokens_seen": 3098542080
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 6.56971770744226e-05,
+      "loss": 2.6606,
+      "theoretical_loss": 3.312535482683127,
+      "tokens_seen": 3098673152
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 6.565440547476476e-05,
+      "loss": 2.5937,
+      "theoretical_loss": 3.3125247804127405,
+      "tokens_seen": 3098804224
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 6.561163387510694e-05,
+      "loss": 2.4754,
+      "theoretical_loss": 3.312514078721769,
+      "tokens_seen": 3098935296
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 6.55688622754491e-05,
+      "loss": 2.5848,
+      "theoretical_loss": 3.312503377610157,
+      "tokens_seen": 3099066368
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 6.552609067579128e-05,
+      "loss": 2.5223,
+      "theoretical_loss": 3.3124926770778487,
+      "tokens_seen": 3099197440
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 6.548331907613346e-05,
+      "loss": 2.5736,
+      "theoretical_loss": 3.3124819771247878,
+      "tokens_seen": 3099328512
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 6.544054747647562e-05,
+      "loss": 2.6126,
+      "theoretical_loss": 3.3124712777509187,
+      "tokens_seen": 3099459584
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 6.539777587681779e-05,
+      "loss": 2.5122,
+      "theoretical_loss": 3.312460578956186,
+      "tokens_seen": 3099590656
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 6.535500427715997e-05,
+      "loss": 2.5951,
+      "theoretical_loss": 3.3124498807405325,
+      "tokens_seen": 3099721728
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 6.531223267750214e-05,
+      "loss": 2.5596,
+      "theoretical_loss": 3.312439183103904,
+      "tokens_seen": 3099852800
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 6.526946107784432e-05,
+      "loss": 2.6898,
+      "theoretical_loss": 3.312428486046244,
+      "tokens_seen": 3099983872
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 6.522668947818649e-05,
+      "loss": 2.7212,
+      "theoretical_loss": 3.312417789567496,
+      "tokens_seen": 3100114944
+    },
+    {
+      "epoch": 0.35,
+      "objective/train/docs_used": 1698465,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 2.276474714279175,
+      "objective/train/theoretical_loss": 3.3124124415451974,
+      "objective/train/tokens_used": 130231776,
+      "theoretical_loss": 3.3124124415451974,
+      "tokens_seen": 3100180480
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 6.518391787852865e-05,
+      "loss": 2.4624,
+      "theoretical_loss": 3.3124070936676056,
+      "tokens_seen": 3100246016
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 6.514114627887083e-05,
+      "loss": 2.5426,
+      "theoretical_loss": 3.312396398346516,
+      "tokens_seen": 3100377088
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 6.509837467921301e-05,
+      "loss": 2.5892,
+      "theoretical_loss": 3.3123857036041717,
+      "tokens_seen": 3100508160
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 6.505560307955519e-05,
+      "loss": 2.6646,
+      "theoretical_loss": 3.3123750094405167,
+      "tokens_seen": 3100639232
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 6.501283147989735e-05,
+      "loss": 2.4117,
+      "theoretical_loss": 3.3123643158554956,
+      "tokens_seen": 3100770304
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 6.497005988023952e-05,
+      "loss": 2.5704,
+      "theoretical_loss": 3.312353622849052,
+      "tokens_seen": 3100901376
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 6.49272882805817e-05,
+      "loss": 2.4199,
+      "theoretical_loss": 3.3123429304211314,
+      "tokens_seen": 3101032448
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 6.488451668092387e-05,
+      "loss": 2.4503,
+      "theoretical_loss": 3.3123322385716767,
+      "tokens_seen": 3101163520
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 6.484174508126605e-05,
+      "loss": 2.528,
+      "theoretical_loss": 3.312321547300633,
+      "tokens_seen": 3101294592
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 6.479897348160822e-05,
+      "loss": 2.5026,
+      "theoretical_loss": 3.3123108566079438,
+      "tokens_seen": 3101425664
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 6.475620188195038e-05,
+      "loss": 2.607,
+      "theoretical_loss": 3.3123001664935545,
+      "tokens_seen": 3101556736
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 6.471343028229256e-05,
+      "loss": 2.5181,
+      "theoretical_loss": 3.3122894769574085,
+      "tokens_seen": 3101687808
+    },
+    {
+      "epoch": 0.36,
+      "objective/train/docs_used": 1699641,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 2.600740671157837,
+      "objective/train/theoretical_loss": 3.3122787879994497,
+      "objective/train/tokens_used": 131870176,
+      "theoretical_loss": 3.3122787879994497,
+      "tokens_seen": 3101818880
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 6.467065868263474e-05,
+      "loss": 2.6134,
+      "theoretical_loss": 3.3122787879994497,
+      "tokens_seen": 3101818880
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 6.46278870829769e-05,
+      "loss": 2.4917,
+      "theoretical_loss": 3.3122680996196237,
+      "tokens_seen": 3101949952
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 6.458511548331908e-05,
+      "loss": 2.6256,
+      "theoretical_loss": 3.312257411817874,
+      "tokens_seen": 3102081024
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 6.454234388366125e-05,
+      "loss": 2.4842,
+      "theoretical_loss": 3.3122467245941447,
+      "tokens_seen": 3102212096
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 6.449957228400342e-05,
+      "loss": 2.5249,
+      "theoretical_loss": 3.312236037948381,
+      "tokens_seen": 3102343168
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 6.44568006843456e-05,
+      "loss": 2.6317,
+      "theoretical_loss": 3.312225351880526,
+      "tokens_seen": 3102474240
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 6.441402908468777e-05,
+      "loss": 2.468,
+      "theoretical_loss": 3.312214666390525,
+      "tokens_seen": 3102605312
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 6.437125748502994e-05,
+      "loss": 2.5314,
+      "theoretical_loss": 3.312203981478322,
+      "tokens_seen": 3102736384
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 6.432848588537211e-05,
+      "loss": 2.6474,
+      "theoretical_loss": 3.3121932971438612,
+      "tokens_seen": 3102867456
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 6.428571428571429e-05,
+      "loss": 2.5816,
+      "theoretical_loss": 3.3121826133870873,
+      "tokens_seen": 3102998528
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 6.424294268605647e-05,
+      "loss": 2.6009,
+      "theoretical_loss": 3.3121719302079446,
+      "tokens_seen": 3103129600
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 6.420017108639863e-05,
+      "loss": 2.5371,
+      "theoretical_loss": 3.3121612476063778,
+      "tokens_seen": 3103260672
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 6.415739948674081e-05,
+      "loss": 2.5445,
+      "theoretical_loss": 3.3121505655823302,
+      "tokens_seen": 3103391744
+    },
+    {
+      "epoch": 0.36,
+      "objective/train/docs_used": 1699977,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 2.1121926307678223,
+      "objective/train/theoretical_loss": 3.312145224786859,
+      "objective/train/tokens_used": 133508576,
+      "theoretical_loss": 3.312145224786859,
+      "tokens_seen": 3103457280
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 6.411462788708297e-05,
+      "loss": 2.5187,
+      "theoretical_loss": 3.312139884135747,
+      "tokens_seen": 3103522816
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 6.407185628742515e-05,
+      "loss": 2.6903,
+      "theoretical_loss": 3.3121292032665726,
+      "tokens_seen": 3103653888
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 6.402908468776733e-05,
+      "loss": 2.5954,
+      "theoretical_loss": 3.3121185229747514,
+      "tokens_seen": 3103784960
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 6.39863130881095e-05,
+      "loss": 2.4992,
+      "theoretical_loss": 3.312107843260227,
+      "tokens_seen": 3103916032
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 6.394354148845167e-05,
+      "loss": 2.4812,
+      "theoretical_loss": 3.312097164122945,
+      "tokens_seen": 3104047104
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 6.390076988879384e-05,
+      "loss": 2.5708,
+      "theoretical_loss": 3.3120864855628493,
+      "tokens_seen": 3104178176
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 6.385799828913602e-05,
+      "loss": 2.5199,
+      "theoretical_loss": 3.3120758075798844,
+      "tokens_seen": 3104309248
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 6.38152266894782e-05,
+      "loss": 2.5963,
+      "theoretical_loss": 3.3120651301739947,
+      "tokens_seen": 3104440320
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 6.377245508982036e-05,
+      "loss": 2.5702,
+      "theoretical_loss": 3.312054453345125,
+      "tokens_seen": 3104571392
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 6.372968349016254e-05,
+      "loss": 2.5606,
+      "theoretical_loss": 3.3120437770932183,
+      "tokens_seen": 3104702464
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 6.36869118905047e-05,
+      "loss": 2.5872,
+      "theoretical_loss": 3.312033101418221,
+      "tokens_seen": 3104833536
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 6.364414029084688e-05,
+      "loss": 2.6235,
+      "theoretical_loss": 3.3120224263200764,
+      "tokens_seen": 3104964608
+    },
+    {
+      "epoch": 0.37,
+      "objective/train/docs_used": 1701288,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 2.725125551223755,
+      "objective/train/theoretical_loss": 3.3120117517987295,
+      "objective/train/tokens_used": 135146976,
+      "theoretical_loss": 3.3120117517987295,
+      "tokens_seen": 3105095680
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 6.360136869118906e-05,
+      "loss": 2.5687,
+      "theoretical_loss": 3.3120117517987295,
+      "tokens_seen": 3105095680
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 6.355859709153122e-05,
+      "loss": 2.5826,
+      "theoretical_loss": 3.312001077854125,
+      "tokens_seen": 3105226752
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 6.35158254918734e-05,
+      "loss": 2.5514,
+      "theoretical_loss": 3.311990404486206,
+      "tokens_seen": 3105357824
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 6.347305389221557e-05,
+      "loss": 2.5869,
+      "theoretical_loss": 3.311979731694919,
+      "tokens_seen": 3105488896
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 6.343028229255775e-05,
+      "loss": 2.6095,
+      "theoretical_loss": 3.311969059480207,
+      "tokens_seen": 3105619968
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 6.338751069289992e-05,
+      "loss": 2.613,
+      "theoretical_loss": 3.311958387842015,
+      "tokens_seen": 3105751040
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 6.334473909324209e-05,
+      "loss": 2.5923,
+      "theoretical_loss": 3.311947716780288,
+      "tokens_seen": 3105882112
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 6.330196749358425e-05,
+      "loss": 2.6335,
+      "theoretical_loss": 3.31193704629497,
+      "tokens_seen": 3106013184
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 6.325919589392643e-05,
+      "loss": 2.5942,
+      "theoretical_loss": 3.311926376386005,
+      "tokens_seen": 3106144256
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 6.321642429426861e-05,
+      "loss": 2.5971,
+      "theoretical_loss": 3.311915707053339,
+      "tokens_seen": 3106275328
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 6.317365269461079e-05,
+      "loss": 2.5642,
+      "theoretical_loss": 3.3119050382969153,
+      "tokens_seen": 3106406400
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 6.313088109495295e-05,
+      "loss": 2.4445,
+      "theoretical_loss": 3.311894370116679,
+      "tokens_seen": 3106537472
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 6.308810949529512e-05,
+      "loss": 2.5959,
+      "theoretical_loss": 3.3118837025125747,
+      "tokens_seen": 3106668544
+    },
+    {
+      "epoch": 0.38,
+      "objective/train/docs_used": 1702491,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 2.138563632965088,
+      "objective/train/theoretical_loss": 3.3118783689265547,
+      "objective/train/tokens_used": 136785376,
+      "theoretical_loss": 3.3118783689265547,
+      "tokens_seen": 3106734080
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 6.30453378956373e-05,
+      "loss": 2.4946,
+      "theoretical_loss": 3.311873035484547,
+      "tokens_seen": 3106799616
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 6.300256629597947e-05,
+      "loss": 2.4598,
+      "theoretical_loss": 3.31186236903254,
+      "tokens_seen": 3106930688
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 6.295979469632165e-05,
+      "loss": 2.5823,
+      "theoretical_loss": 3.311851703156499,
+      "tokens_seen": 3107061760
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 6.291702309666382e-05,
+      "loss": 2.6785,
+      "theoretical_loss": 3.311841037856368,
+      "tokens_seen": 3107192832
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 6.287425149700598e-05,
+      "loss": 2.4439,
+      "theoretical_loss": 3.311830373132092,
+      "tokens_seen": 3107323904
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 6.283147989734816e-05,
+      "loss": 2.5055,
+      "theoretical_loss": 3.3118197089836157,
+      "tokens_seen": 3107454976
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 6.278870829769034e-05,
+      "loss": 2.4456,
+      "theoretical_loss": 3.3118090454108833,
+      "tokens_seen": 3107586048
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 6.274593669803252e-05,
+      "loss": 2.5729,
+      "theoretical_loss": 3.3117983824138397,
+      "tokens_seen": 3107717120
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 6.270316509837468e-05,
+      "loss": 2.5774,
+      "theoretical_loss": 3.311787719992429,
+      "tokens_seen": 3107848192
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 6.266039349871685e-05,
+      "loss": 2.4742,
+      "theoretical_loss": 3.311777058146597,
+      "tokens_seen": 3107979264
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 6.261762189905902e-05,
+      "loss": 2.4432,
+      "theoretical_loss": 3.311766396876288,
+      "tokens_seen": 3108110336
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 6.25748502994012e-05,
+      "loss": 2.5103,
+      "theoretical_loss": 3.3117557361814454,
+      "tokens_seen": 3108241408
+    },
+    {
+      "epoch": 0.38,
+      "objective/train/docs_used": 1702889,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 2.351708173751831,
+      "objective/train/theoretical_loss": 3.3117450760620155,
+      "objective/train/tokens_used": 138423776,
+      "theoretical_loss": 3.3117450760620155,
+      "tokens_seen": 3108372480
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 6.253207869974337e-05,
+      "loss": 2.4282,
+      "theoretical_loss": 3.3117450760620155,
+      "tokens_seen": 3108372480
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 6.248930710008555e-05,
+      "loss": 2.5714,
+      "theoretical_loss": 3.311734416517942,
+      "tokens_seen": 3108503552
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 6.244653550042771e-05,
+      "loss": 2.4235,
+      "theoretical_loss": 3.31172375754917,
+      "tokens_seen": 3108634624
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 6.24037639007699e-05,
+      "loss": 2.6382,
+      "theoretical_loss": 3.311713099155644,
+      "tokens_seen": 3108765696
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 6.236099230111207e-05,
+      "loss": 2.5431,
+      "theoretical_loss": 3.311702441337309,
+      "tokens_seen": 3108896768
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 6.231822070145423e-05,
+      "loss": 2.3186,
+      "theoretical_loss": 3.3116917840941094,
+      "tokens_seen": 3109027840
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 6.227544910179641e-05,
+      "loss": 2.5123,
+      "theoretical_loss": 3.3116811274259903,
+      "tokens_seen": 3109158912
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 6.223267750213857e-05,
+      "loss": 2.5532,
+      "theoretical_loss": 3.3116704713328957,
+      "tokens_seen": 3109289984
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 6.218990590248077e-05,
+      "loss": 2.6137,
+      "theoretical_loss": 3.311659815814771,
+      "tokens_seen": 3109421056
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 6.214713430282293e-05,
+      "loss": 2.4946,
+      "theoretical_loss": 3.311649160871561,
+      "tokens_seen": 3109552128
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 6.21043627031651e-05,
+      "loss": 2.499,
+      "theoretical_loss": 3.31163850650321,
+      "tokens_seen": 3109683200
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 6.206159110350727e-05,
+      "loss": 2.558,
+      "theoretical_loss": 3.311627852709663,
+      "tokens_seen": 3109814272
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 6.201881950384944e-05,
+      "loss": 2.4069,
+      "theoretical_loss": 3.3116171994908647,
+      "tokens_seen": 3109945344
+    },
+    {
+      "epoch": 0.39,
+      "objective/train/docs_used": 1704007,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 2.46962571144104,
+      "objective/train/theoretical_loss": 3.311611873096979,
+      "objective/train/tokens_used": 140062176,
+      "theoretical_loss": 3.311611873096979,
+      "tokens_seen": 3110010880
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 6.197604790419162e-05,
+      "loss": 2.5208,
+      "theoretical_loss": 3.3116065468467597,
+      "tokens_seen": 3110076416
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 6.19332763045338e-05,
+      "loss": 2.5436,
+      "theoretical_loss": 3.311595894777293,
+      "tokens_seen": 3110207488
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 6.189050470487596e-05,
+      "loss": 2.4587,
+      "theoretical_loss": 3.3115852432824093,
+      "tokens_seen": 3110338560
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 6.184773310521814e-05,
+      "loss": 2.6021,
+      "theoretical_loss": 3.311574592362054,
+      "tokens_seen": 3110469632
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 6.18049615055603e-05,
+      "loss": 2.3898,
+      "theoretical_loss": 3.311563942016171,
+      "tokens_seen": 3110600704
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 6.176218990590248e-05,
+      "loss": 2.603,
+      "theoretical_loss": 3.311553292244705,
+      "tokens_seen": 3110731776
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 6.171941830624466e-05,
+      "loss": 2.5603,
+      "theoretical_loss": 3.3115426430476016,
+      "tokens_seen": 3110862848
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 6.167664670658683e-05,
+      "loss": 2.4966,
+      "theoretical_loss": 3.3115319944248056,
+      "tokens_seen": 3110993920
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 6.1633875106929e-05,
+      "loss": 2.6126,
+      "theoretical_loss": 3.311521346376261,
+      "tokens_seen": 3111124992
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 6.159110350727118e-05,
+      "loss": 2.535,
+      "theoretical_loss": 3.311510698901914,
+      "tokens_seen": 3111256064
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 6.154833190761335e-05,
+      "loss": 2.5019,
+      "theoretical_loss": 3.311500052001708,
+      "tokens_seen": 3111387136
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 6.150556030795552e-05,
+      "loss": 2.6325,
+      "theoretical_loss": 3.311489405675588,
+      "tokens_seen": 3111518208
+    },
+    {
+      "epoch": 0.39,
+      "objective/train/docs_used": 1704747,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 2.4722747802734375,
+      "objective/train/theoretical_loss": 3.3114787599235003,
+      "objective/train/tokens_used": 141700576,
+      "theoretical_loss": 3.3114787599235003,
+      "tokens_seen": 3111649280
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 6.146278870829769e-05,
+      "loss": 2.4573,
+      "theoretical_loss": 3.3114787599235003,
+      "tokens_seen": 3111649280
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 6.142001710863987e-05,
+      "loss": 2.6906,
+      "theoretical_loss": 3.3114681147453884,
+      "tokens_seen": 3111780352
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 6.137724550898205e-05,
+      "loss": 2.5696,
+      "theoretical_loss": 3.311457470141198,
+      "tokens_seen": 3111911424
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 6.133447390932421e-05,
+      "loss": 2.3866,
+      "theoretical_loss": 3.311446826110873,
+      "tokens_seen": 3112042496
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 6.129170230966639e-05,
+      "loss": 2.5784,
+      "theoretical_loss": 3.311436182654359,
+      "tokens_seen": 3112173568
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 6.124893071000855e-05,
+      "loss": 2.4109,
+      "theoretical_loss": 3.311425539771601,
+      "tokens_seen": 3112304640
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 6.120615911035072e-05,
+      "loss": 2.5248,
+      "theoretical_loss": 3.3114148974625435,
+      "tokens_seen": 3112435712
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 6.116338751069291e-05,
+      "loss": 2.5607,
+      "theoretical_loss": 3.3114042557271315,
+      "tokens_seen": 3112566784
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 6.112061591103508e-05,
+      "loss": 2.4251,
+      "theoretical_loss": 3.3113936145653105,
+      "tokens_seen": 3112697856
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 6.107784431137725e-05,
+      "loss": 2.6288,
+      "theoretical_loss": 3.3113829739770244,
+      "tokens_seen": 3112828928
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 6.103507271171942e-05,
+      "loss": 2.4883,
+      "theoretical_loss": 3.311372333962219,
+      "tokens_seen": 3112960000
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 6.099230111206159e-05,
+      "loss": 2.5164,
+      "theoretical_loss": 3.311361694520839,
+      "tokens_seen": 3113091072
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 6.094952951240377e-05,
+      "loss": 2.5751,
+      "theoretical_loss": 3.311351055652829,
+      "tokens_seen": 3113222144
+    },
+    {
+      "epoch": 0.4,
+      "objective/train/docs_used": 1705767,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 2.8438971042633057,
+      "objective/train/theoretical_loss": 3.311345736433821,
+      "objective/train/tokens_used": 143338976,
+      "theoretical_loss": 3.311345736433821,
+      "tokens_seen": 3113287680
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 6.090675791274594e-05,
+      "loss": 2.5603,
+      "theoretical_loss": 3.3113404173581342,
+      "tokens_seen": 3113353216
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 6.086398631308812e-05,
+      "loss": 2.6501,
+      "theoretical_loss": 3.3113297796367,
+      "tokens_seen": 3113484288
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 6.082121471343028e-05,
+      "loss": 2.3713,
+      "theoretical_loss": 3.311319142488471,
+      "tokens_seen": 3113615360
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 6.0778443113772454e-05,
+      "loss": 2.6465,
+      "theoretical_loss": 3.311308505913392,
+      "tokens_seen": 3113746432
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 6.073567151411463e-05,
+      "loss": 2.5023,
+      "theoretical_loss": 3.3112978699114084,
+      "tokens_seen": 3113877504
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 6.0692899914456804e-05,
+      "loss": 2.5409,
+      "theoretical_loss": 3.3112872344824646,
+      "tokens_seen": 3114008576
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 6.065012831479898e-05,
+      "loss": 2.5392,
+      "theoretical_loss": 3.3112765996265066,
+      "tokens_seen": 3114139648
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 6.060735671514115e-05,
+      "loss": 2.565,
+      "theoretical_loss": 3.3112659653434786,
+      "tokens_seen": 3114270720
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 6.056458511548332e-05,
+      "loss": 2.4205,
+      "theoretical_loss": 3.3112553316333253,
+      "tokens_seen": 3114401792
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 6.05218135158255e-05,
+      "loss": 2.5678,
+      "theoretical_loss": 3.311244698495993,
+      "tokens_seen": 3114532864
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 6.047904191616767e-05,
+      "loss": 2.5991,
+      "theoretical_loss": 3.3112340659314254,
+      "tokens_seen": 3114663936
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 6.043627031650983e-05,
+      "loss": 2.6641,
+      "theoretical_loss": 3.3112234339395687,
+      "tokens_seen": 3114795008
+    },
+    {
+      "epoch": 0.4,
+      "objective/train/docs_used": 1706383,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 2.5129618644714355,
+      "objective/train/theoretical_loss": 3.311212802520367,
+      "objective/train/tokens_used": 144977376,
+      "theoretical_loss": 3.311212802520367,
+      "tokens_seen": 3114926080
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 6.039349871685202e-05,
+      "loss": 2.4797,
+      "theoretical_loss": 3.311212802520367,
+      "tokens_seen": 3114926080
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 6.035072711719418e-05,
+      "loss": 2.5095,
+      "theoretical_loss": 3.311202171673766,
+      "tokens_seen": 3115057152
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 6.030795551753636e-05,
+      "loss": 2.5513,
+      "theoretical_loss": 3.3111915413997104,
+      "tokens_seen": 3115188224
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 6.026518391787853e-05,
+      "loss": 2.5783,
+      "theoretical_loss": 3.3111809116981457,
+      "tokens_seen": 3115319296
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 6.02224123182207e-05,
+      "loss": 2.4676,
+      "theoretical_loss": 3.3111702825690164,
+      "tokens_seen": 3115450368
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 6.017964071856288e-05,
+      "loss": 2.4629,
+      "theoretical_loss": 3.311159654012268,
+      "tokens_seen": 3115581440
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 6.013686911890505e-05,
+      "loss": 2.6033,
+      "theoretical_loss": 3.311149026027845,
+      "tokens_seen": 3115712512
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 6.0094097519247226e-05,
+      "loss": 2.5076,
+      "theoretical_loss": 3.3111383986156935,
+      "tokens_seen": 3115843584
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 6.00513259195894e-05,
+      "loss": 2.6581,
+      "theoretical_loss": 3.3111277717757583,
+      "tokens_seen": 3115974656
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 6.000855431993156e-05,
+      "loss": 2.6139,
+      "theoretical_loss": 3.3111171455079838,
+      "tokens_seen": 3116105728
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 5.996578272027375e-05,
+      "loss": 2.6152,
+      "theoretical_loss": 3.311106519812316,
+      "tokens_seen": 3116236800
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 5.992301112061591e-05,
+      "loss": 2.6476,
+      "theoretical_loss": 3.3110958946887,
+      "tokens_seen": 3116367872
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 5.988023952095808e-05,
+      "loss": 2.5176,
+      "theoretical_loss": 3.31108527013708,
+      "tokens_seen": 3116498944
+    },
+    {
+      "epoch": 0.41,
+      "objective/train/docs_used": 1707531,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 2.416107654571533,
+      "objective/train/theoretical_loss": 3.3110799580757515,
+      "objective/train/tokens_used": 146615776,
+      "theoretical_loss": 3.3110799580757515,
+      "tokens_seen": 3116564480
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 5.983746792130026e-05,
+      "loss": 2.6398,
+      "theoretical_loss": 3.311074646157402,
+      "tokens_seen": 3116630016
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 5.9794696321642426e-05,
+      "loss": 2.598,
+      "theoretical_loss": 3.311064022749611,
+      "tokens_seen": 3116761088
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 5.975192472198461e-05,
+      "loss": 2.5575,
+      "theoretical_loss": 3.311053399913652,
+      "tokens_seen": 3116892160
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 5.9709153122326776e-05,
+      "loss": 2.6516,
+      "theoretical_loss": 3.3110427776494706,
+      "tokens_seen": 3117023232
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 5.966638152266895e-05,
+      "loss": 2.5544,
+      "theoretical_loss": 3.3110321559570117,
+      "tokens_seen": 3117154304
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 5.9623609923011126e-05,
+      "loss": 2.5245,
+      "theoretical_loss": 3.31102153483622,
+      "tokens_seen": 3117285376
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 5.95808383233533e-05,
+      "loss": 2.5908,
+      "theoretical_loss": 3.3110109142870416,
+      "tokens_seen": 3117416448
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 5.9538066723695476e-05,
+      "loss": 2.5651,
+      "theoretical_loss": 3.311000294309421,
+      "tokens_seen": 3117547520
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 5.949529512403764e-05,
+      "loss": 2.631,
+      "theoretical_loss": 3.310989674903304,
+      "tokens_seen": 3117678592
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 5.945252352437981e-05,
+      "loss": 2.6308,
+      "theoretical_loss": 3.310979056068635,
+      "tokens_seen": 3117809664
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 5.940975192472199e-05,
+      "loss": 2.3692,
+      "theoretical_loss": 3.3109684378053603,
+      "tokens_seen": 3117940736
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 5.936698032506416e-05,
+      "loss": 2.5465,
+      "theoretical_loss": 3.3109578201134244,
+      "tokens_seen": 3118071808
+    },
+    {
+      "epoch": 0.41,
+      "objective/train/docs_used": 1708692,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 2.6952877044677734,
+      "objective/train/theoretical_loss": 3.3109472029927725,
+      "objective/train/tokens_used": 148254176,
+      "theoretical_loss": 3.3109472029927725,
+      "tokens_seen": 3118202880
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 5.932420872540634e-05,
+      "loss": 2.5085,
+      "theoretical_loss": 3.3109472029927725,
+      "tokens_seen": 3118202880
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 5.9281437125748505e-05,
+      "loss": 2.4693,
+      "theoretical_loss": 3.31093658644335,
+      "tokens_seen": 3118333952
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 5.9238665526090676e-05,
+      "loss": 2.5199,
+      "theoretical_loss": 3.3109259704651026,
+      "tokens_seen": 3118465024
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 5.9195893926432855e-05,
+      "loss": 2.5373,
+      "theoretical_loss": 3.3109153550579746,
+      "tokens_seen": 3118596096
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 5.9153122326775026e-05,
+      "loss": 2.625,
+      "theoretical_loss": 3.3109047402219125,
+      "tokens_seen": 3118727168
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 5.911035072711719e-05,
+      "loss": 2.5271,
+      "theoretical_loss": 3.3108941259568607,
+      "tokens_seen": 3118858240
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 5.906757912745937e-05,
+      "loss": 2.4392,
+      "theoretical_loss": 3.3108835122627647,
+      "tokens_seen": 3118989312
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 5.902480752780154e-05,
+      "loss": 2.6036,
+      "theoretical_loss": 3.3108728991395697,
+      "tokens_seen": 3119120384
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 5.898203592814372e-05,
+      "loss": 2.4951,
+      "theoretical_loss": 3.3108622865872213,
+      "tokens_seen": 3119251456
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 5.893926432848589e-05,
+      "loss": 2.5654,
+      "theoretical_loss": 3.3108516746056647,
+      "tokens_seen": 3119382528
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 5.8896492728828055e-05,
+      "loss": 2.4893,
+      "theoretical_loss": 3.310841063194845,
+      "tokens_seen": 3119513600
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 5.8853721129170233e-05,
+      "loss": 2.5437,
+      "theoretical_loss": 3.3108304523547076,
+      "tokens_seen": 3119644672
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 5.8810949529512405e-05,
+      "loss": 2.4567,
+      "theoretical_loss": 3.310819842085198,
+      "tokens_seen": 3119775744
+    },
+    {
+      "epoch": 0.42,
+      "objective/train/docs_used": 1709277,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 2.5585927963256836,
+      "objective/train/theoretical_loss": 3.3108145371644113,
+      "objective/train/tokens_used": 149892576,
+      "theoretical_loss": 3.3108145371644113,
+      "tokens_seen": 3119841280
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 5.876817792985458e-05,
+      "loss": 2.4938,
+      "theoretical_loss": 3.3108092323862617,
+      "tokens_seen": 3119906816
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 5.8725406330196755e-05,
+      "loss": 2.5817,
+      "theoretical_loss": 3.3107986232578437,
+      "tokens_seen": 3120037888
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 5.868263473053892e-05,
+      "loss": 2.5277,
+      "theoretical_loss": 3.310788014699889,
+      "tokens_seen": 3120168960
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 5.86398631308811e-05,
+      "loss": 2.5421,
+      "theoretical_loss": 3.3107774067123437,
+      "tokens_seen": 3120300032
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 5.859709153122327e-05,
+      "loss": 2.566,
+      "theoretical_loss": 3.310766799295153,
+      "tokens_seen": 3120431104
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 5.855431993156544e-05,
+      "loss": 2.7047,
+      "theoretical_loss": 3.3107561924482622,
+      "tokens_seen": 3120562176
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 5.851154833190762e-05,
+      "loss": 2.4278,
+      "theoretical_loss": 3.3107455861716164,
+      "tokens_seen": 3120693248
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 5.8468776732249784e-05,
+      "loss": 2.551,
+      "theoretical_loss": 3.3107349804651616,
+      "tokens_seen": 3120824320
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 5.842600513259196e-05,
+      "loss": 2.4814,
+      "theoretical_loss": 3.3107243753288427,
+      "tokens_seen": 3120955392
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 5.8383233532934134e-05,
+      "loss": 2.5773,
+      "theoretical_loss": 3.310713770762605,
+      "tokens_seen": 3121086464
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 5.8340461933276305e-05,
+      "loss": 2.6013,
+      "theoretical_loss": 3.3107031667663946,
+      "tokens_seen": 3121217536
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 5.8297690333618484e-05,
+      "loss": 2.4748,
+      "theoretical_loss": 3.3106925633401563,
+      "tokens_seen": 3121348608
+    },
+    {
+      "epoch": 0.42,
+      "objective/train/docs_used": 1710494,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 2.6660287380218506,
+      "objective/train/theoretical_loss": 3.3106819604838353,
+      "objective/train/tokens_used": 151530976,
+      "theoretical_loss": 3.3106819604838353,
+      "tokens_seen": 3121479680
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 5.825491873396065e-05,
+      "loss": 2.6014,
+      "theoretical_loss": 3.3106819604838353,
+      "tokens_seen": 3121479680
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 5.8212147134302833e-05,
+      "loss": 2.3807,
+      "theoretical_loss": 3.310671358197378,
+      "tokens_seen": 3121610752
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 5.8169375534645e-05,
+      "loss": 2.5044,
+      "theoretical_loss": 3.3106607564807295,
+      "tokens_seen": 3121741824
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 5.812660393498717e-05,
+      "loss": 2.4961,
+      "theoretical_loss": 3.3106501553338346,
+      "tokens_seen": 3121872896
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 5.808383233532935e-05,
+      "loss": 2.5321,
+      "theoretical_loss": 3.3106395547566394,
+      "tokens_seen": 3122003968
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 5.804106073567151e-05,
+      "loss": 2.6361,
+      "theoretical_loss": 3.3106289547490895,
+      "tokens_seen": 3122135040
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 5.79982891360137e-05,
+      "loss": 2.5194,
+      "theoretical_loss": 3.3106183553111297,
+      "tokens_seen": 3122266112
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 5.795551753635586e-05,
+      "loss": 2.5992,
+      "theoretical_loss": 3.310607756442706,
+      "tokens_seen": 3122397184
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 5.7912745936698034e-05,
+      "loss": 2.4211,
+      "theoretical_loss": 3.3105971581437634,
+      "tokens_seen": 3122528256
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 5.786997433704021e-05,
+      "loss": 2.4752,
+      "theoretical_loss": 3.310586560414248,
+      "tokens_seen": 3122659328
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 5.782720273738238e-05,
+      "loss": 2.5157,
+      "theoretical_loss": 3.3105759632541054,
+      "tokens_seen": 3122790400
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 5.778443113772455e-05,
+      "loss": 2.4493,
+      "theoretical_loss": 3.3105653666632806,
+      "tokens_seen": 3122921472
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 5.774165953806673e-05,
+      "loss": 2.4614,
+      "theoretical_loss": 3.310554770641719,
+      "tokens_seen": 3123052544
+    },
+    {
+      "epoch": 0.43,
+      "objective/train/docs_used": 1711181,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 2.3176870346069336,
+      "objective/train/theoretical_loss": 3.310549472844395,
+      "objective/train/tokens_used": 153169376,
+      "theoretical_loss": 3.310549472844395,
+      "tokens_seen": 3123118080
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 5.76988879384089e-05,
+      "loss": 2.4795,
+      "theoretical_loss": 3.3105441751893667,
+      "tokens_seen": 3123183616
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 5.765611633875108e-05,
+      "loss": 2.5418,
+      "theoretical_loss": 3.310533580306169,
+      "tokens_seen": 3123314688
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 5.761334473909324e-05,
+      "loss": 2.5666,
+      "theoretical_loss": 3.310522985992071,
+      "tokens_seen": 3123445760
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 5.757057313943541e-05,
+      "loss": 2.5606,
+      "theoretical_loss": 3.310512392247019,
+      "tokens_seen": 3123576832
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 5.752780153977759e-05,
+      "loss": 2.6241,
+      "theoretical_loss": 3.310501799070958,
+      "tokens_seen": 3123707904
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 5.748502994011976e-05,
+      "loss": 2.4346,
+      "theoretical_loss": 3.310491206463834,
+      "tokens_seen": 3123838976
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 5.744225834046194e-05,
+      "loss": 2.6249,
+      "theoretical_loss": 3.3104806144255923,
+      "tokens_seen": 3123970048
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 5.739948674080411e-05,
+      "loss": 2.6147,
+      "theoretical_loss": 3.3104700229561783,
+      "tokens_seen": 3124101120
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 5.735671514114628e-05,
+      "loss": 2.6046,
+      "theoretical_loss": 3.310459432055538,
+      "tokens_seen": 3124232192
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 5.7313943541488456e-05,
+      "loss": 2.4672,
+      "theoretical_loss": 3.3104488417236166,
+      "tokens_seen": 3124363264
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 5.727117194183063e-05,
+      "loss": 2.5318,
+      "theoretical_loss": 3.31043825196036,
+      "tokens_seen": 3124494336
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 5.7228400342172805e-05,
+      "loss": 2.433,
+      "theoretical_loss": 3.310427662765714,
+      "tokens_seen": 3124625408
+    },
+    {
+      "epoch": 0.43,
+      "objective/train/docs_used": 1712507,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 2.4671425819396973,
+      "objective/train/theoretical_loss": 3.310417074139624,
+      "objective/train/tokens_used": 154807776,
+      "theoretical_loss": 3.310417074139624,
+      "tokens_seen": 3124756480
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 5.718562874251498e-05,
+      "loss": 2.3719,
+      "theoretical_loss": 3.310417074139624,
+      "tokens_seen": 3124756480
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 5.714285714285714e-05,
+      "loss": 2.3982,
+      "theoretical_loss": 3.3104064860820355,
+      "tokens_seen": 3124887552
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 5.710008554319932e-05,
+      "loss": 2.6333,
+      "theoretical_loss": 3.310395898592894,
+      "tokens_seen": 3125018624
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 5.705731394354149e-05,
+      "loss": 2.4311,
+      "theoretical_loss": 3.310385311672145,
+      "tokens_seen": 3125149696
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 5.7014542343883656e-05,
+      "loss": 2.3717,
+      "theoretical_loss": 3.3103747253197353,
+      "tokens_seen": 3125280768
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 5.697177074422584e-05,
+      "loss": 2.582,
+      "theoretical_loss": 3.310364139535609,
+      "tokens_seen": 3125411840
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 5.6928999144568006e-05,
+      "loss": 2.4568,
+      "theoretical_loss": 3.3103535543197133,
+      "tokens_seen": 3125542912
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 5.6886227544910184e-05,
+      "loss": 2.6903,
+      "theoretical_loss": 3.310342969671993,
+      "tokens_seen": 3125673984
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 5.6843455945252356e-05,
+      "loss": 2.5035,
+      "theoretical_loss": 3.3103323855923934,
+      "tokens_seen": 3125805056
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 5.680068434559452e-05,
+      "loss": 2.5451,
+      "theoretical_loss": 3.3103218020808605,
+      "tokens_seen": 3125936128
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 5.6757912745936706e-05,
+      "loss": 2.4153,
+      "theoretical_loss": 3.3103112191373407,
+      "tokens_seen": 3126067200
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 5.671514114627887e-05,
+      "loss": 2.5702,
+      "theoretical_loss": 3.3103006367617787,
+      "tokens_seen": 3126198272
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 5.667236954662105e-05,
+      "loss": 2.4602,
+      "theoretical_loss": 3.310290054954121,
+      "tokens_seen": 3126329344
+    },
+    {
+      "epoch": 0.44,
+      "objective/train/docs_used": 1713045,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 2.963982105255127,
+      "objective/train/theoretical_loss": 3.310284764263239,
+      "objective/train/tokens_used": 156446176,
+      "theoretical_loss": 3.310284764263239,
+      "tokens_seen": 3126394880
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 5.662959794696322e-05,
+      "loss": 2.487,
+      "theoretical_loss": 3.3102794737143126,
+      "tokens_seen": 3126460416
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 5.6586826347305385e-05,
+      "loss": 2.6109,
+      "theoretical_loss": 3.3102688930423,
+      "tokens_seen": 3126591488
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 5.654405474764757e-05,
+      "loss": 2.5076,
+      "theoretical_loss": 3.3102583129380285,
+      "tokens_seen": 3126722560
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 5.6501283147989735e-05,
+      "loss": 2.4448,
+      "theoretical_loss": 3.3102477334014435,
+      "tokens_seen": 3126853632
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 5.6458511548331906e-05,
+      "loss": 2.5856,
+      "theoretical_loss": 3.3102371544324916,
+      "tokens_seen": 3126984704
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 5.6415739948674085e-05,
+      "loss": 2.3335,
+      "theoretical_loss": 3.3102265760311176,
+      "tokens_seen": 3127115776
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 5.6372968349016256e-05,
+      "loss": 2.4991,
+      "theoretical_loss": 3.310215998197268,
+      "tokens_seen": 3127246848
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 5.6330196749358434e-05,
+      "loss": 2.4256,
+      "theoretical_loss": 3.310205420930888,
+      "tokens_seen": 3127377920
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 5.62874251497006e-05,
+      "loss": 2.4177,
+      "theoretical_loss": 3.310194844231924,
+      "tokens_seen": 3127508992
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 5.624465355004277e-05,
+      "loss": 2.4802,
+      "theoretical_loss": 3.3101842681003215,
+      "tokens_seen": 3127640064
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 5.620188195038495e-05,
+      "loss": 2.5473,
+      "theoretical_loss": 3.310173692536026,
+      "tokens_seen": 3127771136
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 5.615911035072712e-05,
+      "loss": 2.6317,
+      "theoretical_loss": 3.3101631175389836,
+      "tokens_seen": 3127902208
+    },
+    {
+      "epoch": 0.44,
+      "objective/train/docs_used": 1713856,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 2.385249137878418,
+      "objective/train/theoretical_loss": 3.31015254310914,
+      "objective/train/tokens_used": 158084576,
+      "theoretical_loss": 3.31015254310914,
+      "tokens_seen": 3128033280
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 5.61163387510693e-05,
+      "loss": 2.5323,
+      "theoretical_loss": 3.31015254310914,
+      "tokens_seen": 3128033280
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 5.6073567151411464e-05,
+      "loss": 2.4468,
+      "theoretical_loss": 3.310141969246441,
+      "tokens_seen": 3128164352
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 5.6030795551753635e-05,
+      "loss": 2.5161,
+      "theoretical_loss": 3.3101313959508327,
+      "tokens_seen": 3128295424
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 5.5988023952095813e-05,
+      "loss": 2.3882,
+      "theoretical_loss": 3.3101208232222605,
+      "tokens_seen": 3128426496
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 5.5945252352437985e-05,
+      "loss": 2.6025,
+      "theoretical_loss": 3.310110251060671,
+      "tokens_seen": 3128557568
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 5.590248075278016e-05,
+      "loss": 2.6315,
+      "theoretical_loss": 3.3100996794660085,
+      "tokens_seen": 3128688640
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 5.585970915312233e-05,
+      "loss": 2.4923,
+      "theoretical_loss": 3.3100891084382207,
+      "tokens_seen": 3128819712
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 5.58169375534645e-05,
+      "loss": 2.4411,
+      "theoretical_loss": 3.310078537977252,
+      "tokens_seen": 3128950784
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 5.577416595380668e-05,
+      "loss": 2.4036,
+      "theoretical_loss": 3.3100679680830494,
+      "tokens_seen": 3129081856
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 5.573139435414885e-05,
+      "loss": 2.5574,
+      "theoretical_loss": 3.3100573987555575,
+      "tokens_seen": 3129212928
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 5.5688622754491014e-05,
+      "loss": 2.4951,
+      "theoretical_loss": 3.3100468299947234,
+      "tokens_seen": 3129344000
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 5.564585115483319e-05,
+      "loss": 2.4359,
+      "theoretical_loss": 3.3100362618004926,
+      "tokens_seen": 3129475072
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 5.5603079555175364e-05,
+      "loss": 2.4827,
+      "theoretical_loss": 3.3100256941728103,
+      "tokens_seen": 3129606144
+    },
+    {
+      "epoch": 0.45,
+      "objective/train/docs_used": 1714507,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 2.3790676593780518,
+      "objective/train/theoretical_loss": 3.3100204105714086,
+      "objective/train/tokens_used": 159722976,
+      "theoretical_loss": 3.3100204105714086,
+      "tokens_seen": 3129671680
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 5.556030795551754e-05,
+      "loss": 2.4887,
+      "theoretical_loss": 3.3100151271116234,
+      "tokens_seen": 3129737216
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 5.5517536355859714e-05,
+      "loss": 2.475,
+      "theoretical_loss": 3.3100045606168775,
+      "tokens_seen": 3129868288
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 5.547476475620188e-05,
+      "loss": 2.6397,
+      "theoretical_loss": 3.3099939946885186,
+      "tokens_seen": 3129999360
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 5.543199315654406e-05,
+      "loss": 2.5231,
+      "theoretical_loss": 3.309983429326492,
+      "tokens_seen": 3130130432
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 5.538922155688623e-05,
+      "loss": 2.5931,
+      "theoretical_loss": 3.309972864530744,
+      "tokens_seen": 3130261504
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 5.5346449957228407e-05,
+      "loss": 2.5449,
+      "theoretical_loss": 3.309962300301221,
+      "tokens_seen": 3130392576
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 5.530367835757058e-05,
+      "loss": 2.4872,
+      "theoretical_loss": 3.3099517366378683,
+      "tokens_seen": 3130523648
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 5.526090675791274e-05,
+      "loss": 2.5182,
+      "theoretical_loss": 3.3099411735406323,
+      "tokens_seen": 3130654720
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 5.521813515825492e-05,
+      "loss": 2.4474,
+      "theoretical_loss": 3.3099306110094586,
+      "tokens_seen": 3130785792
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 5.517536355859709e-05,
+      "loss": 2.712,
+      "theoretical_loss": 3.3099200490442935,
+      "tokens_seen": 3130916864
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 5.5132591958939264e-05,
+      "loss": 2.5673,
+      "theoretical_loss": 3.3099094876450827,
+      "tokens_seen": 3131047936
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 5.508982035928144e-05,
+      "loss": 2.6539,
+      "theoretical_loss": 3.3098989268117722,
+      "tokens_seen": 3131179008
+    },
+    {
+      "epoch": 0.45,
+      "objective/train/docs_used": 1715427,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 2.553161382675171,
+      "objective/train/theoretical_loss": 3.3098883665443086,
+      "objective/train/tokens_used": 161361376,
+      "theoretical_loss": 3.3098883665443086,
+      "tokens_seen": 3131310080
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 5.504704875962361e-05,
+      "loss": 2.5297,
+      "theoretical_loss": 3.3098883665443086,
+      "tokens_seen": 3131310080
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 5.500427715996579e-05,
+      "loss": 2.5105,
+      "theoretical_loss": 3.309877806842637,
+      "tokens_seen": 3131441152
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 5.496150556030796e-05,
+      "loss": 2.6815,
+      "theoretical_loss": 3.3098672477067037,
+      "tokens_seen": 3131572224
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 5.491873396065013e-05,
+      "loss": 2.5448,
+      "theoretical_loss": 3.309856689136455,
+      "tokens_seen": 3131703296
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 5.487596236099231e-05,
+      "loss": 2.5363,
+      "theoretical_loss": 3.3098461311318363,
+      "tokens_seen": 3131834368
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 5.483319076133447e-05,
+      "loss": 2.6097,
+      "theoretical_loss": 3.3098355736927947,
+      "tokens_seen": 3131965440
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 5.479041916167666e-05,
+      "loss": 2.3725,
+      "theoretical_loss": 3.3098250168192753,
+      "tokens_seen": 3132096512
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 5.474764756201882e-05,
+      "loss": 2.6798,
+      "theoretical_loss": 3.3098144605112245,
+      "tokens_seen": 3132227584
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 5.470487596236099e-05,
+      "loss": 2.4954,
+      "theoretical_loss": 3.3098039047685885,
+      "tokens_seen": 3132358656
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 5.466210436270317e-05,
+      "loss": 2.6217,
+      "theoretical_loss": 3.3097933495913128,
+      "tokens_seen": 3132489728
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 5.4619332763045336e-05,
+      "loss": 2.6219,
+      "theoretical_loss": 3.309782794979344,
+      "tokens_seen": 3132620800
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 5.457656116338752e-05,
+      "loss": 2.4051,
+      "theoretical_loss": 3.309772240932628,
+      "tokens_seen": 3132751872
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 5.4533789563729686e-05,
+      "loss": 2.6075,
+      "theoretical_loss": 3.30976168745111,
+      "tokens_seen": 3132882944
+    },
+    {
+      "epoch": 0.46,
+      "objective/train/docs_used": 1716696,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 2.7211923599243164,
+      "objective/train/theoretical_loss": 3.309756410922285,
+      "objective/train/tokens_used": 162999776,
+      "theoretical_loss": 3.309756410922285,
+      "tokens_seen": 3132948480
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 5.449101796407186e-05,
+      "loss": 2.7641,
+      "theoretical_loss": 3.3097511345347383,
+      "tokens_seen": 3133014016
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 5.4448246364414036e-05,
+      "loss": 2.4953,
+      "theoretical_loss": 3.3097405821834567,
+      "tokens_seen": 3133145088
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 5.44054747647562e-05,
+      "loss": 2.5759,
+      "theoretical_loss": 3.309730030397213,
+      "tokens_seen": 3133276160
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 5.436270316509837e-05,
+      "loss": 2.5484,
+      "theoretical_loss": 3.309719479175952,
+      "tokens_seen": 3133407232
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 5.431993156544055e-05,
+      "loss": 2.594,
+      "theoretical_loss": 3.309708928519621,
+      "tokens_seen": 3133538304
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 5.427715996578272e-05,
+      "loss": 2.589,
+      "theoretical_loss": 3.309698378428165,
+      "tokens_seen": 3133669376
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 5.42343883661249e-05,
+      "loss": 2.4454,
+      "theoretical_loss": 3.3096878289015303,
+      "tokens_seen": 3133800448
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 5.419161676646707e-05,
+      "loss": 2.533,
+      "theoretical_loss": 3.3096772799396637,
+      "tokens_seen": 3133931520
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 5.4148845166809236e-05,
+      "loss": 2.5289,
+      "theoretical_loss": 3.309666731542511,
+      "tokens_seen": 3134062592
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 5.4106073567151415e-05,
+      "loss": 2.4881,
+      "theoretical_loss": 3.3096561837100187,
+      "tokens_seen": 3134193664
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 5.4063301967493586e-05,
+      "loss": 2.5104,
+      "theoretical_loss": 3.3096456364421325,
+      "tokens_seen": 3134324736
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 5.4020530367835764e-05,
+      "loss": 2.5572,
+      "theoretical_loss": 3.3096350897387983,
+      "tokens_seen": 3134455808
+    },
+    {
+      "epoch": 0.47,
+      "objective/train/docs_used": 1717402,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 2.579829692840576,
+      "objective/train/theoretical_loss": 3.309624543599963,
+      "objective/train/tokens_used": 164638176,
+      "theoretical_loss": 3.309624543599963,
+      "tokens_seen": 3134586880
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 5.3977758768177936e-05,
+      "loss": 2.5648,
+      "theoretical_loss": 3.309624543599963,
+      "tokens_seen": 3134586880
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 5.39349871685201e-05,
+      "loss": 2.6332,
+      "theoretical_loss": 3.3096139980255725,
+      "tokens_seen": 3134717952
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 5.389221556886228e-05,
+      "loss": 2.5212,
+      "theoretical_loss": 3.3096034530155727,
+      "tokens_seen": 3134849024
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 5.384944396920445e-05,
+      "loss": 2.6408,
+      "theoretical_loss": 3.3095929085699103,
+      "tokens_seen": 3134980096
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 5.380667236954663e-05,
+      "loss": 2.5513,
+      "theoretical_loss": 3.309582364688531,
+      "tokens_seen": 3135111168
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 5.37639007698888e-05,
+      "loss": 2.5817,
+      "theoretical_loss": 3.3095718213713816,
+      "tokens_seen": 3135242240
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 5.3721129170230965e-05,
+      "loss": 2.5799,
+      "theoretical_loss": 3.309561278618408,
+      "tokens_seen": 3135373312
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 5.367835757057314e-05,
+      "loss": 2.6115,
+      "theoretical_loss": 3.309550736429556,
+      "tokens_seen": 3135504384
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 5.3635585970915315e-05,
+      "loss": 2.5717,
+      "theoretical_loss": 3.3095401948047725,
+      "tokens_seen": 3135635456
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 5.359281437125748e-05,
+      "loss": 2.5582,
+      "theoretical_loss": 3.3095296537440038,
+      "tokens_seen": 3135766528
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 5.3550042771599665e-05,
+      "loss": 2.5156,
+      "theoretical_loss": 3.3095191132471955,
+      "tokens_seen": 3135897600
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 5.350727117194183e-05,
+      "loss": 2.4804,
+      "theoretical_loss": 3.309508573314294,
+      "tokens_seen": 3136028672
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 5.346449957228401e-05,
+      "loss": 2.4863,
+      "theoretical_loss": 3.309498033945246,
+      "tokens_seen": 3136159744
+    },
+    {
+      "epoch": 0.47,
+      "objective/train/docs_used": 1718738,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 2.53719162940979,
+      "objective/train/theoretical_loss": 3.30949276447215,
+      "objective/train/tokens_used": 166276576,
+      "theoretical_loss": 3.30949276447215,
+      "tokens_seen": 3136225280
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 5.342172797262618e-05,
+      "loss": 2.6355,
+      "theoretical_loss": 3.3094874951399977,
+      "tokens_seen": 3136290816
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 5.3378956372968344e-05,
+      "loss": 2.5385,
+      "theoretical_loss": 3.3094769568984947,
+      "tokens_seen": 3136421888
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 5.333618477331053e-05,
+      "loss": 2.5406,
+      "theoretical_loss": 3.3094664192206844,
+      "tokens_seen": 3136552960
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 5.3293413173652694e-05,
+      "loss": 2.4186,
+      "theoretical_loss": 3.309455882106512,
+      "tokens_seen": 3136684032
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 5.325064157399487e-05,
+      "loss": 2.3413,
+      "theoretical_loss": 3.309445345555925,
+      "tokens_seen": 3136815104
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 5.3207869974337044e-05,
+      "loss": 2.5227,
+      "theoretical_loss": 3.3094348095688684,
+      "tokens_seen": 3136946176
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 5.3165098374679215e-05,
+      "loss": 2.3453,
+      "theoretical_loss": 3.3094242741452895,
+      "tokens_seen": 3137077248
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 5.312232677502139e-05,
+      "loss": 2.5183,
+      "theoretical_loss": 3.3094137392851337,
+      "tokens_seen": 3137208320
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 5.307955517536356e-05,
+      "loss": 2.5605,
+      "theoretical_loss": 3.3094032049883486,
+      "tokens_seen": 3137339392
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 5.303678357570573e-05,
+      "loss": 2.3662,
+      "theoretical_loss": 3.309392671254879,
+      "tokens_seen": 3137470464
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 5.299401197604791e-05,
+      "loss": 2.6077,
+      "theoretical_loss": 3.309382138084673,
+      "tokens_seen": 3137601536
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 5.295124037639008e-05,
+      "loss": 2.4003,
+      "theoretical_loss": 3.3093716054776756,
+      "tokens_seen": 3137732608
+    },
+    {
+      "epoch": 0.48,
+      "objective/train/docs_used": 1719408,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 2.7354519367218018,
+      "objective/train/theoretical_loss": 3.3093610734338332,
+      "objective/train/tokens_used": 167914976,
+      "theoretical_loss": 3.3093610734338332,
+      "tokens_seen": 3137863680
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 5.290846877673226e-05,
+      "loss": 2.5897,
+      "theoretical_loss": 3.3093610734338332,
+      "tokens_seen": 3137863680
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 5.286569717707442e-05,
+      "loss": 2.5798,
+      "theoretical_loss": 3.309350541953093,
+      "tokens_seen": 3137994752
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 5.2822925577416594e-05,
+      "loss": 2.4144,
+      "theoretical_loss": 3.3093400110354008,
+      "tokens_seen": 3138125824
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 5.278015397775877e-05,
+      "loss": 2.5288,
+      "theoretical_loss": 3.3093294806807028,
+      "tokens_seen": 3138256896
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 5.2737382378100944e-05,
+      "loss": 2.3532,
+      "theoretical_loss": 3.309318950888946,
+      "tokens_seen": 3138387968
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 5.269461077844312e-05,
+      "loss": 2.4541,
+      "theoretical_loss": 3.3093084216600763,
+      "tokens_seen": 3138519040
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 5.265183917878529e-05,
+      "loss": 2.642,
+      "theoretical_loss": 3.3092978929940404,
+      "tokens_seen": 3138650112
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 5.260906757912746e-05,
+      "loss": 2.5889,
+      "theoretical_loss": 3.309287364890785,
+      "tokens_seen": 3138781184
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 5.256629597946964e-05,
+      "loss": 2.5466,
+      "theoretical_loss": 3.3092768373502555,
+      "tokens_seen": 3138912256
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 5.252352437981181e-05,
+      "loss": 2.4441,
+      "theoretical_loss": 3.3092663103723994,
+      "tokens_seen": 3139043328
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 5.2480752780153986e-05,
+      "loss": 2.5501,
+      "theoretical_loss": 3.3092557839571626,
+      "tokens_seen": 3139174400
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 5.243798118049615e-05,
+      "loss": 2.4633,
+      "theoretical_loss": 3.3092452581044913,
+      "tokens_seen": 3139305472
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 5.239520958083832e-05,
+      "loss": 2.5114,
+      "theoretical_loss": 3.309234732814333,
+      "tokens_seen": 3139436544
+    },
+    {
+      "epoch": 0.48,
+      "objective/train/docs_used": 1720490,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 2.6848251819610596,
+      "objective/train/theoretical_loss": 3.3092294703801786,
+      "objective/train/tokens_used": 169553376,
+      "theoretical_loss": 3.3092294703801786,
+      "tokens_seen": 3139502080
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 5.23524379811805e-05,
+      "loss": 2.558,
+      "theoretical_loss": 3.3092242080866323,
+      "tokens_seen": 3139567616
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 5.230966638152267e-05,
+      "loss": 2.5359,
+      "theoretical_loss": 3.3092136839213375,
+      "tokens_seen": 3139698688
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 5.226689478186484e-05,
+      "loss": 2.5309,
+      "theoretical_loss": 3.3092031603183942,
+      "tokens_seen": 3139829760
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 5.2224123182207016e-05,
+      "loss": 2.5739,
+      "theoretical_loss": 3.309192637277749,
+      "tokens_seen": 3139960832
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 5.218135158254919e-05,
+      "loss": 2.5384,
+      "theoretical_loss": 3.3091821147993485,
+      "tokens_seen": 3140091904
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 5.2138579982891365e-05,
+      "loss": 2.6171,
+      "theoretical_loss": 3.3091715928831396,
+      "tokens_seen": 3140222976
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 5.209580838323354e-05,
+      "loss": 2.6771,
+      "theoretical_loss": 3.3091610715290676,
+      "tokens_seen": 3140354048
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 5.20530367835757e-05,
+      "loss": 2.5835,
+      "theoretical_loss": 3.3091505507370798,
+      "tokens_seen": 3140485120
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 5.201026518391789e-05,
+      "loss": 2.4421,
+      "theoretical_loss": 3.3091400305071232,
+      "tokens_seen": 3140616192
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 5.196749358426005e-05,
+      "loss": 2.5076,
+      "theoretical_loss": 3.309129510839143,
+      "tokens_seen": 3140747264
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 5.192472198460223e-05,
+      "loss": 2.5753,
+      "theoretical_loss": 3.309118991733087,
+      "tokens_seen": 3140878336
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 5.18819503849444e-05,
+      "loss": 2.5797,
+      "theoretical_loss": 3.309108473188901,
+      "tokens_seen": 3141009408
+    },
+    {
+      "epoch": 0.49,
+      "objective/train/docs_used": 1720925,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 2.6266226768493652,
+      "objective/train/theoretical_loss": 3.309097955206532,
+      "objective/train/tokens_used": 171191776,
+      "theoretical_loss": 3.309097955206532,
+      "tokens_seen": 3141140480
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 5.1839178785286566e-05,
+      "loss": 2.5632,
+      "theoretical_loss": 3.309097955206532,
+      "tokens_seen": 3141140480
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 5.179640718562875e-05,
+      "loss": 2.5929,
+      "theoretical_loss": 3.309087437785926,
+      "tokens_seen": 3141271552
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 5.1753635585970916e-05,
+      "loss": 2.5103,
+      "theoretical_loss": 3.30907692092703,
+      "tokens_seen": 3141402624
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 5.171086398631309e-05,
+      "loss": 2.4878,
+      "theoretical_loss": 3.3090664046297906,
+      "tokens_seen": 3141533696
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 5.1668092386655266e-05,
+      "loss": 2.4295,
+      "theoretical_loss": 3.309055888894154,
+      "tokens_seen": 3141664768
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 5.162532078699743e-05,
+      "loss": 2.5133,
+      "theoretical_loss": 3.309045373720067,
+      "tokens_seen": 3141795840
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 5.1582549187339616e-05,
+      "loss": 2.4796,
+      "theoretical_loss": 3.3090348591074763,
+      "tokens_seen": 3141926912
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 5.153977758768178e-05,
+      "loss": 2.4262,
+      "theoretical_loss": 3.3090243450563284,
+      "tokens_seen": 3142057984
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 5.149700598802395e-05,
+      "loss": 2.4909,
+      "theoretical_loss": 3.30901383156657,
+      "tokens_seen": 3142189056
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 5.145423438836613e-05,
+      "loss": 2.3213,
+      "theoretical_loss": 3.3090033186381476,
+      "tokens_seen": 3142320128
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 5.1411462788708295e-05,
+      "loss": 2.4614,
+      "theoretical_loss": 3.3089928062710072,
+      "tokens_seen": 3142451200
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 5.136869118905048e-05,
+      "loss": 2.5389,
+      "theoretical_loss": 3.3089822944650966,
+      "tokens_seen": 3142582272
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 5.1325919589392645e-05,
+      "loss": 2.5776,
+      "theoretical_loss": 3.308971783220362,
+      "tokens_seen": 3142713344
+    },
+    {
+      "epoch": 0.49,
+      "objective/train/docs_used": 1722164,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 2.7013967037200928,
+      "objective/train/theoretical_loss": 3.3089665278084186,
+      "objective/train/tokens_used": 172830176,
+      "theoretical_loss": 3.3089665278084186,
+      "tokens_seen": 3142778880
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 5.1283147989734816e-05,
+      "loss": 2.516,
+      "theoretical_loss": 3.308961272536749,
+      "tokens_seen": 3142844416
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 5.1240376390076994e-05,
+      "loss": 2.4546,
+      "theoretical_loss": 3.308950762414206,
+      "tokens_seen": 3142975488
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 5.119760479041916e-05,
+      "loss": 2.5464,
+      "theoretical_loss": 3.3089402528526786,
+      "tokens_seen": 3143106560
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 5.1154833190761344e-05,
+      "loss": 2.4838,
+      "theoretical_loss": 3.3089297438521137,
+      "tokens_seen": 3143237632
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 5.111206159110351e-05,
+      "loss": 2.5751,
+      "theoretical_loss": 3.308919235412458,
+      "tokens_seen": 3143368704
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 5.106928999144568e-05,
+      "loss": 2.6909,
+      "theoretical_loss": 3.3089087275336575,
+      "tokens_seen": 3143499776
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 5.102651839178786e-05,
+      "loss": 2.6497,
+      "theoretical_loss": 3.30889822021566,
+      "tokens_seen": 3143630848
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 5.098374679213003e-05,
+      "loss": 2.5391,
+      "theoretical_loss": 3.308887713458412,
+      "tokens_seen": 3143761920
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 5.0940975192472195e-05,
+      "loss": 2.3483,
+      "theoretical_loss": 3.308877207261859,
+      "tokens_seen": 3143892992
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 5.089820359281437e-05,
+      "loss": 2.5682,
+      "theoretical_loss": 3.3088667016259494,
+      "tokens_seen": 3144024064
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 5.0855431993156545e-05,
+      "loss": 2.5066,
+      "theoretical_loss": 3.3088561965506287,
+      "tokens_seen": 3144155136
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 5.081266039349872e-05,
+      "loss": 2.6193,
+      "theoretical_loss": 3.3088456920358436,
+      "tokens_seen": 3144286208
+    },
+    {
+      "epoch": 0.5,
+      "objective/train/docs_used": 1723267,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 2.3256311416625977,
+      "objective/train/theoretical_loss": 3.3088351880815416,
+      "objective/train/tokens_used": 174468576,
+      "theoretical_loss": 3.3088351880815416,
+      "tokens_seen": 3144417280
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 5.0769888793840895e-05,
+      "loss": 2.4757,
+      "theoretical_loss": 3.3088351880815416,
+      "tokens_seen": 3144417280
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 5.072711719418306e-05,
+      "loss": 2.4459,
+      "theoretical_loss": 3.308824684687669,
+      "tokens_seen": 3144548352
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 5.068434559452524e-05,
+      "loss": 2.4645,
+      "theoretical_loss": 3.308814181854173,
+      "tokens_seen": 3144679424
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 5.064157399486741e-05,
+      "loss": 2.4541,
+      "theoretical_loss": 3.3088036795809996,
+      "tokens_seen": 3144810496
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 5.059880239520959e-05,
+      "loss": 2.5162,
+      "theoretical_loss": 3.308793177868096,
+      "tokens_seen": 3144941568
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 5.055603079555176e-05,
+      "loss": 2.5396,
+      "theoretical_loss": 3.308782676715409,
+      "tokens_seen": 3145072640
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 5.0513259195893924e-05,
+      "loss": 2.601,
+      "theoretical_loss": 3.308772176122885,
+      "tokens_seen": 3145203712
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 5.04704875962361e-05,
+      "loss": 2.5741,
+      "theoretical_loss": 3.3087616760904712,
+      "tokens_seen": 3145334784
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 5.0427715996578274e-05,
+      "loss": 2.4807,
+      "theoretical_loss": 3.308751176618114,
+      "tokens_seen": 3145465856
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 5.038494439692045e-05,
+      "loss": 2.5737,
+      "theoretical_loss": 3.3087406777057606,
+      "tokens_seen": 3145596928
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 5.0342172797262623e-05,
+      "loss": 2.5546,
+      "theoretical_loss": 3.3087301793533577,
+      "tokens_seen": 3145728000
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 5.029940119760479e-05,
+      "loss": 2.4823,
+      "theoretical_loss": 3.3087196815608517,
+      "tokens_seen": 3145859072
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 5.0256629597946967e-05,
+      "loss": 2.5493,
+      "theoretical_loss": 3.3087091843281895,
+      "tokens_seen": 3145990144
+    },
+    {
+      "epoch": 0.5,
+      "objective/train/docs_used": 1723860,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 2.650022506713867,
+      "objective/train/theoretical_loss": 3.3087039359217836,
+      "objective/train/tokens_used": 176106976,
+      "theoretical_loss": 3.3087039359217836,
+      "tokens_seen": 3146055680
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 5.021385799828914e-05,
+      "loss": 2.7187,
+      "theoretical_loss": 3.3086986876553186,
+      "tokens_seen": 3146121216
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 5.017108639863131e-05,
+      "loss": 2.5579,
+      "theoretical_loss": 3.3086881915421853,
+      "tokens_seen": 3146252288
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 5.012831479897349e-05,
+      "loss": 2.6554,
+      "theoretical_loss": 3.3086776959887363,
+      "tokens_seen": 3146383360
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 5.008554319931565e-05,
+      "loss": 2.6396,
+      "theoretical_loss": 3.3086672009949187,
+      "tokens_seen": 3146514432
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 5.004277159965783e-05,
+      "loss": 2.5456,
+      "theoretical_loss": 3.3086567065606793,
+      "tokens_seen": 3146645504
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 5e-05,
+      "loss": 2.4661,
+      "theoretical_loss": 3.3086462126859653,
+      "tokens_seen": 3146776576
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 4.9957228400342174e-05,
+      "loss": 2.4853,
+      "theoretical_loss": 3.3086357193707228,
+      "tokens_seen": 3146907648
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 4.991445680068435e-05,
+      "loss": 2.4425,
+      "theoretical_loss": 3.308625226614899,
+      "tokens_seen": 3147038720
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 4.987168520102652e-05,
+      "loss": 2.4538,
+      "theoretical_loss": 3.308614734418441,
+      "tokens_seen": 3147169792
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 4.9828913601368695e-05,
+      "loss": 2.3674,
+      "theoretical_loss": 3.3086042427812954,
+      "tokens_seen": 3147300864
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 4.978614200171087e-05,
+      "loss": 2.5229,
+      "theoretical_loss": 3.3085937517034094,
+      "tokens_seen": 3147431936
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 4.974337040205304e-05,
+      "loss": 2.4467,
+      "theoretical_loss": 3.30858326118473,
+      "tokens_seen": 3147563008
+    },
+    {
+      "epoch": 0.51,
+      "objective/train/docs_used": 1724798,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 2.843707323074341,
+      "objective/train/theoretical_loss": 3.308572771225203,
+      "objective/train/tokens_used": 177745376,
+      "theoretical_loss": 3.308572771225203,
+      "tokens_seen": 3147694080
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 4.970059880239521e-05,
+      "loss": 2.7121,
+      "theoretical_loss": 3.308572771225203,
+      "tokens_seen": 3147694080
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 4.965782720273738e-05,
+      "loss": 2.6234,
+      "theoretical_loss": 3.308562281824777,
+      "tokens_seen": 3147825152
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 4.961505560307956e-05,
+      "loss": 2.5533,
+      "theoretical_loss": 3.3085517929833976,
+      "tokens_seen": 3147956224
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 4.957228400342173e-05,
+      "loss": 2.4984,
+      "theoretical_loss": 3.308541304701013,
+      "tokens_seen": 3148087296
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 4.95295124037639e-05,
+      "loss": 2.5731,
+      "theoretical_loss": 3.3085308169775685,
+      "tokens_seen": 3148218368
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 4.9486740804106074e-05,
+      "loss": 2.4481,
+      "theoretical_loss": 3.308520329813012,
+      "tokens_seen": 3148349440
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 4.9443969204448246e-05,
+      "loss": 2.5747,
+      "theoretical_loss": 3.3085098432072906,
+      "tokens_seen": 3148480512
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 4.9401197604790424e-05,
+      "loss": 2.5293,
+      "theoretical_loss": 3.308499357160351,
+      "tokens_seen": 3148611584
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 4.9358426005132596e-05,
+      "loss": 2.6658,
+      "theoretical_loss": 3.3084888716721403,
+      "tokens_seen": 3148742656
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 4.931565440547477e-05,
+      "loss": 2.4781,
+      "theoretical_loss": 3.308478386742605,
+      "tokens_seen": 3148873728
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 4.927288280581694e-05,
+      "loss": 2.4863,
+      "theoretical_loss": 3.3084679023716923,
+      "tokens_seen": 3149004800
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 4.923011120615911e-05,
+      "loss": 2.5368,
+      "theoretical_loss": 3.3084574185593496,
+      "tokens_seen": 3149135872
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 4.918733960650129e-05,
+      "loss": 2.5071,
+      "theoretical_loss": 3.3084469353055237,
+      "tokens_seen": 3149266944
+    },
+    {
+      "epoch": 0.51,
+      "objective/train/docs_used": 1725490,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 2.170764684677124,
+      "objective/train/theoretical_loss": 3.308441693888038,
+      "objective/train/tokens_used": 179383776,
+      "theoretical_loss": 3.308441693888038,
+      "tokens_seen": 3149332480
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 4.914456800684346e-05,
+      "loss": 2.4342,
+      "theoretical_loss": 3.3084364526101613,
+      "tokens_seen": 3149398016
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 4.910179640718563e-05,
+      "loss": 2.5341,
+      "theoretical_loss": 3.30842597047321,
+      "tokens_seen": 3149529088
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 4.90590248075278e-05,
+      "loss": 2.6495,
+      "theoretical_loss": 3.3084154888946156,
+      "tokens_seen": 3149660160
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 4.9016253207869974e-05,
+      "loss": 2.555,
+      "theoretical_loss": 3.3084050078743266,
+      "tokens_seen": 3149791232
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 4.897348160821215e-05,
+      "loss": 2.4809,
+      "theoretical_loss": 3.3083945274122892,
+      "tokens_seen": 3149922304
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 4.893071000855432e-05,
+      "loss": 2.4281,
+      "theoretical_loss": 3.3083840475084507,
+      "tokens_seen": 3150053376
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 4.8887938408896496e-05,
+      "loss": 2.4226,
+      "theoretical_loss": 3.308373568162758,
+      "tokens_seen": 3150184448
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 4.884516680923867e-05,
+      "loss": 2.5657,
+      "theoretical_loss": 3.3083630893751588,
+      "tokens_seen": 3150315520
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 4.8802395209580846e-05,
+      "loss": 2.5738,
+      "theoretical_loss": 3.3083526111455988,
+      "tokens_seen": 3150446592
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 4.875962360992301e-05,
+      "loss": 2.4448,
+      "theoretical_loss": 3.308342133474026,
+      "tokens_seen": 3150577664
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 4.871685201026518e-05,
+      "loss": 2.4711,
+      "theoretical_loss": 3.3083316563603877,
+      "tokens_seen": 3150708736
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 4.867408041060736e-05,
+      "loss": 2.5539,
+      "theoretical_loss": 3.30832117980463,
+      "tokens_seen": 3150839808
+    },
+    {
+      "epoch": 0.52,
+      "objective/train/docs_used": 1726849,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 2.238480806350708,
+      "objective/train/theoretical_loss": 3.3083107038067014,
+      "objective/train/tokens_used": 181022176,
+      "theoretical_loss": 3.3083107038067014,
+      "tokens_seen": 3150970880
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 4.863130881094953e-05,
+      "loss": 2.5334,
+      "theoretical_loss": 3.3083107038067014,
+      "tokens_seen": 3150970880
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 4.858853721129171e-05,
+      "loss": 2.458,
+      "theoretical_loss": 3.3083002283665475,
+      "tokens_seen": 3151101952
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 4.8545765611633875e-05,
+      "loss": 2.5054,
+      "theoretical_loss": 3.3082897534841162,
+      "tokens_seen": 3151233024
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 4.8502994011976046e-05,
+      "loss": 2.52,
+      "theoretical_loss": 3.308279279159355,
+      "tokens_seen": 3151364096
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 4.8460222412318225e-05,
+      "loss": 2.4724,
+      "theoretical_loss": 3.30826880539221,
+      "tokens_seen": 3151495168
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 4.8417450812660396e-05,
+      "loss": 2.5626,
+      "theoretical_loss": 3.308258332182629,
+      "tokens_seen": 3151626240
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 4.837467921300257e-05,
+      "loss": 2.387,
+      "theoretical_loss": 3.308247859530559,
+      "tokens_seen": 3151757312
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 4.833190761334474e-05,
+      "loss": 2.4943,
+      "theoretical_loss": 3.308237387435947,
+      "tokens_seen": 3151888384
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 4.828913601368692e-05,
+      "loss": 2.5211,
+      "theoretical_loss": 3.3082269158987403,
+      "tokens_seen": 3152019456
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 4.824636441402909e-05,
+      "loss": 2.5913,
+      "theoretical_loss": 3.308216444918886,
+      "tokens_seen": 3152150528
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 4.820359281437126e-05,
+      "loss": 2.4218,
+      "theoretical_loss": 3.3082059744963317,
+      "tokens_seen": 3152281600
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 4.816082121471343e-05,
+      "loss": 2.4526,
+      "theoretical_loss": 3.3081955046310236,
+      "tokens_seen": 3152412672
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 4.8118049615055603e-05,
+      "loss": 2.4867,
+      "theoretical_loss": 3.3081850353229094,
+      "tokens_seen": 3152543744
+    },
+    {
+      "epoch": 0.52,
+      "objective/train/docs_used": 1727307,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 2.4184036254882812,
+      "objective/train/theoretical_loss": 3.3081798008777836,
+      "objective/train/tokens_used": 182660576,
+      "theoretical_loss": 3.3081798008777836,
+      "tokens_seen": 3152609280
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 4.807527801539778e-05,
+      "loss": 2.4747,
+      "theoretical_loss": 3.3081745665719366,
+      "tokens_seen": 3152674816
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 4.803250641573995e-05,
+      "loss": 2.4888,
+      "theoretical_loss": 3.308164098378052,
+      "tokens_seen": 3152805888
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 4.798973481608212e-05,
+      "loss": 2.4952,
+      "theoretical_loss": 3.308153630741203,
+      "tokens_seen": 3152936960
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 4.7946963216424296e-05,
+      "loss": 2.505,
+      "theoretical_loss": 3.3081431636613363,
+      "tokens_seen": 3153068032
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 4.790419161676647e-05,
+      "loss": 2.5401,
+      "theoretical_loss": 3.3081326971384,
+      "tokens_seen": 3153199104
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 4.7861420017108646e-05,
+      "loss": 2.5614,
+      "theoretical_loss": 3.308122231172341,
+      "tokens_seen": 3153330176
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 4.781864841745082e-05,
+      "loss": 2.5993,
+      "theoretical_loss": 3.3081117657631056,
+      "tokens_seen": 3153461248
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 4.777587681779299e-05,
+      "loss": 2.6378,
+      "theoretical_loss": 3.3081013009106424,
+      "tokens_seen": 3153592320
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 4.773310521813516e-05,
+      "loss": 2.544,
+      "theoretical_loss": 3.3080908366148973,
+      "tokens_seen": 3153723392
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 4.769033361847733e-05,
+      "loss": 2.4272,
+      "theoretical_loss": 3.308080372875819,
+      "tokens_seen": 3153854464
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 4.764756201881951e-05,
+      "loss": 2.5282,
+      "theoretical_loss": 3.3080699096933537,
+      "tokens_seen": 3153985536
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 4.7604790419161675e-05,
+      "loss": 2.4947,
+      "theoretical_loss": 3.308059447067449,
+      "tokens_seen": 3154116608
+    },
+    {
+      "debugging/Self-BLEU-5": 0.4768917357816163,
+      "debugging/distinct-1-grams": 0.8069642973374905,
+      "debugging/distinct-2-grams": 0.9493042579915046,
+      "debugging/entropy-1-grams": 5.952991944891716,
+      "debugging/entropy-2-grams": 6.680805112573269,
+      "debugging/length": 514.4615384615385,
+      "debugging/num_segments": 13,
+      "debugging/score": 0.008350422819326409,
+      "debugging/score_std": 0.0069101491985588594,
+      "epoch": 0.53,
+      "objective/train/docs_used": 1728625,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 2.800560712814331,
+      "objective/train/theoretical_loss": 3.3080489849980523,
+      "objective/train/tokens_used": 184298976,
+      "theoretical_loss": 3.3080489849980523,
+      "tokens_seen": 3154247680
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 4.7562018819503854e-05,
+      "loss": 2.3931,
+      "theoretical_loss": 3.3080489849980523,
+      "tokens_seen": 3154247680
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 4.7519247219846025e-05,
+      "loss": 2.6028,
+      "theoretical_loss": 3.3080385234851106,
+      "tokens_seen": 3154378752
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 4.74764756201882e-05,
+      "loss": 2.4836,
+      "theoretical_loss": 3.3080280625285714,
+      "tokens_seen": 3154509824
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 4.7433704020530375e-05,
+      "loss": 2.641,
+      "theoretical_loss": 3.308017602128382,
+      "tokens_seen": 3154640896
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 4.739093242087254e-05,
+      "loss": 2.5354,
+      "theoretical_loss": 3.308007142284489,
+      "tokens_seen": 3154771968
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 4.734816082121472e-05,
+      "loss": 2.5539,
+      "theoretical_loss": 3.307996682996841,
+      "tokens_seen": 3154903040
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 4.730538922155689e-05,
+      "loss": 2.4411,
+      "theoretical_loss": 3.3079862242653846,
+      "tokens_seen": 3155034112
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 4.726261762189906e-05,
+      "loss": 2.4431,
+      "theoretical_loss": 3.307975766090067,
+      "tokens_seen": 3155165184
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 4.721984602224123e-05,
+      "loss": 2.3219,
+      "theoretical_loss": 3.3079653084708354,
+      "tokens_seen": 3155296256
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 4.7177074422583404e-05,
+      "loss": 2.5297,
+      "theoretical_loss": 3.3079548514076373,
+      "tokens_seen": 3155427328
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 4.713430282292558e-05,
+      "loss": 2.6636,
+      "theoretical_loss": 3.307944394900421,
+      "tokens_seen": 3155558400
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 4.7091531223267754e-05,
+      "loss": 2.3727,
+      "theoretical_loss": 3.307933938949132,
+      "tokens_seen": 3155689472
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 4.7048759623609925e-05,
+      "loss": 2.6068,
+      "theoretical_loss": 3.307923483553719,
+      "tokens_seen": 3155820544
+    },
+    {
+      "epoch": 0.53,
+      "objective/train/docs_used": 1729266,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 2.3172640800476074,
+      "objective/train/theoretical_loss": 3.3079182560644496,
+      "objective/train/tokens_used": 185937376,
+      "theoretical_loss": 3.3079182560644496,
+      "tokens_seen": 3155886080
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 4.70059880239521e-05,
+      "loss": 2.4148,
+      "theoretical_loss": 3.307913028714129,
+      "tokens_seen": 3155951616
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 4.696321642429427e-05,
+      "loss": 2.6612,
+      "theoretical_loss": 3.3079025744303094,
+      "tokens_seen": 3156082688
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 4.692044482463645e-05,
+      "loss": 2.5739,
+      "theoretical_loss": 3.3078921207022076,
+      "tokens_seen": 3156213760
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 4.687767322497862e-05,
+      "loss": 2.4681,
+      "theoretical_loss": 3.307881667529771,
+      "tokens_seen": 3156344832
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 4.683490162532079e-05,
+      "loss": 2.3788,
+      "theoretical_loss": 3.3078712149129466,
+      "tokens_seen": 3156475904
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 4.679213002566296e-05,
+      "loss": 2.6225,
+      "theoretical_loss": 3.3078607628516825,
+      "tokens_seen": 3156606976
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 4.674935842600513e-05,
+      "loss": 2.564,
+      "theoretical_loss": 3.3078503113459257,
+      "tokens_seen": 3156738048
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 4.670658682634731e-05,
+      "loss": 2.3909,
+      "theoretical_loss": 3.307839860395623,
+      "tokens_seen": 3156869120
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 4.6663815226689476e-05,
+      "loss": 2.4339,
+      "theoretical_loss": 3.307829410000723,
+      "tokens_seen": 3157000192
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 4.6621043627031654e-05,
+      "loss": 2.4966,
+      "theoretical_loss": 3.307818960161173,
+      "tokens_seen": 3157131264
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 4.6578272027373826e-05,
+      "loss": 2.5257,
+      "theoretical_loss": 3.307808510876919,
+      "tokens_seen": 3157262336
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 4.6535500427716e-05,
+      "loss": 2.3732,
+      "theoretical_loss": 3.3077980621479104,
+      "tokens_seen": 3157393408
+    },
+    {
+      "epoch": 0.54,
+      "objective/train/docs_used": 1730350,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 2.4390780925750732,
+      "objective/train/theoretical_loss": 3.307787613974093,
+      "objective/train/tokens_used": 187575776,
+      "theoretical_loss": 3.307787613974093,
+      "tokens_seen": 3157524480
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 4.6492728828058175e-05,
+      "loss": 2.6069,
+      "theoretical_loss": 3.307787613974093,
+      "tokens_seen": 3157524480
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 4.644995722840034e-05,
+      "loss": 2.5569,
+      "theoretical_loss": 3.3077771663554154,
+      "tokens_seen": 3157655552
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 4.640718562874252e-05,
+      "loss": 2.5445,
+      "theoretical_loss": 3.3077667192918243,
+      "tokens_seen": 3157786624
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 4.636441402908469e-05,
+      "loss": 2.1927,
+      "theoretical_loss": 3.307756272783268,
+      "tokens_seen": 3157917696
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 4.632164242942686e-05,
+      "loss": 2.5662,
+      "theoretical_loss": 3.3077458268296933,
+      "tokens_seen": 3158048768
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 4.627887082976903e-05,
+      "loss": 2.4771,
+      "theoretical_loss": 3.3077353814310477,
+      "tokens_seen": 3158179840
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 4.6236099230111205e-05,
+      "loss": 2.5113,
+      "theoretical_loss": 3.3077249365872787,
+      "tokens_seen": 3158310912
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 4.619332763045338e-05,
+      "loss": 2.3738,
+      "theoretical_loss": 3.307714492298334,
+      "tokens_seen": 3158441984
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 4.6150556030795554e-05,
+      "loss": 2.5683,
+      "theoretical_loss": 3.307704048564161,
+      "tokens_seen": 3158573056
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 4.610778443113773e-05,
+      "loss": 2.429,
+      "theoretical_loss": 3.3076936053847072,
+      "tokens_seen": 3158704128
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 4.60650128314799e-05,
+      "loss": 2.5088,
+      "theoretical_loss": 3.3076831627599206,
+      "tokens_seen": 3158835200
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 4.602224123182207e-05,
+      "loss": 2.4902,
+      "theoretical_loss": 3.307672720689748,
+      "tokens_seen": 3158966272
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 4.597946963216425e-05,
+      "loss": 2.2596,
+      "theoretical_loss": 3.3076622791741377,
+      "tokens_seen": 3159097344
+    },
+    {
+      "epoch": 0.54,
+      "objective/train/docs_used": 1730945,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 3.087862730026245,
+      "objective/train/theoretical_loss": 3.3076570586242764,
+      "objective/train/tokens_used": 189214176,
+      "theoretical_loss": 3.3076570586242764,
+      "tokens_seen": 3159162880
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 4.593669803250642e-05,
+      "loss": 2.4972,
+      "theoretical_loss": 3.307651838213036,
+      "tokens_seen": 3159228416
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 4.589392643284859e-05,
+      "loss": 2.6479,
+      "theoretical_loss": 3.3076413978063917,
+      "tokens_seen": 3159359488
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 4.585115483319076e-05,
+      "loss": 2.639,
+      "theoretical_loss": 3.307630957954152,
+      "tokens_seen": 3159490560
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 4.580838323353293e-05,
+      "loss": 2.5289,
+      "theoretical_loss": 3.307620518656264,
+      "tokens_seen": 3159621632
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 4.576561163387511e-05,
+      "loss": 2.3132,
+      "theoretical_loss": 3.307610079912676,
+      "tokens_seen": 3159752704
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 4.572284003421728e-05,
+      "loss": 2.4593,
+      "theoretical_loss": 3.3075996417233346,
+      "tokens_seen": 3159883776
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 4.5680068434559455e-05,
+      "loss": 2.4431,
+      "theoretical_loss": 3.3075892040881887,
+      "tokens_seen": 3160014848
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 4.5637296834901626e-05,
+      "loss": 2.5069,
+      "theoretical_loss": 3.307578767007185,
+      "tokens_seen": 3160145920
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 4.5594525235243804e-05,
+      "loss": 2.4377,
+      "theoretical_loss": 3.307568330480271,
+      "tokens_seen": 3160276992
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 4.5551753635585976e-05,
+      "loss": 2.5466,
+      "theoretical_loss": 3.3075578945073945,
+      "tokens_seen": 3160408064
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 4.550898203592814e-05,
+      "loss": 2.457,
+      "theoretical_loss": 3.3075474590885032,
+      "tokens_seen": 3160539136
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 4.546621043627032e-05,
+      "loss": 2.4887,
+      "theoretical_loss": 3.307537024223545,
+      "tokens_seen": 3160670208
+    },
+    {
+      "epoch": 0.55,
+      "objective/train/docs_used": 1732078,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 2.740237236022949,
+      "objective/train/theoretical_loss": 3.307526589912467,
+      "objective/train/tokens_used": 190852576,
+      "theoretical_loss": 3.307526589912467,
+      "tokens_seen": 3160801280
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 4.542343883661249e-05,
+      "loss": 2.5039,
+      "theoretical_loss": 3.307526589912467,
+      "tokens_seen": 3160801280
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 4.538066723695467e-05,
+      "loss": 2.4584,
+      "theoretical_loss": 3.3075161561552173,
+      "tokens_seen": 3160932352
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 4.5337895637296834e-05,
+      "loss": 2.3226,
+      "theoretical_loss": 3.307505722951743,
+      "tokens_seen": 3161063424
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 4.5295124037639005e-05,
+      "loss": 2.4711,
+      "theoretical_loss": 3.307495290301992,
+      "tokens_seen": 3161194496
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 4.5252352437981183e-05,
+      "loss": 2.5309,
+      "theoretical_loss": 3.307484858205912,
+      "tokens_seen": 3161325568
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 4.5209580838323355e-05,
+      "loss": 2.4896,
+      "theoretical_loss": 3.3074744266634513,
+      "tokens_seen": 3161456640
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 4.516680923866553e-05,
+      "loss": 2.3985,
+      "theoretical_loss": 3.3074639956745564,
+      "tokens_seen": 3161587712
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 4.51240376390077e-05,
+      "loss": 2.5226,
+      "theoretical_loss": 3.3074535652391757,
+      "tokens_seen": 3161718784
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 4.5081266039349876e-05,
+      "loss": 2.3819,
+      "theoretical_loss": 3.3074431353572566,
+      "tokens_seen": 3161849856
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 4.503849443969205e-05,
+      "loss": 2.5918,
+      "theoretical_loss": 3.307432706028747,
+      "tokens_seen": 3161980928
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 4.499572284003422e-05,
+      "loss": 2.5223,
+      "theoretical_loss": 3.307422277253594,
+      "tokens_seen": 3162112000
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 4.495295124037639e-05,
+      "loss": 2.4723,
+      "theoretical_loss": 3.3074118490317463,
+      "tokens_seen": 3162243072
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 4.491017964071856e-05,
+      "loss": 2.4196,
+      "theoretical_loss": 3.307401421363151,
+      "tokens_seen": 3162374144
+    },
+    {
+      "epoch": 0.56,
+      "objective/train/docs_used": 1732556,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 2.6318604946136475,
+      "objective/train/theoretical_loss": 3.3073962077363066,
+      "objective/train/tokens_used": 192490976,
+      "theoretical_loss": 3.3073962077363066,
+      "tokens_seen": 3162439680
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 4.486740804106074e-05,
+      "loss": 2.4152,
+      "theoretical_loss": 3.307390994247756,
+      "tokens_seen": 3162505216
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 4.482463644140291e-05,
+      "loss": 2.4101,
+      "theoretical_loss": 3.3073805676855086,
+      "tokens_seen": 3162636288
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 4.4781864841745084e-05,
+      "loss": 2.5474,
+      "theoretical_loss": 3.3073701416763575,
+      "tokens_seen": 3162767360
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 4.4739093242087255e-05,
+      "loss": 2.5322,
+      "theoretical_loss": 3.3073597162202493,
+      "tokens_seen": 3162898432
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 4.469632164242943e-05,
+      "loss": 2.4196,
+      "theoretical_loss": 3.3073492913171325,
+      "tokens_seen": 3163029504
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 4.4653550042771605e-05,
+      "loss": 2.5852,
+      "theoretical_loss": 3.307338866966955,
+      "tokens_seen": 3163160576
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 4.4610778443113777e-05,
+      "loss": 2.6559,
+      "theoretical_loss": 3.3073284431696632,
+      "tokens_seen": 3163291648
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 4.456800684345595e-05,
+      "loss": 2.5197,
+      "theoretical_loss": 3.3073180199252064,
+      "tokens_seen": 3163422720
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 4.452523524379812e-05,
+      "loss": 2.5187,
+      "theoretical_loss": 3.3073075972335317,
+      "tokens_seen": 3163553792
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 4.448246364414029e-05,
+      "loss": 2.5293,
+      "theoretical_loss": 3.307297175094587,
+      "tokens_seen": 3163684864
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 4.443969204448247e-05,
+      "loss": 2.5596,
+      "theoretical_loss": 3.3072867535083206,
+      "tokens_seen": 3163815936
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 4.439692044482464e-05,
+      "loss": 2.4285,
+      "theoretical_loss": 3.3072763324746792,
+      "tokens_seen": 3163947008
+    },
+    {
+      "epoch": 0.56,
+      "objective/train/docs_used": 1733728,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 2.5950536727905273,
+      "objective/train/theoretical_loss": 3.3072659119936114,
+      "objective/train/tokens_used": 194129376,
+      "theoretical_loss": 3.3072659119936114,
+      "tokens_seen": 3164078080
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 4.435414884516681e-05,
+      "loss": 2.3195,
+      "theoretical_loss": 3.3072659119936114,
+      "tokens_seen": 3164078080
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 4.4311377245508984e-05,
+      "loss": 2.3918,
+      "theoretical_loss": 3.3072554920650648,
+      "tokens_seen": 3164209152
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 4.4268605645851155e-05,
+      "loss": 2.5205,
+      "theoretical_loss": 3.3072450726889873,
+      "tokens_seen": 3164340224
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 4.4225834046193334e-05,
+      "loss": 2.5269,
+      "theoretical_loss": 3.3072346538653266,
+      "tokens_seen": 3164471296
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 4.41830624465355e-05,
+      "loss": 2.3742,
+      "theoretical_loss": 3.307224235594031,
+      "tokens_seen": 3164602368
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 4.414029084687768e-05,
+      "loss": 2.4977,
+      "theoretical_loss": 3.3072138178750476,
+      "tokens_seen": 3164733440
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 4.409751924721985e-05,
+      "loss": 2.365,
+      "theoretical_loss": 3.3072034007083246,
+      "tokens_seen": 3164864512
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 4.405474764756202e-05,
+      "loss": 2.5296,
+      "theoretical_loss": 3.3071929840938097,
+      "tokens_seen": 3164995584
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 4.40119760479042e-05,
+      "loss": 2.4961,
+      "theoretical_loss": 3.307182568031451,
+      "tokens_seen": 3165126656
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 4.396920444824636e-05,
+      "loss": 2.355,
+      "theoretical_loss": 3.307172152521196,
+      "tokens_seen": 3165257728
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 4.392643284858854e-05,
+      "loss": 2.3241,
+      "theoretical_loss": 3.307161737562993,
+      "tokens_seen": 3165388800
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 4.388366124893071e-05,
+      "loss": 2.3619,
+      "theoretical_loss": 3.30715132315679,
+      "tokens_seen": 3165519872
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 4.3840889649272884e-05,
+      "loss": 2.4652,
+      "theoretical_loss": 3.307140909302534,
+      "tokens_seen": 3165650944
+    },
+    {
+      "epoch": 0.57,
+      "objective/train/docs_used": 1734416,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 2.3513553142547607,
+      "objective/train/theoretical_loss": 3.3071357025823707,
+      "objective/train/tokens_used": 195767776,
+      "theoretical_loss": 3.3071357025823707,
+      "tokens_seen": 3165716480
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 4.3798118049615056e-05,
+      "loss": 2.5116,
+      "theoretical_loss": 3.307130496000174,
+      "tokens_seen": 3165782016
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 4.375534644995723e-05,
+      "loss": 2.4344,
+      "theoretical_loss": 3.3071200832496577,
+      "tokens_seen": 3165913088
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 4.3712574850299406e-05,
+      "loss": 2.5286,
+      "theoretical_loss": 3.3071096710509322,
+      "tokens_seen": 3166044160
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 4.366980325064158e-05,
+      "loss": 2.4363,
+      "theoretical_loss": 3.3070992594039463,
+      "tokens_seen": 3166175232
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 4.362703165098375e-05,
+      "loss": 2.4088,
+      "theoretical_loss": 3.307088848308647,
+      "tokens_seen": 3166306304
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 4.358426005132592e-05,
+      "loss": 2.4939,
+      "theoretical_loss": 3.307078437764983,
+      "tokens_seen": 3166437376
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 4.354148845166809e-05,
+      "loss": 2.4801,
+      "theoretical_loss": 3.3070680277729023,
+      "tokens_seen": 3166568448
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 4.349871685201027e-05,
+      "loss": 2.3799,
+      "theoretical_loss": 3.307057618332353,
+      "tokens_seen": 3166699520
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 4.345594525235244e-05,
+      "loss": 2.4907,
+      "theoretical_loss": 3.3070472094432817,
+      "tokens_seen": 3166830592
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 4.341317365269461e-05,
+      "loss": 2.4501,
+      "theoretical_loss": 3.307036801105638,
+      "tokens_seen": 3166961664
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 4.3370402053036785e-05,
+      "loss": 2.3822,
+      "theoretical_loss": 3.3070263933193687,
+      "tokens_seen": 3167092736
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 4.3327630453378956e-05,
+      "loss": 2.6524,
+      "theoretical_loss": 3.307015986084422,
+      "tokens_seen": 3167223808
+    },
+    {
+      "epoch": 0.57,
+      "objective/train/docs_used": 1735648,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 2.1001577377319336,
+      "objective/train/theoretical_loss": 3.3070055794007467,
+      "objective/train/tokens_used": 197406176,
+      "theoretical_loss": 3.3070055794007467,
+      "tokens_seen": 3167354880
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 4.3284858853721134e-05,
+      "loss": 2.3798,
+      "theoretical_loss": 3.3070055794007467,
+      "tokens_seen": 3167354880
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 4.32420872540633e-05,
+      "loss": 2.5349,
+      "theoretical_loss": 3.3069951732682896,
+      "tokens_seen": 3167485952
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 4.319931565440548e-05,
+      "loss": 2.5036,
+      "theoretical_loss": 3.3069847676869997,
+      "tokens_seen": 3167617024
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 4.315654405474765e-05,
+      "loss": 2.5548,
+      "theoretical_loss": 3.306974362656825,
+      "tokens_seen": 3167748096
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 4.311377245508982e-05,
+      "loss": 2.4116,
+      "theoretical_loss": 3.3069639581777124,
+      "tokens_seen": 3167879168
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 4.3071000855432e-05,
+      "loss": 2.551,
+      "theoretical_loss": 3.3069535542496107,
+      "tokens_seen": 3168010240
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 4.3028229255774163e-05,
+      "loss": 2.6564,
+      "theoretical_loss": 3.3069431508724683,
+      "tokens_seen": 3168141312
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 4.298545765611634e-05,
+      "loss": 2.3732,
+      "theoretical_loss": 3.3069327480462323,
+      "tokens_seen": 3168272384
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 4.294268605645851e-05,
+      "loss": 2.4254,
+      "theoretical_loss": 3.3069223457708516,
+      "tokens_seen": 3168403456
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 4.289991445680069e-05,
+      "loss": 2.4169,
+      "theoretical_loss": 3.3069119440462735,
+      "tokens_seen": 3168534528
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 4.2857142857142856e-05,
+      "loss": 2.4854,
+      "theoretical_loss": 3.306901542872447,
+      "tokens_seen": 3168665600
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 4.281437125748503e-05,
+      "loss": 2.4044,
+      "theoretical_loss": 3.306891142249319,
+      "tokens_seen": 3168796672
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 4.2771599657827206e-05,
+      "loss": 2.4397,
+      "theoretical_loss": 3.3068807421768383,
+      "tokens_seen": 3168927744
+    },
+    {
+      "epoch": 0.58,
+      "objective/train/docs_used": 1736895,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 2.7372190952301025,
+      "objective/train/theoretical_loss": 3.306875542347074,
+      "objective/train/tokens_used": 199044576,
+      "theoretical_loss": 3.306875542347074,
+      "tokens_seen": 3168993280
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 4.272882805816938e-05,
+      "loss": 2.5245,
+      "theoretical_loss": 3.306870342654953,
+      "tokens_seen": 3169058816
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 4.2686056458511556e-05,
+      "loss": 2.5318,
+      "theoretical_loss": 3.3068599436836106,
+      "tokens_seen": 3169189888
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 4.264328485885372e-05,
+      "loss": 2.4989,
+      "theoretical_loss": 3.30684954526276,
+      "tokens_seen": 3169320960
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 4.260051325919589e-05,
+      "loss": 2.4893,
+      "theoretical_loss": 3.306839147392348,
+      "tokens_seen": 3169452032
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 4.255774165953807e-05,
+      "loss": 2.4955,
+      "theoretical_loss": 3.3068287500723246,
+      "tokens_seen": 3169583104
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 4.251497005988024e-05,
+      "loss": 2.509,
+      "theoretical_loss": 3.3068183533026367,
+      "tokens_seen": 3169714176
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 4.2472198460222414e-05,
+      "loss": 2.498,
+      "theoretical_loss": 3.3068079570832323,
+      "tokens_seen": 3169845248
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 4.2429426860564585e-05,
+      "loss": 2.4562,
+      "theoretical_loss": 3.30679756141406,
+      "tokens_seen": 3169976320
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 4.238665526090676e-05,
+      "loss": 2.4116,
+      "theoretical_loss": 3.3067871662950674,
+      "tokens_seen": 3170107392
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 4.2343883661248935e-05,
+      "loss": 2.4851,
+      "theoretical_loss": 3.3067767717262035,
+      "tokens_seen": 3170238464
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 4.2301112061591106e-05,
+      "loss": 2.535,
+      "theoretical_loss": 3.3067663777074157,
+      "tokens_seen": 3170369536
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 4.225834046193328e-05,
+      "loss": 2.5879,
+      "theoretical_loss": 3.3067559842386522,
+      "tokens_seen": 3170500608
+    },
+    {
+      "epoch": 0.58,
+      "objective/train/docs_used": 1737490,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 2.7611770629882812,
+      "objective/train/theoretical_loss": 3.306745591319862,
+      "objective/train/tokens_used": 200682976,
+      "theoretical_loss": 3.306745591319862,
+      "tokens_seen": 3170631680
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 4.221556886227545e-05,
+      "loss": 2.5032,
+      "theoretical_loss": 3.306745591319862,
+      "tokens_seen": 3170631680
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 4.217279726261763e-05,
+      "loss": 2.5335,
+      "theoretical_loss": 3.3067351989509914,
+      "tokens_seen": 3170762752
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 4.21300256629598e-05,
+      "loss": 2.4593,
+      "theoretical_loss": 3.3067248071319906,
+      "tokens_seen": 3170893824
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 4.208725406330197e-05,
+      "loss": 2.4957,
+      "theoretical_loss": 3.306714415862807,
+      "tokens_seen": 3171024896
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 4.204448246364414e-05,
+      "loss": 2.5613,
+      "theoretical_loss": 3.3067040251433886,
+      "tokens_seen": 3171155968
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 4.2001710863986314e-05,
+      "loss": 2.4987,
+      "theoretical_loss": 3.3066936349736835,
+      "tokens_seen": 3171287040
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 4.195893926432849e-05,
+      "loss": 2.3298,
+      "theoretical_loss": 3.3066832453536406,
+      "tokens_seen": 3171418112
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 4.191616766467066e-05,
+      "loss": 2.3801,
+      "theoretical_loss": 3.306672856283207,
+      "tokens_seen": 3171549184
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 4.1873396065012835e-05,
+      "loss": 2.5703,
+      "theoretical_loss": 3.306662467762332,
+      "tokens_seen": 3171680256
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 4.183062446535501e-05,
+      "loss": 2.3197,
+      "theoretical_loss": 3.3066520797909633,
+      "tokens_seen": 3171811328
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 4.178785286569718e-05,
+      "loss": 2.48,
+      "theoretical_loss": 3.306641692369049,
+      "tokens_seen": 3171942400
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 4.1745081266039356e-05,
+      "loss": 2.4383,
+      "theoretical_loss": 3.3066313054965377,
+      "tokens_seen": 3172073472
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 4.170230966638152e-05,
+      "loss": 2.4996,
+      "theoretical_loss": 3.3066209191733775,
+      "tokens_seen": 3172204544
+    },
+    {
+      "epoch": 0.59,
+      "objective/train/docs_used": 1738634,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 2.6035115718841553,
+      "objective/train/theoretical_loss": 3.306615726217788,
+      "objective/train/tokens_used": 202321376,
+      "theoretical_loss": 3.306615726217788,
+      "tokens_seen": 3172270080
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 4.16595380667237e-05,
+      "loss": 2.4728,
+      "theoretical_loss": 3.3066105333995166,
+      "tokens_seen": 3172335616
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 4.161676646706587e-05,
+      "loss": 2.4305,
+      "theoretical_loss": 3.306600148174903,
+      "tokens_seen": 3172466688
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 4.157399486740804e-05,
+      "loss": 2.4707,
+      "theoretical_loss": 3.3065897634994856,
+      "tokens_seen": 3172597760
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 4.1531223267750214e-05,
+      "loss": 2.4014,
+      "theoretical_loss": 3.306579379373212,
+      "tokens_seen": 3172728832
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 4.1488451668092386e-05,
+      "loss": 2.5214,
+      "theoretical_loss": 3.306568995796031,
+      "tokens_seen": 3172859904
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 4.1445680068434564e-05,
+      "loss": 2.5563,
+      "theoretical_loss": 3.3065586127678905,
+      "tokens_seen": 3172990976
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 4.1402908468776735e-05,
+      "loss": 2.4841,
+      "theoretical_loss": 3.3065482302887395,
+      "tokens_seen": 3173122048
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 4.136013686911891e-05,
+      "loss": 2.4925,
+      "theoretical_loss": 3.3065378483585253,
+      "tokens_seen": 3173253120
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 4.131736526946108e-05,
+      "loss": 2.5094,
+      "theoretical_loss": 3.3065274669771965,
+      "tokens_seen": 3173384192
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 4.127459366980325e-05,
+      "loss": 2.4376,
+      "theoretical_loss": 3.3065170861447015,
+      "tokens_seen": 3173515264
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 4.123182207014543e-05,
+      "loss": 2.4933,
+      "theoretical_loss": 3.306506705860989,
+      "tokens_seen": 3173646336
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 4.11890504704876e-05,
+      "loss": 2.4276,
+      "theoretical_loss": 3.306496326126007,
+      "tokens_seen": 3173777408
+    },
+    {
+      "epoch": 0.59,
+      "objective/train/docs_used": 1739179,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 2.3437700271606445,
+      "objective/train/theoretical_loss": 3.306485946939704,
+      "objective/train/tokens_used": 203959776,
+      "theoretical_loss": 3.306485946939704,
+      "tokens_seen": 3173908480
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 4.114627887082977e-05,
+      "loss": 2.419,
+      "theoretical_loss": 3.306485946939704,
+      "tokens_seen": 3173908480
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 4.110350727117194e-05,
+      "loss": 2.4699,
+      "theoretical_loss": 3.306475568302028,
+      "tokens_seen": 3174039552
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 4.1060735671514114e-05,
+      "loss": 2.5379,
+      "theoretical_loss": 3.3064651902129274,
+      "tokens_seen": 3174170624
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 4.101796407185629e-05,
+      "loss": 2.6357,
+      "theoretical_loss": 3.3064548126723508,
+      "tokens_seen": 3174301696
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 4.0975192472198464e-05,
+      "loss": 2.5486,
+      "theoretical_loss": 3.3064444356802465,
+      "tokens_seen": 3174432768
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 4.0932420872540636e-05,
+      "loss": 2.4297,
+      "theoretical_loss": 3.3064340592365626,
+      "tokens_seen": 3174563840
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 4.088964927288281e-05,
+      "loss": 2.3582,
+      "theoretical_loss": 3.306423683341248,
+      "tokens_seen": 3174694912
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 4.084687767322498e-05,
+      "loss": 2.3793,
+      "theoretical_loss": 3.3064133079942506,
+      "tokens_seen": 3174825984
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 4.080410607356716e-05,
+      "loss": 2.3722,
+      "theoretical_loss": 3.3064029331955194,
+      "tokens_seen": 3174957056
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 4.076133447390932e-05,
+      "loss": 2.4821,
+      "theoretical_loss": 3.3063925589450016,
+      "tokens_seen": 3175088128
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 4.07185628742515e-05,
+      "loss": 2.4909,
+      "theoretical_loss": 3.3063821852426467,
+      "tokens_seen": 3175219200
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 4.067579127459367e-05,
+      "loss": 2.445,
+      "theoretical_loss": 3.306371812088403,
+      "tokens_seen": 3175350272
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 4.063301967493584e-05,
+      "loss": 2.4686,
+      "theoretical_loss": 3.3063614394822185,
+      "tokens_seen": 3175481344
+    },
+    {
+      "epoch": 0.6,
+      "objective/train/docs_used": 1739867,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 2.4862287044525146,
+      "objective/train/theoretical_loss": 3.3063562533846325,
+      "objective/train/tokens_used": 205598176,
+      "theoretical_loss": 3.3063562533846325,
+      "tokens_seen": 3175546880
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 4.059024807527802e-05,
+      "loss": 2.5223,
+      "theoretical_loss": 3.3063510674240417,
+      "tokens_seen": 3175612416
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 4.0547476475620186e-05,
+      "loss": 2.495,
+      "theoretical_loss": 3.3063406959138213,
+      "tokens_seen": 3175743488
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 4.0504704875962364e-05,
+      "loss": 2.3905,
+      "theoretical_loss": 3.3063303249515057,
+      "tokens_seen": 3175874560
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 4.0461933276304536e-05,
+      "loss": 2.6257,
+      "theoretical_loss": 3.3063199545370425,
+      "tokens_seen": 3176005632
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 4.041916167664671e-05,
+      "loss": 2.4279,
+      "theoretical_loss": 3.3063095846703816,
+      "tokens_seen": 3176136704
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 4.037639007698888e-05,
+      "loss": 2.4832,
+      "theoretical_loss": 3.3062992153514705,
+      "tokens_seen": 3176267776
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 4.033361847733105e-05,
+      "loss": 2.5037,
+      "theoretical_loss": 3.306288846580258,
+      "tokens_seen": 3176398848
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 4.029084687767323e-05,
+      "loss": 2.5226,
+      "theoretical_loss": 3.3062784783566923,
+      "tokens_seen": 3176529920
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 4.02480752780154e-05,
+      "loss": 2.4342,
+      "theoretical_loss": 3.306268110680722,
+      "tokens_seen": 3176660992
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 4.020530367835757e-05,
+      "loss": 2.4567,
+      "theoretical_loss": 3.306257743552296,
+      "tokens_seen": 3176792064
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 4.016253207869974e-05,
+      "loss": 2.4013,
+      "theoretical_loss": 3.306247376971362,
+      "tokens_seen": 3176923136
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 4.0119760479041915e-05,
+      "loss": 2.5258,
+      "theoretical_loss": 3.3062370109378696,
+      "tokens_seen": 3177054208
+    },
+    {
+      "epoch": 0.6,
+      "objective/train/docs_used": 1740434,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 1.7373958826065063,
+      "objective/train/theoretical_loss": 3.306226645451766,
+      "objective/train/tokens_used": 207236576,
+      "theoretical_loss": 3.306226645451766,
+      "tokens_seen": 3177185280
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 4.007698887938409e-05,
+      "loss": 2.3115,
+      "theoretical_loss": 3.306226645451766,
+      "tokens_seen": 3177185280
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 4.0034217279726265e-05,
+      "loss": 2.4377,
+      "theoretical_loss": 3.3062162805130004,
+      "tokens_seen": 3177316352
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 3.9991445680068436e-05,
+      "loss": 2.3801,
+      "theoretical_loss": 3.3062059161215216,
+      "tokens_seen": 3177447424
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 3.994867408041061e-05,
+      "loss": 2.5384,
+      "theoretical_loss": 3.3061955522772775,
+      "tokens_seen": 3177578496
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 3.990590248075278e-05,
+      "loss": 2.4873,
+      "theoretical_loss": 3.306185188980217,
+      "tokens_seen": 3177709568
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 3.986313088109496e-05,
+      "loss": 2.3113,
+      "theoretical_loss": 3.3061748262302886,
+      "tokens_seen": 3177840640
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 3.982035928143712e-05,
+      "loss": 2.5892,
+      "theoretical_loss": 3.306164464027441,
+      "tokens_seen": 3177971712
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 3.97775876817793e-05,
+      "loss": 2.5497,
+      "theoretical_loss": 3.3061541023716225,
+      "tokens_seen": 3178102784
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 3.973481608212147e-05,
+      "loss": 2.3053,
+      "theoretical_loss": 3.3061437412627814,
+      "tokens_seen": 3178233856
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 3.969204448246365e-05,
+      "loss": 2.5014,
+      "theoretical_loss": 3.306133380700867,
+      "tokens_seen": 3178364928
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 3.964927288280582e-05,
+      "loss": 2.671,
+      "theoretical_loss": 3.306123020685827,
+      "tokens_seen": 3178496000
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 3.960650128314799e-05,
+      "loss": 2.5158,
+      "theoretical_loss": 3.306112661217611,
+      "tokens_seen": 3178627072
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 3.9563729683490165e-05,
+      "loss": 2.4479,
+      "theoretical_loss": 3.306102302296167,
+      "tokens_seen": 3178758144
+    },
+    {
+      "epoch": 0.61,
+      "objective/train/docs_used": 1741553,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 2.4893863201141357,
+      "objective/train/theoretical_loss": 3.3060971230404683,
+      "objective/train/tokens_used": 208874976,
+      "theoretical_loss": 3.3060971230404683,
+      "tokens_seen": 3178823680
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 3.9520958083832336e-05,
+      "loss": 2.5002,
+      "theoretical_loss": 3.3060919439214436,
+      "tokens_seen": 3178889216
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 3.9478186484174515e-05,
+      "loss": 2.4046,
+      "theoretical_loss": 3.3060815860933896,
+      "tokens_seen": 3179020288
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 3.943541488451668e-05,
+      "loss": 2.458,
+      "theoretical_loss": 3.306071228811953,
+      "tokens_seen": 3179151360
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 3.939264328485886e-05,
+      "loss": 2.4769,
+      "theoretical_loss": 3.3060608720770834,
+      "tokens_seen": 3179282432
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 3.934987168520103e-05,
+      "loss": 2.4564,
+      "theoretical_loss": 3.306050515888729,
+      "tokens_seen": 3179413504
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 3.93071000855432e-05,
+      "loss": 2.4812,
+      "theoretical_loss": 3.306040160246838,
+      "tokens_seen": 3179544576
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 3.926432848588538e-05,
+      "loss": 2.5644,
+      "theoretical_loss": 3.3060298051513595,
+      "tokens_seen": 3179675648
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 3.9221556886227544e-05,
+      "loss": 2.6354,
+      "theoretical_loss": 3.306019450602242,
+      "tokens_seen": 3179806720
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 3.917878528656972e-05,
+      "loss": 2.4243,
+      "theoretical_loss": 3.306009096599434,
+      "tokens_seen": 3179937792
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 3.9136013686911894e-05,
+      "loss": 2.5855,
+      "theoretical_loss": 3.3059987431428848,
+      "tokens_seen": 3180068864
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 3.9093242087254065e-05,
+      "loss": 2.4582,
+      "theoretical_loss": 3.3059883902325424,
+      "tokens_seen": 3180199936
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 3.905047048759624e-05,
+      "loss": 2.4974,
+      "theoretical_loss": 3.3059780378683556,
+      "tokens_seen": 3180331008
+    },
+    {
+      "epoch": 0.61,
+      "objective/train/docs_used": 1742088,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 2.383901834487915,
+      "objective/train/theoretical_loss": 3.3059676860502734,
+      "objective/train/tokens_used": 210513376,
+      "theoretical_loss": 3.3059676860502734,
+      "tokens_seen": 3180462080
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 3.900769888793841e-05,
+      "loss": 2.5039,
+      "theoretical_loss": 3.3059676860502734,
+      "tokens_seen": 3180462080
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 3.8964927288280587e-05,
+      "loss": 2.5339,
+      "theoretical_loss": 3.305957334778244,
+      "tokens_seen": 3180593152
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 3.892215568862276e-05,
+      "loss": 2.4543,
+      "theoretical_loss": 3.3059469840522167,
+      "tokens_seen": 3180724224
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 3.887938408896493e-05,
+      "loss": 2.5014,
+      "theoretical_loss": 3.3059366338721397,
+      "tokens_seen": 3180855296
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 3.88366124893071e-05,
+      "loss": 2.4249,
+      "theoretical_loss": 3.305926284237962,
+      "tokens_seen": 3180986368
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 3.879384088964927e-05,
+      "loss": 2.4925,
+      "theoretical_loss": 3.305915935149632,
+      "tokens_seen": 3181117440
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 3.875106928999145e-05,
+      "loss": 2.4658,
+      "theoretical_loss": 3.3059055866070985,
+      "tokens_seen": 3181248512
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 3.870829769033362e-05,
+      "loss": 2.4208,
+      "theoretical_loss": 3.3058952386103106,
+      "tokens_seen": 3181379584
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 3.8665526090675794e-05,
+      "loss": 2.4329,
+      "theoretical_loss": 3.305884891159217,
+      "tokens_seen": 3181510656
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 3.8622754491017966e-05,
+      "loss": 2.343,
+      "theoretical_loss": 3.305874544253766,
+      "tokens_seen": 3181641728
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 3.857998289136014e-05,
+      "loss": 2.558,
+      "theoretical_loss": 3.3058641978939063,
+      "tokens_seen": 3181772800
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 3.8537211291702315e-05,
+      "loss": 2.5192,
+      "theoretical_loss": 3.305853852079587,
+      "tokens_seen": 3181903872
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 3.849443969204448e-05,
+      "loss": 2.636,
+      "theoretical_loss": 3.305843506810757,
+      "tokens_seen": 3182034944
+    },
+    {
+      "epoch": 0.62,
+      "objective/train/docs_used": 1743107,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 2.428229570388794,
+      "objective/train/theoretical_loss": 3.3058383343808844,
+      "objective/train/tokens_used": 212151776,
+      "theoretical_loss": 3.3058383343808844,
+      "tokens_seen": 3182100480
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 3.845166809238666e-05,
+      "loss": 2.5503,
+      "theoretical_loss": 3.3058331620873647,
+      "tokens_seen": 3182166016
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 3.840889649272883e-05,
+      "loss": 2.5304,
+      "theoretical_loss": 3.305822817909359,
+      "tokens_seen": 3182297088
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 3.8366124893071e-05,
+      "loss": 2.4096,
+      "theoretical_loss": 3.305812474276689,
+      "tokens_seen": 3182428160
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 3.832335329341318e-05,
+      "loss": 2.5894,
+      "theoretical_loss": 3.305802131189303,
+      "tokens_seen": 3182559232
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 3.8280581693755344e-05,
+      "loss": 2.4894,
+      "theoretical_loss": 3.30579178864715,
+      "tokens_seen": 3182690304
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 3.823781009409752e-05,
+      "loss": 2.5676,
+      "theoretical_loss": 3.3057814466501787,
+      "tokens_seen": 3182821376
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 3.8195038494439694e-05,
+      "loss": 2.5241,
+      "theoretical_loss": 3.3057711051983385,
+      "tokens_seen": 3182952448
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 3.8152266894781866e-05,
+      "loss": 2.5246,
+      "theoretical_loss": 3.3057607642915774,
+      "tokens_seen": 3183083520
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 3.810949529512404e-05,
+      "loss": 2.3917,
+      "theoretical_loss": 3.3057504239298448,
+      "tokens_seen": 3183214592
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 3.806672369546621e-05,
+      "loss": 2.4044,
+      "theoretical_loss": 3.305740084113089,
+      "tokens_seen": 3183345664
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 3.802395209580839e-05,
+      "loss": 2.3885,
+      "theoretical_loss": 3.3057297448412593,
+      "tokens_seen": 3183476736
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 3.798118049615056e-05,
+      "loss": 2.7271,
+      "theoretical_loss": 3.3057194061143047,
+      "tokens_seen": 3183607808
+    },
+    {
+      "epoch": 0.62,
+      "objective/train/docs_used": 1744349,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 2.4869487285614014,
+      "objective/train/theoretical_loss": 3.305709067932173,
+      "objective/train/tokens_used": 213790176,
+      "theoretical_loss": 3.305709067932173,
+      "tokens_seen": 3183738880
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 3.793840889649273e-05,
+      "loss": 2.487,
+      "theoretical_loss": 3.305709067932173,
+      "tokens_seen": 3183738880
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 3.78956372968349e-05,
+      "loss": 2.4626,
+      "theoretical_loss": 3.3056987302948144,
+      "tokens_seen": 3183869952
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 3.785286569717707e-05,
+      "loss": 2.5541,
+      "theoretical_loss": 3.3056883932021774,
+      "tokens_seen": 3184001024
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 3.781009409751925e-05,
+      "loss": 2.4975,
+      "theoretical_loss": 3.30567805665421,
+      "tokens_seen": 3184132096
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 3.776732249786142e-05,
+      "loss": 2.5569,
+      "theoretical_loss": 3.3056677206508622,
+      "tokens_seen": 3184263168
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 3.7724550898203595e-05,
+      "loss": 2.5492,
+      "theoretical_loss": 3.3056573851920827,
+      "tokens_seen": 3184394240
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 3.7681779298545766e-05,
+      "loss": 2.65,
+      "theoretical_loss": 3.3056470502778197,
+      "tokens_seen": 3184525312
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 3.763900769888794e-05,
+      "loss": 2.4132,
+      "theoretical_loss": 3.3056367159080224,
+      "tokens_seen": 3184656384
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 3.7596236099230116e-05,
+      "loss": 2.5916,
+      "theoretical_loss": 3.3056263820826404,
+      "tokens_seen": 3184787456
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 3.755346449957229e-05,
+      "loss": 2.5077,
+      "theoretical_loss": 3.3056160488016215,
+      "tokens_seen": 3184918528
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 3.751069289991446e-05,
+      "loss": 2.485,
+      "theoretical_loss": 3.3056057160649153,
+      "tokens_seen": 3185049600
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 3.746792130025663e-05,
+      "loss": 2.3922,
+      "theoretical_loss": 3.3055953838724705,
+      "tokens_seen": 3185180672
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 3.74251497005988e-05,
+      "loss": 2.4665,
+      "theoretical_loss": 3.3055850522242363,
+      "tokens_seen": 3185311744
+    },
+    {
+      "epoch": 0.63,
+      "objective/train/docs_used": 1745003,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 2.075413227081299,
+      "objective/train/theoretical_loss": 3.3055798866041823,
+      "objective/train/tokens_used": 215428576,
+      "theoretical_loss": 3.3055798866041823,
+      "tokens_seen": 3185377280
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 3.738237810094098e-05,
+      "loss": 2.3933,
+      "theoretical_loss": 3.305574721120162,
+      "tokens_seen": 3185442816
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 3.7339606501283145e-05,
+      "loss": 2.457,
+      "theoretical_loss": 3.3055643905601952,
+      "tokens_seen": 3185573888
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 3.729683490162532e-05,
+      "loss": 2.4476,
+      "theoretical_loss": 3.305554060544286,
+      "tokens_seen": 3185704960
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 3.7254063301967495e-05,
+      "loss": 2.3651,
+      "theoretical_loss": 3.305543731072383,
+      "tokens_seen": 3185836032
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 3.7211291702309666e-05,
+      "loss": 2.642,
+      "theoretical_loss": 3.3055334021444356,
+      "tokens_seen": 3185967104
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 3.7168520102651845e-05,
+      "loss": 2.5106,
+      "theoretical_loss": 3.305523073760392,
+      "tokens_seen": 3186098176
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 3.712574850299401e-05,
+      "loss": 2.4407,
+      "theoretical_loss": 3.305512745920202,
+      "tokens_seen": 3186229248
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 3.708297690333619e-05,
+      "loss": 2.4805,
+      "theoretical_loss": 3.305502418623814,
+      "tokens_seen": 3186360320
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 3.704020530367836e-05,
+      "loss": 2.5066,
+      "theoretical_loss": 3.305492091871177,
+      "tokens_seen": 3186491392
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 3.699743370402054e-05,
+      "loss": 2.4317,
+      "theoretical_loss": 3.3054817656622406,
+      "tokens_seen": 3186622464
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 3.69546621043627e-05,
+      "loss": 2.5524,
+      "theoretical_loss": 3.305471439996953,
+      "tokens_seen": 3186753536
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 3.6911890504704874e-05,
+      "loss": 2.3725,
+      "theoretical_loss": 3.3054611148752637,
+      "tokens_seen": 3186884608
+    },
+    {
+      "epoch": 0.63,
+      "objective/train/docs_used": 1746088,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 2.514112710952759,
+      "objective/train/theoretical_loss": 3.3054507902971215,
+      "objective/train/tokens_used": 217066976,
+      "theoretical_loss": 3.3054507902971215,
+      "tokens_seen": 3187015680
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 3.686911890504705e-05,
+      "loss": 2.5984,
+      "theoretical_loss": 3.3054507902971215,
+      "tokens_seen": 3187015680
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 3.6826347305389224e-05,
+      "loss": 2.4928,
+      "theoretical_loss": 3.305440466262476,
+      "tokens_seen": 3187146752
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 3.6783575705731395e-05,
+      "loss": 2.4952,
+      "theoretical_loss": 3.3054301427712756,
+      "tokens_seen": 3187277824
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 3.6740804106073567e-05,
+      "loss": 2.5138,
+      "theoretical_loss": 3.3054198198234697,
+      "tokens_seen": 3187408896
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 3.6698032506415745e-05,
+      "loss": 2.4385,
+      "theoretical_loss": 3.305409497419007,
+      "tokens_seen": 3187539968
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 3.6655260906757916e-05,
+      "loss": 2.4016,
+      "theoretical_loss": 3.305399175557837,
+      "tokens_seen": 3187671040
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 3.661248930710009e-05,
+      "loss": 2.6873,
+      "theoretical_loss": 3.3053888542399084,
+      "tokens_seen": 3187802112
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 3.656971770744226e-05,
+      "loss": 2.4365,
+      "theoretical_loss": 3.3053785334651704,
+      "tokens_seen": 3187933184
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 3.652694610778443e-05,
+      "loss": 2.5373,
+      "theoretical_loss": 3.305368213233572,
+      "tokens_seen": 3188064256
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 3.648417450812661e-05,
+      "loss": 2.4138,
+      "theoretical_loss": 3.3053578935450627,
+      "tokens_seen": 3188195328
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 3.644140290846878e-05,
+      "loss": 2.4721,
+      "theoretical_loss": 3.3053475743995913,
+      "tokens_seen": 3188326400
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 3.6398631308810946e-05,
+      "loss": 2.4902,
+      "theoretical_loss": 3.305337255797107,
+      "tokens_seen": 3188457472
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 3.6355859709153124e-05,
+      "loss": 2.5323,
+      "theoretical_loss": 3.3053269377375587,
+      "tokens_seen": 3188588544
+    },
+    {
+      "epoch": 0.64,
+      "objective/train/docs_used": 1746515,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 2.3587210178375244,
+      "objective/train/theoretical_loss": 3.3053217789113694,
+      "objective/train/tokens_used": 218705376,
+      "theoretical_loss": 3.3053217789113694,
+      "tokens_seen": 3188654080
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 3.6313088109495295e-05,
+      "loss": 2.5451,
+      "theoretical_loss": 3.3053166202208955,
+      "tokens_seen": 3188719616
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 3.6270316509837474e-05,
+      "loss": 2.6429,
+      "theoretical_loss": 3.3053063032470664,
+      "tokens_seen": 3188850688
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 3.6227544910179645e-05,
+      "loss": 2.7202,
+      "theoretical_loss": 3.305295986816021,
+      "tokens_seen": 3188981760
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 3.618477331052182e-05,
+      "loss": 2.4629,
+      "theoretical_loss": 3.3052856709277085,
+      "tokens_seen": 3189112832
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 3.614200171086399e-05,
+      "loss": 2.46,
+      "theoretical_loss": 3.3052753555820775,
+      "tokens_seen": 3189243904
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 3.609923011120616e-05,
+      "loss": 2.5008,
+      "theoretical_loss": 3.3052650407790773,
+      "tokens_seen": 3189374976
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 3.605645851154834e-05,
+      "loss": 2.3809,
+      "theoretical_loss": 3.3052547265186574,
+      "tokens_seen": 3189506048
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 3.60136869118905e-05,
+      "loss": 2.5,
+      "theoretical_loss": 3.3052444128007665,
+      "tokens_seen": 3189637120
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 3.597091531223268e-05,
+      "loss": 2.5178,
+      "theoretical_loss": 3.305234099625354,
+      "tokens_seen": 3189768192
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 3.592814371257485e-05,
+      "loss": 2.4331,
+      "theoretical_loss": 3.3052237869923693,
+      "tokens_seen": 3189899264
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 3.5885372112917024e-05,
+      "loss": 2.5657,
+      "theoretical_loss": 3.305213474901761,
+      "tokens_seen": 3190030336
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 3.58426005132592e-05,
+      "loss": 2.6933,
+      "theoretical_loss": 3.305203163353479,
+      "tokens_seen": 3190161408
+    },
+    {
+      "epoch": 0.65,
+      "objective/train/docs_used": 1747620,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 2.320441484451294,
+      "objective/train/theoretical_loss": 3.305192852347472,
+      "objective/train/tokens_used": 220343776,
+      "theoretical_loss": 3.305192852347472,
+      "tokens_seen": 3190292480
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 3.579982891360137e-05,
+      "loss": 2.5312,
+      "theoretical_loss": 3.305192852347472,
+      "tokens_seen": 3190292480
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 3.5757057313943545e-05,
+      "loss": 2.4923,
+      "theoretical_loss": 3.3051825418836893,
+      "tokens_seen": 3190423552
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 3.571428571428572e-05,
+      "loss": 2.3546,
+      "theoretical_loss": 3.3051722319620804,
+      "tokens_seen": 3190554624
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 3.567151411462789e-05,
+      "loss": 2.5379,
+      "theoretical_loss": 3.305161922582594,
+      "tokens_seen": 3190685696
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 3.562874251497006e-05,
+      "loss": 2.5875,
+      "theoretical_loss": 3.3051516137451795,
+      "tokens_seen": 3190816768
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 3.558597091531223e-05,
+      "loss": 2.5622,
+      "theoretical_loss": 3.3051413054497867,
+      "tokens_seen": 3190947840
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 3.554319931565441e-05,
+      "loss": 2.5436,
+      "theoretical_loss": 3.305130997696364,
+      "tokens_seen": 3191078912
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 3.550042771599658e-05,
+      "loss": 2.5889,
+      "theoretical_loss": 3.3051206904848613,
+      "tokens_seen": 3191209984
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 3.545765611633875e-05,
+      "loss": 2.5459,
+      "theoretical_loss": 3.3051103838152276,
+      "tokens_seen": 3191341056
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 3.5414884516680924e-05,
+      "loss": 2.4801,
+      "theoretical_loss": 3.3051000776874115,
+      "tokens_seen": 3191472128
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 3.5372112917023096e-05,
+      "loss": 2.5707,
+      "theoretical_loss": 3.3050897721013635,
+      "tokens_seen": 3191603200
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 3.5329341317365274e-05,
+      "loss": 2.5332,
+      "theoretical_loss": 3.3050794670570323,
+      "tokens_seen": 3191734272
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 3.5286569717707446e-05,
+      "loss": 2.584,
+      "theoretical_loss": 3.305069162554367,
+      "tokens_seen": 3191865344
+    },
+    {
+      "epoch": 0.65,
+      "objective/train/docs_used": 1748281,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 3.083997964859009,
+      "objective/train/theoretical_loss": 3.3050640105061433,
+      "objective/train/tokens_used": 221982176,
+      "theoretical_loss": 3.3050640105061433,
+      "tokens_seen": 3191930880
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 3.524379811804962e-05,
+      "loss": 2.7437,
+      "theoretical_loss": 3.3050588585933167,
+      "tokens_seen": 3191996416
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 3.520102651839179e-05,
+      "loss": 2.4889,
+      "theoretical_loss": 3.3050485551738316,
+      "tokens_seen": 3192127488
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 3.515825491873396e-05,
+      "loss": 2.5558,
+      "theoretical_loss": 3.30503825229586,
+      "tokens_seen": 3192258560
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 3.511548331907614e-05,
+      "loss": 2.359,
+      "theoretical_loss": 3.305027949959352,
+      "tokens_seen": 3192389632
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 3.50727117194183e-05,
+      "loss": 2.6027,
+      "theoretical_loss": 3.3050176481642564,
+      "tokens_seen": 3192520704
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 3.502994011976048e-05,
+      "loss": 2.4854,
+      "theoretical_loss": 3.3050073469105223,
+      "tokens_seen": 3192651776
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 3.498716852010265e-05,
+      "loss": 2.3569,
+      "theoretical_loss": 3.3049970461980998,
+      "tokens_seen": 3192782848
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 3.4944396920444825e-05,
+      "loss": 2.437,
+      "theoretical_loss": 3.304986746026938,
+      "tokens_seen": 3192913920
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 3.4901625320787e-05,
+      "loss": 2.4806,
+      "theoretical_loss": 3.304976446396986,
+      "tokens_seen": 3193044992
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 3.485885372112917e-05,
+      "loss": 2.4566,
+      "theoretical_loss": 3.304966147308193,
+      "tokens_seen": 3193176064
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 3.4816082121471346e-05,
+      "loss": 2.5142,
+      "theoretical_loss": 3.3049558487605086,
+      "tokens_seen": 3193307136
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 3.477331052181352e-05,
+      "loss": 2.5212,
+      "theoretical_loss": 3.3049455507538825,
+      "tokens_seen": 3193438208
+    },
+    {
+      "epoch": 0.66,
+      "objective/train/docs_used": 1749492,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 2.5565552711486816,
+      "objective/train/theoretical_loss": 3.304935253288263,
+      "objective/train/tokens_used": 223620576,
+      "theoretical_loss": 3.304935253288263,
+      "tokens_seen": 3193569280
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 3.473053892215569e-05,
+      "loss": 2.5474,
+      "theoretical_loss": 3.304935253288263,
+      "tokens_seen": 3193569280
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 3.468776732249786e-05,
+      "loss": 2.4504,
+      "theoretical_loss": 3.304924956363601,
+      "tokens_seen": 3193700352
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 3.464499572284003e-05,
+      "loss": 2.5253,
+      "theoretical_loss": 3.3049146599798447,
+      "tokens_seen": 3193831424
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 3.460222412318221e-05,
+      "loss": 2.3926,
+      "theoretical_loss": 3.3049043641369438,
+      "tokens_seen": 3193962496
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 3.455945252352438e-05,
+      "loss": 2.4699,
+      "theoretical_loss": 3.304894068834848,
+      "tokens_seen": 3194093568
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 3.4516680923866553e-05,
+      "loss": 2.4117,
+      "theoretical_loss": 3.3048837740735064,
+      "tokens_seen": 3194224640
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 3.4473909324208725e-05,
+      "loss": 2.488,
+      "theoretical_loss": 3.3048734798528683,
+      "tokens_seen": 3194355712
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 3.4431137724550896e-05,
+      "loss": 2.4476,
+      "theoretical_loss": 3.3048631861728834,
+      "tokens_seen": 3194486784
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 3.4388366124893075e-05,
+      "loss": 2.3744,
+      "theoretical_loss": 3.304852893033501,
+      "tokens_seen": 3194617856
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 3.4345594525235246e-05,
+      "loss": 2.5922,
+      "theoretical_loss": 3.3048426004346707,
+      "tokens_seen": 3194748928
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 3.430282292557742e-05,
+      "loss": 2.5774,
+      "theoretical_loss": 3.3048323083763416,
+      "tokens_seen": 3194880000
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 3.426005132591959e-05,
+      "loss": 2.3775,
+      "theoretical_loss": 3.304822016858463,
+      "tokens_seen": 3195011072
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 3.421727972626176e-05,
+      "loss": 2.42,
+      "theoretical_loss": 3.304811725880985,
+      "tokens_seen": 3195142144
+    },
+    {
+      "epoch": 0.66,
+      "objective/train/docs_used": 1749977,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 2.7795517444610596,
+      "objective/train/theoretical_loss": 3.3048065805948803,
+      "objective/train/tokens_used": 225258976,
+      "theoretical_loss": 3.3048065805948803,
+      "tokens_seen": 3195207680
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 3.417450812660394e-05,
+      "loss": 2.5814,
+      "theoretical_loss": 3.304801435443857,
+      "tokens_seen": 3195273216
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 3.413173652694611e-05,
+      "loss": 2.5834,
+      "theoretical_loss": 3.3047911455470276,
+      "tokens_seen": 3195404288
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 3.408896492728828e-05,
+      "loss": 2.5808,
+      "theoretical_loss": 3.3047808561904475,
+      "tokens_seen": 3195535360
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 3.4046193327630454e-05,
+      "loss": 2.607,
+      "theoretical_loss": 3.304770567374065,
+      "tokens_seen": 3195666432
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 3.400342172797263e-05,
+      "loss": 2.4575,
+      "theoretical_loss": 3.3047602790978305,
+      "tokens_seen": 3195797504
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 3.3960650128314803e-05,
+      "loss": 2.4267,
+      "theoretical_loss": 3.304749991361693,
+      "tokens_seen": 3195928576
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 3.391787852865697e-05,
+      "loss": 2.4974,
+      "theoretical_loss": 3.304739704165602,
+      "tokens_seen": 3196059648
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 3.3875106928999147e-05,
+      "loss": 2.5884,
+      "theoretical_loss": 3.304729417509507,
+      "tokens_seen": 3196190720
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 3.383233532934132e-05,
+      "loss": 2.7342,
+      "theoretical_loss": 3.304719131393358,
+      "tokens_seen": 3196321792
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 3.3789563729683496e-05,
+      "loss": 2.4659,
+      "theoretical_loss": 3.304708845817104,
+      "tokens_seen": 3196452864
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 3.374679213002567e-05,
+      "loss": 2.4338,
+      "theoretical_loss": 3.304698560780695,
+      "tokens_seen": 3196583936
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 3.370402053036783e-05,
+      "loss": 2.4709,
+      "theoretical_loss": 3.3046882762840797,
+      "tokens_seen": 3196715008
+    },
+    {
+      "epoch": 0.67,
+      "objective/train/docs_used": 1750966,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 2.7182776927948,
+      "objective/train/theoretical_loss": 3.3046779923272083,
+      "objective/train/tokens_used": 226897376,
+      "theoretical_loss": 3.3046779923272083,
+      "tokens_seen": 3196846080
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 3.366124893071001e-05,
+      "loss": 2.5408,
+      "theoretical_loss": 3.3046779923272083,
+      "tokens_seen": 3196846080
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 3.361847733105218e-05,
+      "loss": 2.4637,
+      "theoretical_loss": 3.30466770891003,
+      "tokens_seen": 3196977152
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 3.357570573139436e-05,
+      "loss": 2.5451,
+      "theoretical_loss": 3.304657426032495,
+      "tokens_seen": 3197108224
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 3.3532934131736525e-05,
+      "loss": 2.5917,
+      "theoretical_loss": 3.304647143694552,
+      "tokens_seen": 3197239296
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 3.3490162532078704e-05,
+      "loss": 2.449,
+      "theoretical_loss": 3.3046368618961512,
+      "tokens_seen": 3197370368
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 3.3447390932420875e-05,
+      "loss": 2.6124,
+      "theoretical_loss": 3.3046265806372417,
+      "tokens_seen": 3197501440
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 3.340461933276305e-05,
+      "loss": 2.4882,
+      "theoretical_loss": 3.3046162999177735,
+      "tokens_seen": 3197632512
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 3.336184773310522e-05,
+      "loss": 2.5891,
+      "theoretical_loss": 3.304606019737696,
+      "tokens_seen": 3197763584
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 3.331907613344739e-05,
+      "loss": 2.5998,
+      "theoretical_loss": 3.304595740096959,
+      "tokens_seen": 3197894656
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 3.327630453378957e-05,
+      "loss": 2.5076,
+      "theoretical_loss": 3.3045854609955114,
+      "tokens_seen": 3198025728
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 3.323353293413174e-05,
+      "loss": 2.5564,
+      "theoretical_loss": 3.304575182433304,
+      "tokens_seen": 3198156800
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 3.319076133447391e-05,
+      "loss": 2.5137,
+      "theoretical_loss": 3.3045649044102854,
+      "tokens_seen": 3198287872
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 3.314798973481608e-05,
+      "loss": 2.6105,
+      "theoretical_loss": 3.3045546269264054,
+      "tokens_seen": 3198418944
+    },
+    {
+      "epoch": 0.67,
+      "objective/train/docs_used": 1752106,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 2.8310353755950928,
+      "objective/train/theoretical_loss": 3.3045494883866264,
+      "objective/train/tokens_used": 228535776,
+      "theoretical_loss": 3.3045494883866264,
+      "tokens_seen": 3198484480
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 3.3105218135158254e-05,
+      "loss": 2.5698,
+      "theoretical_loss": 3.304544349981614,
+      "tokens_seen": 3198550016
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 3.306244653550043e-05,
+      "loss": 2.4674,
+      "theoretical_loss": 3.3045340735758604,
+      "tokens_seen": 3198681088
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 3.3019674935842604e-05,
+      "loss": 2.6941,
+      "theoretical_loss": 3.304523797709094,
+      "tokens_seen": 3198812160
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 3.2976903336184776e-05,
+      "loss": 2.5911,
+      "theoretical_loss": 3.3045135223812654,
+      "tokens_seen": 3198943232
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 3.293413173652695e-05,
+      "loss": 2.5483,
+      "theoretical_loss": 3.304503247592324,
+      "tokens_seen": 3199074304
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 3.289136013686912e-05,
+      "loss": 2.5965,
+      "theoretical_loss": 3.304492973342219,
+      "tokens_seen": 3199205376
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 3.28485885372113e-05,
+      "loss": 2.5635,
+      "theoretical_loss": 3.3044826996309,
+      "tokens_seen": 3199336448
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 3.280581693755347e-05,
+      "loss": 2.4813,
+      "theoretical_loss": 3.304472426458317,
+      "tokens_seen": 3199467520
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 3.276304533789564e-05,
+      "loss": 2.4929,
+      "theoretical_loss": 3.30446215382442,
+      "tokens_seen": 3199598592
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 3.272027373823781e-05,
+      "loss": 2.4621,
+      "theoretical_loss": 3.304451881729158,
+      "tokens_seen": 3199729664
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 3.267750213857998e-05,
+      "loss": 2.4748,
+      "theoretical_loss": 3.304441610172481,
+      "tokens_seen": 3199860736
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 3.263473053892216e-05,
+      "loss": 2.6285,
+      "theoretical_loss": 3.304431339154339,
+      "tokens_seen": 3199991808
+    },
+    {
+      "epoch": 0.68,
+      "objective/train/docs_used": 1752559,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 3.019050359725952,
+      "objective/train/theoretical_loss": 3.304421068674681,
+      "objective/train/tokens_used": 230174176,
+      "theoretical_loss": 3.304421068674681,
+      "tokens_seen": 3200122880
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 3.2591958939264326e-05,
+      "loss": 2.4811,
+      "theoretical_loss": 3.304421068674681,
+      "tokens_seen": 3200122880
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 3.2549187339606504e-05,
+      "loss": 2.4264,
+      "theoretical_loss": 3.3044107987334574,
+      "tokens_seen": 3200253952
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 3.2506415739948676e-05,
+      "loss": 2.5031,
+      "theoretical_loss": 3.304400529330618,
+      "tokens_seen": 3200385024
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 3.246364414029085e-05,
+      "loss": 2.559,
+      "theoretical_loss": 3.3043902604661115,
+      "tokens_seen": 3200516096
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 3.2420872540633026e-05,
+      "loss": 2.4459,
+      "theoretical_loss": 3.3043799921398884,
+      "tokens_seen": 3200647168
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 3.237810094097519e-05,
+      "loss": 2.5738,
+      "theoretical_loss": 3.304369724351899,
+      "tokens_seen": 3200778240
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 3.233532934131737e-05,
+      "loss": 2.4598,
+      "theoretical_loss": 3.3043594571020916,
+      "tokens_seen": 3200909312
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 3.229255774165954e-05,
+      "loss": 2.5795,
+      "theoretical_loss": 3.3043491903904174,
+      "tokens_seen": 3201040384
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 3.224978614200171e-05,
+      "loss": 2.6696,
+      "theoretical_loss": 3.3043389242168253,
+      "tokens_seen": 3201171456
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 3.220701454234388e-05,
+      "loss": 2.4077,
+      "theoretical_loss": 3.304328658581265,
+      "tokens_seen": 3201302528
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 3.2164242942686055e-05,
+      "loss": 2.4908,
+      "theoretical_loss": 3.304318393483687,
+      "tokens_seen": 3201433600
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 3.212147134302823e-05,
+      "loss": 2.4689,
+      "theoretical_loss": 3.30430812892404,
+      "tokens_seen": 3201564672
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 3.2078699743370405e-05,
+      "loss": 2.4558,
+      "theoretical_loss": 3.304297864902275,
+      "tokens_seen": 3201695744
+    },
+    {
+      "epoch": 0.68,
+      "objective/train/docs_used": 1753714,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 2.8162074089050293,
+      "objective/train/theoretical_loss": 3.3042927330930825,
+      "objective/train/tokens_used": 231812576,
+      "theoretical_loss": 3.3042927330930825,
+      "tokens_seen": 3201761280
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 3.2035928143712576e-05,
+      "loss": 2.4873,
+      "theoretical_loss": 3.3042876014183413,
+      "tokens_seen": 3201826816
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 3.199315654405475e-05,
+      "loss": 2.3903,
+      "theoretical_loss": 3.304277338472189,
+      "tokens_seen": 3201957888
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 3.195038494439692e-05,
+      "loss": 2.3911,
+      "theoretical_loss": 3.3042670760637667,
+      "tokens_seen": 3202088960
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 3.19076133447391e-05,
+      "loss": 2.5844,
+      "theoretical_loss": 3.3042568141930255,
+      "tokens_seen": 3202220032
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 3.186484174508127e-05,
+      "loss": 2.5279,
+      "theoretical_loss": 3.3042465528599148,
+      "tokens_seen": 3202351104
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 3.182207014542344e-05,
+      "loss": 2.3653,
+      "theoretical_loss": 3.304236292064384,
+      "tokens_seen": 3202482176
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 3.177929854576561e-05,
+      "loss": 2.5646,
+      "theoretical_loss": 3.304226031806384,
+      "tokens_seen": 3202613248
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 3.1736526946107784e-05,
+      "loss": 2.3691,
+      "theoretical_loss": 3.304215772085864,
+      "tokens_seen": 3202744320
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 3.169375534644996e-05,
+      "loss": 2.3526,
+      "theoretical_loss": 3.304205512902773,
+      "tokens_seen": 3202875392
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 3.1650983746792127e-05,
+      "loss": 2.4674,
+      "theoretical_loss": 3.3041952542570625,
+      "tokens_seen": 3203006464
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 3.1608212147134305e-05,
+      "loss": 2.6753,
+      "theoretical_loss": 3.304184996148681,
+      "tokens_seen": 3203137536
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 3.1565440547476476e-05,
+      "loss": 2.4466,
+      "theoretical_loss": 3.3041747385775793,
+      "tokens_seen": 3203268608
+    },
+    {
+      "epoch": 0.69,
+      "objective/train/docs_used": 1754313,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 2.8113200664520264,
+      "objective/train/theoretical_loss": 3.304164481543707,
+      "objective/train/tokens_used": 233450976,
+      "theoretical_loss": 3.304164481543707,
+      "tokens_seen": 3203399680
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 3.152266894781865e-05,
+      "loss": 2.624,
+      "theoretical_loss": 3.304164481543707,
+      "tokens_seen": 3203399680
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 3.1479897348160826e-05,
+      "loss": 2.4266,
+      "theoretical_loss": 3.3041542250470135,
+      "tokens_seen": 3203530752
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 3.143712574850299e-05,
+      "loss": 2.4957,
+      "theoretical_loss": 3.3041439690874492,
+      "tokens_seen": 3203661824
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 3.139435414884517e-05,
+      "loss": 2.4742,
+      "theoretical_loss": 3.3041337136649642,
+      "tokens_seen": 3203792896
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 3.135158254918734e-05,
+      "loss": 2.5775,
+      "theoretical_loss": 3.3041234587795083,
+      "tokens_seen": 3203923968
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 3.130881094952951e-05,
+      "loss": 2.4736,
+      "theoretical_loss": 3.3041132044310304,
+      "tokens_seen": 3204055040
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 3.1266039349871684e-05,
+      "loss": 2.3845,
+      "theoretical_loss": 3.3041029506194817,
+      "tokens_seen": 3204186112
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 3.1223267750213855e-05,
+      "loss": 2.4658,
+      "theoretical_loss": 3.3040926973448115,
+      "tokens_seen": 3204317184
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 3.1180496150556034e-05,
+      "loss": 2.4869,
+      "theoretical_loss": 3.30408244460697,
+      "tokens_seen": 3204448256
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 3.1137724550898205e-05,
+      "loss": 2.6231,
+      "theoretical_loss": 3.304072192405907,
+      "tokens_seen": 3204579328
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 3.1094952951240383e-05,
+      "loss": 2.5349,
+      "theoretical_loss": 3.3040619407415726,
+      "tokens_seen": 3204710400
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 3.105218135158255e-05,
+      "loss": 2.498,
+      "theoretical_loss": 3.304051689613916,
+      "tokens_seen": 3204841472
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 3.100940975192472e-05,
+      "loss": 2.3935,
+      "theoretical_loss": 3.304041439022888,
+      "tokens_seen": 3204972544
+    },
+    {
+      "epoch": 0.69,
+      "objective/train/docs_used": 1755636,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 2.230454206466675,
+      "objective/train/theoretical_loss": 3.304036313928594,
+      "objective/train/tokens_used": 235089376,
+      "theoretical_loss": 3.304036313928594,
+      "tokens_seen": 3205038080
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 3.09666381522669e-05,
+      "loss": 2.5012,
+      "theoretical_loss": 3.3040311889684384,
+      "tokens_seen": 3205103616
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 3.092386655260907e-05,
+      "loss": 2.5347,
+      "theoretical_loss": 3.3040209394505173,
+      "tokens_seen": 3205234688
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 3.088109495295124e-05,
+      "loss": 2.2631,
+      "theoretical_loss": 3.3040106904690747,
+      "tokens_seen": 3205365760
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 3.083832335329341e-05,
+      "loss": 2.4597,
+      "theoretical_loss": 3.30400044202406,
+      "tokens_seen": 3205496832
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 3.079555175363559e-05,
+      "loss": 2.3887,
+      "theoretical_loss": 3.303990194115423,
+      "tokens_seen": 3205627904
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 3.075278015397776e-05,
+      "loss": 2.4304,
+      "theoretical_loss": 3.303979946743115,
+      "tokens_seen": 3205758976
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 3.0710008554319934e-05,
+      "loss": 2.4462,
+      "theoretical_loss": 3.303969699907085,
+      "tokens_seen": 3205890048
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 3.0667236954662105e-05,
+      "loss": 2.5555,
+      "theoretical_loss": 3.303959453607283,
+      "tokens_seen": 3206021120
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 3.062446535500428e-05,
+      "loss": 2.4496,
+      "theoretical_loss": 3.30394920784366,
+      "tokens_seen": 3206152192
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 3.0581693755346455e-05,
+      "loss": 2.3901,
+      "theoretical_loss": 3.3039389626161646,
+      "tokens_seen": 3206283264
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 3.053892215568863e-05,
+      "loss": 2.5053,
+      "theoretical_loss": 3.3039287179247476,
+      "tokens_seen": 3206414336
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 3.0496150556030795e-05,
+      "loss": 2.4153,
+      "theoretical_loss": 3.303918473769359,
+      "tokens_seen": 3206545408
+    },
+    {
+      "epoch": 0.7,
+      "objective/train/docs_used": 1756084,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 2.25111985206604,
+      "objective/train/theoretical_loss": 3.303908230149949,
+      "objective/train/tokens_used": 236727776,
+      "theoretical_loss": 3.303908230149949,
+      "tokens_seen": 3206676480
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 3.045337895637297e-05,
+      "loss": 2.5711,
+      "theoretical_loss": 3.303908230149949,
+      "tokens_seen": 3206676480
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 3.041060735671514e-05,
+      "loss": 2.4415,
+      "theoretical_loss": 3.3038979870664678,
+      "tokens_seen": 3206807552
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 3.0367835757057316e-05,
+      "loss": 2.5201,
+      "theoretical_loss": 3.303887744518865,
+      "tokens_seen": 3206938624
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 3.032506415739949e-05,
+      "loss": 2.4766,
+      "theoretical_loss": 3.30387750250709,
+      "tokens_seen": 3207069696
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 3.028229255774166e-05,
+      "loss": 2.413,
+      "theoretical_loss": 3.303867261031094,
+      "tokens_seen": 3207200768
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 3.0239520958083834e-05,
+      "loss": 2.5375,
+      "theoretical_loss": 3.303857020090827,
+      "tokens_seen": 3207331840
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 3.019674935842601e-05,
+      "loss": 2.5067,
+      "theoretical_loss": 3.3038467796862387,
+      "tokens_seen": 3207462912
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 3.015397775876818e-05,
+      "loss": 2.4761,
+      "theoretical_loss": 3.3038365398172793,
+      "tokens_seen": 3207593984
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 3.011120615911035e-05,
+      "loss": 2.5521,
+      "theoretical_loss": 3.303826300483899,
+      "tokens_seen": 3207725056
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 3.0068434559452524e-05,
+      "loss": 2.4859,
+      "theoretical_loss": 3.3038160616860477,
+      "tokens_seen": 3207856128
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 3.00256629597947e-05,
+      "loss": 2.4261,
+      "theoretical_loss": 3.3038058234236756,
+      "tokens_seen": 3207987200
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 2.9982891360136873e-05,
+      "loss": 2.5167,
+      "theoretical_loss": 3.303795585696733,
+      "tokens_seen": 3208118272
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 2.994011976047904e-05,
+      "loss": 2.4994,
+      "theoretical_loss": 3.30378534850517,
+      "tokens_seen": 3208249344
+    },
+    {
+      "epoch": 0.7,
+      "objective/train/docs_used": 1757266,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 2.5266647338867188,
+      "objective/train/theoretical_loss": 3.30378023011014,
+      "objective/train/tokens_used": 238366176,
+      "theoretical_loss": 3.30378023011014,
+      "tokens_seen": 3208314880
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 2.9897348160821213e-05,
+      "loss": 2.4878,
+      "theoretical_loss": 3.3037751118489362,
+      "tokens_seen": 3208380416
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 2.9854576561163388e-05,
+      "loss": 2.599,
+      "theoretical_loss": 3.3037648757279827,
+      "tokens_seen": 3208511488
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 2.9811804961505563e-05,
+      "loss": 2.4182,
+      "theoretical_loss": 3.3037546401422584,
+      "tokens_seen": 3208642560
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 2.9769033361847738e-05,
+      "loss": 2.3344,
+      "theoretical_loss": 3.3037444050917144,
+      "tokens_seen": 3208773632
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 2.9726261762189906e-05,
+      "loss": 2.4827,
+      "theoretical_loss": 3.303734170576301,
+      "tokens_seen": 3208904704
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 2.968349016253208e-05,
+      "loss": 2.6349,
+      "theoretical_loss": 3.3037239365959676,
+      "tokens_seen": 3209035776
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 2.9640718562874252e-05,
+      "loss": 2.532,
+      "theoretical_loss": 3.3037137031506645,
+      "tokens_seen": 3209166848
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 2.9597946963216427e-05,
+      "loss": 2.4858,
+      "theoretical_loss": 3.3037034702403423,
+      "tokens_seen": 3209297920
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 2.9555175363558595e-05,
+      "loss": 2.4419,
+      "theoretical_loss": 3.303693237864951,
+      "tokens_seen": 3209428992
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 2.951240376390077e-05,
+      "loss": 2.4977,
+      "theoretical_loss": 3.303683006024441,
+      "tokens_seen": 3209560064
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 2.9469632164242945e-05,
+      "loss": 2.5561,
+      "theoretical_loss": 3.3036727747187618,
+      "tokens_seen": 3209691136
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 2.9426860564585117e-05,
+      "loss": 2.5139,
+      "theoretical_loss": 3.303662543947864,
+      "tokens_seen": 3209822208
+    },
+    {
+      "epoch": 0.71,
+      "objective/train/docs_used": 1757623,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 3.1107144355773926,
+      "objective/train/theoretical_loss": 3.303652313711699,
+      "objective/train/tokens_used": 240004576,
+      "theoretical_loss": 3.303652313711699,
+      "tokens_seen": 3209953280
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 2.938408896492729e-05,
+      "loss": 2.6032,
+      "theoretical_loss": 3.303652313711699,
+      "tokens_seen": 3209953280
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 2.934131736526946e-05,
+      "loss": 2.513,
+      "theoretical_loss": 3.3036420840102148,
+      "tokens_seen": 3210084352
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 2.9298545765611635e-05,
+      "loss": 2.5001,
+      "theoretical_loss": 3.3036318548433634,
+      "tokens_seen": 3210215424
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 2.925577416595381e-05,
+      "loss": 2.3906,
+      "theoretical_loss": 3.303621626211094,
+      "tokens_seen": 3210346496
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 2.921300256629598e-05,
+      "loss": 2.4056,
+      "theoretical_loss": 3.303611398113357,
+      "tokens_seen": 3210477568
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 2.9170230966638153e-05,
+      "loss": 2.643,
+      "theoretical_loss": 3.3036011705501034,
+      "tokens_seen": 3210608640
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 2.9127459366980324e-05,
+      "loss": 2.5167,
+      "theoretical_loss": 3.3035909435212827,
+      "tokens_seen": 3210739712
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 2.90846877673225e-05,
+      "loss": 2.5026,
+      "theoretical_loss": 3.303580717026845,
+      "tokens_seen": 3210870784
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 2.9041916167664674e-05,
+      "loss": 2.5411,
+      "theoretical_loss": 3.3035704910667416,
+      "tokens_seen": 3211001856
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 2.899914456800685e-05,
+      "loss": 2.3603,
+      "theoretical_loss": 3.3035602656409218,
+      "tokens_seen": 3211132928
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 2.8956372968349017e-05,
+      "loss": 2.6508,
+      "theoretical_loss": 3.303550040749336,
+      "tokens_seen": 3211264000
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 2.891360136869119e-05,
+      "loss": 2.509,
+      "theoretical_loss": 3.3035398163919347,
+      "tokens_seen": 3211395072
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 2.8870829769033363e-05,
+      "loss": 2.6896,
+      "theoretical_loss": 3.303529592568668,
+      "tokens_seen": 3211526144
+    },
+    {
+      "epoch": 0.71,
+      "objective/train/docs_used": 1758775,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 2.5142648220062256,
+      "objective/train/theoretical_loss": 3.3035244808573196,
+      "objective/train/tokens_used": 241642976,
+      "theoretical_loss": 3.3035244808573196,
+      "tokens_seen": 3211591680
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 2.882805816937554e-05,
+      "loss": 2.5199,
+      "theoretical_loss": 3.3035193692794866,
+      "tokens_seen": 3211657216
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 2.8785286569717706e-05,
+      "loss": 2.422,
+      "theoretical_loss": 3.3035091465243407,
+      "tokens_seen": 3211788288
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 2.874251497005988e-05,
+      "loss": 2.4424,
+      "theoretical_loss": 3.3034989243031805,
+      "tokens_seen": 3211919360
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 2.8699743370402056e-05,
+      "loss": 2.3157,
+      "theoretical_loss": 3.303488702615956,
+      "tokens_seen": 3212050432
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 2.8656971770744228e-05,
+      "loss": 2.4144,
+      "theoretical_loss": 3.303478481462618,
+      "tokens_seen": 3212181504
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 2.8614200171086403e-05,
+      "loss": 2.5694,
+      "theoretical_loss": 3.3034682608431165,
+      "tokens_seen": 3212312576
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 2.857142857142857e-05,
+      "loss": 2.3805,
+      "theoretical_loss": 3.303458040757402,
+      "tokens_seen": 3212443648
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 2.8528656971770746e-05,
+      "loss": 2.5355,
+      "theoretical_loss": 3.303447821205425,
+      "tokens_seen": 3212574720
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 2.848588537211292e-05,
+      "loss": 2.3711,
+      "theoretical_loss": 3.3034376021871354,
+      "tokens_seen": 3212705792
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 2.8443113772455092e-05,
+      "loss": 2.4546,
+      "theoretical_loss": 3.303427383702484,
+      "tokens_seen": 3212836864
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 2.840034217279726e-05,
+      "loss": 2.3728,
+      "theoretical_loss": 3.303417165751421,
+      "tokens_seen": 3212967936
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 2.8357570573139435e-05,
+      "loss": 2.5899,
+      "theoretical_loss": 3.3034069483338966,
+      "tokens_seen": 3213099008
+    },
+    {
+      "epoch": 0.72,
+      "objective/train/docs_used": 1759176,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 2.225017547607422,
+      "objective/train/theoretical_loss": 3.303396731449862,
+      "objective/train/tokens_used": 243281376,
+      "theoretical_loss": 3.303396731449862,
+      "tokens_seen": 3213230080
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 2.831479897348161e-05,
+      "loss": 2.4874,
+      "theoretical_loss": 3.303396731449862,
+      "tokens_seen": 3213230080
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 2.8272027373823785e-05,
+      "loss": 2.3283,
+      "theoretical_loss": 3.303386515099266,
+      "tokens_seen": 3213361152
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 2.8229255774165953e-05,
+      "loss": 2.5505,
+      "theoretical_loss": 3.3033762992820606,
+      "tokens_seen": 3213492224
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 2.8186484174508128e-05,
+      "loss": 2.5643,
+      "theoretical_loss": 3.303366083998195,
+      "tokens_seen": 3213623296
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 2.81437125748503e-05,
+      "loss": 2.512,
+      "theoretical_loss": 3.3033558692476204,
+      "tokens_seen": 3213754368
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 2.8100940975192475e-05,
+      "loss": 2.5855,
+      "theoretical_loss": 3.303345655030287,
+      "tokens_seen": 3213885440
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 2.805816937553465e-05,
+      "loss": 2.5781,
+      "theoretical_loss": 3.303335441346145,
+      "tokens_seen": 3214016512
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 2.8015397775876818e-05,
+      "loss": 2.4615,
+      "theoretical_loss": 3.3033252281951455,
+      "tokens_seen": 3214147584
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 2.7972626176218992e-05,
+      "loss": 2.4135,
+      "theoretical_loss": 3.3033150155772377,
+      "tokens_seen": 3214278656
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 2.7929854576561164e-05,
+      "loss": 2.5158,
+      "theoretical_loss": 3.303304803492373,
+      "tokens_seen": 3214409728
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 2.788708297690334e-05,
+      "loss": 2.5429,
+      "theoretical_loss": 3.3032945919405017,
+      "tokens_seen": 3214540800
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 2.7844311377245507e-05,
+      "loss": 2.5166,
+      "theoretical_loss": 3.303284380921574,
+      "tokens_seen": 3214671872
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 2.7801539777587682e-05,
+      "loss": 2.5323,
+      "theoretical_loss": 3.303274170435541,
+      "tokens_seen": 3214802944
+    },
+    {
+      "epoch": 0.72,
+      "objective/train/docs_used": 1760371,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 2.3985843658447266,
+      "objective/train/theoretical_loss": 3.3032690653923438,
+      "objective/train/tokens_used": 244919776,
+      "theoretical_loss": 3.3032690653923438,
+      "tokens_seen": 3214868480
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 2.7758768177929857e-05,
+      "loss": 2.3832,
+      "theoretical_loss": 3.3032639604823517,
+      "tokens_seen": 3214934016
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 2.771599657827203e-05,
+      "loss": 2.3234,
+      "theoretical_loss": 3.303253751061958,
+      "tokens_seen": 3215065088
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 2.7673224978614203e-05,
+      "loss": 2.4125,
+      "theoretical_loss": 3.3032435421743105,
+      "tokens_seen": 3215196160
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 2.763045337895637e-05,
+      "loss": 2.4612,
+      "theoretical_loss": 3.3032333338193585,
+      "tokens_seen": 3215327232
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 2.7587681779298546e-05,
+      "loss": 2.5315,
+      "theoretical_loss": 3.3032231259970533,
+      "tokens_seen": 3215458304
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 2.754491017964072e-05,
+      "loss": 2.5663,
+      "theoretical_loss": 3.303212918707345,
+      "tokens_seen": 3215589376
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 2.7502138579982896e-05,
+      "loss": 2.4922,
+      "theoretical_loss": 3.3032027119501843,
+      "tokens_seen": 3215720448
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 2.7459366980325064e-05,
+      "loss": 2.5179,
+      "theoretical_loss": 3.303192505725522,
+      "tokens_seen": 3215851520
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 2.7416595380667236e-05,
+      "loss": 2.4059,
+      "theoretical_loss": 3.303182300033308,
+      "tokens_seen": 3215982592
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 2.737382378100941e-05,
+      "loss": 2.4359,
+      "theoretical_loss": 3.303172094873493,
+      "tokens_seen": 3216113664
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 2.7331052181351586e-05,
+      "loss": 2.4875,
+      "theoretical_loss": 3.3031618902460282,
+      "tokens_seen": 3216244736
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 2.728828058169376e-05,
+      "loss": 2.5865,
+      "theoretical_loss": 3.303151686150863,
+      "tokens_seen": 3216375808
+    },
+    {
+      "epoch": 0.73,
+      "objective/train/docs_used": 1760932,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 2.7095277309417725,
+      "objective/train/theoretical_loss": 3.303141482587949,
+      "objective/train/tokens_used": 246558176,
+      "theoretical_loss": 3.303141482587949,
+      "tokens_seen": 3216506880
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 2.724550898203593e-05,
+      "loss": 2.5145,
+      "theoretical_loss": 3.303141482587949,
+      "tokens_seen": 3216506880
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 2.72027373823781e-05,
+      "loss": 2.541,
+      "theoretical_loss": 3.303131279557236,
+      "tokens_seen": 3216637952
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 2.7159965782720275e-05,
+      "loss": 2.4856,
+      "theoretical_loss": 3.303121077058675,
+      "tokens_seen": 3216769024
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 2.711719418306245e-05,
+      "loss": 2.6048,
+      "theoretical_loss": 3.3031108750922167,
+      "tokens_seen": 3216900096
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 2.7074422583404618e-05,
+      "loss": 2.371,
+      "theoretical_loss": 3.303100673657811,
+      "tokens_seen": 3217031168
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 2.7031650983746793e-05,
+      "loss": 2.5231,
+      "theoretical_loss": 3.303090472755409,
+      "tokens_seen": 3217162240
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 2.6988879384088968e-05,
+      "loss": 2.5842,
+      "theoretical_loss": 3.3030802723849613,
+      "tokens_seen": 3217293312
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 2.694610778443114e-05,
+      "loss": 2.3844,
+      "theoretical_loss": 3.3030700725464177,
+      "tokens_seen": 3217424384
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 2.6903336184773314e-05,
+      "loss": 2.5238,
+      "theoretical_loss": 3.30305987323973,
+      "tokens_seen": 3217555456
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 2.6860564585115482e-05,
+      "loss": 2.4149,
+      "theoretical_loss": 3.303049674464848,
+      "tokens_seen": 3217686528
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 2.6817792985457657e-05,
+      "loss": 2.4585,
+      "theoretical_loss": 3.303039476221723,
+      "tokens_seen": 3217817600
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 2.6775021385799832e-05,
+      "loss": 2.5258,
+      "theoretical_loss": 3.303029278510305,
+      "tokens_seen": 3217948672
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 2.6732249786142004e-05,
+      "loss": 2.3461,
+      "theoretical_loss": 3.3030190813305444,
+      "tokens_seen": 3218079744
+    },
+    {
+      "epoch": 0.74,
+      "objective/train/docs_used": 1762106,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 2.7149224281311035,
+      "objective/train/theoretical_loss": 3.3030139829400205,
+      "objective/train/tokens_used": 248196576,
+      "theoretical_loss": 3.3030139829400205,
+      "tokens_seen": 3218145280
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 2.6689478186484172e-05,
+      "loss": 2.5382,
+      "theoretical_loss": 3.3030088846823924,
+      "tokens_seen": 3218210816
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 2.6646706586826347e-05,
+      "loss": 2.5688,
+      "theoretical_loss": 3.3029986885657996,
+      "tokens_seen": 3218341888
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 2.6603934987168522e-05,
+      "loss": 2.581,
+      "theoretical_loss": 3.3029884929807163,
+      "tokens_seen": 3218472960
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 2.6561163387510697e-05,
+      "loss": 2.3666,
+      "theoretical_loss": 3.3029782979270936,
+      "tokens_seen": 3218604032
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 2.6518391787852865e-05,
+      "loss": 2.3376,
+      "theoretical_loss": 3.302968103404882,
+      "tokens_seen": 3218735104
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 2.647562018819504e-05,
+      "loss": 2.5427,
+      "theoretical_loss": 3.302957909414032,
+      "tokens_seen": 3218866176
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 2.643284858853721e-05,
+      "loss": 2.5189,
+      "theoretical_loss": 3.3029477159544935,
+      "tokens_seen": 3218997248
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 2.6390076988879386e-05,
+      "loss": 2.3899,
+      "theoretical_loss": 3.302937523026219,
+      "tokens_seen": 3219128320
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 2.634730538922156e-05,
+      "loss": 2.3719,
+      "theoretical_loss": 3.3029273306291578,
+      "tokens_seen": 3219259392
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 2.630453378956373e-05,
+      "loss": 2.4855,
+      "theoretical_loss": 3.302917138763261,
+      "tokens_seen": 3219390464
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 2.6261762189905904e-05,
+      "loss": 2.4274,
+      "theoretical_loss": 3.3029069474284793,
+      "tokens_seen": 3219521536
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 2.6218990590248076e-05,
+      "loss": 2.3962,
+      "theoretical_loss": 3.3028967566247633,
+      "tokens_seen": 3219652608
+    },
+    {
+      "epoch": 0.74,
+      "objective/train/docs_used": 1762646,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/original_loss": 2.652979850769043,
+      "objective/train/theoretical_loss": 3.302886566352064,
+      "objective/train/tokens_used": 249834976,
+      "theoretical_loss": 3.302886566352064,
+      "tokens_seen": 3219783680
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 2.617621899059025e-05,
+      "loss": 2.407,
+      "theoretical_loss": 3.302886566352064,
+      "tokens_seen": 3219783680
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 2.613344739093242e-05,
+      "loss": 2.5723,
+      "theoretical_loss": 3.3028763766103317,
+      "tokens_seen": 3219914752
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 2.6090675791274594e-05,
+      "loss": 2.47,
+      "theoretical_loss": 3.3028661873995175,
+      "tokens_seen": 3220045824
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 2.604790419161677e-05,
+      "loss": 2.5141,
+      "theoretical_loss": 3.302855998719572,
+      "tokens_seen": 3220176896
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 2.6005132591958943e-05,
+      "loss": 2.5068,
+      "theoretical_loss": 3.3028458105704455,
+      "tokens_seen": 3220307968
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 2.5962360992301115e-05,
+      "loss": 2.4223,
+      "theoretical_loss": 3.3028356229520894,
+      "tokens_seen": 3220439040
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 2.5919589392643283e-05,
+      "loss": 2.4377,
+      "theoretical_loss": 3.302825435864454,
+      "tokens_seen": 3220570112
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 2.5876817792985458e-05,
+      "loss": 2.5049,
+      "theoretical_loss": 3.3028152493074905,
+      "tokens_seen": 3220701184
+    }
+  ],
+  "max_steps": 2362,
+  "num_train_epochs": 9223372036854775807,
+  "total_flos": 1.17527065657344e+17,
+  "trial_name": null,
+  "trial_params": null
+}