diff --git "a/checkpoint-25177/trainer_state.json" "b/checkpoint-25177/trainer_state.json" new file mode 100644--- /dev/null +++ "b/checkpoint-25177/trainer_state.json" @@ -0,0 +1,187422 @@ +{ + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 0.5, + "global_step": 25177, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "epoch": 0.0, + "learning_rate": 9.92063492063492e-07, + "loss": 10.7939, + "theoretical_loss": 20.81281780154715, + "tokens_seen": 65536 + }, + { + "epoch": 0.0, + "learning_rate": 1.984126984126984e-06, + "loss": 10.8229, + "theoretical_loss": 17.566201104328645, + "tokens_seen": 131072 + }, + { + "epoch": 0.0, + "learning_rate": 2.9761904761904763e-06, + "loss": 10.7533, + "theoretical_loss": 15.939477092836569, + "tokens_seen": 196608 + }, + { + "epoch": 0.0, + "learning_rate": 3.968253968253968e-06, + "loss": 10.4745, + "theoretical_loss": 14.89231675598857, + "tokens_seen": 262144 + }, + { + "epoch": 0.0, + "learning_rate": 4.96031746031746e-06, + "loss": 10.1845, + "theoretical_loss": 14.136216937762974, + "tokens_seen": 327680 + }, + { + "epoch": 0.0, + "learning_rate": 5.9523809523809525e-06, + "loss": 10.2018, + "theoretical_loss": 13.552561472550224, + "tokens_seen": 393216 + }, + { + "epoch": 0.0, + "learning_rate": 6.944444444444444e-06, + "loss": 9.9875, + "theoretical_loss": 13.08180900140119, + "tokens_seen": 458752 + }, + { + "epoch": 0.0, + "learning_rate": 7.936507936507936e-06, + "loss": 9.6161, + "theoretical_loss": 12.690129625483323, + "tokens_seen": 524288 + }, + { + "epoch": 0.0, + "learning_rate": 8.928571428571428e-06, + "loss": 9.6326, + "theoretical_loss": 12.356592463873625, + "tokens_seen": 589824 + }, + { + "epoch": 0.0, + "learning_rate": 9.92063492063492e-06, + "loss": 9.6135, + "theoretical_loss": 12.067412607035077, + "tokens_seen": 655360 + }, + { + "epoch": 0.0, + "learning_rate": 1.0912698412698412e-05, + "loss": 9.5658, + "theoretical_loss": 11.813066231101676, + "tokens_seen": 720896 + }, + { + "epoch": 0.0, + "learning_rate": 1.1904761904761905e-05, + "loss": 9.3743, + "theoretical_loss": 11.586719208706729, + "tokens_seen": 786432 + }, + { + "epoch": 0.0, + "learning_rate": 1.2896825396825396e-05, + "loss": 9.122, + "theoretical_loss": 11.383314140186787, + "tokens_seen": 851968 + }, + { + "epoch": 0.0, + "learning_rate": 1.3888888888888888e-05, + "loss": 9.1061, + "theoretical_loss": 11.199011702111871, + "tokens_seen": 917504 + }, + { + "epoch": 0.0, + "learning_rate": 1.4880952380952381e-05, + "loss": 8.8849, + "theoretical_loss": 11.030833917977912, + "tokens_seen": 983040 + }, + { + "epoch": 0.0, + "learning_rate": 1.5873015873015872e-05, + "loss": 8.9041, + "theoretical_loss": 10.87642808645695, + "tokens_seen": 1048576 + }, + { + "epoch": 0.0, + "learning_rate": 1.6865079365079364e-05, + "loss": 9.0194, + "theoretical_loss": 10.733905740062724, + "tokens_seen": 1114112 + }, + { + "epoch": 0.0, + "learning_rate": 1.7857142857142855e-05, + "loss": 9.1718, + "theoretical_loss": 10.60172987623028, + "tokens_seen": 1179648 + }, + { + "epoch": 0.0, + "learning_rate": 1.884920634920635e-05, + "loss": 9.228, + "theoretical_loss": 10.478634172356642, + "tokens_seen": 1245184 + }, + { + "epoch": 0.0, + "learning_rate": 1.984126984126984e-05, + "loss": 8.9147, + "theoretical_loss": 10.36356394376333, + "tokens_seen": 1310720 + }, + { + "epoch": 0.0, + "learning_rate": 2.0833333333333333e-05, + "loss": 8.6633, + "theoretical_loss": 10.255632220896747, + "tokens_seen": 1376256 + }, + { + "epoch": 0.0, + "learning_rate": 2.1825396825396824e-05, + "loss": 8.9129, + "theoretical_loss": 10.15408655327002, + "tokens_seen": 1441792 + }, + { + "epoch": 0.0, + "learning_rate": 2.2817460317460315e-05, + "loss": 9.1161, + "theoretical_loss": 10.058283561732598, + "tokens_seen": 1507328 + }, + { + "epoch": 0.0, + "learning_rate": 2.380952380952381e-05, + "loss": 9.0175, + "theoretical_loss": 9.967669178840278, + "tokens_seen": 1572864 + }, + { + "epoch": 0.0, + "objective/train/docs_used": 13112, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 8.177153587341309, + "objective/train/theoretical_loss": 9.881763126393109, + "objective/train/tokens_used": 22098400, + "theoretical_loss": 9.881763126393109, + "tokens_seen": 1638400 + }, + { + "epoch": 0.0, + "learning_rate": 2.48015873015873e-05, + "loss": 8.7851, + "theoretical_loss": 9.881763126393109, + "tokens_seen": 1638400 + }, + { + "epoch": 0.0, + "learning_rate": 2.5793650793650793e-05, + "loss": 8.3401, + "theoretical_loss": 9.80014659154056, + "tokens_seen": 1703936 + }, + { + "epoch": 0.0, + "learning_rate": 2.6785714285714284e-05, + "loss": 8.6635, + "theoretical_loss": 9.722452346907446, + "tokens_seen": 1769472 + }, + { + "epoch": 0.0, + "learning_rate": 2.7777777777777776e-05, + "loss": 8.7163, + "theoretical_loss": 9.648356759081546, + "tokens_seen": 1835008 + }, + { + "epoch": 0.0, + "learning_rate": 2.876984126984127e-05, + "loss": 9.0136, + "theoretical_loss": 9.577573271145639, + "tokens_seen": 1900544 + }, + { + "epoch": 0.0, + "learning_rate": 2.9761904761904762e-05, + "loss": 7.8029, + "theoretical_loss": 9.509847046764852, + "tokens_seen": 1966080 + }, + { + "epoch": 0.0, + "learning_rate": 3.075396825396825e-05, + "loss": 8.5743, + "theoretical_loss": 9.444950537631936, + "tokens_seen": 2031616 + }, + { + "epoch": 0.0, + "learning_rate": 3.1746031746031745e-05, + "loss": 8.9376, + "theoretical_loss": 9.382679790910457, + "tokens_seen": 2097152 + }, + { + "epoch": 0.0, + "learning_rate": 3.273809523809524e-05, + "loss": 8.6593, + "theoretical_loss": 9.32285135423398, + "tokens_seen": 2162688 + }, + { + "epoch": 0.0, + "learning_rate": 3.373015873015873e-05, + "loss": 8.9814, + "theoretical_loss": 9.265299666660276, + "tokens_seen": 2228224 + }, + { + "epoch": 0.0, + "learning_rate": 3.472222222222222e-05, + "loss": 8.2663, + "theoretical_loss": 9.209874847444755, + "tokens_seen": 2293760 + }, + { + "epoch": 0.0, + "learning_rate": 3.571428571428571e-05, + "loss": 8.309, + "theoretical_loss": 9.156440812508292, + "tokens_seen": 2359296 + }, + { + "epoch": 0.0, + "learning_rate": 3.670634920634921e-05, + "loss": 8.2446, + "theoretical_loss": 9.10487366241335, + "tokens_seen": 2424832 + }, + { + "epoch": 0.0, + "learning_rate": 3.76984126984127e-05, + "loss": 8.1849, + "theoretical_loss": 9.055060296533734, + "tokens_seen": 2490368 + }, + { + "epoch": 0.0, + "learning_rate": 3.8690476190476195e-05, + "loss": 8.3155, + "theoretical_loss": 9.006897216643829, + "tokens_seen": 2555904 + }, + { + "epoch": 0.0, + "learning_rate": 3.968253968253968e-05, + "loss": 7.9353, + "theoretical_loss": 8.960289489909357, + "tokens_seen": 2621440 + }, + { + "epoch": 0.0, + "learning_rate": 4.067460317460318e-05, + "loss": 8.2886, + "theoretical_loss": 8.915149846640611, + "tokens_seen": 2686976 + }, + { + "epoch": 0.0, + "learning_rate": 4.1666666666666665e-05, + "loss": 8.4622, + "theoretical_loss": 8.871397892478225, + "tokens_seen": 2752512 + }, + { + "epoch": 0.0, + "learning_rate": 4.265873015873016e-05, + "loss": 8.377, + "theoretical_loss": 8.828959418153499, + "tokens_seen": 2818048 + }, + { + "epoch": 0.0, + "learning_rate": 4.365079365079365e-05, + "loss": 8.5444, + "theoretical_loss": 8.787765792778412, + "tokens_seen": 2883584 + }, + { + "epoch": 0.0, + "learning_rate": 4.464285714285714e-05, + "loss": 8.4319, + "theoretical_loss": 8.747753428911455, + "tokens_seen": 2949120 + }, + { + "epoch": 0.0, + "learning_rate": 4.563492063492063e-05, + "loss": 8.3749, + "theoretical_loss": 8.708863309520833, + "tokens_seen": 3014656 + }, + { + "epoch": 0.0, + "learning_rate": 4.6626984126984126e-05, + "loss": 7.8252, + "theoretical_loss": 8.671040568508847, + "tokens_seen": 3080192 + }, + { + "epoch": 0.0, + "learning_rate": 4.761904761904762e-05, + "loss": 7.6797, + "theoretical_loss": 8.634234117735474, + "tokens_seen": 3145728 + }, + { + "epoch": 0.0, + "learning_rate": 4.8611111111111115e-05, + "loss": 7.9398, + "theoretical_loss": 8.598396314536323, + "tokens_seen": 3211264 + }, + { + "epoch": 0.0, + "objective/train/docs_used": 13755, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 7.332268714904785, + "objective/train/theoretical_loss": 8.563482664611069, + "objective/train/tokens_used": 23736800, + "theoretical_loss": 8.563482664611069, + "tokens_seen": 3276800 + }, + { + "epoch": 0.0, + "learning_rate": 4.96031746031746e-05, + "loss": 7.8472, + "theoretical_loss": 8.563482664611069, + "tokens_seen": 3276800 + }, + { + "epoch": 0.0, + "learning_rate": 5.05952380952381e-05, + "loss": 7.7743, + "theoretical_loss": 8.529451555895115, + "tokens_seen": 3342336 + }, + { + "epoch": 0.0, + "learning_rate": 5.1587301587301586e-05, + "loss": 7.6084, + "theoretical_loss": 8.496264019646002, + "tokens_seen": 3407872 + }, + { + "epoch": 0.0, + "learning_rate": 5.257936507936508e-05, + "loss": 7.9436, + "theoretical_loss": 8.463883515497187, + "tokens_seen": 3473408 + }, + { + "epoch": 0.0, + "learning_rate": 5.357142857142857e-05, + "loss": 7.7833, + "theoretical_loss": 8.432275737672779, + "tokens_seen": 3538944 + }, + { + "epoch": 0.0, + "learning_rate": 5.4563492063492063e-05, + "loss": 7.736, + "theoretical_loss": 8.401408439930716, + "tokens_seen": 3604480 + }, + { + "epoch": 0.0, + "learning_rate": 5.555555555555555e-05, + "loss": 7.5034, + "theoretical_loss": 8.371251277120209, + "tokens_seen": 3670016 + }, + { + "epoch": 0.0, + "learning_rate": 5.6547619047619046e-05, + "loss": 7.2029, + "theoretical_loss": 8.341775661511075, + "tokens_seen": 3735552 + }, + { + "epoch": 0.0, + "learning_rate": 5.753968253968254e-05, + "loss": 7.6365, + "theoretical_loss": 8.31295463228533, + "tokens_seen": 3801088 + }, + { + "epoch": 0.0, + "learning_rate": 5.8531746031746036e-05, + "loss": 7.8458, + "theoretical_loss": 8.284762736781182, + "tokens_seen": 3866624 + }, + { + "epoch": 0.0, + "learning_rate": 5.9523809523809524e-05, + "loss": 7.7968, + "theoretical_loss": 8.257175922251864, + "tokens_seen": 3932160 + }, + { + "epoch": 0.0, + "learning_rate": 6.051587301587302e-05, + "loss": 7.6005, + "theoretical_loss": 8.230171437050114, + "tokens_seen": 3997696 + }, + { + "epoch": 0.0, + "learning_rate": 6.15079365079365e-05, + "loss": 8.0861, + "theoretical_loss": 8.20372774027797, + "tokens_seen": 4063232 + }, + { + "epoch": 0.0, + "learning_rate": 6.25e-05, + "loss": 7.4333, + "theoretical_loss": 8.177824419053046, + "tokens_seen": 4128768 + }, + { + "epoch": 0.0, + "learning_rate": 6.349206349206349e-05, + "loss": 7.4983, + "theoretical_loss": 8.152442112639616, + "tokens_seen": 4194304 + }, + { + "epoch": 0.0, + "learning_rate": 6.448412698412699e-05, + "loss": 6.9985, + "theoretical_loss": 8.1275624427775, + "tokens_seen": 4259840 + }, + { + "epoch": 0.0, + "learning_rate": 6.547619047619048e-05, + "loss": 7.4732, + "theoretical_loss": 8.10316794961571, + "tokens_seen": 4325376 + }, + { + "epoch": 0.0, + "learning_rate": 6.646825396825397e-05, + "loss": 7.6423, + "theoretical_loss": 8.07924203272264, + "tokens_seen": 4390912 + }, + { + "epoch": 0.0, + "learning_rate": 6.746031746031745e-05, + "loss": 7.0492, + "theoretical_loss": 8.055768896701416, + "tokens_seen": 4456448 + }, + { + "epoch": 0.0, + "learning_rate": 6.845238095238096e-05, + "loss": 7.2026, + "theoretical_loss": 8.032733500989007, + "tokens_seen": 4521984 + }, + { + "epoch": 0.0, + "learning_rate": 6.944444444444444e-05, + "loss": 7.4128, + "theoretical_loss": 8.010121513461836, + "tokens_seen": 4587520 + }, + { + "epoch": 0.0, + "learning_rate": 7.043650793650793e-05, + "loss": 7.1315, + "theoretical_loss": 7.987919267509379, + "tokens_seen": 4653056 + }, + { + "epoch": 0.0, + "learning_rate": 7.142857142857142e-05, + "loss": 7.3153, + "theoretical_loss": 7.966113722271801, + "tokens_seen": 4718592 + }, + { + "epoch": 0.0, + "learning_rate": 7.242063492063492e-05, + "loss": 6.9838, + "theoretical_loss": 7.944692425767988, + "tokens_seen": 4784128 + }, + { + "epoch": 0.0, + "learning_rate": 7.341269841269842e-05, + "loss": 7.2357, + "theoretical_loss": 7.9236434806675184, + "tokens_seen": 4849664 + }, + { + "epoch": 0.0, + "objective/train/docs_used": 15099, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 6.67203950881958, + "objective/train/theoretical_loss": 7.902955512484067, + "objective/train/tokens_used": 25375200, + "theoretical_loss": 7.902955512484067, + "tokens_seen": 4915200 + }, + { + "epoch": 0.0, + "learning_rate": 7.440476190476191e-05, + "loss": 6.8598, + "theoretical_loss": 7.902955512484067, + "tokens_seen": 4915200 + }, + { + "epoch": 0.0, + "learning_rate": 7.53968253968254e-05, + "loss": 6.8721, + "theoretical_loss": 7.882617639989203, + "tokens_seen": 4980736 + }, + { + "epoch": 0.0, + "learning_rate": 7.63888888888889e-05, + "loss": 7.0343, + "theoretical_loss": 7.862619447664628, + "tokens_seen": 5046272 + }, + { + "epoch": 0.0, + "learning_rate": 7.738095238095239e-05, + "loss": 6.8538, + "theoretical_loss": 7.842950960027937, + "tokens_seen": 5111808 + }, + { + "epoch": 0.0, + "learning_rate": 7.837301587301588e-05, + "loss": 6.8769, + "theoretical_loss": 7.823602617682313, + "tokens_seen": 5177344 + }, + { + "epoch": 0.0, + "learning_rate": 7.936507936507937e-05, + "loss": 6.7274, + "theoretical_loss": 7.804565254954165, + "tokens_seen": 5242880 + }, + { + "epoch": 0.0, + "learning_rate": 8.035714285714287e-05, + "loss": 7.0779, + "theoretical_loss": 7.7858300789950725, + "tokens_seen": 5308416 + }, + { + "epoch": 0.0, + "learning_rate": 8.134920634920635e-05, + "loss": 6.7618, + "theoretical_loss": 7.767388650235364, + "tokens_seen": 5373952 + }, + { + "epoch": 0.0, + "learning_rate": 8.234126984126984e-05, + "loss": 6.664, + "theoretical_loss": 7.749232864086619, + "tokens_seen": 5439488 + }, + { + "epoch": 0.0, + "learning_rate": 8.333333333333333e-05, + "loss": 6.9805, + "theoretical_loss": 7.731354933799318, + "tokens_seen": 5505024 + }, + { + "epoch": 0.0, + "learning_rate": 8.432539682539683e-05, + "loss": 6.8388, + "theoretical_loss": 7.71374737438992, + "tokens_seen": 5570560 + }, + { + "epoch": 0.0, + "learning_rate": 8.531746031746032e-05, + "loss": 6.862, + "theoretical_loss": 7.696402987558934, + "tokens_seen": 5636096 + }, + { + "epoch": 0.0, + "learning_rate": 8.630952380952381e-05, + "loss": 6.7502, + "theoretical_loss": 7.679314847528181, + "tokens_seen": 5701632 + }, + { + "epoch": 0.0, + "learning_rate": 8.73015873015873e-05, + "loss": 6.5972, + "theoretical_loss": 7.662476287731328, + "tokens_seen": 5767168 + }, + { + "epoch": 0.0, + "learning_rate": 8.82936507936508e-05, + "loss": 6.5691, + "theoretical_loss": 7.645880888297279, + "tokens_seen": 5832704 + }, + { + "epoch": 0.0, + "learning_rate": 8.928571428571429e-05, + "loss": 6.6707, + "theoretical_loss": 7.629522464270861, + "tokens_seen": 5898240 + }, + { + "epoch": 0.0, + "learning_rate": 9.027777777777777e-05, + "loss": 6.1653, + "theoretical_loss": 7.613395054519696, + "tokens_seen": 5963776 + }, + { + "epoch": 0.0, + "learning_rate": 9.126984126984126e-05, + "loss": 6.654, + "theoretical_loss": 7.59749291128028, + "tokens_seen": 6029312 + }, + { + "epoch": 0.0, + "learning_rate": 9.226190476190476e-05, + "loss": 6.774, + "theoretical_loss": 7.581810490299888, + "tokens_seen": 6094848 + }, + { + "epoch": 0.0, + "learning_rate": 9.325396825396825e-05, + "loss": 6.6446, + "theoretical_loss": 7.5663424415343705, + "tokens_seen": 6160384 + }, + { + "epoch": 0.0, + "learning_rate": 9.424603174603175e-05, + "loss": 6.5109, + "theoretical_loss": 7.551083600364949, + "tokens_seen": 6225920 + }, + { + "epoch": 0.0, + "learning_rate": 9.523809523809524e-05, + "loss": 6.7098, + "theoretical_loss": 7.536028979299919, + "tokens_seen": 6291456 + }, + { + "epoch": 0.0, + "learning_rate": 9.623015873015874e-05, + "loss": 6.8004, + "theoretical_loss": 7.521173760129762, + "tokens_seen": 6356992 + }, + { + "epoch": 0.0, + "learning_rate": 9.722222222222223e-05, + "loss": 6.3379, + "theoretical_loss": 7.506513286506497, + "tokens_seen": 6422528 + }, + { + "epoch": 0.0, + "learning_rate": 9.821428571428572e-05, + "loss": 6.418, + "theoretical_loss": 7.492043056920249, + "tokens_seen": 6488064 + }, + { + "epoch": 0.0, + "objective/train/docs_used": 15777, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 6.9466657638549805, + "objective/train/theoretical_loss": 7.4777587180480305, + "objective/train/tokens_used": 27013600, + "theoretical_loss": 7.4777587180480305, + "tokens_seen": 6553600 + }, + { + "epoch": 0.0, + "learning_rate": 9.92063492063492e-05, + "loss": 6.4055, + "theoretical_loss": 7.4777587180480305, + "tokens_seen": 6553600 + }, + { + "epoch": 0.0, + "learning_rate": 0.00010019841269841271, + "loss": 6.4957, + "theoretical_loss": 7.463656058451462, + "tokens_seen": 6619136 + }, + { + "epoch": 0.0, + "learning_rate": 0.0001011904761904762, + "loss": 6.3763, + "theoretical_loss": 7.449731002601916, + "tokens_seen": 6684672 + }, + { + "epoch": 0.0, + "learning_rate": 0.00010218253968253968, + "loss": 6.6865, + "theoretical_loss": 7.435979605213019, + "tokens_seen": 6750208 + }, + { + "epoch": 0.0, + "learning_rate": 0.00010317460317460317, + "loss": 6.5651, + "theoretical_loss": 7.422398045861905, + "tokens_seen": 6815744 + }, + { + "epoch": 0.0, + "learning_rate": 0.00010416666666666667, + "loss": 6.5271, + "theoretical_loss": 7.408982623881875, + "tokens_seen": 6881280 + }, + { + "epoch": 0.0, + "learning_rate": 0.00010515873015873016, + "loss": 6.5694, + "theoretical_loss": 7.395729753510345, + "tokens_seen": 6946816 + }, + { + "epoch": 0.0, + "learning_rate": 0.00010615079365079365, + "loss": 6.5701, + "theoretical_loss": 7.3826359592770325, + "tokens_seen": 7012352 + }, + { + "epoch": 0.0, + "learning_rate": 0.00010714285714285714, + "loss": 6.6416, + "theoretical_loss": 7.369697871618373, + "tokens_seen": 7077888 + }, + { + "epoch": 0.0, + "learning_rate": 0.00010813492063492064, + "loss": 6.3985, + "theoretical_loss": 7.3569122227050885, + "tokens_seen": 7143424 + }, + { + "epoch": 0.0, + "learning_rate": 0.00010912698412698413, + "loss": 6.2035, + "theoretical_loss": 7.3442758424706875, + "tokens_seen": 7208960 + }, + { + "epoch": 0.0, + "learning_rate": 0.00011011904761904761, + "loss": 6.4969, + "theoretical_loss": 7.331785654829519, + "tokens_seen": 7274496 + }, + { + "epoch": 0.0, + "learning_rate": 0.0001111111111111111, + "loss": 6.2016, + "theoretical_loss": 7.319438674073677, + "tokens_seen": 7340032 + }, + { + "epoch": 0.0, + "learning_rate": 0.0001121031746031746, + "loss": 6.4215, + "theoretical_loss": 7.307232001438824, + "tokens_seen": 7405568 + }, + { + "epoch": 0.0, + "learning_rate": 0.00011309523809523809, + "loss": 6.6454, + "theoretical_loss": 7.295162821829564, + "tokens_seen": 7471104 + }, + { + "epoch": 0.0, + "learning_rate": 0.00011408730158730158, + "loss": 6.363, + "theoretical_loss": 7.283228400695652, + "tokens_seen": 7536640 + }, + { + "epoch": 0.0, + "learning_rate": 0.00011507936507936508, + "loss": 6.1649, + "theoretical_loss": 7.271426081050832, + "tokens_seen": 7602176 + }, + { + "epoch": 0.0, + "learning_rate": 0.00011607142857142858, + "loss": 6.4188, + "theoretical_loss": 7.259753280626623, + "tokens_seen": 7667712 + }, + { + "epoch": 0.0, + "learning_rate": 0.00011706349206349207, + "loss": 6.5098, + "theoretical_loss": 7.24820748915387, + "tokens_seen": 7733248 + }, + { + "epoch": 0.0, + "learning_rate": 0.00011805555555555556, + "loss": 5.921, + "theoretical_loss": 7.236786265765262, + "tokens_seen": 7798784 + }, + { + "epoch": 0.0, + "learning_rate": 0.00011904761904761905, + "loss": 6.0876, + "theoretical_loss": 7.225487236512497, + "tokens_seen": 7864320 + }, + { + "epoch": 0.0, + "learning_rate": 0.00012003968253968255, + "loss": 6.6213, + "theoretical_loss": 7.21430809199212, + "tokens_seen": 7929856 + }, + { + "epoch": 0.0, + "learning_rate": 0.00012103174603174604, + "loss": 6.1411, + "theoretical_loss": 7.2032465850744005, + "tokens_seen": 7995392 + }, + { + "epoch": 0.0, + "learning_rate": 0.00012202380952380953, + "loss": 6.3507, + "theoretical_loss": 7.192300528730015, + "tokens_seen": 8060928 + }, + { + "epoch": 0.0, + "learning_rate": 0.000123015873015873, + "loss": 6.3251, + "theoretical_loss": 7.1814677939495155, + "tokens_seen": 8126464 + }, + { + "epoch": 0.0, + "objective/train/docs_used": 17166, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 6.246005535125732, + "objective/train/theoretical_loss": 7.1707463077509646, + "objective/train/tokens_used": 28652000, + "theoretical_loss": 7.1707463077509646, + "tokens_seen": 8192000 + }, + { + "epoch": 0.0, + "learning_rate": 0.0001240079365079365, + "loss": 6.4107, + "theoretical_loss": 7.1707463077509646, + "tokens_seen": 8192000 + }, + { + "epoch": 0.0, + "learning_rate": 0.000125, + "loss": 6.2091, + "theoretical_loss": 7.160134051271272, + "tokens_seen": 8257536 + }, + { + "epoch": 0.0, + "learning_rate": 0.0001259920634920635, + "loss": 6.2459, + "theoretical_loss": 7.149629057937138, + "tokens_seen": 8323072 + }, + { + "epoch": 0.0, + "learning_rate": 0.00012698412698412698, + "loss": 6.219, + "theoretical_loss": 7.139229411711638, + "tokens_seen": 8388608 + }, + { + "epoch": 0.0, + "learning_rate": 0.00012797619047619048, + "loss": 6.0529, + "theoretical_loss": 7.128933245412794, + "tokens_seen": 8454144 + }, + { + "epoch": 0.0, + "learning_rate": 0.00012896825396825398, + "loss": 6.052, + "theoretical_loss": 7.118738739100616, + "tokens_seen": 8519680 + }, + { + "epoch": 0.0, + "learning_rate": 0.00012996031746031748, + "loss": 6.3182, + "theoretical_loss": 7.1086441185293445, + "tokens_seen": 8585216 + }, + { + "epoch": 0.0, + "learning_rate": 0.00013095238095238096, + "loss": 6.2583, + "theoretical_loss": 7.09864765366177, + "tokens_seen": 8650752 + }, + { + "epoch": 0.0, + "learning_rate": 0.00013194444444444446, + "loss": 6.345, + "theoretical_loss": 7.088747657242693, + "tokens_seen": 8716288 + }, + { + "epoch": 0.0, + "learning_rate": 0.00013293650793650793, + "loss": 6.1682, + "theoretical_loss": 7.078942483428749, + "tokens_seen": 8781824 + }, + { + "epoch": 0.0, + "learning_rate": 0.00013392857142857144, + "loss": 6.1602, + "theoretical_loss": 7.069230526471966, + "tokens_seen": 8847360 + }, + { + "epoch": 0.0, + "learning_rate": 0.0001349206349206349, + "loss": 5.9716, + "theoretical_loss": 7.059610219454568, + "tokens_seen": 8912896 + }, + { + "epoch": 0.0, + "learning_rate": 0.0001359126984126984, + "loss": 6.1343, + "theoretical_loss": 7.0500800330726685, + "tokens_seen": 8978432 + }, + { + "epoch": 0.0, + "learning_rate": 0.0001369047619047619, + "loss": 6.1326, + "theoretical_loss": 7.040638474466625, + "tokens_seen": 9043968 + }, + { + "epoch": 0.0, + "learning_rate": 0.00013789682539682541, + "loss": 6.178, + "theoretical_loss": 7.031284086095933, + "tokens_seen": 9109504 + }, + { + "epoch": 0.0, + "learning_rate": 0.0001388888888888889, + "loss": 5.9041, + "theoretical_loss": 7.022015444656678, + "tokens_seen": 9175040 + }, + { + "epoch": 0.0, + "learning_rate": 0.0001398809523809524, + "loss": 6.1053, + "theoretical_loss": 7.012831160039609, + "tokens_seen": 9240576 + }, + { + "epoch": 0.0, + "learning_rate": 0.00014087301587301586, + "loss": 6.3541, + "theoretical_loss": 7.003729874327071, + "tokens_seen": 9306112 + }, + { + "epoch": 0.0, + "learning_rate": 0.00014186507936507937, + "loss": 6.2626, + "theoretical_loss": 6.994710260827057, + "tokens_seen": 9371648 + }, + { + "epoch": 0.0, + "learning_rate": 0.00014285714285714284, + "loss": 6.1362, + "theoretical_loss": 6.98577102314278, + "tokens_seen": 9437184 + }, + { + "epoch": 0.0, + "learning_rate": 0.00014384920634920634, + "loss": 6.2595, + "theoretical_loss": 6.976910894276189, + "tokens_seen": 9502720 + }, + { + "epoch": 0.0, + "learning_rate": 0.00014484126984126984, + "loss": 6.3398, + "theoretical_loss": 6.968128635764015, + "tokens_seen": 9568256 + }, + { + "epoch": 0.0, + "learning_rate": 0.00014583333333333335, + "loss": 6.3496, + "theoretical_loss": 6.959423036844894, + "tokens_seen": 9633792 + }, + { + "epoch": 0.0, + "learning_rate": 0.00014682539682539685, + "loss": 6.1089, + "theoretical_loss": 6.950792913656309, + "tokens_seen": 9699328 + }, + { + "epoch": 0.0, + "learning_rate": 0.00014781746031746032, + "loss": 6.1017, + "theoretical_loss": 6.942237108460029, + "tokens_seen": 9764864 + }, + { + "epoch": 0.0, + "objective/train/docs_used": 17861, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 5.305968761444092, + "objective/train/theoretical_loss": 6.9337544888949, + "objective/train/tokens_used": 30290400, + "theoretical_loss": 6.9337544888949, + "tokens_seen": 9830400 + }, + { + "epoch": 0.0, + "learning_rate": 0.00014880952380952382, + "loss": 5.706, + "theoretical_loss": 6.9337544888949, + "tokens_seen": 9830400 + }, + { + "epoch": 0.0, + "learning_rate": 0.0001498015873015873, + "loss": 6.0193, + "theoretical_loss": 6.925343947255817, + "tokens_seen": 9895936 + }, + { + "epoch": 0.0, + "learning_rate": 0.0001507936507936508, + "loss": 6.1426, + "theoretical_loss": 6.917004399797798, + "tokens_seen": 9961472 + }, + { + "epoch": 0.0, + "learning_rate": 0.00015178571428571427, + "loss": 6.2143, + "theoretical_loss": 6.908734786064147, + "tokens_seen": 10027008 + }, + { + "epoch": 0.0, + "learning_rate": 0.0001527777777777778, + "loss": 6.0097, + "theoretical_loss": 6.900534068237688, + "tokens_seen": 10092544 + }, + { + "epoch": 0.0, + "learning_rate": 0.00015376984126984128, + "loss": 5.8791, + "theoretical_loss": 6.89240123051416, + "tokens_seen": 10158080 + }, + { + "epoch": 0.0, + "learning_rate": 0.00015476190476190478, + "loss": 5.6443, + "theoretical_loss": 6.884335278496871, + "tokens_seen": 10223616 + }, + { + "epoch": 0.0, + "learning_rate": 0.00015575396825396825, + "loss": 6.0154, + "theoretical_loss": 6.87633523861175, + "tokens_seen": 10289152 + }, + { + "epoch": 0.0, + "learning_rate": 0.00015674603174603175, + "loss": 6.0797, + "theoretical_loss": 6.868400157541997, + "tokens_seen": 10354688 + }, + { + "epoch": 0.0, + "learning_rate": 0.00015773809523809523, + "loss": 5.9418, + "theoretical_loss": 6.860529101681551, + "tokens_seen": 10420224 + }, + { + "epoch": 0.0, + "learning_rate": 0.00015873015873015873, + "loss": 5.8387, + "theoretical_loss": 6.85272115660663, + "tokens_seen": 10485760 + }, + { + "epoch": 0.0, + "learning_rate": 0.0001597222222222222, + "loss": 6.0951, + "theoretical_loss": 6.844975426564642, + "tokens_seen": 10551296 + }, + { + "epoch": 0.0, + "learning_rate": 0.00016071428571428573, + "loss": 5.8754, + "theoretical_loss": 6.8372910339797945, + "tokens_seen": 10616832 + }, + { + "epoch": 0.0, + "learning_rate": 0.0001617063492063492, + "loss": 5.8277, + "theoretical_loss": 6.829667118974749, + "tokens_seen": 10682368 + }, + { + "epoch": 0.0, + "learning_rate": 0.0001626984126984127, + "loss": 6.1055, + "theoretical_loss": 6.8221028389077185, + "tokens_seen": 10747904 + }, + { + "epoch": 0.0, + "learning_rate": 0.00016369047619047618, + "loss": 6.02, + "theoretical_loss": 6.814597367924395, + "tokens_seen": 10813440 + }, + { + "epoch": 0.0, + "learning_rate": 0.00016468253968253969, + "loss": 5.8527, + "theoretical_loss": 6.807149896524181, + "tokens_seen": 10878976 + }, + { + "epoch": 0.0, + "learning_rate": 0.00016567460317460316, + "loss": 5.8451, + "theoretical_loss": 6.799759631140145, + "tokens_seen": 10944512 + }, + { + "epoch": 0.0, + "learning_rate": 0.00016666666666666666, + "loss": 5.8543, + "theoretical_loss": 6.7924257937322245, + "tokens_seen": 11010048 + }, + { + "epoch": 0.0, + "learning_rate": 0.00016765873015873016, + "loss": 5.8349, + "theoretical_loss": 6.785147621393148, + "tokens_seen": 11075584 + }, + { + "epoch": 0.0, + "learning_rate": 0.00016865079365079366, + "loss": 5.6144, + "theoretical_loss": 6.777924365966638, + "tokens_seen": 11141120 + }, + { + "epoch": 0.0, + "learning_rate": 0.00016964285714285717, + "loss": 5.6628, + "theoretical_loss": 6.770755293677423, + "tokens_seen": 11206656 + }, + { + "epoch": 0.0, + "learning_rate": 0.00017063492063492064, + "loss": 5.7785, + "theoretical_loss": 6.763639684772625, + "tokens_seen": 11272192 + }, + { + "epoch": 0.0, + "learning_rate": 0.00017162698412698414, + "loss": 5.8836, + "theoretical_loss": 6.756576833174123, + "tokens_seen": 11337728 + }, + { + "epoch": 0.0, + "learning_rate": 0.00017261904761904762, + "loss": 5.9801, + "theoretical_loss": 6.749566046141486, + "tokens_seen": 11403264 + }, + { + "epoch": 0.0, + "objective/train/docs_used": 18493, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 6.053536415100098, + "objective/train/theoretical_loss": 6.7426066439450905, + "objective/train/tokens_used": 31928800, + "theoretical_loss": 6.7426066439450905, + "tokens_seen": 11468800 + }, + { + "epoch": 0.0, + "learning_rate": 0.00017361111111111112, + "loss": 5.8632, + "theoretical_loss": 6.7426066439450905, + "tokens_seen": 11468800 + }, + { + "epoch": 0.0, + "learning_rate": 0.0001746031746031746, + "loss": 5.9225, + "theoretical_loss": 6.735697959549075, + "tokens_seen": 11534336 + }, + { + "epoch": 0.0, + "learning_rate": 0.0001755952380952381, + "loss": 5.8973, + "theoretical_loss": 6.728839338303761, + "tokens_seen": 11599872 + }, + { + "epoch": 0.0, + "learning_rate": 0.0001765873015873016, + "loss": 6.04, + "theoretical_loss": 6.722030137647226, + "tokens_seen": 11665408 + }, + { + "epoch": 0.0, + "learning_rate": 0.0001775793650793651, + "loss": 5.5697, + "theoretical_loss": 6.715269726815689, + "tokens_seen": 11730944 + }, + { + "epoch": 0.0, + "learning_rate": 0.00017857142857142857, + "loss": 6.092, + "theoretical_loss": 6.7085574865624125, + "tokens_seen": 11796480 + }, + { + "epoch": 0.0, + "learning_rate": 0.00017956349206349207, + "loss": 5.1821, + "theoretical_loss": 6.701892808884824, + "tokens_seen": 11862016 + }, + { + "epoch": 0.0, + "learning_rate": 0.00018055555555555555, + "loss": 5.4202, + "theoretical_loss": 6.695275096759559, + "tokens_seen": 11927552 + }, + { + "epoch": 0.0, + "learning_rate": 0.00018154761904761905, + "loss": 5.8922, + "theoretical_loss": 6.68870376388518, + "tokens_seen": 11993088 + }, + { + "epoch": 0.0, + "learning_rate": 0.00018253968253968252, + "loss": 5.8933, + "theoretical_loss": 6.682178234432274, + "tokens_seen": 12058624 + }, + { + "epoch": 0.0, + "learning_rate": 0.00018353174603174602, + "loss": 5.7001, + "theoretical_loss": 6.675697942800715, + "tokens_seen": 12124160 + }, + { + "epoch": 0.0, + "learning_rate": 0.00018452380952380953, + "loss": 6.0192, + "theoretical_loss": 6.669262333383815, + "tokens_seen": 12189696 + }, + { + "epoch": 0.0, + "learning_rate": 0.00018551587301587303, + "loss": 5.9503, + "theoretical_loss": 6.662870860339158, + "tokens_seen": 12255232 + }, + { + "epoch": 0.0, + "learning_rate": 0.0001865079365079365, + "loss": 5.8466, + "theoretical_loss": 6.656522987365879, + "tokens_seen": 12320768 + }, + { + "epoch": 0.0, + "learning_rate": 0.0001875, + "loss": 5.7919, + "theoretical_loss": 6.6502181874881705, + "tokens_seen": 12386304 + }, + { + "epoch": 0.0, + "learning_rate": 0.0001884920634920635, + "loss": 5.7367, + "theoretical_loss": 6.643955942844831, + "tokens_seen": 12451840 + }, + { + "epoch": 0.0, + "learning_rate": 0.00018948412698412698, + "loss": 5.9055, + "theoretical_loss": 6.637735744484626, + "tokens_seen": 12517376 + }, + { + "epoch": 0.0, + "learning_rate": 0.00019047619047619048, + "loss": 5.5262, + "theoretical_loss": 6.631557092167304, + "tokens_seen": 12582912 + }, + { + "epoch": 0.0, + "learning_rate": 0.00019146825396825398, + "loss": 5.7919, + "theoretical_loss": 6.625419494170049, + "tokens_seen": 12648448 + }, + { + "epoch": 0.0, + "learning_rate": 0.00019246031746031748, + "loss": 5.512, + "theoretical_loss": 6.619322467099223, + "tokens_seen": 12713984 + }, + { + "epoch": 0.0, + "learning_rate": 0.00019345238095238096, + "loss": 5.7394, + "theoretical_loss": 6.613265535707211, + "tokens_seen": 12779520 + }, + { + "epoch": 0.0, + "learning_rate": 0.00019444444444444446, + "loss": 5.5762, + "theoretical_loss": 6.607248232714213, + "tokens_seen": 12845056 + }, + { + "epoch": 0.0, + "learning_rate": 0.00019543650793650793, + "loss": 5.7096, + "theoretical_loss": 6.60127009863481, + "tokens_seen": 12910592 + }, + { + "epoch": 0.0, + "learning_rate": 0.00019642857142857144, + "loss": 5.6571, + "theoretical_loss": 6.59533068160918, + "tokens_seen": 12976128 + }, + { + "epoch": 0.0, + "learning_rate": 0.0001974206349206349, + "loss": 5.5167, + "theoretical_loss": 6.589429537238785, + "tokens_seen": 13041664 + }, + { + "epoch": 0.0, + "objective/train/docs_used": 19731, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 5.5551981925964355, + "objective/train/theoretical_loss": 6.583566228426414, + "objective/train/tokens_used": 33567200, + "theoretical_loss": 6.583566228426414, + "tokens_seen": 13107200 + }, + { + "epoch": 0.0, + "learning_rate": 0.0001984126984126984, + "loss": 5.7675, + "theoretical_loss": 6.583566228426414, + "tokens_seen": 13107200 + }, + { + "epoch": 0.0, + "learning_rate": 0.00019940476190476191, + "loss": 5.9385, + "theoretical_loss": 6.5777403252204305, + "tokens_seen": 13172736 + }, + { + "epoch": 0.0, + "learning_rate": 0.00020039682539682542, + "loss": 5.7506, + "theoretical_loss": 6.571951404663098, + "tokens_seen": 13238272 + }, + { + "epoch": 0.0, + "learning_rate": 0.0002013888888888889, + "loss": 5.6647, + "theoretical_loss": 6.566199050642863, + "tokens_seen": 13303808 + }, + { + "epoch": 0.0, + "learning_rate": 0.0002023809523809524, + "loss": 5.8159, + "theoretical_loss": 6.560482853750463, + "tokens_seen": 13369344 + }, + { + "epoch": 0.0, + "learning_rate": 0.00020337301587301587, + "loss": 5.6132, + "theoretical_loss": 6.554802411138745, + "tokens_seen": 13434880 + }, + { + "epoch": 0.0, + "learning_rate": 0.00020436507936507937, + "loss": 5.6516, + "theoretical_loss": 6.549157326386091, + "tokens_seen": 13500416 + }, + { + "epoch": 0.0, + "learning_rate": 0.00020535714285714284, + "loss": 5.8625, + "theoretical_loss": 6.54354720936333, + "tokens_seen": 13565952 + }, + { + "epoch": 0.0, + "learning_rate": 0.00020634920634920634, + "loss": 5.8153, + "theoretical_loss": 6.537971676104026, + "tokens_seen": 13631488 + }, + { + "epoch": 0.0, + "learning_rate": 0.00020734126984126985, + "loss": 5.6829, + "theoretical_loss": 6.532430348678068, + "tokens_seen": 13697024 + }, + { + "epoch": 0.0, + "learning_rate": 0.00020833333333333335, + "loss": 5.5882, + "theoretical_loss": 6.5269228550684195, + "tokens_seen": 13762560 + }, + { + "epoch": 0.0, + "learning_rate": 0.00020932539682539685, + "loss": 5.682, + "theoretical_loss": 6.521448829050978, + "tokens_seen": 13828096 + }, + { + "epoch": 0.0, + "learning_rate": 0.00021031746031746032, + "loss": 5.8469, + "theoretical_loss": 6.516007910077416, + "tokens_seen": 13893632 + }, + { + "epoch": 0.0, + "learning_rate": 0.00021130952380952382, + "loss": 5.7414, + "theoretical_loss": 6.51059974316095, + "tokens_seen": 13959168 + }, + { + "epoch": 0.0, + "learning_rate": 0.0002123015873015873, + "loss": 5.8944, + "theoretical_loss": 6.50522397876491, + "tokens_seen": 14024704 + }, + { + "epoch": 0.0, + "learning_rate": 0.0002132936507936508, + "loss": 5.6079, + "theoretical_loss": 6.499880272694068, + "tokens_seen": 14090240 + }, + { + "epoch": 0.0, + "learning_rate": 0.00021428571428571427, + "loss": 5.7272, + "theoretical_loss": 6.494568285988618, + "tokens_seen": 14155776 + }, + { + "epoch": 0.0, + "learning_rate": 0.0002152777777777778, + "loss": 6.0081, + "theoretical_loss": 6.489287684820745, + "tokens_seen": 14221312 + }, + { + "epoch": 0.0, + "learning_rate": 0.00021626984126984128, + "loss": 5.6363, + "theoretical_loss": 6.484038140393699, + "tokens_seen": 14286848 + }, + { + "epoch": 0.0, + "learning_rate": 0.00021726190476190478, + "loss": 5.2918, + "theoretical_loss": 6.4788193288433105, + "tokens_seen": 14352384 + }, + { + "epoch": 0.0, + "learning_rate": 0.00021825396825396825, + "loss": 5.6286, + "theoretical_loss": 6.473630931141869, + "tokens_seen": 14417920 + }, + { + "epoch": 0.0, + "learning_rate": 0.00021924603174603176, + "loss": 5.5647, + "theoretical_loss": 6.468472633004308, + "tokens_seen": 14483456 + }, + { + "epoch": 0.0, + "learning_rate": 0.00022023809523809523, + "loss": 5.4526, + "theoretical_loss": 6.463344124796616, + "tokens_seen": 14548992 + }, + { + "epoch": 0.0, + "learning_rate": 0.00022123015873015873, + "loss": 5.6384, + "theoretical_loss": 6.45824510144643, + "tokens_seen": 14614528 + }, + { + "epoch": 0.0, + "learning_rate": 0.0002222222222222222, + "loss": 5.5404, + "theoretical_loss": 6.45317526235573, + "tokens_seen": 14680064 + }, + { + "epoch": 0.0, + "objective/train/docs_used": 20428, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 5.2156500816345215, + "objective/train/theoretical_loss": 6.448134311315593, + "objective/train/tokens_used": 35205600, + "theoretical_loss": 6.448134311315593, + "tokens_seen": 14745600 + }, + { + "epoch": 0.0, + "learning_rate": 0.00022321428571428573, + "loss": 5.3624, + "theoretical_loss": 6.448134311315593, + "tokens_seen": 14745600 + }, + { + "epoch": 0.0, + "learning_rate": 0.0002242063492063492, + "loss": 5.6652, + "theoretical_loss": 6.443121956422939, + "tokens_seen": 14811136 + }, + { + "epoch": 0.0, + "learning_rate": 0.0002251984126984127, + "loss": 5.823, + "theoretical_loss": 6.438137909999214, + "tokens_seen": 14876672 + }, + { + "epoch": 0.0, + "learning_rate": 0.00022619047619047618, + "loss": 5.4108, + "theoretical_loss": 6.433181888510964, + "tokens_seen": 14942208 + }, + { + "epoch": 0.0, + "learning_rate": 0.00022718253968253969, + "loss": 5.8993, + "theoretical_loss": 6.428253612492239, + "tokens_seen": 15007744 + }, + { + "epoch": 0.0, + "learning_rate": 0.00022817460317460316, + "loss": 5.6564, + "theoretical_loss": 6.4233528064687855, + "tokens_seen": 15073280 + }, + { + "epoch": 0.0, + "learning_rate": 0.00022916666666666666, + "loss": 5.5352, + "theoretical_loss": 6.418479198883969, + "tokens_seen": 15138816 + }, + { + "epoch": 0.0, + "learning_rate": 0.00023015873015873016, + "loss": 5.3828, + "theoretical_loss": 6.413632522026391, + "tokens_seen": 15204352 + }, + { + "epoch": 0.0, + "learning_rate": 0.00023115079365079367, + "loss": 5.4627, + "theoretical_loss": 6.40881251195914, + "tokens_seen": 15269888 + }, + { + "epoch": 0.0, + "learning_rate": 0.00023214285714285717, + "loss": 5.7049, + "theoretical_loss": 6.404018908450656, + "tokens_seen": 15335424 + }, + { + "epoch": 0.0, + "learning_rate": 0.00023313492063492064, + "loss": 5.8795, + "theoretical_loss": 6.399251454907132, + "tokens_seen": 15400960 + }, + { + "epoch": 0.0, + "learning_rate": 0.00023412698412698414, + "loss": 5.5257, + "theoretical_loss": 6.394509898306452, + "tokens_seen": 15466496 + }, + { + "epoch": 0.0, + "learning_rate": 0.00023511904761904762, + "loss": 5.4516, + "theoretical_loss": 6.389793989133574, + "tokens_seen": 15532032 + }, + { + "epoch": 0.0, + "learning_rate": 0.00023611111111111112, + "loss": 5.4694, + "theoretical_loss": 6.385103481317387, + "tokens_seen": 15597568 + }, + { + "epoch": 0.0, + "learning_rate": 0.0002371031746031746, + "loss": 5.415, + "theoretical_loss": 6.380438132168923, + "tokens_seen": 15663104 + }, + { + "epoch": 0.0, + "learning_rate": 0.0002380952380952381, + "loss": 5.502, + "theoretical_loss": 6.375797702320966, + "tokens_seen": 15728640 + }, + { + "epoch": 0.0, + "learning_rate": 0.0002390873015873016, + "loss": 5.676, + "theoretical_loss": 6.371181955668966, + "tokens_seen": 15794176 + }, + { + "epoch": 0.0, + "learning_rate": 0.0002400793650793651, + "loss": 5.3495, + "theoretical_loss": 6.366590659313248, + "tokens_seen": 15859712 + }, + { + "epoch": 0.0, + "learning_rate": 0.00024107142857142857, + "loss": 5.4804, + "theoretical_loss": 6.36202358350248, + "tokens_seen": 15925248 + }, + { + "epoch": 0.0, + "learning_rate": 0.00024206349206349207, + "loss": 5.5344, + "theoretical_loss": 6.357480501578371, + "tokens_seen": 15990784 + }, + { + "epoch": 0.0, + "learning_rate": 0.00024305555555555555, + "loss": 5.3079, + "theoretical_loss": 6.352961189921553, + "tokens_seen": 16056320 + }, + { + "epoch": 0.0, + "learning_rate": 0.00024404761904761905, + "loss": 5.2554, + "theoretical_loss": 6.348465427898629, + "tokens_seen": 16121856 + }, + { + "epoch": 0.0, + "learning_rate": 0.00024503968253968255, + "loss": 5.7887, + "theoretical_loss": 6.343992997810366, + "tokens_seen": 16187392 + }, + { + "epoch": 0.0, + "learning_rate": 0.000246031746031746, + "loss": 5.7003, + "theoretical_loss": 6.33954368484097, + "tokens_seen": 16252928 + }, + { + "epoch": 0.0, + "learning_rate": 0.00024702380952380955, + "loss": 5.7018, + "theoretical_loss": 6.33511727700846, + "tokens_seen": 16318464 + }, + { + "epoch": 0.0, + "objective/train/docs_used": 21641, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 5.011041641235352, + "objective/train/theoretical_loss": 6.330713565116083, + "objective/train/tokens_used": 36844000, + "theoretical_loss": 6.330713565116083, + "tokens_seen": 16384000 + }, + { + "epoch": 0.0, + "learning_rate": 0.000248015873015873, + "loss": 5.5501, + "theoretical_loss": 6.330713565116083, + "tokens_seen": 16384000 + }, + { + "epoch": 0.0, + "learning_rate": 0.0002490079365079365, + "loss": 5.6897, + "theoretical_loss": 6.326332342704751, + "tokens_seen": 16449536 + }, + { + "epoch": 0.01, + "learning_rate": 0.00025, + "loss": 5.4021, + "theoretical_loss": 6.32197340600647, + "tokens_seen": 16515072 + }, + { + "epoch": 0.01, + "learning_rate": 0.0002509920634920635, + "loss": 5.1547, + "theoretical_loss": 6.3176365538987636, + "tokens_seen": 16580608 + }, + { + "epoch": 0.01, + "learning_rate": 0.000251984126984127, + "loss": 5.5003, + "theoretical_loss": 6.313321587860021, + "tokens_seen": 16646144 + }, + { + "epoch": 0.01, + "learning_rate": 0.00025297619047619046, + "loss": 5.0617, + "theoretical_loss": 6.309028311925785, + "tokens_seen": 16711680 + }, + { + "epoch": 0.01, + "learning_rate": 0.00025396825396825396, + "loss": 5.4583, + "theoretical_loss": 6.304756532645939, + "tokens_seen": 16777216 + }, + { + "epoch": 0.01, + "learning_rate": 0.00025496031746031746, + "loss": 5.5747, + "theoretical_loss": 6.300506059042775, + "tokens_seen": 16842752 + }, + { + "epoch": 0.01, + "learning_rate": 0.00025595238095238096, + "loss": 5.4609, + "theoretical_loss": 6.296276702569918, + "tokens_seen": 16908288 + }, + { + "epoch": 0.01, + "learning_rate": 0.0002569444444444444, + "loss": 5.4644, + "theoretical_loss": 6.292068277072099, + "tokens_seen": 16973824 + }, + { + "epoch": 0.01, + "learning_rate": 0.00025793650793650796, + "loss": 5.1387, + "theoretical_loss": 6.28788059874573, + "tokens_seen": 17039360 + }, + { + "epoch": 0.01, + "learning_rate": 0.00025892857142857146, + "loss": 5.3902, + "theoretical_loss": 6.283713486100297, + "tokens_seen": 17104896 + }, + { + "epoch": 0.01, + "learning_rate": 0.00025992063492063497, + "loss": 5.5279, + "theoretical_loss": 6.279566759920507, + "tokens_seen": 17170432 + }, + { + "epoch": 0.01, + "learning_rate": 0.0002609126984126984, + "loss": 5.2058, + "theoretical_loss": 6.275440243229228, + "tokens_seen": 17235968 + }, + { + "epoch": 0.01, + "learning_rate": 0.0002619047619047619, + "loss": 5.5622, + "theoretical_loss": 6.271333761251142, + "tokens_seen": 17301504 + }, + { + "epoch": 0.01, + "learning_rate": 0.0002628968253968254, + "loss": 5.472, + "theoretical_loss": 6.267247141377137, + "tokens_seen": 17367040 + }, + { + "epoch": 0.01, + "learning_rate": 0.0002638888888888889, + "loss": 5.462, + "theoretical_loss": 6.2631802131294085, + "tokens_seen": 17432576 + }, + { + "epoch": 0.01, + "learning_rate": 0.00026488095238095237, + "loss": 5.5232, + "theoretical_loss": 6.259132808127246, + "tokens_seen": 17498112 + }, + { + "epoch": 0.01, + "learning_rate": 0.00026587301587301587, + "loss": 5.2239, + "theoretical_loss": 6.255104760053497, + "tokens_seen": 17563648 + }, + { + "epoch": 0.01, + "learning_rate": 0.00026686507936507937, + "loss": 5.5605, + "theoretical_loss": 6.251095904621689, + "tokens_seen": 17629184 + }, + { + "epoch": 0.01, + "learning_rate": 0.00026785714285714287, + "loss": 5.4604, + "theoretical_loss": 6.247106079543801, + "tokens_seen": 17694720 + }, + { + "epoch": 0.01, + "learning_rate": 0.0002688492063492063, + "loss": 5.348, + "theoretical_loss": 6.243135124498652, + "tokens_seen": 17760256 + }, + { + "epoch": 0.01, + "learning_rate": 0.0002698412698412698, + "loss": 5.4695, + "theoretical_loss": 6.239182881100916, + "tokens_seen": 17825792 + }, + { + "epoch": 0.01, + "learning_rate": 0.0002708333333333333, + "loss": 5.3292, + "theoretical_loss": 6.235249192870732, + "tokens_seen": 17891328 + }, + { + "epoch": 0.01, + "learning_rate": 0.0002718253968253968, + "loss": 5.3658, + "theoretical_loss": 6.231333905203899, + "tokens_seen": 17956864 + }, + { + "epoch": 0.01, + "objective/train/docs_used": 22253, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 5.1402482986450195, + "objective/train/theoretical_loss": 6.227436865342643, + "objective/train/tokens_used": 38482400, + "theoretical_loss": 6.227436865342643, + "tokens_seen": 18022400 + }, + { + "epoch": 0.01, + "learning_rate": 0.0002728174603174603, + "loss": 5.5694, + "theoretical_loss": 6.227436865342643, + "tokens_seen": 18022400 + }, + { + "epoch": 0.01, + "learning_rate": 0.0002738095238095238, + "loss": 5.3122, + "theoretical_loss": 6.223557922346955, + "tokens_seen": 18087936 + }, + { + "epoch": 0.01, + "learning_rate": 0.0002748015873015873, + "loss": 5.5297, + "theoretical_loss": 6.219696927066456, + "tokens_seen": 18153472 + }, + { + "epoch": 0.01, + "learning_rate": 0.00027579365079365083, + "loss": 5.3266, + "theoretical_loss": 6.215853732112821, + "tokens_seen": 18219008 + }, + { + "epoch": 0.01, + "learning_rate": 0.00027678571428571433, + "loss": 5.4493, + "theoretical_loss": 6.212028191832702, + "tokens_seen": 18284544 + }, + { + "epoch": 0.01, + "learning_rate": 0.0002777777777777778, + "loss": 5.0321, + "theoretical_loss": 6.208220162281178, + "tokens_seen": 18350080 + }, + { + "epoch": 0.01, + "learning_rate": 0.0002787698412698413, + "loss": 5.3719, + "theoretical_loss": 6.204429501195701, + "tokens_seen": 18415616 + }, + { + "epoch": 0.01, + "learning_rate": 0.0002797619047619048, + "loss": 5.454, + "theoretical_loss": 6.20065606797053, + "tokens_seen": 18481152 + }, + { + "epoch": 0.01, + "learning_rate": 0.0002807539682539683, + "loss": 5.4319, + "theoretical_loss": 6.19689972363164, + "tokens_seen": 18546688 + }, + { + "epoch": 0.01, + "learning_rate": 0.00028174603174603173, + "loss": 5.2545, + "theoretical_loss": 6.1931603308120975, + "tokens_seen": 18612224 + }, + { + "epoch": 0.01, + "learning_rate": 0.00028273809523809523, + "loss": 4.9311, + "theoretical_loss": 6.189437753727901, + "tokens_seen": 18677760 + }, + { + "epoch": 0.01, + "learning_rate": 0.00028373015873015873, + "loss": 5.164, + "theoretical_loss": 6.185731858154261, + "tokens_seen": 18743296 + }, + { + "epoch": 0.01, + "learning_rate": 0.00028472222222222223, + "loss": 5.3031, + "theoretical_loss": 6.182042511402313, + "tokens_seen": 18808832 + }, + { + "epoch": 0.01, + "learning_rate": 0.0002857142857142857, + "loss": 5.2461, + "theoretical_loss": 6.17836958229627, + "tokens_seen": 18874368 + }, + { + "epoch": 0.01, + "learning_rate": 0.0002867063492063492, + "loss": 5.3154, + "theoretical_loss": 6.1747129411509825, + "tokens_seen": 18939904 + }, + { + "epoch": 0.01, + "learning_rate": 0.0002876984126984127, + "loss": 5.067, + "theoretical_loss": 6.171072459749913, + "tokens_seen": 19005440 + }, + { + "epoch": 0.01, + "learning_rate": 0.0002886904761904762, + "loss": 5.2993, + "theoretical_loss": 6.1674480113235095, + "tokens_seen": 19070976 + }, + { + "epoch": 0.01, + "learning_rate": 0.0002896825396825397, + "loss": 5.2298, + "theoretical_loss": 6.163839470527964, + "tokens_seen": 19136512 + }, + { + "epoch": 0.01, + "learning_rate": 0.0002906746031746032, + "loss": 5.6501, + "theoretical_loss": 6.160246713424372, + "tokens_seen": 19202048 + }, + { + "epoch": 0.01, + "learning_rate": 0.0002916666666666667, + "loss": 5.2417, + "theoretical_loss": 6.156669617458243, + "tokens_seen": 19267584 + }, + { + "epoch": 0.01, + "learning_rate": 0.0002926587301587302, + "loss": 5.3279, + "theoretical_loss": 6.153108061439397, + "tokens_seen": 19333120 + }, + { + "epoch": 0.01, + "learning_rate": 0.0002936507936507937, + "loss": 5.5752, + "theoretical_loss": 6.149561925522211, + "tokens_seen": 19398656 + }, + { + "epoch": 0.01, + "learning_rate": 0.00029464285714285714, + "loss": 5.1687, + "theoretical_loss": 6.146031091186222, + "tokens_seen": 19464192 + }, + { + "epoch": 0.01, + "learning_rate": 0.00029563492063492064, + "loss": 5.4203, + "theoretical_loss": 6.142515441217064, + "tokens_seen": 19529728 + }, + { + "epoch": 0.01, + "learning_rate": 0.00029662698412698414, + "loss": 5.2936, + "theoretical_loss": 6.1390148596877605, + "tokens_seen": 19595264 + }, + { + "epoch": 0.01, + "objective/train/docs_used": 23612, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 5.7043352127075195, + "objective/train/theoretical_loss": 6.135529231940326, + "objective/train/tokens_used": 40120800, + "theoretical_loss": 6.135529231940326, + "tokens_seen": 19660800 + }, + { + "epoch": 0.01, + "learning_rate": 0.00029761904761904765, + "loss": 5.3328, + "theoretical_loss": 6.135529231940326, + "tokens_seen": 19660800 + }, + { + "epoch": 0.01, + "learning_rate": 0.0002986111111111111, + "loss": 5.3103, + "theoretical_loss": 6.132058444567705, + "tokens_seen": 19726336 + }, + { + "epoch": 0.01, + "learning_rate": 0.0002996031746031746, + "loss": 5.3164, + "theoretical_loss": 6.128602385396022, + "tokens_seen": 19791872 + }, + { + "epoch": 0.01, + "learning_rate": 0.0003005952380952381, + "loss": 5.2793, + "theoretical_loss": 6.125160943467138, + "tokens_seen": 19857408 + }, + { + "epoch": 0.01, + "learning_rate": 0.0003015873015873016, + "loss": 5.4137, + "theoretical_loss": 6.121734009021521, + "tokens_seen": 19922944 + }, + { + "epoch": 0.01, + "learning_rate": 0.00030257936507936505, + "loss": 5.2687, + "theoretical_loss": 6.118321473481398, + "tokens_seen": 19988480 + }, + { + "epoch": 0.01, + "learning_rate": 0.00030357142857142855, + "loss": 5.3738, + "theoretical_loss": 6.114923229434213, + "tokens_seen": 20054016 + }, + { + "epoch": 0.01, + "learning_rate": 0.00030456349206349205, + "loss": 5.2758, + "theoretical_loss": 6.111539170616359, + "tokens_seen": 20119552 + }, + { + "epoch": 0.01, + "learning_rate": 0.0003055555555555556, + "loss": 5.1104, + "theoretical_loss": 6.108169191897195, + "tokens_seen": 20185088 + }, + { + "epoch": 0.01, + "learning_rate": 0.00030654761904761905, + "loss": 4.9461, + "theoretical_loss": 6.104813189263336, + "tokens_seen": 20250624 + }, + { + "epoch": 0.01, + "learning_rate": 0.00030753968253968255, + "loss": 5.3188, + "theoretical_loss": 6.101471059803204, + "tokens_seen": 20316160 + }, + { + "epoch": 0.01, + "learning_rate": 0.00030853174603174605, + "loss": 5.3442, + "theoretical_loss": 6.098142701691856, + "tokens_seen": 20381696 + }, + { + "epoch": 0.01, + "learning_rate": 0.00030952380952380956, + "loss": 5.2134, + "theoretical_loss": 6.094828014176053, + "tokens_seen": 20447232 + }, + { + "epoch": 0.01, + "learning_rate": 0.000310515873015873, + "loss": 5.171, + "theoretical_loss": 6.091526897559593, + "tokens_seen": 20512768 + }, + { + "epoch": 0.01, + "learning_rate": 0.0003115079365079365, + "loss": 5.2667, + "theoretical_loss": 6.088239253188885, + "tokens_seen": 20578304 + }, + { + "epoch": 0.01, + "learning_rate": 0.0003125, + "loss": 5.3781, + "theoretical_loss": 6.084964983438763, + "tokens_seen": 20643840 + }, + { + "epoch": 0.01, + "learning_rate": 0.0003134920634920635, + "loss": 5.1914, + "theoretical_loss": 6.0817039916985465, + "tokens_seen": 20709376 + }, + { + "epoch": 0.01, + "learning_rate": 0.000314484126984127, + "loss": 5.0682, + "theoretical_loss": 6.078456182358325, + "tokens_seen": 20774912 + }, + { + "epoch": 0.01, + "learning_rate": 0.00031547619047619046, + "loss": 5.2664, + "theoretical_loss": 6.075221460795472, + "tokens_seen": 20840448 + }, + { + "epoch": 0.01, + "learning_rate": 0.00031646825396825396, + "loss": 5.2372, + "theoretical_loss": 6.071999733361386, + "tokens_seen": 20905984 + }, + { + "epoch": 0.01, + "learning_rate": 0.00031746031746031746, + "loss": 5.4249, + "theoretical_loss": 6.068790907368448, + "tokens_seen": 20971520 + }, + { + "epoch": 0.01, + "learning_rate": 0.00031845238095238096, + "loss": 5.1234, + "theoretical_loss": 6.0655948910771915, + "tokens_seen": 21037056 + }, + { + "epoch": 0.01, + "learning_rate": 0.0003194444444444444, + "loss": 5.2364, + "theoretical_loss": 6.062411593683687, + "tokens_seen": 21102592 + }, + { + "epoch": 0.01, + "learning_rate": 0.00032043650793650796, + "loss": 5.075, + "theoretical_loss": 6.059240925307134, + "tokens_seen": 21168128 + }, + { + "epoch": 0.01, + "learning_rate": 0.00032142857142857147, + "loss": 5.291, + "theoretical_loss": 6.056082796977648, + "tokens_seen": 21233664 + }, + { + "epoch": 0.01, + "objective/train/docs_used": 24330, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 5.891261577606201, + "objective/train/theoretical_loss": 6.052937120624258, + "objective/train/tokens_used": 41759200, + "theoretical_loss": 6.052937120624258, + "tokens_seen": 21299200 + }, + { + "epoch": 0.01, + "learning_rate": 0.00032242063492063497, + "loss": 5.39, + "theoretical_loss": 6.052937120624258, + "tokens_seen": 21299200 + }, + { + "epoch": 0.01, + "learning_rate": 0.0003234126984126984, + "loss": 5.2284, + "theoretical_loss": 6.049803809063083, + "tokens_seen": 21364736 + }, + { + "epoch": 0.01, + "learning_rate": 0.0003244047619047619, + "loss": 5.19, + "theoretical_loss": 6.0466827759857145, + "tokens_seen": 21430272 + }, + { + "epoch": 0.01, + "learning_rate": 0.0003253968253968254, + "loss": 5.2948, + "theoretical_loss": 6.04357393594778, + "tokens_seen": 21495808 + }, + { + "epoch": 0.01, + "learning_rate": 0.0003263888888888889, + "loss": 5.2322, + "theoretical_loss": 6.040477204357686, + "tokens_seen": 21561344 + }, + { + "epoch": 0.01, + "learning_rate": 0.00032738095238095237, + "loss": 5.1235, + "theoretical_loss": 6.037392497465552, + "tokens_seen": 21626880 + }, + { + "epoch": 0.01, + "learning_rate": 0.00032837301587301587, + "loss": 5.4208, + "theoretical_loss": 6.034319732352309, + "tokens_seen": 21692416 + }, + { + "epoch": 0.01, + "learning_rate": 0.00032936507936507937, + "loss": 5.3908, + "theoretical_loss": 6.031258826918979, + "tokens_seen": 21757952 + }, + { + "epoch": 0.01, + "learning_rate": 0.00033035714285714287, + "loss": 5.3411, + "theoretical_loss": 6.0282096998761245, + "tokens_seen": 21823488 + }, + { + "epoch": 0.01, + "learning_rate": 0.0003313492063492063, + "loss": 5.2287, + "theoretical_loss": 6.025172270733464, + "tokens_seen": 21889024 + }, + { + "epoch": 0.01, + "learning_rate": 0.0003323412698412698, + "loss": 5.0525, + "theoretical_loss": 6.0221464597896475, + "tokens_seen": 21954560 + }, + { + "epoch": 0.01, + "learning_rate": 0.0003333333333333333, + "loss": 5.0482, + "theoretical_loss": 6.0191321881221995, + "tokens_seen": 22020096 + }, + { + "epoch": 0.01, + "learning_rate": 0.0003343253968253968, + "loss": 5.1841, + "theoretical_loss": 6.016129377577614, + "tokens_seen": 22085632 + }, + { + "epoch": 0.01, + "learning_rate": 0.0003353174603174603, + "loss": 5.2679, + "theoretical_loss": 6.01313795076161, + "tokens_seen": 22151168 + }, + { + "epoch": 0.01, + "learning_rate": 0.0003363095238095238, + "loss": 5.3621, + "theoretical_loss": 6.010157831029533, + "tokens_seen": 22216704 + }, + { + "epoch": 0.01, + "learning_rate": 0.00033730158730158733, + "loss": 5.1528, + "theoretical_loss": 6.007188942476907, + "tokens_seen": 22282240 + }, + { + "epoch": 0.01, + "learning_rate": 0.00033829365079365083, + "loss": 5.04, + "theoretical_loss": 6.0042312099301425, + "tokens_seen": 22347776 + }, + { + "epoch": 0.01, + "learning_rate": 0.00033928571428571433, + "loss": 5.3237, + "theoretical_loss": 6.001284558937368, + "tokens_seen": 22413312 + }, + { + "epoch": 0.01, + "learning_rate": 0.0003402777777777778, + "loss": 5.1166, + "theoretical_loss": 5.998348915759426, + "tokens_seen": 22478848 + }, + { + "epoch": 0.01, + "learning_rate": 0.0003412698412698413, + "loss": 5.1229, + "theoretical_loss": 5.995424207360987, + "tokens_seen": 22544384 + }, + { + "epoch": 0.01, + "learning_rate": 0.0003422619047619048, + "loss": 5.236, + "theoretical_loss": 5.992510361401818, + "tokens_seen": 22609920 + }, + { + "epoch": 0.01, + "learning_rate": 0.0003432539682539683, + "loss": 4.9905, + "theoretical_loss": 5.989607306228168, + "tokens_seen": 22675456 + }, + { + "epoch": 0.01, + "learning_rate": 0.00034424603174603173, + "loss": 5.0993, + "theoretical_loss": 5.986714970864292, + "tokens_seen": 22740992 + }, + { + "epoch": 0.01, + "learning_rate": 0.00034523809523809523, + "loss": 5.0155, + "theoretical_loss": 5.983833285004112, + "tokens_seen": 22806528 + }, + { + "epoch": 0.01, + "learning_rate": 0.00034623015873015873, + "loss": 5.0538, + "theoretical_loss": 5.980962179002983, + "tokens_seen": 22872064 + }, + { + "epoch": 0.01, + "objective/train/docs_used": 24983, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 5.53542423248291, + "objective/train/theoretical_loss": 5.978101583869607, + "objective/train/tokens_used": 43397600, + "theoretical_loss": 5.978101583869607, + "tokens_seen": 22937600 + }, + { + "epoch": 0.01, + "learning_rate": 0.00034722222222222224, + "loss": 5.211, + "theoretical_loss": 5.978101583869607, + "tokens_seen": 22937600 + }, + { + "epoch": 0.01, + "learning_rate": 0.0003482142857142857, + "loss": 5.1569, + "theoretical_loss": 5.975251431258057, + "tokens_seen": 23003136 + }, + { + "epoch": 0.01, + "learning_rate": 0.0003492063492063492, + "loss": 5.1745, + "theoretical_loss": 5.972411653459913, + "tokens_seen": 23068672 + }, + { + "epoch": 0.01, + "learning_rate": 0.0003501984126984127, + "loss": 5.2682, + "theoretical_loss": 5.9695821833965335, + "tokens_seen": 23134208 + }, + { + "epoch": 0.01, + "learning_rate": 0.0003511904761904762, + "loss": 4.9742, + "theoretical_loss": 5.966762954611432, + "tokens_seen": 23199744 + }, + { + "epoch": 0.01, + "learning_rate": 0.0003521825396825397, + "loss": 4.9819, + "theoretical_loss": 5.963953901262764, + "tokens_seen": 23265280 + }, + { + "epoch": 0.01, + "learning_rate": 0.0003531746031746032, + "loss": 4.833, + "theoretical_loss": 5.961154958115937, + "tokens_seen": 23330816 + }, + { + "epoch": 0.01, + "learning_rate": 0.0003541666666666667, + "loss": 4.9403, + "theoretical_loss": 5.958366060536315, + "tokens_seen": 23396352 + }, + { + "epoch": 0.01, + "learning_rate": 0.0003551587301587302, + "loss": 5.2518, + "theoretical_loss": 5.955587144482044, + "tokens_seen": 23461888 + }, + { + "epoch": 0.01, + "learning_rate": 0.0003561507936507937, + "loss": 5.0226, + "theoretical_loss": 5.952818146496978, + "tokens_seen": 23527424 + }, + { + "epoch": 0.01, + "learning_rate": 0.00035714285714285714, + "loss": 5.1747, + "theoretical_loss": 5.950059003703704, + "tokens_seen": 23592960 + }, + { + "epoch": 0.01, + "learning_rate": 0.00035813492063492064, + "loss": 4.9185, + "theoretical_loss": 5.94730965379668, + "tokens_seen": 23658496 + }, + { + "epoch": 0.01, + "learning_rate": 0.00035912698412698415, + "loss": 5.2702, + "theoretical_loss": 5.944570035035458, + "tokens_seen": 23724032 + }, + { + "epoch": 0.01, + "learning_rate": 0.00036011904761904765, + "loss": 5.2239, + "theoretical_loss": 5.941840086238027, + "tokens_seen": 23789568 + }, + { + "epoch": 0.01, + "learning_rate": 0.0003611111111111111, + "loss": 5.088, + "theoretical_loss": 5.939119746774228, + "tokens_seen": 23855104 + }, + { + "epoch": 0.01, + "learning_rate": 0.0003621031746031746, + "loss": 5.3164, + "theoretical_loss": 5.936408956559284, + "tokens_seen": 23920640 + }, + { + "epoch": 0.01, + "learning_rate": 0.0003630952380952381, + "loss": 5.0086, + "theoretical_loss": 5.933707656047414, + "tokens_seen": 23986176 + }, + { + "epoch": 0.01, + "learning_rate": 0.0003640873015873016, + "loss": 5.1381, + "theoretical_loss": 5.93101578622554, + "tokens_seen": 24051712 + }, + { + "epoch": 0.01, + "learning_rate": 0.00036507936507936505, + "loss": 5.2379, + "theoretical_loss": 5.928333288607086, + "tokens_seen": 24117248 + }, + { + "epoch": 0.01, + "learning_rate": 0.00036607142857142855, + "loss": 5.1632, + "theoretical_loss": 5.925660105225867, + "tokens_seen": 24182784 + }, + { + "epoch": 0.01, + "learning_rate": 0.00036706349206349205, + "loss": 5.2687, + "theoretical_loss": 5.92299617863006, + "tokens_seen": 24248320 + }, + { + "epoch": 0.01, + "learning_rate": 0.0003680555555555556, + "loss": 5.0602, + "theoretical_loss": 5.920341451876267, + "tokens_seen": 24313856 + }, + { + "epoch": 0.01, + "learning_rate": 0.00036904761904761905, + "loss": 5.1337, + "theoretical_loss": 5.9176958685236585, + "tokens_seen": 24379392 + }, + { + "epoch": 0.01, + "learning_rate": 0.00037003968253968255, + "loss": 5.0732, + "theoretical_loss": 5.9150593726282015, + "tokens_seen": 24444928 + }, + { + "epoch": 0.01, + "learning_rate": 0.00037103174603174606, + "loss": 5.1471, + "theoretical_loss": 5.912431908736972, + "tokens_seen": 24510464 + }, + { + "epoch": 0.01, + "objective/train/docs_used": 26439, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 4.719700813293457, + "objective/train/theoretical_loss": 5.909813421882534, + "objective/train/tokens_used": 45036000, + "theoretical_loss": 5.909813421882534, + "tokens_seen": 24576000 + }, + { + "epoch": 0.01, + "learning_rate": 0.00037202380952380956, + "loss": 4.8254, + "theoretical_loss": 5.909813421882534, + "tokens_seen": 24576000 + }, + { + "epoch": 0.01, + "learning_rate": 0.000373015873015873, + "loss": 5.0976, + "theoretical_loss": 5.907203857577422, + "tokens_seen": 24641536 + }, + { + "epoch": 0.01, + "learning_rate": 0.0003740079365079365, + "loss": 4.874, + "theoretical_loss": 5.9046031618086765, + "tokens_seen": 24707072 + }, + { + "epoch": 0.01, + "learning_rate": 0.000375, + "loss": 4.9545, + "theoretical_loss": 5.902011281032472, + "tokens_seen": 24772608 + }, + { + "epoch": 0.01, + "learning_rate": 0.0003759920634920635, + "loss": 5.3031, + "theoretical_loss": 5.899428162168808, + "tokens_seen": 24838144 + }, + { + "epoch": 0.01, + "learning_rate": 0.000376984126984127, + "loss": 5.1298, + "theoretical_loss": 5.896853752596286, + "tokens_seen": 24903680 + }, + { + "epoch": 0.01, + "learning_rate": 0.00037797619047619046, + "loss": 5.123, + "theoretical_loss": 5.894288000146949, + "tokens_seen": 24969216 + }, + { + "epoch": 0.01, + "learning_rate": 0.00037896825396825396, + "loss": 5.0995, + "theoretical_loss": 5.891730853101199, + "tokens_seen": 25034752 + }, + { + "epoch": 0.01, + "learning_rate": 0.00037996031746031746, + "loss": 5.0041, + "theoretical_loss": 5.88918226018278, + "tokens_seen": 25100288 + }, + { + "epoch": 0.01, + "learning_rate": 0.00038095238095238096, + "loss": 5.2597, + "theoretical_loss": 5.8866421705538325, + "tokens_seen": 25165824 + }, + { + "epoch": 0.01, + "learning_rate": 0.0003819444444444444, + "loss": 5.0365, + "theoretical_loss": 5.8841105338100155, + "tokens_seen": 25231360 + }, + { + "epoch": 0.01, + "learning_rate": 0.00038293650793650797, + "loss": 4.9195, + "theoretical_loss": 5.881587299975694, + "tokens_seen": 25296896 + }, + { + "epoch": 0.01, + "learning_rate": 0.00038392857142857147, + "loss": 5.3162, + "theoretical_loss": 5.8790724194991935, + "tokens_seen": 25362432 + }, + { + "epoch": 0.01, + "learning_rate": 0.00038492063492063497, + "loss": 4.8924, + "theoretical_loss": 5.876565843248124, + "tokens_seen": 25427968 + }, + { + "epoch": 0.01, + "learning_rate": 0.0003859126984126984, + "loss": 5.2532, + "theoretical_loss": 5.8740675225047525, + "tokens_seen": 25493504 + }, + { + "epoch": 0.01, + "learning_rate": 0.0003869047619047619, + "loss": 5.1018, + "theoretical_loss": 5.871577408961457, + "tokens_seen": 25559040 + }, + { + "epoch": 0.01, + "learning_rate": 0.0003878968253968254, + "loss": 4.942, + "theoretical_loss": 5.869095454716231, + "tokens_seen": 25624576 + }, + { + "epoch": 0.01, + "learning_rate": 0.0003888888888888889, + "loss": 5.0305, + "theoretical_loss": 5.866621612268246, + "tokens_seen": 25690112 + }, + { + "epoch": 0.01, + "learning_rate": 0.00038988095238095237, + "loss": 5.2363, + "theoretical_loss": 5.864155834513486, + "tokens_seen": 25755648 + }, + { + "epoch": 0.01, + "learning_rate": 0.00039087301587301587, + "loss": 4.9397, + "theoretical_loss": 5.8616980747404295, + "tokens_seen": 25821184 + }, + { + "epoch": 0.01, + "learning_rate": 0.00039186507936507937, + "loss": 5.3477, + "theoretical_loss": 5.859248286625787, + "tokens_seen": 25886720 + }, + { + "epoch": 0.01, + "learning_rate": 0.0003928571428571429, + "loss": 5.3025, + "theoretical_loss": 5.856806424230314, + "tokens_seen": 25952256 + }, + { + "epoch": 0.01, + "learning_rate": 0.0003938492063492063, + "loss": 5.0934, + "theoretical_loss": 5.854372441994654, + "tokens_seen": 26017792 + }, + { + "epoch": 0.01, + "learning_rate": 0.0003948412698412698, + "loss": 5.0046, + "theoretical_loss": 5.851946294735258, + "tokens_seen": 26083328 + }, + { + "epoch": 0.01, + "learning_rate": 0.0003958333333333333, + "loss": 5.0881, + "theoretical_loss": 5.849527937640345, + "tokens_seen": 26148864 + }, + { + "epoch": 0.01, + "objective/train/docs_used": 27060, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 5.078693866729736, + "objective/train/theoretical_loss": 5.8471173262659235, + "objective/train/tokens_used": 46674400, + "theoretical_loss": 5.8471173262659235, + "tokens_seen": 26214400 + }, + { + "epoch": 0.01, + "learning_rate": 0.0003968253968253968, + "loss": 4.8549, + "theoretical_loss": 5.8471173262659235, + "tokens_seen": 26214400 + }, + { + "epoch": 0.01, + "learning_rate": 0.0003978174603174603, + "loss": 4.9043, + "theoretical_loss": 5.84471441653186, + "tokens_seen": 26279936 + }, + { + "epoch": 0.01, + "learning_rate": 0.00039880952380952383, + "loss": 5.0871, + "theoretical_loss": 5.842319164718004, + "tokens_seen": 26345472 + }, + { + "epoch": 0.01, + "learning_rate": 0.00039980158730158733, + "loss": 5.2712, + "theoretical_loss": 5.83993152746036, + "tokens_seen": 26411008 + }, + { + "epoch": 0.01, + "learning_rate": 0.00040079365079365083, + "loss": 4.9069, + "theoretical_loss": 5.83755146174731, + "tokens_seen": 26476544 + }, + { + "epoch": 0.01, + "learning_rate": 0.00040178571428571433, + "loss": 4.9664, + "theoretical_loss": 5.835178924915889, + "tokens_seen": 26542080 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004027777777777778, + "loss": 4.982, + "theoretical_loss": 5.832813874648102, + "tokens_seen": 26607616 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004037698412698413, + "loss": 5.1504, + "theoretical_loss": 5.8304562689673, + "tokens_seen": 26673152 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004047619047619048, + "loss": 4.8655, + "theoretical_loss": 5.828106066234588, + "tokens_seen": 26738688 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004057539682539683, + "loss": 4.887, + "theoretical_loss": 5.825763225145295, + "tokens_seen": 26804224 + }, + { + "epoch": 0.01, + "learning_rate": 0.00040674603174603173, + "loss": 4.8631, + "theoretical_loss": 5.823427704725473, + "tokens_seen": 26869760 + }, + { + "epoch": 0.01, + "learning_rate": 0.00040773809523809523, + "loss": 5.0702, + "theoretical_loss": 5.82109946432846, + "tokens_seen": 26935296 + }, + { + "epoch": 0.01, + "learning_rate": 0.00040873015873015874, + "loss": 4.8857, + "theoretical_loss": 5.818778463631473, + "tokens_seen": 27000832 + }, + { + "epoch": 0.01, + "learning_rate": 0.00040972222222222224, + "loss": 5.1942, + "theoretical_loss": 5.816464662632243, + "tokens_seen": 27066368 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004107142857142857, + "loss": 4.7942, + "theoretical_loss": 5.8141580216457065, + "tokens_seen": 27131904 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004117063492063492, + "loss": 5.0626, + "theoretical_loss": 5.811858501300729, + "tokens_seen": 27197440 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004126984126984127, + "loss": 5.1755, + "theoretical_loss": 5.809566062536868, + "tokens_seen": 27262976 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004136904761904762, + "loss": 4.8807, + "theoretical_loss": 5.807280666601191, + "tokens_seen": 27328512 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004146825396825397, + "loss": 5.0093, + "theoretical_loss": 5.805002275045111, + "tokens_seen": 27394048 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004156746031746032, + "loss": 4.8892, + "theoretical_loss": 5.8027308497212875, + "tokens_seen": 27459584 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004166666666666667, + "loss": 5.1595, + "theoretical_loss": 5.800466352780546, + "tokens_seen": 27525120 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004176587301587302, + "loss": 4.8036, + "theoretical_loss": 5.798208746668847, + "tokens_seen": 27590656 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004186507936507937, + "loss": 4.7298, + "theoretical_loss": 5.795957994124291, + "tokens_seen": 27656192 + }, + { + "epoch": 0.01, + "learning_rate": 0.00041964285714285714, + "loss": 4.9627, + "theoretical_loss": 5.7937140581741575, + "tokens_seen": 27721728 + }, + { + "epoch": 0.01, + "learning_rate": 0.00042063492063492065, + "loss": 4.9128, + "theoretical_loss": 5.791476902131985, + "tokens_seen": 27787264 + }, + { + "epoch": 0.01, + "objective/train/docs_used": 28453, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 4.822299003601074, + "objective/train/theoretical_loss": 5.789246489594688, + "objective/train/tokens_used": 48312800, + "theoretical_loss": 5.789246489594688, + "tokens_seen": 27852800 + }, + { + "epoch": 0.01, + "learning_rate": 0.00042162698412698415, + "loss": 4.8374, + "theoretical_loss": 5.789246489594688, + "tokens_seen": 27852800 + }, + { + "epoch": 0.01, + "learning_rate": 0.00042261904761904765, + "loss": 5.1795, + "theoretical_loss": 5.787022784439701, + "tokens_seen": 27918336 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004236111111111111, + "loss": 5.2375, + "theoretical_loss": 5.784805750822171, + "tokens_seen": 27983872 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004246031746031746, + "loss": 4.6077, + "theoretical_loss": 5.782595353172176, + "tokens_seen": 28049408 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004255952380952381, + "loss": 4.8647, + "theoretical_loss": 5.780391556191977, + "tokens_seen": 28114944 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004265873015873016, + "loss": 4.6522, + "theoretical_loss": 5.778194324853311, + "tokens_seen": 28180480 + }, + { + "epoch": 0.01, + "learning_rate": 0.00042757936507936505, + "loss": 4.8762, + "theoretical_loss": 5.776003624394711, + "tokens_seen": 28246016 + }, + { + "epoch": 0.01, + "learning_rate": 0.00042857142857142855, + "loss": 5.1233, + "theoretical_loss": 5.773819420318858, + "tokens_seen": 28311552 + }, + { + "epoch": 0.01, + "learning_rate": 0.00042956349206349205, + "loss": 4.6334, + "theoretical_loss": 5.771641678389971, + "tokens_seen": 28377088 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004305555555555556, + "loss": 4.9368, + "theoretical_loss": 5.769470364631225, + "tokens_seen": 28442624 + }, + { + "epoch": 0.01, + "learning_rate": 0.00043154761904761905, + "loss": 5.0081, + "theoretical_loss": 5.767305445322201, + "tokens_seen": 28508160 + }, + { + "epoch": 0.01, + "learning_rate": 0.00043253968253968256, + "loss": 4.6543, + "theoretical_loss": 5.765146886996363, + "tokens_seen": 28573696 + }, + { + "epoch": 0.01, + "learning_rate": 0.00043353174603174606, + "loss": 5.1211, + "theoretical_loss": 5.762994656438579, + "tokens_seen": 28639232 + }, + { + "epoch": 0.01, + "learning_rate": 0.00043452380952380956, + "loss": 4.7534, + "theoretical_loss": 5.760848720682651, + "tokens_seen": 28704768 + }, + { + "epoch": 0.01, + "learning_rate": 0.000435515873015873, + "loss": 5.2379, + "theoretical_loss": 5.758709047008894, + "tokens_seen": 28770304 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004365079365079365, + "loss": 5.1209, + "theoretical_loss": 5.756575602941732, + "tokens_seen": 28835840 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004375, + "loss": 5.1614, + "theoretical_loss": 5.75444835624733, + "tokens_seen": 28901376 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004384920634920635, + "loss": 4.608, + "theoretical_loss": 5.752327274931249, + "tokens_seen": 28966912 + }, + { + "epoch": 0.01, + "learning_rate": 0.000439484126984127, + "loss": 5.2104, + "theoretical_loss": 5.750212327236129, + "tokens_seen": 29032448 + }, + { + "epoch": 0.01, + "learning_rate": 0.00044047619047619046, + "loss": 5.3587, + "theoretical_loss": 5.7481034816394105, + "tokens_seen": 29097984 + }, + { + "epoch": 0.01, + "learning_rate": 0.00044146825396825396, + "loss": 4.918, + "theoretical_loss": 5.7460007068510635, + "tokens_seen": 29163520 + }, + { + "epoch": 0.01, + "learning_rate": 0.00044246031746031746, + "loss": 4.8193, + "theoretical_loss": 5.74390397181136, + "tokens_seen": 29229056 + }, + { + "epoch": 0.01, + "learning_rate": 0.00044345238095238096, + "loss": 5.0172, + "theoretical_loss": 5.741813245688668, + "tokens_seen": 29294592 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004444444444444444, + "loss": 5.1065, + "theoretical_loss": 5.739728497877267, + "tokens_seen": 29360128 + }, + { + "epoch": 0.01, + "learning_rate": 0.00044543650793650797, + "loss": 4.9811, + "theoretical_loss": 5.737649697995197, + "tokens_seen": 29425664 + }, + { + "epoch": 0.01, + "objective/train/docs_used": 29115, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 5.337010860443115, + "objective/train/theoretical_loss": 5.7355768158821245, + "objective/train/tokens_used": 49951200, + "theoretical_loss": 5.7355768158821245, + "tokens_seen": 29491200 + }, + { + "epoch": 0.01, + "learning_rate": 0.00044642857142857147, + "loss": 4.8053, + "theoretical_loss": 5.7355768158821245, + "tokens_seen": 29491200 + }, + { + "epoch": 0.01, + "learning_rate": 0.00044742063492063497, + "loss": 5.1626, + "theoretical_loss": 5.73350982159724, + "tokens_seen": 29556736 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004484126984126984, + "loss": 5.083, + "theoretical_loss": 5.731448685417178, + "tokens_seen": 29622272 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004494047619047619, + "loss": 4.7883, + "theoretical_loss": 5.729393377833956, + "tokens_seen": 29687808 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004503968253968254, + "loss": 5.1663, + "theoretical_loss": 5.7273438695529535, + "tokens_seen": 29753344 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004513888888888889, + "loss": 5.0705, + "theoretical_loss": 5.725300131490888, + "tokens_seen": 29818880 + }, + { + "epoch": 0.01, + "learning_rate": 0.00045238095238095237, + "loss": 4.7906, + "theoretical_loss": 5.7232621347738455, + "tokens_seen": 29884416 + }, + { + "epoch": 0.01, + "learning_rate": 0.00045337301587301587, + "loss": 4.8778, + "theoretical_loss": 5.721229850735305, + "tokens_seen": 29949952 + }, + { + "epoch": 0.01, + "learning_rate": 0.00045436507936507937, + "loss": 5.0545, + "theoretical_loss": 5.719203250914208, + "tokens_seen": 30015488 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004553571428571429, + "loss": 4.7269, + "theoretical_loss": 5.717182307053037, + "tokens_seen": 30081024 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004563492063492063, + "loss": 5.2302, + "theoretical_loss": 5.715166991095922, + "tokens_seen": 30146560 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004573412698412698, + "loss": 4.7812, + "theoretical_loss": 5.713157275186761, + "tokens_seen": 30212096 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004583333333333333, + "loss": 4.4907, + "theoretical_loss": 5.71115313166738, + "tokens_seen": 30277632 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004593253968253968, + "loss": 4.9072, + "theoretical_loss": 5.709154533075688, + "tokens_seen": 30343168 + }, + { + "epoch": 0.01, + "learning_rate": 0.00046031746031746033, + "loss": 4.9624, + "theoretical_loss": 5.707161452143879, + "tokens_seen": 30408704 + }, + { + "epoch": 0.01, + "learning_rate": 0.00046130952380952383, + "loss": 4.8523, + "theoretical_loss": 5.7051738617966326, + "tokens_seen": 30474240 + }, + { + "epoch": 0.01, + "learning_rate": 0.00046230158730158733, + "loss": 4.7722, + "theoretical_loss": 5.7031917351493515, + "tokens_seen": 30539776 + }, + { + "epoch": 0.01, + "learning_rate": 0.00046329365079365083, + "loss": 5.0354, + "theoretical_loss": 5.701215045506411, + "tokens_seen": 30605312 + }, + { + "epoch": 0.01, + "learning_rate": 0.00046428571428571433, + "loss": 4.9273, + "theoretical_loss": 5.699243766359421, + "tokens_seen": 30670848 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004652777777777778, + "loss": 5.0123, + "theoretical_loss": 5.697277871385534, + "tokens_seen": 30736384 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004662698412698413, + "loss": 5.2502, + "theoretical_loss": 5.695317334445736, + "tokens_seen": 30801920 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004672619047619048, + "loss": 5.0945, + "theoretical_loss": 5.693362129583184, + "tokens_seen": 30867456 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004682539682539683, + "loss": 4.8974, + "theoretical_loss": 5.691412231021549, + "tokens_seen": 30932992 + }, + { + "epoch": 0.01, + "learning_rate": 0.00046924603174603173, + "loss": 4.7934, + "theoretical_loss": 5.689467613163388, + "tokens_seen": 30998528 + }, + { + "epoch": 0.01, + "learning_rate": 0.00047023809523809523, + "loss": 4.8138, + "theoretical_loss": 5.687528250588518, + "tokens_seen": 31064064 + }, + { + "epoch": 0.01, + "objective/train/docs_used": 30152, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 5.014135837554932, + "objective/train/theoretical_loss": 5.6855941180524265, + "objective/train/tokens_used": 51589600, + "theoretical_loss": 5.6855941180524265, + "tokens_seen": 31129600 + }, + { + "epoch": 0.01, + "learning_rate": 0.00047123015873015874, + "loss": 4.8102, + "theoretical_loss": 5.6855941180524265, + "tokens_seen": 31129600 + }, + { + "epoch": 0.01, + "learning_rate": 0.00047222222222222224, + "loss": 4.8164, + "theoretical_loss": 5.683665190484683, + "tokens_seen": 31195136 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004732142857142857, + "loss": 4.8358, + "theoretical_loss": 5.681741442987381, + "tokens_seen": 31260672 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004742063492063492, + "loss": 4.843, + "theoretical_loss": 5.679822850833591, + "tokens_seen": 31326208 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004751984126984127, + "loss": 4.7202, + "theoretical_loss": 5.677909389465831, + "tokens_seen": 31391744 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004761904761904762, + "loss": 4.8043, + "theoretical_loss": 5.676001034494554, + "tokens_seen": 31457280 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004771825396825397, + "loss": 4.8315, + "theoretical_loss": 5.674097761696653, + "tokens_seen": 31522816 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004781746031746032, + "loss": 4.783, + "theoretical_loss": 5.672199547013983, + "tokens_seen": 31588352 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004791666666666667, + "loss": 5.0941, + "theoretical_loss": 5.670306366551898, + "tokens_seen": 31653888 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004801587301587302, + "loss": 4.9059, + "theoretical_loss": 5.6684181965778, + "tokens_seen": 31719424 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004811507936507937, + "loss": 4.9222, + "theoretical_loss": 5.666535013519715, + "tokens_seen": 31784960 + }, + { + "epoch": 0.01, + "learning_rate": 0.00048214285714285715, + "loss": 4.7335, + "theoretical_loss": 5.6646567939648715, + "tokens_seen": 31850496 + }, + { + "epoch": 0.01, + "learning_rate": 0.00048313492063492065, + "loss": 4.9942, + "theoretical_loss": 5.6627835146583045, + "tokens_seen": 31916032 + }, + { + "epoch": 0.01, + "learning_rate": 0.00048412698412698415, + "loss": 4.8914, + "theoretical_loss": 5.660915152501465, + "tokens_seen": 31981568 + }, + { + "epoch": 0.01, + "learning_rate": 0.00048511904761904765, + "loss": 4.7158, + "theoretical_loss": 5.659051684550857, + "tokens_seen": 32047104 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004861111111111111, + "loss": 4.8467, + "theoretical_loss": 5.657193088016677, + "tokens_seen": 32112640 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004871031746031746, + "loss": 5.1659, + "theoretical_loss": 5.655339340261474, + "tokens_seen": 32178176 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004880952380952381, + "loss": 4.8791, + "theoretical_loss": 5.653490418798825, + "tokens_seen": 32243712 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004890873015873016, + "loss": 4.8231, + "theoretical_loss": 5.651646301292022, + "tokens_seen": 32309248 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004900793650793651, + "loss": 4.8529, + "theoretical_loss": 5.649806965552774, + "tokens_seen": 32374784 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004910714285714286, + "loss": 5.112, + "theoretical_loss": 5.6479723895399205, + "tokens_seen": 32440320 + }, + { + "epoch": 0.01, + "learning_rate": 0.000492063492063492, + "loss": 4.8277, + "theoretical_loss": 5.6461425513581665, + "tokens_seen": 32505856 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004930555555555556, + "loss": 4.7104, + "theoretical_loss": 5.6443174292568195, + "tokens_seen": 32571392 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004940476190476191, + "loss": 4.6015, + "theoretical_loss": 5.6424970016285485, + "tokens_seen": 32636928 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004950396825396826, + "loss": 4.836, + "theoretical_loss": 5.640681247008156, + "tokens_seen": 32702464 + }, + { + "epoch": 0.01, + "objective/train/docs_used": 30758, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 5.134499549865723, + "objective/train/theoretical_loss": 5.638870144071353, + "objective/train/tokens_used": 53228000, + "theoretical_loss": 5.638870144071353, + "tokens_seen": 32768000 + }, + { + "epoch": 0.01, + "learning_rate": 0.000496031746031746, + "loss": 5.0989, + "theoretical_loss": 5.638870144071353, + "tokens_seen": 32768000 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004970238095238095, + "loss": 4.8818, + "theoretical_loss": 5.637063671633564, + "tokens_seen": 32833536 + }, + { + "epoch": 0.01, + "learning_rate": 0.000498015873015873, + "loss": 4.6817, + "theoretical_loss": 5.635261808648728, + "tokens_seen": 32899072 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004990079365079365, + "loss": 4.7503, + "theoretical_loss": 5.6334645342081195, + "tokens_seen": 32964608 + }, + { + "epoch": 0.01, + "learning_rate": 0.0005, + "loss": 4.8838, + "theoretical_loss": 5.631671827539186, + "tokens_seen": 33030144 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004999899699097292, + "loss": 4.8347, + "theoretical_loss": 5.629883668004389, + "tokens_seen": 33095680 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004999799398194584, + "loss": 4.8479, + "theoretical_loss": 5.628100035100061, + "tokens_seen": 33161216 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004999699097291876, + "loss": 4.8031, + "theoretical_loss": 5.626320908455279, + "tokens_seen": 33226752 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004999598796389167, + "loss": 4.7294, + "theoretical_loss": 5.6245462678307385, + "tokens_seen": 33292288 + }, + { + "epoch": 0.01, + "learning_rate": 0.000499949849548646, + "loss": 4.9359, + "theoretical_loss": 5.622776093117652, + "tokens_seen": 33357824 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004999398194583751, + "loss": 4.6613, + "theoretical_loss": 5.621010364336651, + "tokens_seen": 33423360 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004999297893681044, + "loss": 4.9132, + "theoretical_loss": 5.619249061636698, + "tokens_seen": 33488896 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004999197592778335, + "loss": 4.7372, + "theoretical_loss": 5.61749216529402, + "tokens_seen": 33554432 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004999097291875627, + "loss": 5.0197, + "theoretical_loss": 5.615739655711037, + "tokens_seen": 33619968 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004998996990972919, + "loss": 4.8952, + "theoretical_loss": 5.61399151341532, + "tokens_seen": 33685504 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004998896690070211, + "loss": 4.8491, + "theoretical_loss": 5.6122477190585425, + "tokens_seen": 33751040 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004998796389167503, + "loss": 4.7787, + "theoretical_loss": 5.610508253415453, + "tokens_seen": 33816576 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004998696088264795, + "loss": 4.8971, + "theoretical_loss": 5.6087730973828585, + "tokens_seen": 33882112 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004998595787362087, + "loss": 4.9652, + "theoretical_loss": 5.6070422319786095, + "tokens_seen": 33947648 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004998495486459378, + "loss": 5.0351, + "theoretical_loss": 5.605315638340606, + "tokens_seen": 34013184 + }, + { + "epoch": 0.01, + "learning_rate": 0.000499839518555667, + "loss": 4.9092, + "theoretical_loss": 5.603593297725807, + "tokens_seen": 34078720 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004998294884653962, + "loss": 4.8017, + "theoretical_loss": 5.601875191509249, + "tokens_seen": 34144256 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004998194583751254, + "loss": 4.724, + "theoretical_loss": 5.600161301183084, + "tokens_seen": 34209792 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004998094282848546, + "loss": 4.949, + "theoretical_loss": 5.598451608355614, + "tokens_seen": 34275328 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004997993981945837, + "loss": 4.9808, + "theoretical_loss": 5.596746094750342, + "tokens_seen": 34340864 + }, + { + "epoch": 0.01, + "objective/train/docs_used": 31355, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 4.490021705627441, + "objective/train/theoretical_loss": 5.595044742205037, + "objective/train/tokens_used": 54866400, + "theoretical_loss": 5.595044742205037, + "tokens_seen": 34406400 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004997893681043129, + "loss": 4.7752, + "theoretical_loss": 5.595044742205037, + "tokens_seen": 34406400 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004997793380140421, + "loss": 5.022, + "theoretical_loss": 5.5933475326707995, + "tokens_seen": 34471936 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004997693079237714, + "loss": 5.0329, + "theoretical_loss": 5.591654448211143, + "tokens_seen": 34537472 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004997592778335005, + "loss": 4.9142, + "theoretical_loss": 5.589965471001077, + "tokens_seen": 34603008 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004997492477432298, + "loss": 4.7151, + "theoretical_loss": 5.5882805833262115, + "tokens_seen": 34668544 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004997392176529588, + "loss": 4.9032, + "theoretical_loss": 5.586599767581859, + "tokens_seen": 34734080 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004997291875626881, + "loss": 4.5613, + "theoretical_loss": 5.584923006272151, + "tokens_seen": 34799616 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004997191574724173, + "loss": 4.7843, + "theoretical_loss": 5.583250282009159, + "tokens_seen": 34865152 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004997091273821465, + "loss": 4.9458, + "theoretical_loss": 5.581581577512031, + "tokens_seen": 34930688 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004996990972918757, + "loss": 4.7184, + "theoretical_loss": 5.579916875606134, + "tokens_seen": 34996224 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004996890672016048, + "loss": 4.9108, + "theoretical_loss": 5.578256159222196, + "tokens_seen": 35061760 + }, + { + "epoch": 0.01, + "learning_rate": 0.000499679037111334, + "loss": 4.973, + "theoretical_loss": 5.576599411395472, + "tokens_seen": 35127296 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004996690070210632, + "loss": 4.805, + "theoretical_loss": 5.574946615264906, + "tokens_seen": 35192832 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004996589769307924, + "loss": 4.7397, + "theoretical_loss": 5.5732977540723105, + "tokens_seen": 35258368 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004996489468405216, + "loss": 4.8079, + "theoretical_loss": 5.571652811161542, + "tokens_seen": 35323904 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004996389167502507, + "loss": 4.7984, + "theoretical_loss": 5.570011769977693, + "tokens_seen": 35389440 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004996288866599799, + "loss": 4.6185, + "theoretical_loss": 5.568374614066299, + "tokens_seen": 35454976 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004996188565697091, + "loss": 4.7032, + "theoretical_loss": 5.566741327072535, + "tokens_seen": 35520512 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004996088264794383, + "loss": 4.6364, + "theoretical_loss": 5.565111892740433, + "tokens_seen": 35586048 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004995987963891675, + "loss": 4.7849, + "theoretical_loss": 5.563486294912105, + "tokens_seen": 35651584 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004995887662988968, + "loss": 4.5396, + "theoretical_loss": 5.56186451752697, + "tokens_seen": 35717120 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004995787362086258, + "loss": 4.6833, + "theoretical_loss": 5.560246544620993, + "tokens_seen": 35782656 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004995687061183551, + "loss": 4.9462, + "theoretical_loss": 5.558632360325929, + "tokens_seen": 35848192 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004995586760280842, + "loss": 4.8909, + "theoretical_loss": 5.557021948868571, + "tokens_seen": 35913728 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004995486459378135, + "loss": 4.5196, + "theoretical_loss": 5.555415294570011, + "tokens_seen": 35979264 + }, + { + "epoch": 0.01, + "objective/train/docs_used": 32385, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 4.769467830657959, + "objective/train/theoretical_loss": 5.553812381844907, + "objective/train/tokens_used": 56504800, + "theoretical_loss": 5.553812381844907, + "tokens_seen": 36044800 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004995386158475427, + "loss": 4.8084, + "theoretical_loss": 5.553812381844907, + "tokens_seen": 36044800 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004995285857572718, + "loss": 4.7244, + "theoretical_loss": 5.552213195200755, + "tokens_seen": 36110336 + }, + { + "epoch": 0.01, + "learning_rate": 0.000499518555667001, + "loss": 4.9177, + "theoretical_loss": 5.550617719237167, + "tokens_seen": 36175872 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004995085255767302, + "loss": 4.8871, + "theoretical_loss": 5.549025938645155, + "tokens_seen": 36241408 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004994984954864594, + "loss": 4.386, + "theoretical_loss": 5.547437838206435, + "tokens_seen": 36306944 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004994884653961886, + "loss": 4.881, + "theoretical_loss": 5.545853402792717, + "tokens_seen": 36372480 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004994784353059178, + "loss": 5.0445, + "theoretical_loss": 5.544272617365014, + "tokens_seen": 36438016 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004994684052156469, + "loss": 4.5503, + "theoretical_loss": 5.542695466972956, + "tokens_seen": 36503552 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004994583751253761, + "loss": 4.6811, + "theoretical_loss": 5.541121936754111, + "tokens_seen": 36569088 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004994483450351053, + "loss": 4.653, + "theoretical_loss": 5.539552011933312, + "tokens_seen": 36634624 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004994383149448345, + "loss": 4.797, + "theoretical_loss": 5.537985677821986, + "tokens_seen": 36700160 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004994282848545637, + "loss": 4.4853, + "theoretical_loss": 5.536422919817495, + "tokens_seen": 36765696 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004994182547642928, + "loss": 4.6355, + "theoretical_loss": 5.5348637234024824, + "tokens_seen": 36831232 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004994082246740221, + "loss": 4.6619, + "theoretical_loss": 5.53330807414422, + "tokens_seen": 36896768 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004993981945837512, + "loss": 4.6313, + "theoretical_loss": 5.5317559576939725, + "tokens_seen": 36962304 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004993881644934805, + "loss": 4.7326, + "theoretical_loss": 5.530207359786353, + "tokens_seen": 37027840 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004993781344032096, + "loss": 4.6594, + "theoretical_loss": 5.5286622662386975, + "tokens_seen": 37093376 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004993681043129389, + "loss": 4.8116, + "theoretical_loss": 5.52712066295044, + "tokens_seen": 37158912 + }, + { + "epoch": 0.01, + "learning_rate": 0.000499358074222668, + "loss": 4.6854, + "theoretical_loss": 5.525582535902489, + "tokens_seen": 37224448 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004993480441323972, + "loss": 4.7265, + "theoretical_loss": 5.524047871156618, + "tokens_seen": 37289984 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004993380140421264, + "loss": 4.7761, + "theoretical_loss": 5.52251665485486, + "tokens_seen": 37355520 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004993279839518556, + "loss": 4.6545, + "theoretical_loss": 5.520988873218897, + "tokens_seen": 37421056 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004993179538615848, + "loss": 4.5957, + "theoretical_loss": 5.519464512549478, + "tokens_seen": 37486592 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004993079237713139, + "loss": 4.7608, + "theoretical_loss": 5.5179435592258095, + "tokens_seen": 37552128 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004992978936810431, + "loss": 4.455, + "theoretical_loss": 5.516425999704987, + "tokens_seen": 37617664 + }, + { + "epoch": 0.01, + "objective/train/docs_used": 32883, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 4.969899654388428, + "objective/train/theoretical_loss": 5.514911820521407, + "objective/train/tokens_used": 58143200, + "theoretical_loss": 5.514911820521407, + "tokens_seen": 37683200 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004992878635907723, + "loss": 4.7932, + "theoretical_loss": 5.514911820521407, + "tokens_seen": 37683200 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004992778335005015, + "loss": 4.6854, + "theoretical_loss": 5.5134010082861895, + "tokens_seen": 37748736 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004992678034102307, + "loss": 4.8306, + "theoretical_loss": 5.511893549686616, + "tokens_seen": 37814272 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004992577733199598, + "loss": 4.9362, + "theoretical_loss": 5.51038943148556, + "tokens_seen": 37879808 + }, + { + "epoch": 0.01, + "learning_rate": 0.000499247743229689, + "loss": 4.7838, + "theoretical_loss": 5.508888640520928, + "tokens_seen": 37945344 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004992377131394183, + "loss": 4.9437, + "theoretical_loss": 5.50739116370511, + "tokens_seen": 38010880 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004992276830491475, + "loss": 4.8481, + "theoretical_loss": 5.505896988024423, + "tokens_seen": 38076416 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004992176529588767, + "loss": 4.7686, + "theoretical_loss": 5.5044061005385725, + "tokens_seen": 38141952 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004992076228686059, + "loss": 4.734, + "theoretical_loss": 5.502918488380116, + "tokens_seen": 38207488 + }, + { + "epoch": 0.01, + "learning_rate": 0.000499197592778335, + "loss": 4.6296, + "theoretical_loss": 5.501434138753918, + "tokens_seen": 38273024 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004991875626880642, + "loss": 4.4811, + "theoretical_loss": 5.499953038936635, + "tokens_seen": 38338560 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004991775325977934, + "loss": 4.7157, + "theoretical_loss": 5.498475176276176, + "tokens_seen": 38404096 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004991675025075226, + "loss": 4.5808, + "theoretical_loss": 5.497000538191195, + "tokens_seen": 38469632 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004991574724172518, + "loss": 4.8695, + "theoretical_loss": 5.495529112170568, + "tokens_seen": 38535168 + }, + { + "epoch": 0.01, + "learning_rate": 0.000499147442326981, + "loss": 4.9383, + "theoretical_loss": 5.494060885772887, + "tokens_seen": 38600704 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004991374122367101, + "loss": 4.5244, + "theoretical_loss": 5.492595846625951, + "tokens_seen": 38666240 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004991273821464393, + "loss": 4.8075, + "theoretical_loss": 5.491133982426266, + "tokens_seen": 38731776 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004991173520561685, + "loss": 4.5661, + "theoretical_loss": 5.489675280938547, + "tokens_seen": 38797312 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004991073219658977, + "loss": 4.722, + "theoretical_loss": 5.488219729995227, + "tokens_seen": 38862848 + }, + { + "epoch": 0.01, + "learning_rate": 0.000499097291875627, + "loss": 4.7978, + "theoretical_loss": 5.486767317495966, + "tokens_seen": 38928384 + }, + { + "epoch": 0.01, + "learning_rate": 0.000499087261785356, + "loss": 4.8233, + "theoretical_loss": 5.48531803140717, + "tokens_seen": 38993920 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004990772316950853, + "loss": 4.6414, + "theoretical_loss": 5.483871859761511, + "tokens_seen": 39059456 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004990672016048144, + "loss": 4.8695, + "theoretical_loss": 5.482428790657449, + "tokens_seen": 39124992 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004990571715145437, + "loss": 4.6583, + "theoretical_loss": 5.480988812258763, + "tokens_seen": 39190528 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004990471414242729, + "loss": 4.6252, + "theoretical_loss": 5.479551912794086, + "tokens_seen": 39256064 + }, + { + "epoch": 0.01, + "objective/train/docs_used": 34166, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 4.586700439453125, + "objective/train/theoretical_loss": 5.478118080556438, + "objective/train/tokens_used": 59781600, + "theoretical_loss": 5.478118080556438, + "tokens_seen": 39321600 + }, + { + "epoch": 0.01, + "learning_rate": 0.000499037111334002, + "loss": 4.7212, + "theoretical_loss": 5.478118080556438, + "tokens_seen": 39321600 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004990270812437312, + "loss": 4.8059, + "theoretical_loss": 5.476687303902768, + "tokens_seen": 39387136 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004990170511534604, + "loss": 4.4909, + "theoretical_loss": 5.475259571253502, + "tokens_seen": 39452672 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004990070210631896, + "loss": 4.4263, + "theoretical_loss": 5.473834871092089, + "tokens_seen": 39518208 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004989969909729188, + "loss": 4.5858, + "theoretical_loss": 5.4724131919645576, + "tokens_seen": 39583744 + }, + { + "epoch": 0.01, + "learning_rate": 0.000498986960882648, + "loss": 4.5064, + "theoretical_loss": 5.470994522479069, + "tokens_seen": 39649280 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004989769307923771, + "loss": 4.9843, + "theoretical_loss": 5.4695788513054815, + "tokens_seen": 39714816 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004989669007021063, + "loss": 4.8083, + "theoretical_loss": 5.468166167174912, + "tokens_seen": 39780352 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004989568706118355, + "loss": 4.7379, + "theoretical_loss": 5.466756458879306, + "tokens_seen": 39845888 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004989468405215647, + "loss": 4.8945, + "theoretical_loss": 5.465349715271013, + "tokens_seen": 39911424 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004989368104312939, + "loss": 4.615, + "theoretical_loss": 5.463945925262355, + "tokens_seen": 39976960 + }, + { + "epoch": 0.01, + "learning_rate": 0.000498926780341023, + "loss": 4.6061, + "theoretical_loss": 5.462545077825214, + "tokens_seen": 40042496 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004989167502507523, + "loss": 4.7573, + "theoretical_loss": 5.461147161990611, + "tokens_seen": 40108032 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004989067201604814, + "loss": 4.8811, + "theoretical_loss": 5.459752166848292, + "tokens_seen": 40173568 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004988966900702107, + "loss": 4.8454, + "theoretical_loss": 5.458360081546321, + "tokens_seen": 40239104 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004988866599799398, + "loss": 4.5857, + "theoretical_loss": 5.456970895290674, + "tokens_seen": 40304640 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004988766298896691, + "loss": 4.5957, + "theoretical_loss": 5.455584597344835, + "tokens_seen": 40370176 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004988665997993982, + "loss": 4.6182, + "theoretical_loss": 5.454201177029395, + "tokens_seen": 40435712 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004988565697091274, + "loss": 4.6703, + "theoretical_loss": 5.452820623721662, + "tokens_seen": 40501248 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004988465396188566, + "loss": 4.6941, + "theoretical_loss": 5.45144292685526, + "tokens_seen": 40566784 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004988365095285858, + "loss": 4.1535, + "theoretical_loss": 5.450068075919752, + "tokens_seen": 40632320 + }, + { + "epoch": 0.01, + "learning_rate": 0.000498826479438315, + "loss": 4.803, + "theoretical_loss": 5.44869606046024, + "tokens_seen": 40697856 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004988164493480441, + "loss": 4.5452, + "theoretical_loss": 5.447326870076996, + "tokens_seen": 40763392 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004988064192577733, + "loss": 4.4677, + "theoretical_loss": 5.445960494425072, + "tokens_seen": 40828928 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004987963891675025, + "loss": 4.7102, + "theoretical_loss": 5.444596923213931, + "tokens_seen": 40894464 + }, + { + "epoch": 0.01, + "objective/train/docs_used": 34866, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 4.587603569030762, + "objective/train/theoretical_loss": 5.443236146207074, + "objective/train/tokens_used": 61420000, + "theoretical_loss": 5.443236146207074, + "tokens_seen": 40960000 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004987863590772317, + "loss": 4.5605, + "theoretical_loss": 5.443236146207074, + "tokens_seen": 40960000 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004987763289869609, + "loss": 4.7196, + "theoretical_loss": 5.441878153221662, + "tokens_seen": 41025536 + }, + { + "epoch": 0.01, + "learning_rate": 0.00049876629889669, + "loss": 4.6618, + "theoretical_loss": 5.440522934128164, + "tokens_seen": 41091072 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004987562688064192, + "loss": 4.6099, + "theoretical_loss": 5.439170478849976, + "tokens_seen": 41156608 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004987462387161484, + "loss": 4.8384, + "theoretical_loss": 5.437820777363078, + "tokens_seen": 41222144 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004987362086258777, + "loss": 4.5836, + "theoretical_loss": 5.4364738196956655, + "tokens_seen": 41287680 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004987261785356068, + "loss": 4.773, + "theoretical_loss": 5.435129595927794, + "tokens_seen": 41353216 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004987161484453361, + "loss": 4.6999, + "theoretical_loss": 5.433788096191039, + "tokens_seen": 41418752 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004987061183550651, + "loss": 4.6353, + "theoretical_loss": 5.432449310668134, + "tokens_seen": 41484288 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004986960882647944, + "loss": 4.657, + "theoretical_loss": 5.4311132295926345, + "tokens_seen": 41549824 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004986860581745236, + "loss": 4.7683, + "theoretical_loss": 5.42977984324857, + "tokens_seen": 41615360 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004986760280842528, + "loss": 4.6404, + "theoretical_loss": 5.428449141970107, + "tokens_seen": 41680896 + }, + { + "epoch": 0.01, + "learning_rate": 0.000498665997993982, + "loss": 4.7387, + "theoretical_loss": 5.427121116141212, + "tokens_seen": 41746432 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004986559679037111, + "loss": 4.6191, + "theoretical_loss": 5.42579575619531, + "tokens_seen": 41811968 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004986459378134403, + "loss": 4.8279, + "theoretical_loss": 5.424473052614967, + "tokens_seen": 41877504 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004986359077231695, + "loss": 4.5567, + "theoretical_loss": 5.423152995931552, + "tokens_seen": 41943040 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004986258776328987, + "loss": 4.4154, + "theoretical_loss": 5.421835576724906, + "tokens_seen": 42008576 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004986158475426279, + "loss": 4.3609, + "theoretical_loss": 5.420520785623031, + "tokens_seen": 42074112 + }, + { + "epoch": 0.01, + "learning_rate": 0.000498605817452357, + "loss": 4.5976, + "theoretical_loss": 5.4192086133017625, + "tokens_seen": 42139648 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004985957873620862, + "loss": 4.8015, + "theoretical_loss": 5.417899050484451, + "tokens_seen": 42205184 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004985857572718154, + "loss": 4.6452, + "theoretical_loss": 5.416592087941646, + "tokens_seen": 42270720 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004985757271815446, + "loss": 4.6363, + "theoretical_loss": 5.415287716490787, + "tokens_seen": 42336256 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004985656970912738, + "loss": 4.4392, + "theoretical_loss": 5.413985926995892, + "tokens_seen": 42401792 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004985556670010031, + "loss": 4.4929, + "theoretical_loss": 5.412686710367245, + "tokens_seen": 42467328 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004985456369107321, + "loss": 4.5347, + "theoretical_loss": 5.411390057561097, + "tokens_seen": 42532864 + }, + { + "epoch": 0.01, + "objective/train/docs_used": 35868, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 4.4660491943359375, + "objective/train/theoretical_loss": 5.410095959579362, + "objective/train/tokens_used": 63058400, + "theoretical_loss": 5.410095959579362, + "tokens_seen": 42598400 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004985356068204614, + "loss": 4.5438, + "theoretical_loss": 5.410095959579362, + "tokens_seen": 42598400 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004985255767301905, + "loss": 4.6672, + "theoretical_loss": 5.408804407469308, + "tokens_seen": 42663936 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004985155466399198, + "loss": 4.7192, + "theoretical_loss": 5.407515392323276, + "tokens_seen": 42729472 + }, + { + "epoch": 0.01, + "learning_rate": 0.000498505516549649, + "loss": 4.6645, + "theoretical_loss": 5.406228905278368, + "tokens_seen": 42795008 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004984954864593782, + "loss": 4.5282, + "theoretical_loss": 5.404944937516161, + "tokens_seen": 42860544 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004984854563691073, + "loss": 4.6349, + "theoretical_loss": 5.403663480262418, + "tokens_seen": 42926080 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004984754262788365, + "loss": 4.6171, + "theoretical_loss": 5.402384524786797, + "tokens_seen": 42991616 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004984653961885657, + "loss": 4.1495, + "theoretical_loss": 5.401108062402562, + "tokens_seen": 43057152 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004984553660982949, + "loss": 4.7403, + "theoretical_loss": 5.399834084466306, + "tokens_seen": 43122688 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004984453360080241, + "loss": 4.544, + "theoretical_loss": 5.398562582377666, + "tokens_seen": 43188224 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004984353059177532, + "loss": 4.1721, + "theoretical_loss": 5.397293547579041, + "tokens_seen": 43253760 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004984252758274825, + "loss": 4.5259, + "theoretical_loss": 5.396026971555319, + "tokens_seen": 43319296 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004984152457372116, + "loss": 4.5817, + "theoretical_loss": 5.394762845833601, + "tokens_seen": 43384832 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004984052156469409, + "loss": 4.8222, + "theoretical_loss": 5.393501161982926, + "tokens_seen": 43450368 + }, + { + "epoch": 0.01, + "learning_rate": 0.00049839518555667, + "loss": 4.6328, + "theoretical_loss": 5.392241911614005, + "tokens_seen": 43515904 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004983851554663993, + "loss": 4.6985, + "theoretical_loss": 5.390985086378949, + "tokens_seen": 43581440 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004983751253761284, + "loss": 4.5976, + "theoretical_loss": 5.389730677971002, + "tokens_seen": 43646976 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004983650952858576, + "loss": 4.6173, + "theoretical_loss": 5.388478678124285, + "tokens_seen": 43712512 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004983550651955868, + "loss": 4.6766, + "theoretical_loss": 5.387229078613521, + "tokens_seen": 43778048 + }, + { + "epoch": 0.01, + "learning_rate": 0.000498345035105316, + "loss": 4.5231, + "theoretical_loss": 5.385981871253785, + "tokens_seen": 43843584 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004983350050150452, + "loss": 4.394, + "theoretical_loss": 5.384737047900243, + "tokens_seen": 43909120 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004983249749247743, + "loss": 4.349, + "theoretical_loss": 5.3834946004478965, + "tokens_seen": 43974656 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004983149448345035, + "loss": 4.3686, + "theoretical_loss": 5.382254520831328, + "tokens_seen": 44040192 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004983049147442327, + "loss": 4.6781, + "theoretical_loss": 5.381016801024449, + "tokens_seen": 44105728 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004982948846539619, + "loss": 4.3143, + "theoretical_loss": 5.379781433040252, + "tokens_seen": 44171264 + }, + { + "epoch": 0.01, + "objective/train/docs_used": 36544, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 4.51654052734375, + "objective/train/theoretical_loss": 5.378548408930558, + "objective/train/tokens_used": 64696800, + "theoretical_loss": 5.378548408930558, + "tokens_seen": 44236800 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004982848545636911, + "loss": 4.6362, + "theoretical_loss": 5.378548408930558, + "tokens_seen": 44236800 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004982748244734202, + "loss": 4.3143, + "theoretical_loss": 5.377317720785777, + "tokens_seen": 44302336 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004982647943831494, + "loss": 4.5732, + "theoretical_loss": 5.37608936073466, + "tokens_seen": 44367872 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004982547642928786, + "loss": 4.5403, + "theoretical_loss": 5.374863320944057, + "tokens_seen": 44433408 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004982447342026079, + "loss": 4.2759, + "theoretical_loss": 5.373639593618675, + "tokens_seen": 44498944 + }, + { + "epoch": 0.01, + "learning_rate": 0.000498234704112337, + "loss": 4.4902, + "theoretical_loss": 5.372418171000847, + "tokens_seen": 44564480 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004982246740220663, + "loss": 4.6849, + "theoretical_loss": 5.371199045370283, + "tokens_seen": 44630016 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004982146439317953, + "loss": 4.5875, + "theoretical_loss": 5.369982209043851, + "tokens_seen": 44695552 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004982046138415246, + "loss": 4.8658, + "theoretical_loss": 5.368767654375327, + "tokens_seen": 44761088 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004981945837512538, + "loss": 4.5279, + "theoretical_loss": 5.367555373755179, + "tokens_seen": 44826624 + }, + { + "epoch": 0.01, + "learning_rate": 0.000498184553660983, + "loss": 4.639, + "theoretical_loss": 5.366345359610327, + "tokens_seen": 44892160 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004981745235707122, + "loss": 4.7532, + "theoretical_loss": 5.365137604403923, + "tokens_seen": 44957696 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004981644934804413, + "loss": 4.7074, + "theoretical_loss": 5.363932100635117, + "tokens_seen": 45023232 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004981544633901705, + "loss": 4.5738, + "theoretical_loss": 5.362728840838843, + "tokens_seen": 45088768 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004981444332998997, + "loss": 4.5549, + "theoretical_loss": 5.361527817585586, + "tokens_seen": 45154304 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004981344032096289, + "loss": 4.3221, + "theoretical_loss": 5.360329023481169, + "tokens_seen": 45219840 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004981243731193581, + "loss": 4.7314, + "theoretical_loss": 5.359132451166534, + "tokens_seen": 45285376 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004981143430290873, + "loss": 4.3606, + "theoretical_loss": 5.357938093317518, + "tokens_seen": 45350912 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004981043129388164, + "loss": 4.4498, + "theoretical_loss": 5.356745942644645, + "tokens_seen": 45416448 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004980942828485456, + "loss": 4.6391, + "theoretical_loss": 5.355555991892905, + "tokens_seen": 45481984 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004980842527582748, + "loss": 4.6191, + "theoretical_loss": 5.35436823384155, + "tokens_seen": 45547520 + }, + { + "epoch": 0.01, + "learning_rate": 0.000498074222668004, + "loss": 4.3579, + "theoretical_loss": 5.353182661303873, + "tokens_seen": 45613056 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004980641925777333, + "loss": 4.3796, + "theoretical_loss": 5.35199926712701, + "tokens_seen": 45678592 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004980541624874623, + "loss": 4.6802, + "theoretical_loss": 5.350818044191721, + "tokens_seen": 45744128 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004980441323971916, + "loss": 4.4053, + "theoretical_loss": 5.349638985412193, + "tokens_seen": 45809664 + }, + { + "epoch": 0.01, + "objective/train/docs_used": 37915, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.7989251613616943, + "objective/train/theoretical_loss": 5.348462083735834, + "objective/train/tokens_used": 66335200, + "theoretical_loss": 5.348462083735834, + "tokens_seen": 45875200 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004980341023069207, + "loss": 4.471, + "theoretical_loss": 5.348462083735834, + "tokens_seen": 45875200 + }, + { + "epoch": 0.01, + "learning_rate": 0.00049802407221665, + "loss": 4.3087, + "theoretical_loss": 5.347287332143064, + "tokens_seen": 45940736 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004980140421263792, + "loss": 4.5543, + "theoretical_loss": 5.346114723647119, + "tokens_seen": 46006272 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004980040120361084, + "loss": 4.5818, + "theoretical_loss": 5.344944251293852, + "tokens_seen": 46071808 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004979939819458375, + "loss": 4.5992, + "theoretical_loss": 5.343775908161532, + "tokens_seen": 46137344 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004979839518555667, + "loss": 4.6285, + "theoretical_loss": 5.342609687360644, + "tokens_seen": 46202880 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004979739217652959, + "loss": 4.573, + "theoretical_loss": 5.341445582033705, + "tokens_seen": 46268416 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004979638916750251, + "loss": 4.6819, + "theoretical_loss": 5.3402835853550545, + "tokens_seen": 46333952 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004979538615847543, + "loss": 4.4549, + "theoretical_loss": 5.339123690530673, + "tokens_seen": 46399488 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004979438314944834, + "loss": 4.4187, + "theoretical_loss": 5.337965890797989, + "tokens_seen": 46465024 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004979338014042126, + "loss": 4.3822, + "theoretical_loss": 5.336810179425685, + "tokens_seen": 46530560 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004979237713139418, + "loss": 4.4009, + "theoretical_loss": 5.335656549713516, + "tokens_seen": 46596096 + }, + { + "epoch": 0.01, + "learning_rate": 0.000497913741223671, + "loss": 4.5233, + "theoretical_loss": 5.334504994992115, + "tokens_seen": 46661632 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004979037111334002, + "loss": 4.4209, + "theoretical_loss": 5.333355508622814, + "tokens_seen": 46727168 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004978936810431293, + "loss": 4.7283, + "theoretical_loss": 5.332208083997459, + "tokens_seen": 46792704 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004978836509528586, + "loss": 4.3806, + "theoretical_loss": 5.33106271453822, + "tokens_seen": 46858240 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004978736208625877, + "loss": 4.6191, + "theoretical_loss": 5.329919393697422, + "tokens_seen": 46923776 + }, + { + "epoch": 0.01, + "learning_rate": 0.000497863590772317, + "loss": 4.3761, + "theoretical_loss": 5.328778114957351, + "tokens_seen": 46989312 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004978535606820461, + "loss": 4.4733, + "theoretical_loss": 5.327638871830089, + "tokens_seen": 47054848 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004978435305917754, + "loss": 4.3985, + "theoretical_loss": 5.326501657857326, + "tokens_seen": 47120384 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004978335005015045, + "loss": 4.3121, + "theoretical_loss": 5.32536646661019, + "tokens_seen": 47185920 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004978234704112337, + "loss": 4.5639, + "theoretical_loss": 5.324233291689069, + "tokens_seen": 47251456 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004978134403209629, + "loss": 4.5488, + "theoretical_loss": 5.323102126723439, + "tokens_seen": 47316992 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004978034102306921, + "loss": 4.5908, + "theoretical_loss": 5.321972965371691, + "tokens_seen": 47382528 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004977933801404213, + "loss": 4.6264, + "theoretical_loss": 5.320845801320959, + "tokens_seen": 47448064 + }, + { + "epoch": 0.01, + "objective/train/docs_used": 38651, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 4.225734710693359, + "objective/train/theoretical_loss": 5.319720628286955, + "objective/train/tokens_used": 67973600, + "theoretical_loss": 5.319720628286955, + "tokens_seen": 47513600 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004977833500501504, + "loss": 4.4201, + "theoretical_loss": 5.319720628286955, + "tokens_seen": 47513600 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004977733199598796, + "loss": 4.3315, + "theoretical_loss": 5.318597440013795, + "tokens_seen": 47579136 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004977632898696088, + "loss": 4.4611, + "theoretical_loss": 5.317476230273831, + "tokens_seen": 47644672 + }, + { + "epoch": 0.01, + "learning_rate": 0.000497753259779338, + "loss": 4.422, + "theoretical_loss": 5.316356992867491, + "tokens_seen": 47710208 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004977432296890672, + "loss": 4.4259, + "theoretical_loss": 5.31523972162311, + "tokens_seen": 47775744 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004977331995987965, + "loss": 4.5782, + "theoretical_loss": 5.314124410396767, + "tokens_seen": 47841280 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004977231695085255, + "loss": 4.5894, + "theoretical_loss": 5.31301105307212, + "tokens_seen": 47906816 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004977131394182548, + "loss": 4.4378, + "theoretical_loss": 5.311899643560251, + "tokens_seen": 47972352 + }, + { + "epoch": 0.01, + "learning_rate": 0.000497703109327984, + "loss": 4.4976, + "theoretical_loss": 5.310790175799497, + "tokens_seen": 48037888 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004976930792377132, + "loss": 4.4905, + "theoretical_loss": 5.3096826437553, + "tokens_seen": 48103424 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004976830491474424, + "loss": 4.6138, + "theoretical_loss": 5.308577041420046, + "tokens_seen": 48168960 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004976730190571715, + "loss": 4.4043, + "theoretical_loss": 5.3074733628129005, + "tokens_seen": 48234496 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004976629889669007, + "loss": 4.3806, + "theoretical_loss": 5.3063716019796665, + "tokens_seen": 48300032 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004976529588766299, + "loss": 4.6855, + "theoretical_loss": 5.305271752992619, + "tokens_seen": 48365568 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004976429287863591, + "loss": 4.5758, + "theoretical_loss": 5.304173809950358, + "tokens_seen": 48431104 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004976328986960883, + "loss": 4.3697, + "theoretical_loss": 5.303077766977653, + "tokens_seen": 48496640 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004976228686058175, + "loss": 4.3663, + "theoretical_loss": 5.3019836182252895, + "tokens_seen": 48562176 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004976128385155466, + "loss": 4.3629, + "theoretical_loss": 5.300891357869929, + "tokens_seen": 48627712 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004976028084252758, + "loss": 4.3293, + "theoretical_loss": 5.299800980113945, + "tokens_seen": 48693248 + }, + { + "epoch": 0.01, + "learning_rate": 0.000497592778335005, + "loss": 4.4344, + "theoretical_loss": 5.298712479185288, + "tokens_seen": 48758784 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004975827482447342, + "loss": 4.3147, + "theoretical_loss": 5.297625849337331, + "tokens_seen": 48824320 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004975727181544635, + "loss": 4.1611, + "theoretical_loss": 5.296541084848727, + "tokens_seen": 48889856 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004975626880641925, + "loss": 4.4476, + "theoretical_loss": 5.295458180023262, + "tokens_seen": 48955392 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004975526579739218, + "loss": 4.471, + "theoretical_loss": 5.294377129189715, + "tokens_seen": 49020928 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004975426278836509, + "loss": 4.4177, + "theoretical_loss": 5.293297926701706, + "tokens_seen": 49086464 + }, + { + "epoch": 0.01, + "objective/train/docs_used": 39958, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 4.2063398361206055, + "objective/train/theoretical_loss": 5.292220566937567, + "objective/train/tokens_used": 69612000, + "theoretical_loss": 5.292220566937567, + "tokens_seen": 49152000 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004975325977933802, + "loss": 4.2325, + "theoretical_loss": 5.292220566937567, + "tokens_seen": 49152000 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004975225677031094, + "loss": 4.3137, + "theoretical_loss": 5.29114504430019, + "tokens_seen": 49217536 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004975125376128386, + "loss": 4.2022, + "theoretical_loss": 5.290071353216895, + "tokens_seen": 49283072 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004975025075225677, + "loss": 4.355, + "theoretical_loss": 5.288999488139284, + "tokens_seen": 49348608 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004974924774322969, + "loss": 4.0324, + "theoretical_loss": 5.28792944354311, + "tokens_seen": 49414144 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004974824473420261, + "loss": 4.3288, + "theoretical_loss": 5.286861213928137, + "tokens_seen": 49479680 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004974724172517553, + "loss": 4.5284, + "theoretical_loss": 5.285794793817999, + "tokens_seen": 49545216 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004974623871614845, + "loss": 4.054, + "theoretical_loss": 5.284730177760077, + "tokens_seen": 49610752 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004974523570712136, + "loss": 4.6041, + "theoretical_loss": 5.283667360325351, + "tokens_seen": 49676288 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004974423269809428, + "loss": 4.3015, + "theoretical_loss": 5.2826063361082785, + "tokens_seen": 49741824 + }, + { + "epoch": 0.02, + "learning_rate": 0.000497432296890672, + "loss": 4.1752, + "theoretical_loss": 5.281547099726654, + "tokens_seen": 49807360 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004974222668004012, + "loss": 4.4212, + "theoretical_loss": 5.280489645821483, + "tokens_seen": 49872896 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004974122367101304, + "loss": 4.4892, + "theoretical_loss": 5.279433969056848, + "tokens_seen": 49938432 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004974022066198595, + "loss": 4.5725, + "theoretical_loss": 5.278380064119782, + "tokens_seen": 50003968 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004973921765295888, + "loss": 4.2418, + "theoretical_loss": 5.277327925720137, + "tokens_seen": 50069504 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004973821464393179, + "loss": 4.3186, + "theoretical_loss": 5.276277548590457, + "tokens_seen": 50135040 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004973721163490472, + "loss": 4.2967, + "theoretical_loss": 5.275228927485855, + "tokens_seen": 50200576 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004973620862587763, + "loss": 4.383, + "theoretical_loss": 5.2741820571838804, + "tokens_seen": 50266112 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004973520561685056, + "loss": 4.5176, + "theoretical_loss": 5.273136932484399, + "tokens_seen": 50331648 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004973420260782347, + "loss": 4.4906, + "theoretical_loss": 5.272093548209467, + "tokens_seen": 50397184 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004973319959879639, + "loss": 4.316, + "theoretical_loss": 5.271051899203207, + "tokens_seen": 50462720 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004973219658976931, + "loss": 4.4176, + "theoretical_loss": 5.270011980331685, + "tokens_seen": 50528256 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004973119358074223, + "loss": 4.356, + "theoretical_loss": 5.268973786482794, + "tokens_seen": 50593792 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004973019057171515, + "loss": 4.0698, + "theoretical_loss": 5.267937312566123, + "tokens_seen": 50659328 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004972918756268806, + "loss": 4.6618, + "theoretical_loss": 5.266902553512847, + "tokens_seen": 50724864 + }, + { + "epoch": 0.02, + "objective/train/docs_used": 40584, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 4.020310401916504, + "objective/train/theoretical_loss": 5.265869504275602, + "objective/train/tokens_used": 71250400, + "theoretical_loss": 5.265869504275602, + "tokens_seen": 50790400 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004972818455366098, + "loss": 4.3522, + "theoretical_loss": 5.265869504275602, + "tokens_seen": 50790400 + }, + { + "epoch": 0.02, + "learning_rate": 0.000497271815446339, + "loss": 4.1165, + "theoretical_loss": 5.264838159828369, + "tokens_seen": 50855936 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004972617853560682, + "loss": 4.4441, + "theoretical_loss": 5.263808515166355, + "tokens_seen": 50921472 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004972517552657974, + "loss": 4.4003, + "theoretical_loss": 5.262780565305875, + "tokens_seen": 50987008 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004972417251755266, + "loss": 4.3305, + "theoretical_loss": 5.261754305284241, + "tokens_seen": 51052544 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004972316950852557, + "loss": 4.3432, + "theoretical_loss": 5.260729730159641, + "tokens_seen": 51118080 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004972216649949849, + "loss": 4.3882, + "theoretical_loss": 5.259706835011027, + "tokens_seen": 51183616 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004972116349047142, + "loss": 4.4182, + "theoretical_loss": 5.2586856149380035, + "tokens_seen": 51249152 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004972016048144433, + "loss": 4.3873, + "theoretical_loss": 5.257666065060709, + "tokens_seen": 51314688 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004971915747241726, + "loss": 4.4506, + "theoretical_loss": 5.256648180519708, + "tokens_seen": 51380224 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004971815446339017, + "loss": 4.0715, + "theoretical_loss": 5.255631956475881, + "tokens_seen": 51445760 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004971715145436309, + "loss": 4.5679, + "theoretical_loss": 5.25461738811031, + "tokens_seen": 51511296 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004971614844533601, + "loss": 4.3788, + "theoretical_loss": 5.25360447062417, + "tokens_seen": 51576832 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004971514543630893, + "loss": 4.5692, + "theoretical_loss": 5.252593199238619, + "tokens_seen": 51642368 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004971414242728185, + "loss": 4.0635, + "theoretical_loss": 5.2515835691946915, + "tokens_seen": 51707904 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004971313941825477, + "loss": 4.3527, + "theoretical_loss": 5.2505755757531904, + "tokens_seen": 51773440 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004971213640922768, + "loss": 4.6217, + "theoretical_loss": 5.24956921419458, + "tokens_seen": 51838976 + }, + { + "epoch": 0.02, + "learning_rate": 0.000497111334002006, + "loss": 3.8833, + "theoretical_loss": 5.248564479818876, + "tokens_seen": 51904512 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004971013039117352, + "loss": 4.4347, + "theoretical_loss": 5.247561367945544, + "tokens_seen": 51970048 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004970912738214644, + "loss": 4.6102, + "theoretical_loss": 5.246559873913396, + "tokens_seen": 52035584 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004970812437311936, + "loss": 4.2971, + "theoretical_loss": 5.245559993080484, + "tokens_seen": 52101120 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004970712136409227, + "loss": 4.5074, + "theoretical_loss": 5.24456172082399, + "tokens_seen": 52166656 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004970611835506519, + "loss": 4.1422, + "theoretical_loss": 5.243565052540136, + "tokens_seen": 52232192 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004970511534603811, + "loss": 4.4427, + "theoretical_loss": 5.242569983644074, + "tokens_seen": 52297728 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004970411233701103, + "loss": 4.4521, + "theoretical_loss": 5.241576509569784, + "tokens_seen": 52363264 + }, + { + "epoch": 0.02, + "objective/train/docs_used": 41941, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 4.287793159484863, + "objective/train/theoretical_loss": 5.240584625769978, + "objective/train/tokens_used": 72888800, + "theoretical_loss": 5.240584625769978, + "tokens_seen": 52428800 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004970310932798396, + "loss": 4.4051, + "theoretical_loss": 5.240584625769978, + "tokens_seen": 52428800 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004970210631895686, + "loss": 4.3876, + "theoretical_loss": 5.239594327715992, + "tokens_seen": 52494336 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004970110330992979, + "loss": 4.4232, + "theoretical_loss": 5.238605610897698, + "tokens_seen": 52559872 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004970010030090271, + "loss": 4.2457, + "theoretical_loss": 5.237618470823394, + "tokens_seen": 52625408 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004969909729187563, + "loss": 4.6063, + "theoretical_loss": 5.2366329030197125, + "tokens_seen": 52690944 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004969809428284855, + "loss": 4.4803, + "theoretical_loss": 5.235648903031521, + "tokens_seen": 52756480 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004969709127382147, + "loss": 4.2552, + "theoretical_loss": 5.2346664664218245, + "tokens_seen": 52822016 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004969608826479438, + "loss": 3.9794, + "theoretical_loss": 5.233685588771669, + "tokens_seen": 52887552 + }, + { + "epoch": 0.02, + "learning_rate": 0.000496950852557673, + "loss": 4.2071, + "theoretical_loss": 5.232706265680049, + "tokens_seen": 52953088 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004969408224674022, + "loss": 4.4012, + "theoretical_loss": 5.231728492763811, + "tokens_seen": 53018624 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004969307923771314, + "loss": 4.346, + "theoretical_loss": 5.230752265657554, + "tokens_seen": 53084160 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004969207622868606, + "loss": 4.423, + "theoretical_loss": 5.229777580013545, + "tokens_seen": 53149696 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004969107321965897, + "loss": 4.3319, + "theoretical_loss": 5.228804431501619, + "tokens_seen": 53215232 + }, + { + "epoch": 0.02, + "learning_rate": 0.000496900702106319, + "loss": 4.3358, + "theoretical_loss": 5.227832815809087, + "tokens_seen": 53280768 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004968906720160481, + "loss": 4.2815, + "theoretical_loss": 5.226862728640651, + "tokens_seen": 53346304 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004968806419257774, + "loss": 4.2718, + "theoretical_loss": 5.2258941657183, + "tokens_seen": 53411840 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004968706118355065, + "loss": 4.2877, + "theoretical_loss": 5.2249271227812315, + "tokens_seen": 53477376 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004968605817452358, + "loss": 4.218, + "theoretical_loss": 5.223961595585755, + "tokens_seen": 53542912 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004968505516549649, + "loss": 4.2902, + "theoretical_loss": 5.222997579905204, + "tokens_seen": 53608448 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004968405215646941, + "loss": 4.3241, + "theoretical_loss": 5.222035071529845, + "tokens_seen": 53673984 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004968304914744233, + "loss": 4.3438, + "theoretical_loss": 5.2210740662667945, + "tokens_seen": 53739520 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004968204613841525, + "loss": 4.241, + "theoretical_loss": 5.220114559939923, + "tokens_seen": 53805056 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004968104312938817, + "loss": 4.4428, + "theoretical_loss": 5.219156548389775, + "tokens_seen": 53870592 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004968004012036108, + "loss": 4.1872, + "theoretical_loss": 5.218200027473481, + "tokens_seen": 53936128 + }, + { + "epoch": 0.02, + "learning_rate": 0.00049679037111334, + "loss": 4.5032, + "theoretical_loss": 5.217244993064664, + "tokens_seen": 54001664 + }, + { + "epoch": 0.02, + "objective/train/docs_used": 42708, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 4.48874568939209, + "objective/train/theoretical_loss": 5.216291441053366, + "objective/train/tokens_used": 74527200, + "theoretical_loss": 5.216291441053366, + "tokens_seen": 54067200 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004967803410230692, + "loss": 4.2594, + "theoretical_loss": 5.216291441053366, + "tokens_seen": 54067200 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004967703109327984, + "loss": 4.4085, + "theoretical_loss": 5.215339367345955, + "tokens_seen": 54132736 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004967602808425276, + "loss": 4.0497, + "theoretical_loss": 5.214388767865036, + "tokens_seen": 54198272 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004967502507522568, + "loss": 4.1594, + "theoretical_loss": 5.2134396385493815, + "tokens_seen": 54263808 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004967402206619859, + "loss": 4.1365, + "theoretical_loss": 5.212491975353835, + "tokens_seen": 54329344 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004967301905717151, + "loss": 4.0098, + "theoretical_loss": 5.211545774249233, + "tokens_seen": 54394880 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004967201604814444, + "loss": 4.3299, + "theoretical_loss": 5.210601031222324, + "tokens_seen": 54460416 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004967101303911735, + "loss": 4.2013, + "theoretical_loss": 5.209657742275683, + "tokens_seen": 54525952 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004967001003009028, + "loss": 4.3072, + "theoretical_loss": 5.208715903427631, + "tokens_seen": 54591488 + }, + { + "epoch": 0.02, + "learning_rate": 0.000496690070210632, + "loss": 4.1325, + "theoretical_loss": 5.207775510712159, + "tokens_seen": 54657024 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004966800401203611, + "loss": 4.2651, + "theoretical_loss": 5.2068365601788384, + "tokens_seen": 54722560 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004966700100300903, + "loss": 4.0526, + "theoretical_loss": 5.205899047892753, + "tokens_seen": 54788096 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004966599799398195, + "loss": 4.0566, + "theoretical_loss": 5.2049629699344075, + "tokens_seen": 54853632 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004966499498495487, + "loss": 4.4428, + "theoretical_loss": 5.204028322399658, + "tokens_seen": 54919168 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004966399197592779, + "loss": 4.0228, + "theoretical_loss": 5.203095101399628, + "tokens_seen": 54984704 + }, + { + "epoch": 0.02, + "learning_rate": 0.000496629889669007, + "loss": 4.1882, + "theoretical_loss": 5.202163303060633, + "tokens_seen": 55050240 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004966198595787362, + "loss": 4.2514, + "theoretical_loss": 5.201232923524104, + "tokens_seen": 55115776 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004966098294884654, + "loss": 4.2252, + "theoretical_loss": 5.20030395894651, + "tokens_seen": 55181312 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004965997993981946, + "loss": 4.3983, + "theoretical_loss": 5.199376405499277, + "tokens_seen": 55246848 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004965897693079238, + "loss": 4.0428, + "theoretical_loss": 5.198450259368721, + "tokens_seen": 55312384 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004965797392176529, + "loss": 4.3083, + "theoretical_loss": 5.197525516755965, + "tokens_seen": 55377920 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004965697091273821, + "loss": 4.3357, + "theoretical_loss": 5.196602173876867, + "tokens_seen": 55443456 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004965596790371113, + "loss": 4.2347, + "theoretical_loss": 5.195680226961947, + "tokens_seen": 55508992 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004965496489468405, + "loss": 4.1677, + "theoretical_loss": 5.194759672256309, + "tokens_seen": 55574528 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004965396188565698, + "loss": 4.173, + "theoretical_loss": 5.19384050601957, + "tokens_seen": 55640064 + }, + { + "epoch": 0.02, + "objective/train/docs_used": 44116, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 4.753946304321289, + "objective/train/theoretical_loss": 5.192922724525789, + "objective/train/tokens_used": 76165600, + "theoretical_loss": 5.192922724525789, + "tokens_seen": 55705600 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004965295887662988, + "loss": 4.3512, + "theoretical_loss": 5.192922724525789, + "tokens_seen": 55705600 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004965195586760281, + "loss": 3.9272, + "theoretical_loss": 5.19200632406339, + "tokens_seen": 55771136 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004965095285857573, + "loss": 3.7975, + "theoretical_loss": 5.19109130093509, + "tokens_seen": 55836672 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004964994984954865, + "loss": 4.2532, + "theoretical_loss": 5.190177651457833, + "tokens_seen": 55902208 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004964894684052157, + "loss": 4.2843, + "theoretical_loss": 5.189265371962712, + "tokens_seen": 55967744 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004964794383149449, + "loss": 4.0108, + "theoretical_loss": 5.188354458794902, + "tokens_seen": 56033280 + }, + { + "epoch": 0.02, + "learning_rate": 0.000496469408224674, + "loss": 3.9393, + "theoretical_loss": 5.187444908313586, + "tokens_seen": 56098816 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004964593781344032, + "loss": 3.695, + "theoretical_loss": 5.186536716891892, + "tokens_seen": 56164352 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004964493480441324, + "loss": 4.4434, + "theoretical_loss": 5.185629880916814, + "tokens_seen": 56229888 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004964393179538616, + "loss": 4.3129, + "theoretical_loss": 5.18472439678915, + "tokens_seen": 56295424 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004964292878635908, + "loss": 4.4823, + "theoretical_loss": 5.18382026092343, + "tokens_seen": 56360960 + }, + { + "epoch": 0.02, + "learning_rate": 0.00049641925777332, + "loss": 4.4461, + "theoretical_loss": 5.182917469747851, + "tokens_seen": 56426496 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004964092276830491, + "loss": 3.8878, + "theoretical_loss": 5.182016019704204, + "tokens_seen": 56492032 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004963991975927783, + "loss": 4.2078, + "theoretical_loss": 5.1811159072478095, + "tokens_seen": 56557568 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004963891675025075, + "loss": 4.0676, + "theoretical_loss": 5.180217128847451, + "tokens_seen": 56623104 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004963791374122367, + "loss": 4.317, + "theoretical_loss": 5.17931968098531, + "tokens_seen": 56688640 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004963691073219659, + "loss": 4.2516, + "theoretical_loss": 5.178423560156894, + "tokens_seen": 56754176 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004963590772316951, + "loss": 4.1935, + "theoretical_loss": 5.177528762870973, + "tokens_seen": 56819712 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004963490471414242, + "loss": 4.1931, + "theoretical_loss": 5.176635285649521, + "tokens_seen": 56885248 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004963390170511535, + "loss": 4.2398, + "theoretical_loss": 5.175743125027638, + "tokens_seen": 56950784 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004963289869608827, + "loss": 4.0872, + "theoretical_loss": 5.174852277553498, + "tokens_seen": 57016320 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004963189568706119, + "loss": 4.097, + "theoretical_loss": 5.173962739788276, + "tokens_seen": 57081856 + }, + { + "epoch": 0.02, + "learning_rate": 0.000496308926780341, + "loss": 4.1401, + "theoretical_loss": 5.17307450830609, + "tokens_seen": 57147392 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004962988966900702, + "loss": 4.0611, + "theoretical_loss": 5.172187579693933, + "tokens_seen": 57212928 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004962888665997994, + "loss": 4.2488, + "theoretical_loss": 5.1713019505516105, + "tokens_seen": 57278464 + }, + { + "epoch": 0.02, + "objective/train/docs_used": 44698, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.731816291809082, + "objective/train/theoretical_loss": 5.170417617491682, + "objective/train/tokens_used": 77804000, + "theoretical_loss": 5.170417617491682, + "tokens_seen": 57344000 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004962788365095286, + "loss": 4.0988, + "theoretical_loss": 5.170417617491682, + "tokens_seen": 57344000 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004962688064192578, + "loss": 4.0473, + "theoretical_loss": 5.169534577139395, + "tokens_seen": 57409536 + }, + { + "epoch": 0.02, + "learning_rate": 0.000496258776328987, + "loss": 4.3844, + "theoretical_loss": 5.168652826132623, + "tokens_seen": 57475072 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004962487462387161, + "loss": 4.3921, + "theoretical_loss": 5.167772361121805, + "tokens_seen": 57540608 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004962387161484453, + "loss": 4.1329, + "theoretical_loss": 5.166893178769884, + "tokens_seen": 57606144 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004962286860581746, + "loss": 4.168, + "theoretical_loss": 5.1660152757522475, + "tokens_seen": 57671680 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004962186559679037, + "loss": 4.2402, + "theoretical_loss": 5.165138648756665, + "tokens_seen": 57737216 + }, + { + "epoch": 0.02, + "learning_rate": 0.000496208625877633, + "loss": 4.1176, + "theoretical_loss": 5.164263294483226, + "tokens_seen": 57802752 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004961985957873621, + "loss": 4.5991, + "theoretical_loss": 5.163389209644287, + "tokens_seen": 57868288 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004961885656970913, + "loss": 4.1507, + "theoretical_loss": 5.162516390964408, + "tokens_seen": 57933824 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004961785356068205, + "loss": 4.4015, + "theoretical_loss": 5.1616448351802875, + "tokens_seen": 57999360 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004961685055165497, + "loss": 4.0985, + "theoretical_loss": 5.160774539040716, + "tokens_seen": 58064896 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004961584754262789, + "loss": 4.1273, + "theoretical_loss": 5.159905499306511, + "tokens_seen": 58130432 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004961484453360081, + "loss": 4.0293, + "theoretical_loss": 5.159037712750455, + "tokens_seen": 58195968 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004961384152457372, + "loss": 4.2195, + "theoretical_loss": 5.158171176157245, + "tokens_seen": 58261504 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004961283851554664, + "loss": 4.1098, + "theoretical_loss": 5.157305886323435, + "tokens_seen": 58327040 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004961183550651956, + "loss": 3.9099, + "theoretical_loss": 5.156441840057371, + "tokens_seen": 58392576 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004961083249749248, + "loss": 4.3777, + "theoretical_loss": 5.155579034179144, + "tokens_seen": 58458112 + }, + { + "epoch": 0.02, + "learning_rate": 0.000496098294884654, + "loss": 4.3101, + "theoretical_loss": 5.15471746552053, + "tokens_seen": 58523648 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004960882647943831, + "loss": 4.4277, + "theoretical_loss": 5.153857130924929, + "tokens_seen": 58589184 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004960782347041123, + "loss": 4.2158, + "theoretical_loss": 5.1529980272473175, + "tokens_seen": 58654720 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004960682046138415, + "loss": 4.2427, + "theoretical_loss": 5.152140151354191, + "tokens_seen": 58720256 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004960581745235707, + "loss": 4.1727, + "theoretical_loss": 5.151283500123505, + "tokens_seen": 58785792 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004960481444333, + "loss": 4.2338, + "theoretical_loss": 5.150428070444621, + "tokens_seen": 58851328 + }, + { + "epoch": 0.02, + "learning_rate": 0.000496038114343029, + "loss": 4.1125, + "theoretical_loss": 5.149573859218261, + "tokens_seen": 58916864 + }, + { + "epoch": 0.02, + "objective/train/docs_used": 46033, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.4300920963287354, + "objective/train/theoretical_loss": 5.1487208633564405, + "objective/train/tokens_used": 79442400, + "theoretical_loss": 5.1487208633564405, + "tokens_seen": 58982400 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004960280842527583, + "loss": 3.7513, + "theoretical_loss": 5.1487208633564405, + "tokens_seen": 58982400 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004960180541624875, + "loss": 4.1882, + "theoretical_loss": 5.147869079782423, + "tokens_seen": 59047936 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004960080240722167, + "loss": 4.3007, + "theoretical_loss": 5.147018505430666, + "tokens_seen": 59113472 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004959979939819459, + "loss": 4.1023, + "theoretical_loss": 5.146169137246765, + "tokens_seen": 59179008 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004959879638916751, + "loss": 4.1611, + "theoretical_loss": 5.145320972187402, + "tokens_seen": 59244544 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004959779338014042, + "loss": 4.2972, + "theoretical_loss": 5.144474007220293, + "tokens_seen": 59310080 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004959679037111334, + "loss": 4.0221, + "theoretical_loss": 5.143628239324139, + "tokens_seen": 59375616 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004959578736208626, + "loss": 3.9921, + "theoretical_loss": 5.142783665488567, + "tokens_seen": 59441152 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004959478435305918, + "loss": 3.9438, + "theoretical_loss": 5.1419402827140885, + "tokens_seen": 59506688 + }, + { + "epoch": 0.02, + "learning_rate": 0.000495937813440321, + "loss": 4.27, + "theoretical_loss": 5.141098088012036, + "tokens_seen": 59572224 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004959277833500501, + "loss": 4.2665, + "theoretical_loss": 5.140257078404524, + "tokens_seen": 59637760 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004959177532597793, + "loss": 4.3008, + "theoretical_loss": 5.13941725092439, + "tokens_seen": 59703296 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004959077231695085, + "loss": 4.2344, + "theoretical_loss": 5.138578602615146, + "tokens_seen": 59768832 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004958976930792377, + "loss": 4.0326, + "theoretical_loss": 5.137741130530934, + "tokens_seen": 59834368 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004958876629889669, + "loss": 3.9377, + "theoretical_loss": 5.1369048317364685, + "tokens_seen": 59899904 + }, + { + "epoch": 0.02, + "learning_rate": 0.000495877632898696, + "loss": 4.1991, + "theoretical_loss": 5.13606970330699, + "tokens_seen": 59965440 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004958676028084253, + "loss": 4.3388, + "theoretical_loss": 5.135235742328217, + "tokens_seen": 60030976 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004958575727181544, + "loss": 4.0614, + "theoretical_loss": 5.134402945896297, + "tokens_seen": 60096512 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004958475426278837, + "loss": 4.2957, + "theoretical_loss": 5.133571311117755, + "tokens_seen": 60162048 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004958375125376129, + "loss": 4.2508, + "theoretical_loss": 5.132740835109448, + "tokens_seen": 60227584 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004958274824473421, + "loss": 4.0624, + "theoretical_loss": 5.131911514998518, + "tokens_seen": 60293120 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004958174523570712, + "loss": 4.2751, + "theoretical_loss": 5.131083347922338, + "tokens_seen": 60358656 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004958074222668004, + "loss": 4.369, + "theoretical_loss": 5.130256331028474, + "tokens_seen": 60424192 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004957973921765296, + "loss": 3.7853, + "theoretical_loss": 5.129430461474628, + "tokens_seen": 60489728 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004957873620862588, + "loss": 4.2569, + "theoretical_loss": 5.128605736428597, + "tokens_seen": 60555264 + }, + { + "epoch": 0.02, + "objective/train/docs_used": 46630, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.7558958530426025, + "objective/train/theoretical_loss": 5.127782153068225, + "objective/train/tokens_used": 81080800, + "theoretical_loss": 5.127782153068225, + "tokens_seen": 60620800 + }, + { + "epoch": 0.02, + "learning_rate": 0.000495777331995988, + "loss": 3.8794, + "theoretical_loss": 5.127782153068225, + "tokens_seen": 60620800 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004957673019057172, + "loss": 3.9246, + "theoretical_loss": 5.126959708581356, + "tokens_seen": 60686336 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004957572718154463, + "loss": 3.7762, + "theoretical_loss": 5.1261384001657895, + "tokens_seen": 60751872 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004957472417251755, + "loss": 4.0968, + "theoretical_loss": 5.125318225029231, + "tokens_seen": 60817408 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004957372116349047, + "loss": 4.2332, + "theoretical_loss": 5.124499180389249, + "tokens_seen": 60882944 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004957271815446339, + "loss": 3.8761, + "theoretical_loss": 5.12368126347323, + "tokens_seen": 60948480 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004957171514543631, + "loss": 4.3047, + "theoretical_loss": 5.122864471518334, + "tokens_seen": 61014016 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004957071213640923, + "loss": 3.9338, + "theoretical_loss": 5.122048801771443, + "tokens_seen": 61079552 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004956970912738214, + "loss": 4.0911, + "theoretical_loss": 5.121234251489128, + "tokens_seen": 61145088 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004956870611835507, + "loss": 4.2958, + "theoretical_loss": 5.120420817937591, + "tokens_seen": 61210624 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004956770310932798, + "loss": 4.0011, + "theoretical_loss": 5.119608498392633, + "tokens_seen": 61276160 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004956670010030091, + "loss": 4.1588, + "theoretical_loss": 5.118797290139605, + "tokens_seen": 61341696 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004956569709127383, + "loss": 4.2555, + "theoretical_loss": 5.117987190473361, + "tokens_seen": 61407232 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004956469408224674, + "loss": 4.2073, + "theoretical_loss": 5.1171781966982195, + "tokens_seen": 61472768 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004956369107321966, + "loss": 4.0754, + "theoretical_loss": 5.116370306127921, + "tokens_seen": 61538304 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004956268806419258, + "loss": 4.185, + "theoretical_loss": 5.11556351608558, + "tokens_seen": 61603840 + }, + { + "epoch": 0.02, + "learning_rate": 0.000495616850551655, + "loss": 3.9227, + "theoretical_loss": 5.114757823903647, + "tokens_seen": 61669376 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004956068204613842, + "loss": 4.1175, + "theoretical_loss": 5.113953226923864, + "tokens_seen": 61734912 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004955967903711133, + "loss": 4.2411, + "theoretical_loss": 5.113149722497221, + "tokens_seen": 61800448 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004955867602808425, + "loss": 3.9858, + "theoretical_loss": 5.112347307983919, + "tokens_seen": 61865984 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004955767301905717, + "loss": 4.01, + "theoretical_loss": 5.111545980753322, + "tokens_seen": 61931520 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004955667001003009, + "loss": 3.9622, + "theoretical_loss": 5.110745738183919, + "tokens_seen": 61997056 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004955566700100301, + "loss": 4.2031, + "theoretical_loss": 5.109946577663284, + "tokens_seen": 62062592 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004955466399197592, + "loss": 4.2128, + "theoretical_loss": 5.109148496588032, + "tokens_seen": 62128128 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004955366098294884, + "loss": 4.2731, + "theoretical_loss": 5.108351492363779, + "tokens_seen": 62193664 + }, + { + "epoch": 0.02, + "objective/train/docs_used": 47896, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 4.220624923706055, + "objective/train/theoretical_loss": 5.107555562405102, + "objective/train/tokens_used": 82719200, + "theoretical_loss": 5.107555562405102, + "tokens_seen": 62259200 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004955265797392177, + "loss": 4.1668, + "theoretical_loss": 5.107555562405102, + "tokens_seen": 62259200 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004955165496489468, + "loss": 4.0892, + "theoretical_loss": 5.106760704135499, + "tokens_seen": 62324736 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004955065195586761, + "loss": 4.1425, + "theoretical_loss": 5.105966914987349, + "tokens_seen": 62390272 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004954964894684052, + "loss": 4.0582, + "theoretical_loss": 5.1051741924018685, + "tokens_seen": 62455808 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004954864593781344, + "loss": 4.0617, + "theoretical_loss": 5.10438253382908, + "tokens_seen": 62521344 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004954764292878636, + "loss": 3.8203, + "theoretical_loss": 5.103591936727762, + "tokens_seen": 62586880 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004954663991975928, + "loss": 3.9825, + "theoretical_loss": 5.102802398565418, + "tokens_seen": 62652416 + }, + { + "epoch": 0.02, + "learning_rate": 0.000495456369107322, + "loss": 3.8991, + "theoretical_loss": 5.102013916818235, + "tokens_seen": 62717952 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004954463390170512, + "loss": 3.9726, + "theoretical_loss": 5.101226488971042, + "tokens_seen": 62783488 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004954363089267803, + "loss": 4.0116, + "theoretical_loss": 5.100440112517276, + "tokens_seen": 62849024 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004954262788365095, + "loss": 3.729, + "theoretical_loss": 5.09965478495894, + "tokens_seen": 62914560 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004954162487462387, + "loss": 3.7828, + "theoretical_loss": 5.098870503806567, + "tokens_seen": 62980096 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004954062186559679, + "loss": 3.8274, + "theoretical_loss": 5.09808726657918, + "tokens_seen": 63045632 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004953961885656971, + "loss": 3.9166, + "theoretical_loss": 5.097305070804255, + "tokens_seen": 63111168 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004953861584754263, + "loss": 4.2024, + "theoretical_loss": 5.096523914017688, + "tokens_seen": 63176704 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004953761283851555, + "loss": 4.0619, + "theoretical_loss": 5.095743793763747, + "tokens_seen": 63242240 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004953660982948846, + "loss": 4.0306, + "theoretical_loss": 5.094964707595047, + "tokens_seen": 63307776 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004953560682046139, + "loss": 4.0187, + "theoretical_loss": 5.094186653072505, + "tokens_seen": 63373312 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004953460381143431, + "loss": 4.0173, + "theoretical_loss": 5.093409627765306, + "tokens_seen": 63438848 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004953360080240723, + "loss": 3.9407, + "theoretical_loss": 5.092633629250866, + "tokens_seen": 63504384 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004953259779338014, + "loss": 3.85, + "theoretical_loss": 5.091858655114796, + "tokens_seen": 63569920 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004953159478435306, + "loss": 4.0891, + "theoretical_loss": 5.091084702950868, + "tokens_seen": 63635456 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004953059177532598, + "loss": 4.1001, + "theoretical_loss": 5.090311770360971, + "tokens_seen": 63700992 + }, + { + "epoch": 0.02, + "learning_rate": 0.000495295887662989, + "loss": 4.0473, + "theoretical_loss": 5.089539854955088, + "tokens_seen": 63766528 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004952858575727182, + "loss": 3.9194, + "theoretical_loss": 5.088768954351249, + "tokens_seen": 63832064 + }, + { + "epoch": 0.02, + "objective/train/docs_used": 48551, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.955749034881592, + "objective/train/theoretical_loss": 5.087999066175502, + "objective/train/tokens_used": 84357600, + "theoretical_loss": 5.087999066175502, + "tokens_seen": 63897600 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004952758274824474, + "loss": 4.0786, + "theoretical_loss": 5.087999066175502, + "tokens_seen": 63897600 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004952657973921765, + "loss": 4.0172, + "theoretical_loss": 5.0872301880618735, + "tokens_seen": 63963136 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004952557673019057, + "loss": 3.9146, + "theoretical_loss": 5.086462317652341, + "tokens_seen": 64028672 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004952457372116349, + "loss": 3.9419, + "theoretical_loss": 5.085695452596788, + "tokens_seen": 64094208 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004952357071213641, + "loss": 3.9896, + "theoretical_loss": 5.084929590552976, + "tokens_seen": 64159744 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004952256770310933, + "loss": 3.9745, + "theoretical_loss": 5.0841647291865115, + "tokens_seen": 64225280 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004952156469408225, + "loss": 3.8429, + "theoretical_loss": 5.083400866170806, + "tokens_seen": 64290816 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004952056168505516, + "loss": 4.1516, + "theoretical_loss": 5.082637999187046, + "tokens_seen": 64356352 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004951955867602809, + "loss": 4.0323, + "theoretical_loss": 5.081876125924159, + "tokens_seen": 64421888 + }, + { + "epoch": 0.02, + "learning_rate": 0.00049518555667001, + "loss": 3.9024, + "theoretical_loss": 5.0811152440787755, + "tokens_seen": 64487424 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004951755265797393, + "loss": 3.9658, + "theoretical_loss": 5.0803553513552036, + "tokens_seen": 64552960 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004951654964894685, + "loss": 3.8413, + "theoretical_loss": 5.079596445465386, + "tokens_seen": 64618496 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004951554663991976, + "loss": 3.6003, + "theoretical_loss": 5.078838524128878, + "tokens_seen": 64684032 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004951454363089268, + "loss": 3.9108, + "theoretical_loss": 5.078081585072802, + "tokens_seen": 64749568 + }, + { + "epoch": 0.02, + "learning_rate": 0.000495135406218656, + "loss": 3.7839, + "theoretical_loss": 5.077325626031826, + "tokens_seen": 64815104 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004951253761283852, + "loss": 3.8686, + "theoretical_loss": 5.076570644748123, + "tokens_seen": 64880640 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004951153460381144, + "loss": 3.9367, + "theoretical_loss": 5.075816638971341, + "tokens_seen": 64946176 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004951053159478435, + "loss": 3.8991, + "theoretical_loss": 5.075063606458576, + "tokens_seen": 65011712 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004950952858575727, + "loss": 3.9725, + "theoretical_loss": 5.074311544974331, + "tokens_seen": 65077248 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004950852557673019, + "loss": 3.958, + "theoretical_loss": 5.07356045229049, + "tokens_seen": 65142784 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004950752256770311, + "loss": 3.8858, + "theoretical_loss": 5.072810326186285, + "tokens_seen": 65208320 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004950651955867603, + "loss": 3.6259, + "theoretical_loss": 5.072061164448261, + "tokens_seen": 65273856 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004950551654964894, + "loss": 4.0376, + "theoretical_loss": 5.071312964870252, + "tokens_seen": 65339392 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004950451354062186, + "loss": 3.8566, + "theoretical_loss": 5.070565725253344, + "tokens_seen": 65404928 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004950351053159479, + "loss": 4.1127, + "theoretical_loss": 5.069819443405842, + "tokens_seen": 65470464 + }, + { + "epoch": 0.02, + "objective/train/docs_used": 49128, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 4.046652317047119, + "objective/train/theoretical_loss": 5.069074117143246, + "objective/train/tokens_used": 85996000, + "theoretical_loss": 5.069074117143246, + "tokens_seen": 65536000 + }, + { + "epoch": 0.02, + "learning_rate": 0.000495025075225677, + "loss": 3.8614, + "theoretical_loss": 5.069074117143246, + "tokens_seen": 65536000 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004950150451354063, + "loss": 3.8885, + "theoretical_loss": 5.068329744288216, + "tokens_seen": 65601536 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004950050150451354, + "loss": 4.1403, + "theoretical_loss": 5.067586322670541, + "tokens_seen": 65667072 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004949949849548646, + "loss": 3.6575, + "theoretical_loss": 5.0668438501271105, + "tokens_seen": 65732608 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004949849548645938, + "loss": 3.7839, + "theoretical_loss": 5.066102324501883, + "tokens_seen": 65798144 + }, + { + "epoch": 0.02, + "learning_rate": 0.000494974924774323, + "loss": 3.9434, + "theoretical_loss": 5.065361743645855, + "tokens_seen": 65863680 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004949648946840522, + "loss": 3.8805, + "theoretical_loss": 5.064622105417033, + "tokens_seen": 65929216 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004949548645937814, + "loss": 3.8876, + "theoretical_loss": 5.063883407680405, + "tokens_seen": 65994752 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004949448345035105, + "loss": 3.731, + "theoretical_loss": 5.063145648307904, + "tokens_seen": 66060288 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004949348044132397, + "loss": 3.7655, + "theoretical_loss": 5.062408825178388, + "tokens_seen": 66125824 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004949247743229689, + "loss": 3.7157, + "theoretical_loss": 5.061672936177604, + "tokens_seen": 66191360 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004949147442326981, + "loss": 3.8121, + "theoretical_loss": 5.06093797919816, + "tokens_seen": 66256896 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004949047141424273, + "loss": 3.692, + "theoretical_loss": 5.060203952139497, + "tokens_seen": 66322432 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004948946840521565, + "loss": 3.8701, + "theoretical_loss": 5.059470852907861, + "tokens_seen": 66387968 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004948846539618856, + "loss": 4.131, + "theoretical_loss": 5.0587386794162725, + "tokens_seen": 66453504 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004948746238716148, + "loss": 3.766, + "theoretical_loss": 5.058007429584498, + "tokens_seen": 66519040 + }, + { + "epoch": 0.02, + "learning_rate": 0.000494864593781344, + "loss": 3.6942, + "theoretical_loss": 5.057277101339023, + "tokens_seen": 66584576 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004948545636910733, + "loss": 3.7047, + "theoretical_loss": 5.056547692613021, + "tokens_seen": 66650112 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004948445336008024, + "loss": 3.4421, + "theoretical_loss": 5.055819201346331, + "tokens_seen": 66715648 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004948345035105316, + "loss": 4.0847, + "theoretical_loss": 5.055091625485421, + "tokens_seen": 66781184 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004948244734202607, + "loss": 3.8288, + "theoretical_loss": 5.054364962983367, + "tokens_seen": 66846720 + }, + { + "epoch": 0.02, + "learning_rate": 0.00049481444332999, + "loss": 3.774, + "theoretical_loss": 5.053639211799824, + "tokens_seen": 66912256 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004948044132397192, + "loss": 3.8226, + "theoretical_loss": 5.052914369900997, + "tokens_seen": 66977792 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004947943831494484, + "loss": 3.8495, + "theoretical_loss": 5.052190435259614, + "tokens_seen": 67043328 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004947843530591776, + "loss": 4.14, + "theoretical_loss": 5.051467405854897, + "tokens_seen": 67108864 + }, + { + "epoch": 0.02, + "objective/train/docs_used": 50658, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.563347578048706, + "objective/train/theoretical_loss": 5.05074527967254, + "objective/train/tokens_used": 87634400, + "theoretical_loss": 5.05074527967254, + "tokens_seen": 67174400 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004947743229689067, + "loss": 3.7245, + "theoretical_loss": 5.05074527967254, + "tokens_seen": 67174400 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004947642928786359, + "loss": 3.7286, + "theoretical_loss": 5.050024054704677, + "tokens_seen": 67239936 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004947542627883651, + "loss": 3.8564, + "theoretical_loss": 5.049303728949859, + "tokens_seen": 67305472 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004947442326980943, + "loss": 3.9023, + "theoretical_loss": 5.048584300413019, + "tokens_seen": 67371008 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004947342026078235, + "loss": 3.7712, + "theoretical_loss": 5.04786576710546, + "tokens_seen": 67436544 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004947241725175527, + "loss": 3.9541, + "theoretical_loss": 5.0471481270448155, + "tokens_seen": 67502080 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004947141424272818, + "loss": 4.0528, + "theoretical_loss": 5.046431378255027, + "tokens_seen": 67567616 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004947041123370111, + "loss": 4.0139, + "theoretical_loss": 5.045715518766322, + "tokens_seen": 67633152 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004946940822467402, + "loss": 3.9368, + "theoretical_loss": 5.0450005466151815, + "tokens_seen": 67698688 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004946840521564695, + "loss": 3.873, + "theoretical_loss": 5.044286459844319, + "tokens_seen": 67764224 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004946740220661987, + "loss": 3.7419, + "theoretical_loss": 5.043573256502652, + "tokens_seen": 67829760 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004946639919759278, + "loss": 3.9483, + "theoretical_loss": 5.0428609346452795, + "tokens_seen": 67895296 + }, + { + "epoch": 0.02, + "learning_rate": 0.000494653961885657, + "loss": 3.6408, + "theoretical_loss": 5.042149492333452, + "tokens_seen": 67960832 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004946439317953862, + "loss": 3.7635, + "theoretical_loss": 5.041438927634549, + "tokens_seen": 68026368 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004946339017051154, + "loss": 3.8077, + "theoretical_loss": 5.040729238622053, + "tokens_seen": 68091904 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004946238716148446, + "loss": 3.9545, + "theoretical_loss": 5.040020423375525, + "tokens_seen": 68157440 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004946138415245737, + "loss": 3.7098, + "theoretical_loss": 5.039312479980579, + "tokens_seen": 68222976 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004946038114343029, + "loss": 4.1081, + "theoretical_loss": 5.038605406528857, + "tokens_seen": 68288512 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004945937813440321, + "loss": 3.8545, + "theoretical_loss": 5.037899201118005, + "tokens_seen": 68354048 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004945837512537613, + "loss": 4.0689, + "theoretical_loss": 5.037193861851646, + "tokens_seen": 68419584 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004945737211634905, + "loss": 3.6816, + "theoretical_loss": 5.03648938683936, + "tokens_seen": 68485120 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004945636910732196, + "loss": 3.7314, + "theoretical_loss": 5.035785774196654, + "tokens_seen": 68550656 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004945536609829488, + "loss": 3.792, + "theoretical_loss": 5.035083022044944, + "tokens_seen": 68616192 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004945436308926781, + "loss": 3.9028, + "theoretical_loss": 5.034381128511525, + "tokens_seen": 68681728 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004945336008024072, + "loss": 3.8524, + "theoretical_loss": 5.0336800917295506, + "tokens_seen": 68747264 + }, + { + "epoch": 0.02, + "objective/train/docs_used": 51428, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.2750582695007324, + "objective/train/theoretical_loss": 5.032979909838007, + "objective/train/tokens_used": 89272800, + "theoretical_loss": 5.032979909838007, + "tokens_seen": 68812800 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004945235707121365, + "loss": 3.7239, + "theoretical_loss": 5.032979909838007, + "tokens_seen": 68812800 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004945135406218656, + "loss": 3.913, + "theoretical_loss": 5.032280580981691, + "tokens_seen": 68878336 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004945035105315948, + "loss": 3.8049, + "theoretical_loss": 5.031582103311187, + "tokens_seen": 68943872 + }, + { + "epoch": 0.02, + "learning_rate": 0.000494493480441324, + "loss": 3.7734, + "theoretical_loss": 5.030884474982842, + "tokens_seen": 69009408 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004944834503510532, + "loss": 3.9018, + "theoretical_loss": 5.030187694158739, + "tokens_seen": 69074944 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004944734202607824, + "loss": 3.8771, + "theoretical_loss": 5.02949175900668, + "tokens_seen": 69140480 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004944633901705116, + "loss": 3.835, + "theoretical_loss": 5.028796667700159, + "tokens_seen": 69206016 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004944533600802407, + "loss": 3.862, + "theoretical_loss": 5.0281024184183405, + "tokens_seen": 69271552 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004944433299899699, + "loss": 3.9933, + "theoretical_loss": 5.0274090093460355, + "tokens_seen": 69337088 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004944332998996991, + "loss": 3.8217, + "theoretical_loss": 5.026716438673677, + "tokens_seen": 69402624 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004944232698094283, + "loss": 3.9852, + "theoretical_loss": 5.0260247045973045, + "tokens_seen": 69468160 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004944132397191575, + "loss": 3.8171, + "theoretical_loss": 5.02533380531853, + "tokens_seen": 69533696 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004944032096288867, + "loss": 3.7941, + "theoretical_loss": 5.024643739044526, + "tokens_seen": 69599232 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004943931795386158, + "loss": 3.9112, + "theoretical_loss": 5.023954503987998, + "tokens_seen": 69664768 + }, + { + "epoch": 0.02, + "learning_rate": 0.000494383149448345, + "loss": 3.7425, + "theoretical_loss": 5.023266098367161, + "tokens_seen": 69730304 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004943731193580742, + "loss": 3.8296, + "theoretical_loss": 5.022578520405721, + "tokens_seen": 69795840 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004943630892678035, + "loss": 3.7659, + "theoretical_loss": 5.0218917683328534, + "tokens_seen": 69861376 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004943530591775326, + "loss": 3.737, + "theoretical_loss": 5.021205840383175, + "tokens_seen": 69926912 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004943430290872618, + "loss": 3.9746, + "theoretical_loss": 5.020520734796728, + "tokens_seen": 69992448 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004943329989969909, + "loss": 3.7522, + "theoretical_loss": 5.019836449818957, + "tokens_seen": 70057984 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004943229689067202, + "loss": 3.5721, + "theoretical_loss": 5.019152983700687, + "tokens_seen": 70123520 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004943129388164494, + "loss": 3.5567, + "theoretical_loss": 5.018470334698101, + "tokens_seen": 70189056 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004943029087261786, + "loss": 3.6431, + "theoretical_loss": 5.01778850107272, + "tokens_seen": 70254592 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004942928786359078, + "loss": 3.9104, + "theoretical_loss": 5.017107481091379, + "tokens_seen": 70320128 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004942828485456369, + "loss": 3.8892, + "theoretical_loss": 5.016427273026212, + "tokens_seen": 70385664 + }, + { + "epoch": 0.02, + "objective/train/docs_used": 52769, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.426807403564453, + "objective/train/theoretical_loss": 5.015747875154622, + "objective/train/tokens_used": 90911200, + "theoretical_loss": 5.015747875154622, + "tokens_seen": 70451200 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004942728184553661, + "loss": 3.657, + "theoretical_loss": 5.015747875154622, + "tokens_seen": 70451200 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004942627883650953, + "loss": 3.9685, + "theoretical_loss": 5.015069285759269, + "tokens_seen": 70516736 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004942527582748245, + "loss": 3.6661, + "theoretical_loss": 5.01439150312804, + "tokens_seen": 70582272 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004942427281845537, + "loss": 3.8004, + "theoretical_loss": 5.0137145255540405, + "tokens_seen": 70647808 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004942326980942828, + "loss": 3.8299, + "theoretical_loss": 5.013038351335559, + "tokens_seen": 70713344 + }, + { + "epoch": 0.02, + "learning_rate": 0.000494222668004012, + "loss": 3.8034, + "theoretical_loss": 5.012362978776057, + "tokens_seen": 70778880 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004942126379137412, + "loss": 3.8363, + "theoretical_loss": 5.011688406184147, + "tokens_seen": 70844416 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004942026078234704, + "loss": 3.933, + "theoretical_loss": 5.011014631873566, + "tokens_seen": 70909952 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004941925777331996, + "loss": 3.6524, + "theoretical_loss": 5.010341654163167, + "tokens_seen": 70975488 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004941825476429289, + "loss": 3.6585, + "theoretical_loss": 5.009669471376882, + "tokens_seen": 71041024 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004941725175526579, + "loss": 3.6262, + "theoretical_loss": 5.008998081843721, + "tokens_seen": 71106560 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004941624874623872, + "loss": 3.9818, + "theoretical_loss": 5.008327483897736, + "tokens_seen": 71172096 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004941524573721163, + "loss": 3.8184, + "theoretical_loss": 5.00765767587801, + "tokens_seen": 71237632 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004941424272818456, + "loss": 3.8068, + "theoretical_loss": 5.006988656128635, + "tokens_seen": 71303168 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004941323971915748, + "loss": 3.81, + "theoretical_loss": 5.006320422998691, + "tokens_seen": 71368704 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004941223671013039, + "loss": 4.1206, + "theoretical_loss": 5.00565297484223, + "tokens_seen": 71434240 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004941123370110331, + "loss": 3.8121, + "theoretical_loss": 5.004986310018252, + "tokens_seen": 71499776 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004941023069207623, + "loss": 3.5212, + "theoretical_loss": 5.004320426890686, + "tokens_seen": 71565312 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004940922768304915, + "loss": 3.7905, + "theoretical_loss": 5.003655323828376, + "tokens_seen": 71630848 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004940822467402207, + "loss": 3.7737, + "theoretical_loss": 5.002990999205057, + "tokens_seen": 71696384 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004940722166499498, + "loss": 4.0285, + "theoretical_loss": 5.002327451399335, + "tokens_seen": 71761920 + }, + { + "epoch": 0.02, + "learning_rate": 0.000494062186559679, + "loss": 3.6866, + "theoretical_loss": 5.001664678794671, + "tokens_seen": 71827456 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004940521564694082, + "loss": 3.624, + "theoretical_loss": 5.001002679779363, + "tokens_seen": 71892992 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004940421263791374, + "loss": 3.9906, + "theoretical_loss": 5.0003414527465235, + "tokens_seen": 71958528 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004940320962888666, + "loss": 3.9518, + "theoretical_loss": 4.99968099609406, + "tokens_seen": 72024064 + }, + { + "epoch": 0.02, + "objective/train/docs_used": 53229, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.645538091659546, + "objective/train/theoretical_loss": 4.999021308224664, + "objective/train/tokens_used": 92549600, + "theoretical_loss": 4.999021308224664, + "tokens_seen": 72089600 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004940220661985958, + "loss": 3.9344, + "theoretical_loss": 4.999021308224664, + "tokens_seen": 72089600 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004940120361083249, + "loss": 3.6264, + "theoretical_loss": 4.998362387545782, + "tokens_seen": 72155136 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004940020060180542, + "loss": 3.9681, + "theoretical_loss": 4.997704232469606, + "tokens_seen": 72220672 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004939919759277834, + "loss": 3.8812, + "theoretical_loss": 4.997046841413049, + "tokens_seen": 72286208 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004939819458375126, + "loss": 4.1244, + "theoretical_loss": 4.996390212797728, + "tokens_seen": 72351744 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004939719157472418, + "loss": 3.891, + "theoretical_loss": 4.995734345049949, + "tokens_seen": 72417280 + }, + { + "epoch": 0.02, + "learning_rate": 0.000493961885656971, + "loss": 3.4783, + "theoretical_loss": 4.995079236600686, + "tokens_seen": 72482816 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004939518555667001, + "loss": 3.5267, + "theoretical_loss": 4.994424885885564, + "tokens_seen": 72548352 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004939418254764293, + "loss": 3.9523, + "theoretical_loss": 4.993771291344839, + "tokens_seen": 72613888 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004939317953861585, + "loss": 3.701, + "theoretical_loss": 4.993118451423381, + "tokens_seen": 72679424 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004939217652958877, + "loss": 4.0333, + "theoretical_loss": 4.992466364570659, + "tokens_seen": 72744960 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004939117352056169, + "loss": 3.7307, + "theoretical_loss": 4.991815029240721, + "tokens_seen": 72810496 + }, + { + "epoch": 0.02, + "learning_rate": 0.000493901705115346, + "loss": 3.9911, + "theoretical_loss": 4.991164443892175, + "tokens_seen": 72876032 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004938916750250752, + "loss": 3.7698, + "theoretical_loss": 4.990514606988173, + "tokens_seen": 72941568 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004938816449348044, + "loss": 3.8859, + "theoretical_loss": 4.989865516996396, + "tokens_seen": 73007104 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004938716148445337, + "loss": 3.8093, + "theoretical_loss": 4.98921717238903, + "tokens_seen": 73072640 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004938615847542628, + "loss": 3.7279, + "theoretical_loss": 4.988569571642756, + "tokens_seen": 73138176 + }, + { + "epoch": 0.02, + "learning_rate": 0.000493851554663992, + "loss": 3.5668, + "theoretical_loss": 4.98792271323873, + "tokens_seen": 73203712 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004938415245737211, + "loss": 3.7708, + "theoretical_loss": 4.9872765956625615, + "tokens_seen": 73269248 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004938314944834504, + "loss": 3.6155, + "theoretical_loss": 4.9866312174043035, + "tokens_seen": 73334784 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004938214643931796, + "loss": 3.7596, + "theoretical_loss": 4.9859865769584335, + "tokens_seen": 73400320 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004938114343029088, + "loss": 3.95, + "theoretical_loss": 4.9853426728238315, + "tokens_seen": 73465856 + }, + { + "epoch": 0.02, + "learning_rate": 0.000493801404212638, + "loss": 3.9783, + "theoretical_loss": 4.984699503503771, + "tokens_seen": 73531392 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004937913741223671, + "loss": 3.6667, + "theoretical_loss": 4.984057067505898, + "tokens_seen": 73596928 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004937813440320963, + "loss": 3.884, + "theoretical_loss": 4.9834153633422105, + "tokens_seen": 73662464 + }, + { + "epoch": 0.02, + "objective/train/docs_used": 53865, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.7793586254119873, + "objective/train/theoretical_loss": 4.982774389529053, + "objective/train/tokens_used": 94188000, + "theoretical_loss": 4.982774389529053, + "tokens_seen": 73728000 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004937713139418255, + "loss": 3.7914, + "theoretical_loss": 4.982774389529053, + "tokens_seen": 73728000 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004937612838515547, + "loss": 3.939, + "theoretical_loss": 4.9821341445870875, + "tokens_seen": 73793536 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004937512537612839, + "loss": 3.8374, + "theoretical_loss": 4.981494627041286, + "tokens_seen": 73859072 + }, + { + "epoch": 0.02, + "learning_rate": 0.000493741223671013, + "loss": 3.7232, + "theoretical_loss": 4.98085583542091, + "tokens_seen": 73924608 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004937311935807422, + "loss": 3.7298, + "theoretical_loss": 4.980217768259496, + "tokens_seen": 73990144 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004937211634904714, + "loss": 3.8428, + "theoretical_loss": 4.979580424094836, + "tokens_seen": 74055680 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004937111334002006, + "loss": 3.6349, + "theoretical_loss": 4.978943801468967, + "tokens_seen": 74121216 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004937011033099298, + "loss": 3.7722, + "theoretical_loss": 4.978307898928149, + "tokens_seen": 74186752 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004936910732196591, + "loss": 3.4667, + "theoretical_loss": 4.977672715022855, + "tokens_seen": 74252288 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004936810431293881, + "loss": 3.7226, + "theoretical_loss": 4.97703824830775, + "tokens_seen": 74317824 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004936710130391174, + "loss": 3.7371, + "theoretical_loss": 4.976404497341676, + "tokens_seen": 74383360 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004936609829488465, + "loss": 3.7971, + "theoretical_loss": 4.975771460687641, + "tokens_seen": 74448896 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004936509528585758, + "loss": 3.6863, + "theoretical_loss": 4.975139136912794, + "tokens_seen": 74514432 + }, + { + "epoch": 0.02, + "learning_rate": 0.000493640922768305, + "loss": 3.8574, + "theoretical_loss": 4.974507524588424, + "tokens_seen": 74579968 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004936308926780341, + "loss": 3.6154, + "theoretical_loss": 4.973876622289927, + "tokens_seen": 74645504 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004936208625877633, + "loss": 3.5371, + "theoretical_loss": 4.973246428596802, + "tokens_seen": 74711040 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004936108324974925, + "loss": 3.7658, + "theoretical_loss": 4.972616942092634, + "tokens_seen": 74776576 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004936008024072217, + "loss": 3.5425, + "theoretical_loss": 4.971988161365077, + "tokens_seen": 74842112 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004935907723169509, + "loss": 3.6816, + "theoretical_loss": 4.9713600850058395, + "tokens_seen": 74907648 + }, + { + "epoch": 0.02, + "learning_rate": 0.00049358074222668, + "loss": 3.6898, + "theoretical_loss": 4.970732711610667, + "tokens_seen": 74973184 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004935707121364092, + "loss": 3.665, + "theoretical_loss": 4.97010603977933, + "tokens_seen": 75038720 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004935606820461384, + "loss": 3.709, + "theoretical_loss": 4.96948006811561, + "tokens_seen": 75104256 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004935506519558676, + "loss": 3.6697, + "theoretical_loss": 4.968854795227281, + "tokens_seen": 75169792 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004935406218655968, + "loss": 3.3706, + "theoretical_loss": 4.968230219726093, + "tokens_seen": 75235328 + }, + { + "epoch": 0.02, + "learning_rate": 0.000493530591775326, + "loss": 3.4165, + "theoretical_loss": 4.967606340227765, + "tokens_seen": 75300864 + }, + { + "epoch": 0.02, + "objective/train/docs_used": 54881, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.313640832901001, + "objective/train/theoretical_loss": 4.966983155351962, + "objective/train/tokens_used": 95826400, + "theoretical_loss": 4.966983155351962, + "tokens_seen": 75366400 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004935205616850551, + "loss": 3.5647, + "theoretical_loss": 4.966983155351962, + "tokens_seen": 75366400 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004935105315947844, + "loss": 3.8484, + "theoretical_loss": 4.966360663722287, + "tokens_seen": 75431936 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004935005015045135, + "loss": 3.6449, + "theoretical_loss": 4.96573886396626, + "tokens_seen": 75497472 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004934904714142428, + "loss": 3.5781, + "theoretical_loss": 4.965117754715307, + "tokens_seen": 75563008 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004934804413239719, + "loss": 3.7969, + "theoretical_loss": 4.964497334604748, + "tokens_seen": 75628544 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004934704112337011, + "loss": 3.6926, + "theoretical_loss": 4.963877602273776, + "tokens_seen": 75694080 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004934603811434303, + "loss": 3.5501, + "theoretical_loss": 4.963258556365449, + "tokens_seen": 75759616 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004934503510531595, + "loss": 3.7861, + "theoretical_loss": 4.962640195526673, + "tokens_seen": 75825152 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004934403209628887, + "loss": 3.5143, + "theoretical_loss": 4.962022518408183, + "tokens_seen": 75890688 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004934302908726179, + "loss": 3.338, + "theoretical_loss": 4.96140552366454, + "tokens_seen": 75956224 + }, + { + "epoch": 0.02, + "learning_rate": 0.000493420260782347, + "loss": 3.7435, + "theoretical_loss": 4.9607892099541075, + "tokens_seen": 76021760 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004934102306920762, + "loss": 4.039, + "theoretical_loss": 4.9601735759390415, + "tokens_seen": 76087296 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004934002006018054, + "loss": 3.8466, + "theoretical_loss": 4.959558620285274, + "tokens_seen": 76152832 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004933901705115346, + "loss": 3.8819, + "theoretical_loss": 4.958944341662502, + "tokens_seen": 76218368 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004933801404212638, + "loss": 3.8339, + "theoretical_loss": 4.958330738744172, + "tokens_seen": 76283904 + }, + { + "epoch": 0.02, + "learning_rate": 0.000493370110330993, + "loss": 3.5672, + "theoretical_loss": 4.957717810207466, + "tokens_seen": 76349440 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004933600802407221, + "loss": 3.7583, + "theoretical_loss": 4.957105554733289, + "tokens_seen": 76414976 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004933500501504513, + "loss": 3.3945, + "theoretical_loss": 4.956493971006253, + "tokens_seen": 76480512 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004933400200601805, + "loss": 3.9953, + "theoretical_loss": 4.955883057714669, + "tokens_seen": 76546048 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004933299899699098, + "loss": 3.4204, + "theoretical_loss": 4.955272813550524, + "tokens_seen": 76611584 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004933199598796389, + "loss": 3.6761, + "theoretical_loss": 4.954663237209477, + "tokens_seen": 76677120 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004933099297893682, + "loss": 3.6519, + "theoretical_loss": 4.954054327390841, + "tokens_seen": 76742656 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004932998996990972, + "loss": 3.7586, + "theoretical_loss": 4.9534460827975675, + "tokens_seen": 76808192 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004932898696088265, + "loss": 3.7217, + "theoretical_loss": 4.952838502136241, + "tokens_seen": 76873728 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004932798395185557, + "loss": 3.5, + "theoretical_loss": 4.952231584117056, + "tokens_seen": 76939264 + }, + { + "epoch": 0.02, + "objective/train/docs_used": 55424, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.8973569869995117, + "objective/train/theoretical_loss": 4.951625327453812, + "objective/train/tokens_used": 97464800, + "theoretical_loss": 4.951625327453812, + "tokens_seen": 77004800 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004932698094282849, + "loss": 3.7802, + "theoretical_loss": 4.951625327453812, + "tokens_seen": 77004800 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004932597793380141, + "loss": 3.7315, + "theoretical_loss": 4.951019730863894, + "tokens_seen": 77070336 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004932497492477432, + "loss": 3.7994, + "theoretical_loss": 4.950414793068266, + "tokens_seen": 77135872 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004932397191574724, + "loss": 3.858, + "theoretical_loss": 4.94981051279145, + "tokens_seen": 77201408 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004932296890672016, + "loss": 3.4736, + "theoretical_loss": 4.94920688876152, + "tokens_seen": 77266944 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004932196589769308, + "loss": 3.5399, + "theoretical_loss": 4.948603919710088, + "tokens_seen": 77332480 + }, + { + "epoch": 0.02, + "learning_rate": 0.00049320962888666, + "loss": 3.8766, + "theoretical_loss": 4.948001604372287, + "tokens_seen": 77398016 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004931995987963893, + "loss": 3.8451, + "theoretical_loss": 4.947399941486762, + "tokens_seen": 77463552 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004931895687061183, + "loss": 3.557, + "theoretical_loss": 4.946798929795658, + "tokens_seen": 77529088 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004931795386158476, + "loss": 3.8453, + "theoretical_loss": 4.946198568044602, + "tokens_seen": 77594624 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004931695085255767, + "loss": 3.6556, + "theoretical_loss": 4.945598854982698, + "tokens_seen": 77660160 + }, + { + "epoch": 0.02, + "learning_rate": 0.000493159478435306, + "loss": 3.5839, + "theoretical_loss": 4.944999789362508, + "tokens_seen": 77725696 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004931494483450352, + "loss": 3.7148, + "theoretical_loss": 4.944401369940043, + "tokens_seen": 77791232 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004931394182547643, + "loss": 3.7053, + "theoretical_loss": 4.9438035954747495, + "tokens_seen": 77856768 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004931293881644935, + "loss": 3.8975, + "theoretical_loss": 4.9432064647294975, + "tokens_seen": 77922304 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004931193580742227, + "loss": 3.5107, + "theoretical_loss": 4.942609976470566, + "tokens_seen": 77987840 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004931093279839519, + "loss": 3.8659, + "theoretical_loss": 4.942014129467637, + "tokens_seen": 78053376 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004930992978936811, + "loss": 3.6884, + "theoretical_loss": 4.941418922493774, + "tokens_seen": 78118912 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004930892678034102, + "loss": 3.4485, + "theoretical_loss": 4.940824354325419, + "tokens_seen": 78184448 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004930792377131394, + "loss": 3.7364, + "theoretical_loss": 4.940230423742372, + "tokens_seen": 78249984 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004930692076228686, + "loss": 3.7948, + "theoretical_loss": 4.939637129527789, + "tokens_seen": 78315520 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004930591775325978, + "loss": 3.5197, + "theoretical_loss": 4.939044470468156, + "tokens_seen": 78381056 + }, + { + "epoch": 0.02, + "learning_rate": 0.000493049147442327, + "loss": 3.4606, + "theoretical_loss": 4.938452445353294, + "tokens_seen": 78446592 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004930391173520562, + "loss": 3.8595, + "theoretical_loss": 4.937861052976332, + "tokens_seen": 78512128 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004930290872617853, + "loss": 3.5134, + "theoretical_loss": 4.937270292133704, + "tokens_seen": 78577664 + }, + { + "epoch": 0.02, + "objective/train/docs_used": 56825, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.54001784324646, + "objective/train/theoretical_loss": 4.9366801616251355, + "objective/train/tokens_used": 99103200, + "theoretical_loss": 4.9366801616251355, + "tokens_seen": 78643200 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004930190571715146, + "loss": 3.728, + "theoretical_loss": 4.9366801616251355, + "tokens_seen": 78643200 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004930090270812437, + "loss": 3.9168, + "theoretical_loss": 4.93609066025363, + "tokens_seen": 78708736 + }, + { + "epoch": 0.02, + "learning_rate": 0.000492998996990973, + "loss": 3.8331, + "theoretical_loss": 4.935501786825457, + "tokens_seen": 78774272 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004929889669007021, + "loss": 3.6185, + "theoretical_loss": 4.934913540150143, + "tokens_seen": 78839808 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004929789368104313, + "loss": 3.478, + "theoretical_loss": 4.934325919040461, + "tokens_seen": 78905344 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004929689067201605, + "loss": 3.5593, + "theoretical_loss": 4.933738922312413, + "tokens_seen": 78970880 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004929588766298897, + "loss": 3.7501, + "theoretical_loss": 4.933152548785222, + "tokens_seen": 79036416 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004929488465396189, + "loss": 3.7266, + "theoretical_loss": 4.932566797281324, + "tokens_seen": 79101952 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004929388164493481, + "loss": 3.7454, + "theoretical_loss": 4.931981666626351, + "tokens_seen": 79167488 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004929287863590773, + "loss": 3.7306, + "theoretical_loss": 4.931397155649121, + "tokens_seen": 79233024 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004929187562688064, + "loss": 3.491, + "theoretical_loss": 4.930813263181631, + "tokens_seen": 79298560 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004929087261785356, + "loss": 4.0916, + "theoretical_loss": 4.93022998805904, + "tokens_seen": 79364096 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004928986960882648, + "loss": 3.5256, + "theoretical_loss": 4.929647329119659, + "tokens_seen": 79429632 + }, + { + "epoch": 0.02, + "learning_rate": 0.000492888665997994, + "loss": 3.534, + "theoretical_loss": 4.9290652852049455, + "tokens_seen": 79495168 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004928786359077232, + "loss": 3.5497, + "theoretical_loss": 4.928483855159485, + "tokens_seen": 79560704 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004928686058174523, + "loss": 3.4912, + "theoretical_loss": 4.927903037830983, + "tokens_seen": 79626240 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004928585757271815, + "loss": 3.7479, + "theoretical_loss": 4.9273228320702565, + "tokens_seen": 79691776 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004928485456369107, + "loss": 3.7224, + "theoretical_loss": 4.926743236731218, + "tokens_seen": 79757312 + }, + { + "epoch": 0.02, + "learning_rate": 0.00049283851554664, + "loss": 3.8575, + "theoretical_loss": 4.926164250670868, + "tokens_seen": 79822848 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004928284854563691, + "loss": 3.5499, + "theoretical_loss": 4.925585872749284, + "tokens_seen": 79888384 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004928184553660984, + "loss": 3.6421, + "theoretical_loss": 4.925008101829608, + "tokens_seen": 79953920 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004928084252758274, + "loss": 3.7839, + "theoretical_loss": 4.9244309367780374, + "tokens_seen": 80019456 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004927983951855567, + "loss": 3.6961, + "theoretical_loss": 4.923854376463816, + "tokens_seen": 80084992 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004927883650952859, + "loss": 3.8682, + "theoretical_loss": 4.923278419759217, + "tokens_seen": 80150528 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004927783350050151, + "loss": 3.7426, + "theoretical_loss": 4.92270306553954, + "tokens_seen": 80216064 + }, + { + "epoch": 0.02, + "objective/train/docs_used": 57296, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 4.011537075042725, + "objective/train/theoretical_loss": 4.922128312683096, + "objective/train/tokens_used": 100741600, + "theoretical_loss": 4.922128312683096, + "tokens_seen": 80281600 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004927683049147443, + "loss": 3.9468, + "theoretical_loss": 4.922128312683096, + "tokens_seen": 80281600 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004927582748244734, + "loss": 3.3915, + "theoretical_loss": 4.921554160071194, + "tokens_seen": 80347136 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004927482447342026, + "loss": 3.7573, + "theoretical_loss": 4.920980606588142, + "tokens_seen": 80412672 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004927382146439318, + "loss": 3.773, + "theoretical_loss": 4.920407651121222, + "tokens_seen": 80478208 + }, + { + "epoch": 0.02, + "learning_rate": 0.000492728184553661, + "loss": 3.4928, + "theoretical_loss": 4.919835292560689, + "tokens_seen": 80543744 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004927181544633902, + "loss": 3.4883, + "theoretical_loss": 4.919263529799759, + "tokens_seen": 80609280 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004927081243731193, + "loss": 3.5591, + "theoretical_loss": 4.918692361734598, + "tokens_seen": 80674816 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004926980942828485, + "loss": 3.5709, + "theoretical_loss": 4.91812178726431, + "tokens_seen": 80740352 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004926880641925777, + "loss": 3.5894, + "theoretical_loss": 4.917551805290929, + "tokens_seen": 80805888 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004926780341023069, + "loss": 3.7812, + "theoretical_loss": 4.916982414719408, + "tokens_seen": 80871424 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004926680040120361, + "loss": 3.5679, + "theoretical_loss": 4.9164136144576105, + "tokens_seen": 80936960 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004926579739217654, + "loss": 3.8754, + "theoretical_loss": 4.915845403416299, + "tokens_seen": 81002496 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004926479438314944, + "loss": 3.6267, + "theoretical_loss": 4.915277780509124, + "tokens_seen": 81068032 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004926379137412237, + "loss": 3.6105, + "theoretical_loss": 4.914710744652614, + "tokens_seen": 81133568 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004926278836509528, + "loss": 3.3855, + "theoretical_loss": 4.914144294766169, + "tokens_seen": 81199104 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004926178535606821, + "loss": 3.5679, + "theoretical_loss": 4.913578429772047, + "tokens_seen": 81264640 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004926078234704113, + "loss": 3.604, + "theoretical_loss": 4.913013148595355, + "tokens_seen": 81330176 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004925977933801404, + "loss": 3.6109, + "theoretical_loss": 4.912448450164041, + "tokens_seen": 81395712 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004925877632898696, + "loss": 3.6396, + "theoretical_loss": 4.91188433340888, + "tokens_seen": 81461248 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004925777331995988, + "loss": 3.3967, + "theoretical_loss": 4.911320797263471, + "tokens_seen": 81526784 + }, + { + "epoch": 0.02, + "learning_rate": 0.000492567703109328, + "loss": 3.5214, + "theoretical_loss": 4.910757840664219, + "tokens_seen": 81592320 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004925576730190572, + "loss": 3.6187, + "theoretical_loss": 4.910195462550334, + "tokens_seen": 81657856 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004925476429287864, + "loss": 3.5343, + "theoretical_loss": 4.909633661863811, + "tokens_seen": 81723392 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004925376128385155, + "loss": 3.726, + "theoretical_loss": 4.909072437549434, + "tokens_seen": 81788928 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004925275827482447, + "loss": 3.5363, + "theoretical_loss": 4.908511788554753, + "tokens_seen": 81854464 + }, + { + "epoch": 0.02, + "objective/train/docs_used": 58605, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 4.065797805786133, + "objective/train/theoretical_loss": 4.907951713830082, + "objective/train/tokens_used": 102380000, + "theoretical_loss": 4.907951713830082, + "tokens_seen": 81920000 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004925175526579739, + "loss": 3.9274, + "theoretical_loss": 4.907951713830082, + "tokens_seen": 81920000 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004925075225677031, + "loss": 3.5137, + "theoretical_loss": 4.907392212328489, + "tokens_seen": 81985536 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004924974924774323, + "loss": 3.9053, + "theoretical_loss": 4.906833283005785, + "tokens_seen": 82051072 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004924874623871615, + "loss": 3.7878, + "theoretical_loss": 4.906274924820515, + "tokens_seen": 82116608 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004924774322968907, + "loss": 3.5195, + "theoretical_loss": 4.90571713673395, + "tokens_seen": 82182144 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004924674022066199, + "loss": 3.5976, + "theoretical_loss": 4.905159917710073, + "tokens_seen": 82247680 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004924573721163491, + "loss": 3.8818, + "theoretical_loss": 4.904603266715578, + "tokens_seen": 82313216 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004924473420260783, + "loss": 3.6514, + "theoretical_loss": 4.904047182719854, + "tokens_seen": 82378752 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004924373119358075, + "loss": 3.6532, + "theoretical_loss": 4.903491664694977, + "tokens_seen": 82444288 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004924272818455366, + "loss": 3.5049, + "theoretical_loss": 4.902936711615702, + "tokens_seen": 82509824 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004924172517552658, + "loss": 3.2734, + "theoretical_loss": 4.902382322459456, + "tokens_seen": 82575360 + }, + { + "epoch": 0.03, + "learning_rate": 0.000492407221664995, + "loss": 3.5766, + "theoretical_loss": 4.901828496206322, + "tokens_seen": 82640896 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004923971915747242, + "loss": 3.8859, + "theoretical_loss": 4.90127523183904, + "tokens_seen": 82706432 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004923871614844534, + "loss": 3.7175, + "theoretical_loss": 4.900722528342988, + "tokens_seen": 82771968 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004923771313941825, + "loss": 3.3675, + "theoretical_loss": 4.900170384706181, + "tokens_seen": 82837504 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004923671013039117, + "loss": 3.6223, + "theoretical_loss": 4.899618799919256, + "tokens_seen": 82903040 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004923570712136409, + "loss": 3.4173, + "theoretical_loss": 4.899067772975469, + "tokens_seen": 82968576 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004923470411233702, + "loss": 3.8806, + "theoretical_loss": 4.898517302870679, + "tokens_seen": 83034112 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004923370110330993, + "loss": 3.5682, + "theoretical_loss": 4.897967388603346, + "tokens_seen": 83099648 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004923269809428286, + "loss": 3.9113, + "theoretical_loss": 4.897418029174519, + "tokens_seen": 83165184 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004923169508525576, + "loss": 3.5243, + "theoretical_loss": 4.896869223587828, + "tokens_seen": 83230720 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004923069207622869, + "loss": 3.1796, + "theoretical_loss": 4.896320970849472, + "tokens_seen": 83296256 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004922968906720161, + "loss": 3.4955, + "theoretical_loss": 4.895773269968219, + "tokens_seen": 83361792 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004922868605817453, + "loss": 3.519, + "theoretical_loss": 4.895226119955386, + "tokens_seen": 83427328 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004922768304914745, + "loss": 3.6079, + "theoretical_loss": 4.894679519824841, + "tokens_seen": 83492864 + }, + { + "epoch": 0.03, + "objective/train/docs_used": 59310, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.792731285095215, + "objective/train/theoretical_loss": 4.894133468592984, + "objective/train/tokens_used": 104018400, + "theoretical_loss": 4.894133468592984, + "tokens_seen": 83558400 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004922668004012036, + "loss": 3.7535, + "theoretical_loss": 4.894133468592984, + "tokens_seen": 83558400 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004922567703109328, + "loss": 3.6677, + "theoretical_loss": 4.8935879652787495, + "tokens_seen": 83623936 + }, + { + "epoch": 0.03, + "learning_rate": 0.000492246740220662, + "loss": 3.6477, + "theoretical_loss": 4.893043008903591, + "tokens_seen": 83689472 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004922367101303912, + "loss": 3.6256, + "theoretical_loss": 4.892498598491473, + "tokens_seen": 83755008 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004922266800401204, + "loss": 3.6847, + "theoretical_loss": 4.891954733068863, + "tokens_seen": 83820544 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004922166499498495, + "loss": 3.933, + "theoretical_loss": 4.891411411664727, + "tokens_seen": 83886080 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004922066198595787, + "loss": 3.2213, + "theoretical_loss": 4.890868633310515, + "tokens_seen": 83951616 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004921965897693079, + "loss": 3.7656, + "theoretical_loss": 4.890326397040158, + "tokens_seen": 84017152 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004921865596790371, + "loss": 3.6064, + "theoretical_loss": 4.889784701890056, + "tokens_seen": 84082688 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004921765295887663, + "loss": 3.6597, + "theoretical_loss": 4.8892435468990705, + "tokens_seen": 84148224 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004921664994984956, + "loss": 3.8026, + "theoretical_loss": 4.88870293110852, + "tokens_seen": 84213760 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004921564694082246, + "loss": 3.6698, + "theoretical_loss": 4.888162853562166, + "tokens_seen": 84279296 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004921464393179539, + "loss": 3.5072, + "theoretical_loss": 4.88762331330621, + "tokens_seen": 84344832 + }, + { + "epoch": 0.03, + "learning_rate": 0.000492136409227683, + "loss": 3.6201, + "theoretical_loss": 4.88708430938928, + "tokens_seen": 84410368 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004921263791374123, + "loss": 3.4406, + "theoretical_loss": 4.8865458408624285, + "tokens_seen": 84475904 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004921163490471415, + "loss": 3.429, + "theoretical_loss": 4.8860079067791204, + "tokens_seen": 84541440 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004921063189568706, + "loss": 3.5253, + "theoretical_loss": 4.885470506195227, + "tokens_seen": 84606976 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004920962888665998, + "loss": 3.6049, + "theoretical_loss": 4.884933638169014, + "tokens_seen": 84672512 + }, + { + "epoch": 0.03, + "learning_rate": 0.000492086258776329, + "loss": 3.5217, + "theoretical_loss": 4.88439730176114, + "tokens_seen": 84738048 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004920762286860582, + "loss": 3.3955, + "theoretical_loss": 4.883861496034644, + "tokens_seen": 84803584 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004920661985957874, + "loss": 3.3955, + "theoretical_loss": 4.88332622005494, + "tokens_seen": 84869120 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004920561685055166, + "loss": 3.7759, + "theoretical_loss": 4.8827914728898065, + "tokens_seen": 84934656 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004920461384152457, + "loss": 3.6524, + "theoretical_loss": 4.88225725360938, + "tokens_seen": 85000192 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004920361083249749, + "loss": 3.4876, + "theoretical_loss": 4.881723561286149, + "tokens_seen": 85065728 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004920260782347041, + "loss": 3.7061, + "theoretical_loss": 4.881190394994943, + "tokens_seen": 85131264 + }, + { + "epoch": 0.03, + "objective/train/docs_used": 60492, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.4165782928466797, + "objective/train/theoretical_loss": 4.880657753812926, + "objective/train/tokens_used": 105656800, + "theoretical_loss": 4.880657753812926, + "tokens_seen": 85196800 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004920160481444333, + "loss": 3.6404, + "theoretical_loss": 4.880657753812926, + "tokens_seen": 85196800 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004920060180541625, + "loss": 3.6319, + "theoretical_loss": 4.880125636819594, + "tokens_seen": 85262336 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004919959879638916, + "loss": 3.6408, + "theoretical_loss": 4.879594043096755, + "tokens_seen": 85327872 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004919859578736209, + "loss": 3.5499, + "theoretical_loss": 4.879062971728534, + "tokens_seen": 85393408 + }, + { + "epoch": 0.03, + "learning_rate": 0.00049197592778335, + "loss": 3.4626, + "theoretical_loss": 4.87853242180136, + "tokens_seen": 85458944 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004919658976930793, + "loss": 3.8156, + "theoretical_loss": 4.878002392403959, + "tokens_seen": 85524480 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004919558676028084, + "loss": 3.7497, + "theoretical_loss": 4.877472882627343, + "tokens_seen": 85590016 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004919458375125377, + "loss": 3.5891, + "theoretical_loss": 4.8769438915648085, + "tokens_seen": 85655552 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004919358074222668, + "loss": 3.7017, + "theoretical_loss": 4.876415418311928, + "tokens_seen": 85721088 + }, + { + "epoch": 0.03, + "learning_rate": 0.000491925777331996, + "loss": 3.4434, + "theoretical_loss": 4.875887461966537, + "tokens_seen": 85786624 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004919157472417252, + "loss": 3.5096, + "theoretical_loss": 4.875360021628733, + "tokens_seen": 85852160 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004919057171514544, + "loss": 3.4735, + "theoretical_loss": 4.874833096400865, + "tokens_seen": 85917696 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004918956870611836, + "loss": 3.6177, + "theoretical_loss": 4.874306685387525, + "tokens_seen": 85983232 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004918856569709127, + "loss": 3.2892, + "theoretical_loss": 4.873780787695547, + "tokens_seen": 86048768 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004918756268806419, + "loss": 3.8418, + "theoretical_loss": 4.87325540243399, + "tokens_seen": 86114304 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004918655967903711, + "loss": 3.5582, + "theoretical_loss": 4.872730528714139, + "tokens_seen": 86179840 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004918555667001003, + "loss": 3.6623, + "theoretical_loss": 4.872206165649493, + "tokens_seen": 86245376 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004918455366098295, + "loss": 3.7298, + "theoretical_loss": 4.871682312355761, + "tokens_seen": 86310912 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004918355065195586, + "loss": 3.5796, + "theoretical_loss": 4.871158967950852, + "tokens_seen": 86376448 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004918254764292878, + "loss": 3.7591, + "theoretical_loss": 4.870636131554869, + "tokens_seen": 86441984 + }, + { + "epoch": 0.03, + "learning_rate": 0.000491815446339017, + "loss": 3.7784, + "theoretical_loss": 4.8701138022901045, + "tokens_seen": 86507520 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004918054162487463, + "loss": 3.5475, + "theoretical_loss": 4.869591979281028, + "tokens_seen": 86573056 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004917953861584754, + "loss": 3.5037, + "theoretical_loss": 4.8690706616542805, + "tokens_seen": 86638592 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004917853560682047, + "loss": 3.4068, + "theoretical_loss": 4.868549848538675, + "tokens_seen": 86704128 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004917753259779337, + "loss": 3.3105, + "theoretical_loss": 4.868029539065176, + "tokens_seen": 86769664 + }, + { + "epoch": 0.03, + "objective/train/docs_used": 61005, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.347909450531006, + "objective/train/theoretical_loss": 4.867509732366907, + "objective/train/tokens_used": 107295200, + "theoretical_loss": 4.867509732366907, + "tokens_seen": 86835200 + }, + { + "epoch": 0.03, + "learning_rate": 0.000491765295887663, + "loss": 3.5795, + "theoretical_loss": 4.867509732366907, + "tokens_seen": 86835200 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004917552657973922, + "loss": 3.6275, + "theoretical_loss": 4.866990427579129, + "tokens_seen": 86900736 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004917452357071214, + "loss": 3.6857, + "theoretical_loss": 4.866471623839248, + "tokens_seen": 86966272 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004917352056168506, + "loss": 3.3089, + "theoretical_loss": 4.8659533202867955, + "tokens_seen": 87031808 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004917251755265797, + "loss": 3.4599, + "theoretical_loss": 4.86543551606343, + "tokens_seen": 87097344 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004917151454363089, + "loss": 3.3786, + "theoretical_loss": 4.864918210312927, + "tokens_seen": 87162880 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004917051153460381, + "loss": 3.6019, + "theoretical_loss": 4.864401402181173, + "tokens_seen": 87228416 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004916950852557673, + "loss": 3.6129, + "theoretical_loss": 4.863885090816158, + "tokens_seen": 87293952 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004916850551654965, + "loss": 3.2057, + "theoretical_loss": 4.863369275367968, + "tokens_seen": 87359488 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004916750250752258, + "loss": 3.4842, + "theoretical_loss": 4.862853954988781, + "tokens_seen": 87425024 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004916649949849548, + "loss": 3.7183, + "theoretical_loss": 4.862339128832857, + "tokens_seen": 87490560 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004916549648946841, + "loss": 3.3177, + "theoretical_loss": 4.861824796056533, + "tokens_seen": 87556096 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004916449348044132, + "loss": 3.8011, + "theoretical_loss": 4.861310955818219, + "tokens_seen": 87621632 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004916349047141425, + "loss": 3.5816, + "theoretical_loss": 4.860797607278385, + "tokens_seen": 87687168 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004916248746238717, + "loss": 3.4481, + "theoretical_loss": 4.86028474959956, + "tokens_seen": 87752704 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004916148445336008, + "loss": 3.5639, + "theoretical_loss": 4.859772381946323, + "tokens_seen": 87818240 + }, + { + "epoch": 0.03, + "learning_rate": 0.00049160481444333, + "loss": 3.7939, + "theoretical_loss": 4.859260503485298, + "tokens_seen": 87883776 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004915947843530592, + "loss": 3.5931, + "theoretical_loss": 4.858749113385144, + "tokens_seen": 87949312 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004915847542627884, + "loss": 3.6997, + "theoretical_loss": 4.858238210816554, + "tokens_seen": 88014848 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004915747241725176, + "loss": 3.5188, + "theoretical_loss": 4.8577277949522415, + "tokens_seen": 88080384 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004915646940822468, + "loss": 3.5349, + "theoretical_loss": 4.857217864966943, + "tokens_seen": 88145920 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004915546639919759, + "loss": 3.7718, + "theoretical_loss": 4.856708420037402, + "tokens_seen": 88211456 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004915446339017051, + "loss": 3.5069, + "theoretical_loss": 4.8561994593423705, + "tokens_seen": 88276992 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004915346038114343, + "loss": 3.8552, + "theoretical_loss": 4.8556909820625975, + "tokens_seen": 88342528 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004915245737211635, + "loss": 3.8561, + "theoretical_loss": 4.855182987380823, + "tokens_seen": 88408064 + }, + { + "epoch": 0.03, + "objective/train/docs_used": 61597, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.7536520957946777, + "objective/train/theoretical_loss": 4.854675474481779, + "objective/train/tokens_used": 108933600, + "theoretical_loss": 4.854675474481779, + "tokens_seen": 88473600 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004915145436308927, + "loss": 3.7015, + "theoretical_loss": 4.854675474481779, + "tokens_seen": 88473600 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004915045135406218, + "loss": 3.477, + "theoretical_loss": 4.8541684425521705, + "tokens_seen": 88539136 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004914944834503511, + "loss": 3.6159, + "theoretical_loss": 4.85366189078068, + "tokens_seen": 88604672 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004914844533600802, + "loss": 3.3941, + "theoretical_loss": 4.853155818357957, + "tokens_seen": 88670208 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004914744232698095, + "loss": 3.4676, + "theoretical_loss": 4.852650224476609, + "tokens_seen": 88735744 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004914643931795386, + "loss": 3.437, + "theoretical_loss": 4.852145108331205, + "tokens_seen": 88801280 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004914543630892679, + "loss": 3.5644, + "theoretical_loss": 4.851640469118255, + "tokens_seen": 88866816 + }, + { + "epoch": 0.03, + "learning_rate": 0.000491444332998997, + "loss": 3.6868, + "theoretical_loss": 4.851136306036219, + "tokens_seen": 88932352 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004914343029087262, + "loss": 3.6481, + "theoretical_loss": 4.850632618285486, + "tokens_seen": 88997888 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004914242728184554, + "loss": 3.714, + "theoretical_loss": 4.850129405068383, + "tokens_seen": 89063424 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004914142427281846, + "loss": 3.7309, + "theoretical_loss": 4.849626665589156, + "tokens_seen": 89128960 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004914042126379138, + "loss": 3.6042, + "theoretical_loss": 4.849124399053969, + "tokens_seen": 89194496 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004913941825476429, + "loss": 3.7706, + "theoretical_loss": 4.8486226046709024, + "tokens_seen": 89260032 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004913841524573721, + "loss": 3.4477, + "theoretical_loss": 4.8481212816499415, + "tokens_seen": 89325568 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004913741223671013, + "loss": 3.5249, + "theoretical_loss": 4.847620429202967, + "tokens_seen": 89391104 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004913640922768305, + "loss": 3.6239, + "theoretical_loss": 4.847120046543763, + "tokens_seen": 89456640 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004913540621865597, + "loss": 3.6447, + "theoretical_loss": 4.846620132887992, + "tokens_seen": 89522176 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004913440320962888, + "loss": 3.6128, + "theoretical_loss": 4.8461206874532055, + "tokens_seen": 89587712 + }, + { + "epoch": 0.03, + "learning_rate": 0.000491334002006018, + "loss": 3.7001, + "theoretical_loss": 4.845621709458831, + "tokens_seen": 89653248 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004913239719157472, + "loss": 3.8381, + "theoretical_loss": 4.845123198126162, + "tokens_seen": 89718784 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004913139418254765, + "loss": 3.5021, + "theoretical_loss": 4.844625152678364, + "tokens_seen": 89784320 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004913039117352056, + "loss": 3.503, + "theoretical_loss": 4.844127572340455, + "tokens_seen": 89849856 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004912938816449349, + "loss": 3.786, + "theoretical_loss": 4.84363045633931, + "tokens_seen": 89915392 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004912838515546639, + "loss": 3.3234, + "theoretical_loss": 4.843133803903651, + "tokens_seen": 89980928 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004912738214643932, + "loss": 3.4571, + "theoretical_loss": 4.84263761426404, + "tokens_seen": 90046464 + }, + { + "epoch": 0.03, + "objective/train/docs_used": 63098, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.570539712905884, + "objective/train/theoretical_loss": 4.842141886652876, + "objective/train/tokens_used": 110572000, + "theoretical_loss": 4.842141886652876, + "tokens_seen": 90112000 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004912637913741224, + "loss": 3.509, + "theoretical_loss": 4.842141886652876, + "tokens_seen": 90112000 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004912537612838516, + "loss": 3.2298, + "theoretical_loss": 4.841646620304388, + "tokens_seen": 90177536 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004912437311935808, + "loss": 3.3318, + "theoretical_loss": 4.841151814454632, + "tokens_seen": 90243072 + }, + { + "epoch": 0.03, + "learning_rate": 0.00049123370110331, + "loss": 3.2128, + "theoretical_loss": 4.840657468341476, + "tokens_seen": 90308608 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004912236710130391, + "loss": 3.4158, + "theoretical_loss": 4.84016358120461, + "tokens_seen": 90374144 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004912136409227683, + "loss": 3.4662, + "theoretical_loss": 4.839670152285526, + "tokens_seen": 90439680 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004912036108324975, + "loss": 3.5356, + "theoretical_loss": 4.8391771808275195, + "tokens_seen": 90505216 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004911935807422267, + "loss": 3.5223, + "theoretical_loss": 4.838684666075682, + "tokens_seen": 90570752 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004911835506519559, + "loss": 3.4911, + "theoretical_loss": 4.838192607276896, + "tokens_seen": 90636288 + }, + { + "epoch": 0.03, + "learning_rate": 0.000491173520561685, + "loss": 3.4955, + "theoretical_loss": 4.837701003679829, + "tokens_seen": 90701824 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004911634904714142, + "loss": 3.5728, + "theoretical_loss": 4.8372098545349305, + "tokens_seen": 90767360 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004911534603811434, + "loss": 3.372, + "theoretical_loss": 4.836719159094422, + "tokens_seen": 90832896 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004911434302908726, + "loss": 3.9461, + "theoretical_loss": 4.836228916612292, + "tokens_seen": 90898432 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004911334002006019, + "loss": 3.5182, + "theoretical_loss": 4.835739126344298, + "tokens_seen": 90963968 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004911233701103309, + "loss": 3.4907, + "theoretical_loss": 4.8352497875479505, + "tokens_seen": 91029504 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004911133400200602, + "loss": 3.5289, + "theoretical_loss": 4.834760899482514, + "tokens_seen": 91095040 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004911033099297893, + "loss": 3.6779, + "theoretical_loss": 4.834272461409001, + "tokens_seen": 91160576 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004910932798395186, + "loss": 3.5982, + "theoretical_loss": 4.833784472590165, + "tokens_seen": 91226112 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004910832497492478, + "loss": 3.663, + "theoretical_loss": 4.833296932290495, + "tokens_seen": 91291648 + }, + { + "epoch": 0.03, + "learning_rate": 0.000491073219658977, + "loss": 3.5075, + "theoretical_loss": 4.832809839776213, + "tokens_seen": 91357184 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004910631895687061, + "loss": 3.7654, + "theoretical_loss": 4.832323194315265, + "tokens_seen": 91422720 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004910531594784353, + "loss": 3.8083, + "theoretical_loss": 4.831836995177319, + "tokens_seen": 91488256 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004910431293881645, + "loss": 3.3865, + "theoretical_loss": 4.831351241633756, + "tokens_seen": 91553792 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004910330992978937, + "loss": 3.5372, + "theoretical_loss": 4.8308659329576695, + "tokens_seen": 91619328 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004910230692076229, + "loss": 3.7713, + "theoretical_loss": 4.830381068423856, + "tokens_seen": 91684864 + }, + { + "epoch": 0.03, + "objective/train/docs_used": 63840, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.5062811374664307, + "objective/train/theoretical_loss": 4.8298966473088125, + "objective/train/tokens_used": 112210400, + "theoretical_loss": 4.8298966473088125, + "tokens_seen": 91750400 + }, + { + "epoch": 0.03, + "learning_rate": 0.000491013039117352, + "loss": 3.475, + "theoretical_loss": 4.8298966473088125, + "tokens_seen": 91750400 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004910030090270812, + "loss": 3.3786, + "theoretical_loss": 4.829412668890729, + "tokens_seen": 91815936 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004909929789368104, + "loss": 3.4434, + "theoretical_loss": 4.8289291324494865, + "tokens_seen": 91881472 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004909829488465397, + "loss": 3.3395, + "theoretical_loss": 4.828446037266647, + "tokens_seen": 91947008 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004909729187562688, + "loss": 3.7328, + "theoretical_loss": 4.827963382625454, + "tokens_seen": 92012544 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004909628886659981, + "loss": 3.539, + "theoretical_loss": 4.827481167810825, + "tokens_seen": 92078080 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004909528585757272, + "loss": 3.6901, + "theoretical_loss": 4.826999392109344, + "tokens_seen": 92143616 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004909428284854564, + "loss": 3.333, + "theoretical_loss": 4.826518054809259, + "tokens_seen": 92209152 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004909327983951856, + "loss": 3.5156, + "theoretical_loss": 4.826037155200478, + "tokens_seen": 92274688 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004909227683049148, + "loss": 3.8303, + "theoretical_loss": 4.825556692574562, + "tokens_seen": 92340224 + }, + { + "epoch": 0.03, + "learning_rate": 0.000490912738214644, + "loss": 3.7307, + "theoretical_loss": 4.825076666224717, + "tokens_seen": 92405760 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004909027081243731, + "loss": 3.4392, + "theoretical_loss": 4.824597075445799, + "tokens_seen": 92471296 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004908926780341023, + "loss": 3.4594, + "theoretical_loss": 4.824117919534297, + "tokens_seen": 92536832 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004908826479438315, + "loss": 3.5734, + "theoretical_loss": 4.823639197788334, + "tokens_seen": 92602368 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004908726178535607, + "loss": 3.542, + "theoretical_loss": 4.823160909507665, + "tokens_seen": 92667904 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004908625877632899, + "loss": 3.5347, + "theoretical_loss": 4.822683053993664, + "tokens_seen": 92733440 + }, + { + "epoch": 0.03, + "learning_rate": 0.000490852557673019, + "loss": 3.527, + "theoretical_loss": 4.822205630549329, + "tokens_seen": 92798976 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004908425275827482, + "loss": 3.5631, + "theoretical_loss": 4.821728638479267, + "tokens_seen": 92864512 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004908324974924774, + "loss": 3.3204, + "theoretical_loss": 4.821252077089696, + "tokens_seen": 92930048 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004908224674022067, + "loss": 3.5134, + "theoretical_loss": 4.820775945688437, + "tokens_seen": 92995584 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004908124373119358, + "loss": 3.4795, + "theoretical_loss": 4.820300243584913, + "tokens_seen": 93061120 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004908024072216651, + "loss": 3.4721, + "theoretical_loss": 4.819824970090138, + "tokens_seen": 93126656 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004907923771313941, + "loss": 3.3702, + "theoretical_loss": 4.819350124516717, + "tokens_seen": 93192192 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004907823470411234, + "loss": 3.8509, + "theoretical_loss": 4.818875706178841, + "tokens_seen": 93257728 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004907723169508526, + "loss": 3.4149, + "theoretical_loss": 4.818401714392279, + "tokens_seen": 93323264 + }, + { + "epoch": 0.03, + "objective/train/docs_used": 65224, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.4774487018585205, + "objective/train/theoretical_loss": 4.817928148474378, + "objective/train/tokens_used": 113848800, + "theoretical_loss": 4.817928148474378, + "tokens_seen": 93388800 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004907622868605818, + "loss": 3.545, + "theoretical_loss": 4.817928148474378, + "tokens_seen": 93388800 + }, + { + "epoch": 0.03, + "learning_rate": 0.000490752256770311, + "loss": 3.4755, + "theoretical_loss": 4.817455007744052, + "tokens_seen": 93454336 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004907422266800401, + "loss": 3.3377, + "theoretical_loss": 4.816982291521785, + "tokens_seen": 93519872 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004907321965897693, + "loss": 3.5366, + "theoretical_loss": 4.816509999129618, + "tokens_seen": 93585408 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004907221664994985, + "loss": 3.4763, + "theoretical_loss": 4.816038129891151, + "tokens_seen": 93650944 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004907121364092277, + "loss": 3.4128, + "theoretical_loss": 4.815566683131536, + "tokens_seen": 93716480 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004907021063189569, + "loss": 3.6753, + "theoretical_loss": 4.815095658177472, + "tokens_seen": 93782016 + }, + { + "epoch": 0.03, + "learning_rate": 0.000490692076228686, + "loss": 3.6325, + "theoretical_loss": 4.814625054357199, + "tokens_seen": 93847552 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004906820461384152, + "loss": 3.5163, + "theoretical_loss": 4.814154871000497, + "tokens_seen": 93913088 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004906720160481444, + "loss": 3.2526, + "theoretical_loss": 4.813685107438679, + "tokens_seen": 93978624 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004906619859578736, + "loss": 3.5083, + "theoretical_loss": 4.813215763004585, + "tokens_seen": 94044160 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004906519558676028, + "loss": 3.659, + "theoretical_loss": 4.812746837032582, + "tokens_seen": 94109696 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004906419257773321, + "loss": 3.4951, + "theoretical_loss": 4.812278328858554, + "tokens_seen": 94175232 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004906318956870611, + "loss": 3.4599, + "theoretical_loss": 4.811810237819904, + "tokens_seen": 94240768 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004906218655967904, + "loss": 3.0896, + "theoretical_loss": 4.81134256325554, + "tokens_seen": 94306304 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004906118355065195, + "loss": 3.5088, + "theoretical_loss": 4.810875304505881, + "tokens_seen": 94371840 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004906018054162488, + "loss": 3.859, + "theoretical_loss": 4.810408460912846, + "tokens_seen": 94437376 + }, + { + "epoch": 0.03, + "learning_rate": 0.000490591775325978, + "loss": 3.4523, + "theoretical_loss": 4.809942031819853, + "tokens_seen": 94502912 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004905817452357072, + "loss": 3.5658, + "theoretical_loss": 4.809476016571809, + "tokens_seen": 94568448 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004905717151454363, + "loss": 3.2486, + "theoretical_loss": 4.809010414515113, + "tokens_seen": 94633984 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004905616850551655, + "loss": 3.5983, + "theoretical_loss": 4.808545224997644, + "tokens_seen": 94699520 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004905516549648947, + "loss": 3.8398, + "theoretical_loss": 4.808080447368766, + "tokens_seen": 94765056 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004905416248746239, + "loss": 3.4788, + "theoretical_loss": 4.807616080979315, + "tokens_seen": 94830592 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004905315947843531, + "loss": 3.5319, + "theoretical_loss": 4.807152125181597, + "tokens_seen": 94896128 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004905215646940822, + "loss": 3.5209, + "theoretical_loss": 4.806688579329387, + "tokens_seen": 94961664 + }, + { + "epoch": 0.03, + "objective/train/docs_used": 65494, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.7538790702819824, + "objective/train/theoretical_loss": 4.8062254427779205, + "objective/train/tokens_used": 115487200, + "theoretical_loss": 4.8062254427779205, + "tokens_seen": 95027200 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004905115346038114, + "loss": 3.4629, + "theoretical_loss": 4.8062254427779205, + "tokens_seen": 95027200 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004905015045135406, + "loss": 3.5269, + "theoretical_loss": 4.80576271488389, + "tokens_seen": 95092736 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004904914744232698, + "loss": 3.6043, + "theoretical_loss": 4.805300395005444, + "tokens_seen": 95158272 + }, + { + "epoch": 0.03, + "learning_rate": 0.000490481444332999, + "loss": 3.3754, + "theoretical_loss": 4.804838482502181, + "tokens_seen": 95223808 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004904714142427281, + "loss": 3.5558, + "theoretical_loss": 4.8043769767351385, + "tokens_seen": 95289344 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004904613841524574, + "loss": 3.4119, + "theoretical_loss": 4.8039158770668005, + "tokens_seen": 95354880 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004904513540621865, + "loss": 3.704, + "theoretical_loss": 4.803455182861087, + "tokens_seen": 95420416 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004904413239719158, + "loss": 3.5812, + "theoretical_loss": 4.802994893483348, + "tokens_seen": 95485952 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004904312938816449, + "loss": 3.6426, + "theoretical_loss": 4.802535008300364, + "tokens_seen": 95551488 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004904212637913742, + "loss": 3.4963, + "theoretical_loss": 4.802075526680335, + "tokens_seen": 95617024 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004904112337011033, + "loss": 3.3356, + "theoretical_loss": 4.801616447992888, + "tokens_seen": 95682560 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004904012036108325, + "loss": 3.6295, + "theoretical_loss": 4.801157771609061, + "tokens_seen": 95748096 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004903911735205617, + "loss": 3.5396, + "theoretical_loss": 4.8006994969013, + "tokens_seen": 95813632 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004903811434302909, + "loss": 3.6436, + "theoretical_loss": 4.800241623243467, + "tokens_seen": 95879168 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004903711133400201, + "loss": 3.5692, + "theoretical_loss": 4.799784150010819, + "tokens_seen": 95944704 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004903610832497492, + "loss": 3.3848, + "theoretical_loss": 4.799327076580017, + "tokens_seen": 96010240 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004903510531594784, + "loss": 3.5816, + "theoretical_loss": 4.798870402329115, + "tokens_seen": 96075776 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004903410230692076, + "loss": 3.6585, + "theoretical_loss": 4.798414126637558, + "tokens_seen": 96141312 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004903309929789368, + "loss": 3.7904, + "theoretical_loss": 4.797958248886179, + "tokens_seen": 96206848 + }, + { + "epoch": 0.03, + "learning_rate": 0.000490320962888666, + "loss": 3.5071, + "theoretical_loss": 4.797502768457193, + "tokens_seen": 96272384 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004903109327983952, + "loss": 3.7765, + "theoretical_loss": 4.797047684734192, + "tokens_seen": 96337920 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004903009027081243, + "loss": 3.4468, + "theoretical_loss": 4.796592997102147, + "tokens_seen": 96403456 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004902908726178535, + "loss": 3.3868, + "theoretical_loss": 4.796138704947397, + "tokens_seen": 96468992 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004902808425275828, + "loss": 3.7522, + "theoretical_loss": 4.795684807657649, + "tokens_seen": 96534528 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004902708124373119, + "loss": 3.6975, + "theoretical_loss": 4.795231304621968, + "tokens_seen": 96600064 + }, + { + "epoch": 0.03, + "objective/train/docs_used": 66055, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.685988664627075, + "objective/train/theoretical_loss": 4.794778195230787, + "objective/train/tokens_used": 117125600, + "theoretical_loss": 4.794778195230787, + "tokens_seen": 96665600 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004902607823470412, + "loss": 3.6337, + "theoretical_loss": 4.794778195230787, + "tokens_seen": 96665600 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004902507522567703, + "loss": 3.6599, + "theoretical_loss": 4.794325478875885, + "tokens_seen": 96731136 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004902407221664995, + "loss": 3.6031, + "theoretical_loss": 4.793873154950399, + "tokens_seen": 96796672 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004902306920762287, + "loss": 3.387, + "theoretical_loss": 4.793421222848808, + "tokens_seen": 96862208 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004902206619859579, + "loss": 3.5348, + "theoretical_loss": 4.7929696819669365, + "tokens_seen": 96927744 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004902106318956871, + "loss": 3.6871, + "theoretical_loss": 4.792518531701948, + "tokens_seen": 96993280 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004902006018054163, + "loss": 3.6698, + "theoretical_loss": 4.792067771452341, + "tokens_seen": 97058816 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004901905717151454, + "loss": 3.7043, + "theoretical_loss": 4.791617400617948, + "tokens_seen": 97124352 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004901805416248746, + "loss": 3.4935, + "theoretical_loss": 4.791167418599925, + "tokens_seen": 97189888 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004901705115346038, + "loss": 3.6633, + "theoretical_loss": 4.790717824800755, + "tokens_seen": 97255424 + }, + { + "epoch": 0.03, + "learning_rate": 0.000490160481444333, + "loss": 3.7065, + "theoretical_loss": 4.790268618624239, + "tokens_seen": 97320960 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004901504513540623, + "loss": 3.3378, + "theoretical_loss": 4.789819799475499, + "tokens_seen": 97386496 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004901404212637913, + "loss": 3.5149, + "theoretical_loss": 4.789371366760961, + "tokens_seen": 97452032 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004901303911735206, + "loss": 3.7281, + "theoretical_loss": 4.788923319888369, + "tokens_seen": 97517568 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004901203610832497, + "loss": 3.4859, + "theoretical_loss": 4.788475658266766, + "tokens_seen": 97583104 + }, + { + "epoch": 0.03, + "learning_rate": 0.000490110330992979, + "loss": 3.5342, + "theoretical_loss": 4.788028381306497, + "tokens_seen": 97648640 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004901003009027082, + "loss": 3.5459, + "theoretical_loss": 4.787581488419207, + "tokens_seen": 97714176 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004900902708124374, + "loss": 3.6066, + "theoretical_loss": 4.787134979017832, + "tokens_seen": 97779712 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004900802407221665, + "loss": 3.616, + "theoretical_loss": 4.786688852516599, + "tokens_seen": 97845248 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004900702106318957, + "loss": 3.6978, + "theoretical_loss": 4.786243108331024, + "tokens_seen": 97910784 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004900601805416249, + "loss": 3.2786, + "theoretical_loss": 4.7857977458779, + "tokens_seen": 97976320 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004900501504513541, + "loss": 3.6792, + "theoretical_loss": 4.785352764575304, + "tokens_seen": 98041856 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004900401203610833, + "loss": 3.3067, + "theoretical_loss": 4.784908163842585, + "tokens_seen": 98107392 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004900300902708124, + "loss": 3.5479, + "theoretical_loss": 4.784463943100367, + "tokens_seen": 98172928 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004900200601805416, + "loss": 3.6372, + "theoretical_loss": 4.7840201017705395, + "tokens_seen": 98238464 + }, + { + "epoch": 0.03, + "objective/train/docs_used": 66835, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.976943254470825, + "objective/train/theoretical_loss": 4.783576639276257, + "objective/train/tokens_used": 118764000, + "theoretical_loss": 4.783576639276257, + "tokens_seen": 98304000 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004900100300902708, + "loss": 3.6349, + "theoretical_loss": 4.783576639276257, + "tokens_seen": 98304000 + }, + { + "epoch": 0.03, + "learning_rate": 0.00049, + "loss": 3.4033, + "theoretical_loss": 4.783133555041934, + "tokens_seen": 98369536 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004899899699097292, + "loss": 3.5037, + "theoretical_loss": 4.782690848493245, + "tokens_seen": 98435072 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004899799398194583, + "loss": 3.3758, + "theoretical_loss": 4.7822485190571165, + "tokens_seen": 98500608 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004899699097291876, + "loss": 3.4468, + "theoretical_loss": 4.781806566161723, + "tokens_seen": 98566144 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004899598796389167, + "loss": 3.7458, + "theoretical_loss": 4.781364989236488, + "tokens_seen": 98631680 + }, + { + "epoch": 0.03, + "learning_rate": 0.000489949849548646, + "loss": 3.6344, + "theoretical_loss": 4.78092378771208, + "tokens_seen": 98697216 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004899398194583751, + "loss": 3.8223, + "theoretical_loss": 4.780482961020402, + "tokens_seen": 98762752 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004899297893681044, + "loss": 3.6466, + "theoretical_loss": 4.780042508594596, + "tokens_seen": 98828288 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004899197592778335, + "loss": 3.5809, + "theoretical_loss": 4.779602429869035, + "tokens_seen": 98893824 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004899097291875627, + "loss": 3.3735, + "theoretical_loss": 4.779162724279324, + "tokens_seen": 98959360 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004898996990972919, + "loss": 3.6141, + "theoretical_loss": 4.7787233912622895, + "tokens_seen": 99024896 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004898896690070211, + "loss": 3.3622, + "theoretical_loss": 4.778284430255981, + "tokens_seen": 99090432 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004898796389167503, + "loss": 3.6091, + "theoretical_loss": 4.77784584069967, + "tokens_seen": 99155968 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004898696088264794, + "loss": 3.559, + "theoretical_loss": 4.777407622033838, + "tokens_seen": 99221504 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004898595787362086, + "loss": 3.4953, + "theoretical_loss": 4.776969773700181, + "tokens_seen": 99287040 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004898495486459378, + "loss": 3.6499, + "theoretical_loss": 4.776532295141601, + "tokens_seen": 99352576 + }, + { + "epoch": 0.03, + "learning_rate": 0.000489839518555667, + "loss": 3.864, + "theoretical_loss": 4.776095185802211, + "tokens_seen": 99418112 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004898294884653962, + "loss": 3.4611, + "theoretical_loss": 4.775658445127318, + "tokens_seen": 99483648 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004898194583751254, + "loss": 3.4514, + "theoretical_loss": 4.775222072563429, + "tokens_seen": 99549184 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004898094282848545, + "loss": 3.4742, + "theoretical_loss": 4.7747860675582485, + "tokens_seen": 99614720 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004897993981945837, + "loss": 3.5126, + "theoretical_loss": 4.77435042956067, + "tokens_seen": 99680256 + }, + { + "epoch": 0.03, + "learning_rate": 0.000489789368104313, + "loss": 3.5352, + "theoretical_loss": 4.773915158020776, + "tokens_seen": 99745792 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004897793380140421, + "loss": 3.5878, + "theoretical_loss": 4.773480252389831, + "tokens_seen": 99811328 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004897693079237714, + "loss": 3.6071, + "theoretical_loss": 4.773045712120284, + "tokens_seen": 99876864 + }, + { + "epoch": 0.03, + "objective/train/docs_used": 67525, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.3745110034942627, + "objective/train/theoretical_loss": 4.77261153666576, + "objective/train/tokens_used": 120402400, + "theoretical_loss": 4.77261153666576, + "tokens_seen": 99942400 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004897592778335005, + "loss": 3.4922, + "theoretical_loss": 4.77261153666576, + "tokens_seen": 99942400 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004897492477432297, + "loss": 3.4046, + "theoretical_loss": 4.772177725481062, + "tokens_seen": 100007936 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004897392176529589, + "loss": 3.4785, + "theoretical_loss": 4.77174427802216, + "tokens_seen": 100073472 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004897291875626881, + "loss": 3.5484, + "theoretical_loss": 4.771311193746191, + "tokens_seen": 100139008 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004897191574724173, + "loss": 3.5031, + "theoretical_loss": 4.770878472111465, + "tokens_seen": 100204544 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004897091273821465, + "loss": 3.5108, + "theoretical_loss": 4.770446112577445, + "tokens_seen": 100270080 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004896990972918756, + "loss": 3.5791, + "theoretical_loss": 4.770014114604756, + "tokens_seen": 100335616 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004896890672016048, + "loss": 3.461, + "theoretical_loss": 4.769582477655177, + "tokens_seen": 100401152 + }, + { + "epoch": 0.03, + "learning_rate": 0.000489679037111334, + "loss": 3.584, + "theoretical_loss": 4.769151201191641, + "tokens_seen": 100466688 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004896690070210632, + "loss": 3.665, + "theoretical_loss": 4.768720284678228, + "tokens_seen": 100532224 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004896589769307924, + "loss": 3.6066, + "theoretical_loss": 4.768289727580161, + "tokens_seen": 100597760 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004896489468405215, + "loss": 3.6137, + "theoretical_loss": 4.767859529363809, + "tokens_seen": 100663296 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004896389167502507, + "loss": 3.5131, + "theoretical_loss": 4.767429689496682, + "tokens_seen": 100728832 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004896288866599799, + "loss": 3.4929, + "theoretical_loss": 4.767000207447417, + "tokens_seen": 100794368 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004896188565697091, + "loss": 3.3646, + "theoretical_loss": 4.766571082685794, + "tokens_seen": 100859904 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004896088264794384, + "loss": 3.361, + "theoretical_loss": 4.766142314682716, + "tokens_seen": 100925440 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004895987963891674, + "loss": 3.4871, + "theoretical_loss": 4.765713902910214, + "tokens_seen": 100990976 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004895887662988967, + "loss": 3.4064, + "theoretical_loss": 4.765285846841444, + "tokens_seen": 101056512 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004895787362086259, + "loss": 3.6241, + "theoretical_loss": 4.76485814595068, + "tokens_seen": 101122048 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004895687061183551, + "loss": 3.464, + "theoretical_loss": 4.764430799713314, + "tokens_seen": 101187584 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004895586760280843, + "loss": 3.6504, + "theoretical_loss": 4.764003807605853, + "tokens_seen": 101253120 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004895486459378135, + "loss": 3.3051, + "theoretical_loss": 4.763577169105912, + "tokens_seen": 101318656 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004895386158475426, + "loss": 3.3959, + "theoretical_loss": 4.763150883692218, + "tokens_seen": 101384192 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004895285857572718, + "loss": 3.4399, + "theoretical_loss": 4.762724950844598, + "tokens_seen": 101449728 + }, + { + "epoch": 0.03, + "learning_rate": 0.000489518555667001, + "loss": 3.7508, + "theoretical_loss": 4.762299370043984, + "tokens_seen": 101515264 + }, + { + "epoch": 0.03, + "objective/train/docs_used": 68617, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.551746129989624, + "objective/train/theoretical_loss": 4.761874140772408, + "objective/train/tokens_used": 122040800, + "theoretical_loss": 4.761874140772408, + "tokens_seen": 101580800 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004895085255767302, + "loss": 3.5575, + "theoretical_loss": 4.761874140772408, + "tokens_seen": 101580800 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004894984954864594, + "loss": 3.431, + "theoretical_loss": 4.761449262512993, + "tokens_seen": 101646336 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004894884653961885, + "loss": 3.5373, + "theoretical_loss": 4.761024734749958, + "tokens_seen": 101711872 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004894784353059178, + "loss": 3.769, + "theoretical_loss": 4.76060055696861, + "tokens_seen": 101777408 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004894684052156469, + "loss": 3.8113, + "theoretical_loss": 4.760176728655345, + "tokens_seen": 101842944 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004894583751253762, + "loss": 3.3997, + "theoretical_loss": 4.75975324929764, + "tokens_seen": 101908480 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004894483450351053, + "loss": 3.2377, + "theoretical_loss": 4.759330118384053, + "tokens_seen": 101974016 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004894383149448346, + "loss": 3.5344, + "theoretical_loss": 4.758907335404221, + "tokens_seen": 102039552 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004894282848545637, + "loss": 3.5613, + "theoretical_loss": 4.758484899848854, + "tokens_seen": 102105088 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004894182547642929, + "loss": 3.6486, + "theoretical_loss": 4.7580628112097365, + "tokens_seen": 102170624 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004894082246740221, + "loss": 3.5358, + "theoretical_loss": 4.7576410689797175, + "tokens_seen": 102236160 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004893981945837513, + "loss": 3.7585, + "theoretical_loss": 4.757219672652717, + "tokens_seen": 102301696 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004893881644934805, + "loss": 3.4001, + "theoretical_loss": 4.756798621723712, + "tokens_seen": 102367232 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004893781344032096, + "loss": 3.5012, + "theoretical_loss": 4.756377915688748, + "tokens_seen": 102432768 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004893681043129388, + "loss": 3.459, + "theoretical_loss": 4.755957554044917, + "tokens_seen": 102498304 + }, + { + "epoch": 0.03, + "learning_rate": 0.000489358074222668, + "loss": 3.3259, + "theoretical_loss": 4.755537536290373, + "tokens_seen": 102563840 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004893480441323972, + "loss": 3.5541, + "theoretical_loss": 4.755117861924321, + "tokens_seen": 102629376 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004893380140421264, + "loss": 3.4538, + "theoretical_loss": 4.754698530447009, + "tokens_seen": 102694912 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004893279839518556, + "loss": 3.4182, + "theoretical_loss": 4.754279541359738, + "tokens_seen": 102760448 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004893179538615847, + "loss": 3.4899, + "theoretical_loss": 4.753860894164845, + "tokens_seen": 102825984 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004893079237713139, + "loss": 3.6181, + "theoretical_loss": 4.75344258836571, + "tokens_seen": 102891520 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004892978936810432, + "loss": 3.5218, + "theoretical_loss": 4.753024623466752, + "tokens_seen": 102957056 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004892878635907723, + "loss": 3.322, + "theoretical_loss": 4.752606998973421, + "tokens_seen": 103022592 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004892778335005016, + "loss": 3.5191, + "theoretical_loss": 4.752189714392202, + "tokens_seen": 103088128 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004892678034102307, + "loss": 3.6298, + "theoretical_loss": 4.7517727692306035, + "tokens_seen": 103153664 + }, + { + "epoch": 0.03, + "objective/train/docs_used": 69265, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.557532548904419, + "objective/train/theoretical_loss": 4.751356162997164, + "objective/train/tokens_used": 123679200, + "theoretical_loss": 4.751356162997164, + "tokens_seen": 103219200 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004892577733199599, + "loss": 3.5667, + "theoretical_loss": 4.751356162997164, + "tokens_seen": 103219200 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004892477432296891, + "loss": 3.5473, + "theoretical_loss": 4.750939895201443, + "tokens_seen": 103284736 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004892377131394183, + "loss": 3.5528, + "theoretical_loss": 4.750523965354024, + "tokens_seen": 103350272 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004892276830491475, + "loss": 3.7633, + "theoretical_loss": 4.750108372966501, + "tokens_seen": 103415808 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004892176529588767, + "loss": 3.4533, + "theoretical_loss": 4.749693117551491, + "tokens_seen": 103481344 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004892076228686058, + "loss": 3.6005, + "theoretical_loss": 4.749278198622617, + "tokens_seen": 103546880 + }, + { + "epoch": 0.03, + "learning_rate": 0.000489197592778335, + "loss": 3.5253, + "theoretical_loss": 4.748863615694514, + "tokens_seen": 103612416 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004891875626880642, + "loss": 3.3038, + "theoretical_loss": 4.748449368282822, + "tokens_seen": 103677952 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004891775325977934, + "loss": 3.6261, + "theoretical_loss": 4.748035455904185, + "tokens_seen": 103743488 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004891675025075226, + "loss": 3.4179, + "theoretical_loss": 4.747621878076252, + "tokens_seen": 103809024 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004891574724172517, + "loss": 3.4603, + "theoretical_loss": 4.747208634317664, + "tokens_seen": 103874560 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004891474423269809, + "loss": 3.6181, + "theoretical_loss": 4.746795724148061, + "tokens_seen": 103940096 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004891374122367101, + "loss": 3.6406, + "theoretical_loss": 4.746383147088078, + "tokens_seen": 104005632 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004891273821464393, + "loss": 3.6562, + "theoretical_loss": 4.745970902659338, + "tokens_seen": 104071168 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004891173520561686, + "loss": 3.2095, + "theoretical_loss": 4.745558990384451, + "tokens_seen": 104136704 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004891073219658976, + "loss": 3.764, + "theoretical_loss": 4.7451474097870125, + "tokens_seen": 104202240 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004890972918756269, + "loss": 3.4928, + "theoretical_loss": 4.744736160391602, + "tokens_seen": 104267776 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004890872617853561, + "loss": 3.5882, + "theoretical_loss": 4.744325241723777, + "tokens_seen": 104333312 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004890772316950853, + "loss": 3.5186, + "theoretical_loss": 4.743914653310073, + "tokens_seen": 104398848 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004890672016048145, + "loss": 3.6067, + "theoretical_loss": 4.743504394678, + "tokens_seen": 104464384 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004890571715145437, + "loss": 3.4479, + "theoretical_loss": 4.743094465356039, + "tokens_seen": 104529920 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004890471414242728, + "loss": 3.5836, + "theoretical_loss": 4.742684864873641, + "tokens_seen": 104595456 + }, + { + "epoch": 0.03, + "learning_rate": 0.000489037111334002, + "loss": 3.2653, + "theoretical_loss": 4.742275592761223, + "tokens_seen": 104660992 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004890270812437312, + "loss": 3.5848, + "theoretical_loss": 4.741866648550168, + "tokens_seen": 104726528 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004890170511534604, + "loss": 3.5183, + "theoretical_loss": 4.741458031772817, + "tokens_seen": 104792064 + }, + { + "epoch": 0.03, + "objective/train/docs_used": 70556, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.4074082374572754, + "objective/train/theoretical_loss": 4.741049741962473, + "objective/train/tokens_used": 125317600, + "theoretical_loss": 4.741049741962473, + "tokens_seen": 104857600 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004890070210631896, + "loss": 3.4412, + "theoretical_loss": 4.741049741962473, + "tokens_seen": 104857600 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004889969909729187, + "loss": 3.6531, + "theoretical_loss": 4.740641778653395, + "tokens_seen": 104923136 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004889869608826479, + "loss": 3.3668, + "theoretical_loss": 4.740234141380794, + "tokens_seen": 104988672 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004889769307923771, + "loss": 3.6094, + "theoretical_loss": 4.739826829680833, + "tokens_seen": 105054208 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004889669007021063, + "loss": 3.3713, + "theoretical_loss": 4.739419843090626, + "tokens_seen": 105119744 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004889568706118355, + "loss": 3.3276, + "theoretical_loss": 4.739013181148229, + "tokens_seen": 105185280 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004889468405215647, + "loss": 3.3537, + "theoretical_loss": 4.738606843392644, + "tokens_seen": 105250816 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004889368104312939, + "loss": 3.4267, + "theoretical_loss": 4.738200829363815, + "tokens_seen": 105316352 + }, + { + "epoch": 0.03, + "learning_rate": 0.000488926780341023, + "loss": 3.4477, + "theoretical_loss": 4.737795138602624, + "tokens_seen": 105381888 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004889167502507523, + "loss": 3.3341, + "theoretical_loss": 4.737389770650887, + "tokens_seen": 105447424 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004889067201604815, + "loss": 3.6777, + "theoretical_loss": 4.736984725051357, + "tokens_seen": 105512960 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004888966900702107, + "loss": 3.4695, + "theoretical_loss": 4.736580001347717, + "tokens_seen": 105578496 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004888866599799398, + "loss": 3.4454, + "theoretical_loss": 4.736175599084576, + "tokens_seen": 105644032 + }, + { + "epoch": 0.03, + "learning_rate": 0.000488876629889669, + "loss": 3.4974, + "theoretical_loss": 4.735771517807473, + "tokens_seen": 105709568 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004888665997993982, + "loss": 3.2598, + "theoretical_loss": 4.735367757062869, + "tokens_seen": 105775104 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004888565697091274, + "loss": 3.4849, + "theoretical_loss": 4.734964316398148, + "tokens_seen": 105840640 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004888465396188566, + "loss": 3.578, + "theoretical_loss": 4.734561195361609, + "tokens_seen": 105906176 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004888365095285858, + "loss": 3.5233, + "theoretical_loss": 4.734158393502471, + "tokens_seen": 105971712 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004888264794383149, + "loss": 3.518, + "theoretical_loss": 4.733755910370867, + "tokens_seen": 106037248 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004888164493480441, + "loss": 3.3379, + "theoretical_loss": 4.73335374551784, + "tokens_seen": 106102784 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004888064192577733, + "loss": 3.386, + "theoretical_loss": 4.732951898495341, + "tokens_seen": 106168320 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004887963891675025, + "loss": 3.4922, + "theoretical_loss": 4.7325503688562325, + "tokens_seen": 106233856 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004887863590772317, + "loss": 3.197, + "theoretical_loss": 4.732149156154276, + "tokens_seen": 106299392 + }, + { + "epoch": 0.03, + "learning_rate": 0.000488776328986961, + "loss": 3.5311, + "theoretical_loss": 4.731748259944139, + "tokens_seen": 106364928 + }, + { + "epoch": 0.03, + "learning_rate": 0.00048876629889669, + "loss": 3.5092, + "theoretical_loss": 4.731347679781386, + "tokens_seen": 106430464 + }, + { + "epoch": 0.03, + "objective/train/docs_used": 70950, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 4.163817882537842, + "objective/train/theoretical_loss": 4.730947415222481, + "objective/train/tokens_used": 126956000, + "theoretical_loss": 4.730947415222481, + "tokens_seen": 106496000 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004887562688064193, + "loss": 3.4369, + "theoretical_loss": 4.730947415222481, + "tokens_seen": 106496000 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004887462387161484, + "loss": 3.446, + "theoretical_loss": 4.730547465824781, + "tokens_seen": 106561536 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004887362086258777, + "loss": 3.5716, + "theoretical_loss": 4.730147831146537, + "tokens_seen": 106627072 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004887261785356069, + "loss": 3.4779, + "theoretical_loss": 4.72974851074689, + "tokens_seen": 106692608 + }, + { + "epoch": 0.03, + "learning_rate": 0.000488716148445336, + "loss": 3.3394, + "theoretical_loss": 4.729349504185867, + "tokens_seen": 106758144 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004887061183550652, + "loss": 3.5933, + "theoretical_loss": 4.728950811024383, + "tokens_seen": 106823680 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004886960882647944, + "loss": 3.5744, + "theoretical_loss": 4.7285524308242355, + "tokens_seen": 106889216 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004886860581745236, + "loss": 3.3917, + "theoretical_loss": 4.728154363148102, + "tokens_seen": 106954752 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004886760280842528, + "loss": 3.4257, + "theoretical_loss": 4.72775660755954, + "tokens_seen": 107020288 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004886659979939819, + "loss": 3.497, + "theoretical_loss": 4.72735916362298, + "tokens_seen": 107085824 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004886559679037111, + "loss": 3.6518, + "theoretical_loss": 4.7269620309037315, + "tokens_seen": 107151360 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004886459378134403, + "loss": 3.7949, + "theoretical_loss": 4.726565208967973, + "tokens_seen": 107216896 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004886359077231695, + "loss": 3.2897, + "theoretical_loss": 4.726168697382751, + "tokens_seen": 107282432 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004886258776328988, + "loss": 3.5038, + "theoretical_loss": 4.725772495715983, + "tokens_seen": 107347968 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004886158475426278, + "loss": 3.4895, + "theoretical_loss": 4.725376603536446, + "tokens_seen": 107413504 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004886058174523571, + "loss": 3.4177, + "theoretical_loss": 4.724981020413787, + "tokens_seen": 107479040 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004885957873620863, + "loss": 3.5275, + "theoretical_loss": 4.724585745918505, + "tokens_seen": 107544576 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004885857572718155, + "loss": 3.3219, + "theoretical_loss": 4.7241907796219635, + "tokens_seen": 107610112 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004885757271815447, + "loss": 3.3796, + "theoretical_loss": 4.723796121096381, + "tokens_seen": 107675648 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004885656970912739, + "loss": 3.7864, + "theoretical_loss": 4.723401769914824, + "tokens_seen": 107741184 + }, + { + "epoch": 0.03, + "learning_rate": 0.000488555667001003, + "loss": 3.6293, + "theoretical_loss": 4.723007725651219, + "tokens_seen": 107806720 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004885456369107322, + "loss": 3.3832, + "theoretical_loss": 4.722613987880335, + "tokens_seen": 107872256 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004885356068204614, + "loss": 3.4382, + "theoretical_loss": 4.722220556177792, + "tokens_seen": 107937792 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004885255767301906, + "loss": 3.9011, + "theoretical_loss": 4.721827430120053, + "tokens_seen": 108003328 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004885155466399198, + "loss": 3.4064, + "theoretical_loss": 4.721434609284424, + "tokens_seen": 108068864 + }, + { + "epoch": 0.03, + "objective/train/docs_used": 72030, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.506429433822632, + "objective/train/theoretical_loss": 4.721042093249051, + "objective/train/tokens_used": 128594400, + "theoretical_loss": 4.721042093249051, + "tokens_seen": 108134400 + }, + { + "epoch": 0.03, + "learning_rate": 0.000488505516549649, + "loss": 3.6657, + "theoretical_loss": 4.721042093249051, + "tokens_seen": 108134400 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004884954864593781, + "loss": 3.4562, + "theoretical_loss": 4.720649881592919, + "tokens_seen": 108199936 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004884854563691073, + "loss": 3.6244, + "theoretical_loss": 4.7202579738958494, + "tokens_seen": 108265472 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004884754262788365, + "loss": 3.5235, + "theoretical_loss": 4.7198663697384955, + "tokens_seen": 108331008 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004884653961885657, + "loss": 3.5054, + "theoretical_loss": 4.719475068702346, + "tokens_seen": 108396544 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004884553660982949, + "loss": 3.5308, + "theoretical_loss": 4.719084070369714, + "tokens_seen": 108462080 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004884453360080241, + "loss": 3.4538, + "theoretical_loss": 4.718693374323747, + "tokens_seen": 108527616 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004884353059177532, + "loss": 3.6474, + "theoretical_loss": 4.718302980148412, + "tokens_seen": 108593152 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004884252758274825, + "loss": 3.3907, + "theoretical_loss": 4.717912887428501, + "tokens_seen": 108658688 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004884152457372117, + "loss": 3.4183, + "theoretical_loss": 4.717523095749626, + "tokens_seen": 108724224 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004884052156469409, + "loss": 3.2727, + "theoretical_loss": 4.717133604698222, + "tokens_seen": 108789760 + }, + { + "epoch": 0.03, + "learning_rate": 0.00048839518555667, + "loss": 3.4586, + "theoretical_loss": 4.7167444138615355, + "tokens_seen": 108855296 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004883851554663992, + "loss": 3.3661, + "theoretical_loss": 4.716355522827633, + "tokens_seen": 108920832 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004883751253761284, + "loss": 3.673, + "theoretical_loss": 4.715966931185388, + "tokens_seen": 108986368 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004883650952858576, + "loss": 3.3665, + "theoretical_loss": 4.715578638524491, + "tokens_seen": 109051904 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004883550651955868, + "loss": 3.7658, + "theoretical_loss": 4.715190644435435, + "tokens_seen": 109117440 + }, + { + "epoch": 0.03, + "learning_rate": 0.000488345035105316, + "loss": 3.4991, + "theoretical_loss": 4.714802948509522, + "tokens_seen": 109182976 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004883350050150451, + "loss": 3.4763, + "theoretical_loss": 4.71441555033886, + "tokens_seen": 109248512 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004883249749247743, + "loss": 3.4869, + "theoretical_loss": 4.714028449516356, + "tokens_seen": 109314048 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004883149448345035, + "loss": 3.3071, + "theoretical_loss": 4.713641645635718, + "tokens_seen": 109379584 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004883049147442327, + "loss": 3.662, + "theoretical_loss": 4.713255138291454, + "tokens_seen": 109445120 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004882948846539619, + "loss": 3.6421, + "theoretical_loss": 4.712868927078868, + "tokens_seen": 109510656 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004882848545636911, + "loss": 3.1988, + "theoretical_loss": 4.712483011594056, + "tokens_seen": 109576192 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004882748244734203, + "loss": 3.3546, + "theoretical_loss": 4.7120973914339075, + "tokens_seen": 109641728 + }, + { + "epoch": 0.03, + "learning_rate": 0.00048826479438314946, + "loss": 3.221, + "theoretical_loss": 4.7117120661961005, + "tokens_seen": 109707264 + }, + { + "epoch": 0.03, + "objective/train/docs_used": 72698, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.053509473800659, + "objective/train/theoretical_loss": 4.711327035479103, + "objective/train/tokens_used": 130232800, + "theoretical_loss": 4.711327035479103, + "tokens_seen": 109772800 + }, + { + "epoch": 0.03, + "learning_rate": 0.00048825476429287864, + "loss": 3.3917, + "theoretical_loss": 4.711327035479103, + "tokens_seen": 109772800 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004882447342026078, + "loss": 3.5134, + "theoretical_loss": 4.710942298882169, + "tokens_seen": 109838336 + }, + { + "epoch": 0.03, + "learning_rate": 0.000488234704112337, + "loss": 3.4893, + "theoretical_loss": 4.710557856005335, + "tokens_seen": 109903872 + }, + { + "epoch": 0.03, + "learning_rate": 0.00048822467402206624, + "loss": 3.5654, + "theoretical_loss": 4.710173706449419, + "tokens_seen": 109969408 + }, + { + "epoch": 0.03, + "learning_rate": 0.00048821464393179536, + "loss": 3.4611, + "theoretical_loss": 4.709789849816021, + "tokens_seen": 110034944 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004882046138415246, + "loss": 3.4322, + "theoretical_loss": 4.7094062857075185, + "tokens_seen": 110100480 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004881945837512537, + "loss": 3.4607, + "theoretical_loss": 4.709023013727063, + "tokens_seen": 110166016 + }, + { + "epoch": 0.03, + "learning_rate": 0.00048818455366098296, + "loss": 3.2409, + "theoretical_loss": 4.708640033478584, + "tokens_seen": 110231552 + }, + { + "epoch": 0.03, + "learning_rate": 0.00048817452357071214, + "loss": 3.4819, + "theoretical_loss": 4.708257344566778, + "tokens_seen": 110297088 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004881644934804413, + "loss": 3.796, + "theoretical_loss": 4.7078749465971175, + "tokens_seen": 110362624 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004881544633901705, + "loss": 3.6937, + "theoretical_loss": 4.707492839175837, + "tokens_seen": 110428160 + }, + { + "epoch": 0.03, + "learning_rate": 0.00048814443329989974, + "loss": 3.404, + "theoretical_loss": 4.707111021909941, + "tokens_seen": 110493696 + }, + { + "epoch": 0.03, + "learning_rate": 0.00048813440320962887, + "loss": 3.5102, + "theoretical_loss": 4.706729494407197, + "tokens_seen": 110559232 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004881243731193581, + "loss": 3.3806, + "theoretical_loss": 4.706348256276138, + "tokens_seen": 110624768 + }, + { + "epoch": 0.03, + "learning_rate": 0.00048811434302908723, + "loss": 3.5207, + "theoretical_loss": 4.705967307126051, + "tokens_seen": 110690304 + }, + { + "epoch": 0.03, + "learning_rate": 0.00048810431293881646, + "loss": 3.453, + "theoretical_loss": 4.705586646566987, + "tokens_seen": 110755840 + }, + { + "epoch": 0.03, + "learning_rate": 0.00048809428284854564, + "loss": 3.5387, + "theoretical_loss": 4.705206274209751, + "tokens_seen": 110821376 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004880842527582748, + "loss": 3.4418, + "theoretical_loss": 4.704826189665905, + "tokens_seen": 110886912 + }, + { + "epoch": 0.03, + "learning_rate": 0.000488074222668004, + "loss": 3.6384, + "theoretical_loss": 4.704446392547759, + "tokens_seen": 110952448 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004880641925777332, + "loss": 3.0001, + "theoretical_loss": 4.7040668824683785, + "tokens_seen": 111017984 + }, + { + "epoch": 0.03, + "learning_rate": 0.00048805416248746237, + "loss": 3.6658, + "theoretical_loss": 4.7036876590415755, + "tokens_seen": 111083520 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004880441323971916, + "loss": 3.3768, + "theoretical_loss": 4.7033087218819105, + "tokens_seen": 111149056 + }, + { + "epoch": 0.03, + "learning_rate": 0.00048803410230692073, + "loss": 3.6358, + "theoretical_loss": 4.7029300706046895, + "tokens_seen": 111214592 + }, + { + "epoch": 0.03, + "learning_rate": 0.00048802407221664997, + "loss": 3.4117, + "theoretical_loss": 4.702551704825957, + "tokens_seen": 111280128 + }, + { + "epoch": 0.03, + "learning_rate": 0.00048801404212637915, + "loss": 3.4253, + "theoretical_loss": 4.702173624162507, + "tokens_seen": 111345664 + }, + { + "epoch": 0.03, + "objective/train/docs_used": 74080, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.3660080432891846, + "objective/train/theoretical_loss": 4.701795828231866, + "objective/train/tokens_used": 131871200, + "theoretical_loss": 4.701795828231866, + "tokens_seen": 111411200 + }, + { + "epoch": 0.03, + "learning_rate": 0.00048800401203610833, + "loss": 3.2861, + "theoretical_loss": 4.701795828231866, + "tokens_seen": 111411200 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004879939819458375, + "loss": 3.3705, + "theoretical_loss": 4.701418316652299, + "tokens_seen": 111476736 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004879839518555667, + "loss": 3.5817, + "theoretical_loss": 4.701041089042813, + "tokens_seen": 111542272 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004879739217652959, + "loss": 3.5806, + "theoretical_loss": 4.700664145023142, + "tokens_seen": 111607808 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004879638916750251, + "loss": 3.1888, + "theoretical_loss": 4.700287484213753, + "tokens_seen": 111673344 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004879538615847543, + "loss": 3.516, + "theoretical_loss": 4.699911106235849, + "tokens_seen": 111738880 + }, + { + "epoch": 0.03, + "learning_rate": 0.00048794383149448347, + "loss": 3.1074, + "theoretical_loss": 4.6995350107113545, + "tokens_seen": 111804416 + }, + { + "epoch": 0.03, + "learning_rate": 0.00048793380140421265, + "loss": 3.1644, + "theoretical_loss": 4.699159197262922, + "tokens_seen": 111869952 + }, + { + "epoch": 0.03, + "learning_rate": 0.00048792377131394183, + "loss": 3.5934, + "theoretical_loss": 4.698783665513934, + "tokens_seen": 111935488 + }, + { + "epoch": 0.03, + "learning_rate": 0.00048791374122367107, + "loss": 3.6202, + "theoretical_loss": 4.698408415088491, + "tokens_seen": 112001024 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004879037111334002, + "loss": 3.5673, + "theoretical_loss": 4.698033445611415, + "tokens_seen": 112066560 + }, + { + "epoch": 0.03, + "learning_rate": 0.00048789368104312943, + "loss": 3.2708, + "theoretical_loss": 4.6976587567082495, + "tokens_seen": 112132096 + }, + { + "epoch": 0.03, + "learning_rate": 0.00048788365095285856, + "loss": 3.3664, + "theoretical_loss": 4.697284348005253, + "tokens_seen": 112197632 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004878736208625878, + "loss": 3.4398, + "theoretical_loss": 4.696910219129402, + "tokens_seen": 112263168 + }, + { + "epoch": 0.03, + "learning_rate": 0.00048786359077231697, + "loss": 3.2821, + "theoretical_loss": 4.696536369708386, + "tokens_seen": 112328704 + }, + { + "epoch": 0.03, + "learning_rate": 0.00048785356068204615, + "loss": 3.7733, + "theoretical_loss": 4.696162799370606, + "tokens_seen": 112394240 + }, + { + "epoch": 0.03, + "learning_rate": 0.00048784353059177533, + "loss": 3.0105, + "theoretical_loss": 4.695789507745176, + "tokens_seen": 112459776 + }, + { + "epoch": 0.03, + "learning_rate": 0.00048783350050150457, + "loss": 3.2715, + "theoretical_loss": 4.695416494461917, + "tokens_seen": 112525312 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004878234704112337, + "loss": 3.5602, + "theoretical_loss": 4.695043759151353, + "tokens_seen": 112590848 + }, + { + "epoch": 0.03, + "learning_rate": 0.00048781344032096293, + "loss": 3.2464, + "theoretical_loss": 4.694671301444722, + "tokens_seen": 112656384 + }, + { + "epoch": 0.03, + "learning_rate": 0.00048780341023069206, + "loss": 3.3212, + "theoretical_loss": 4.694299120973957, + "tokens_seen": 112721920 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004877933801404213, + "loss": 3.4155, + "theoretical_loss": 4.693927217371698, + "tokens_seen": 112787456 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004877833500501505, + "loss": 3.45, + "theoretical_loss": 4.693555590271282, + "tokens_seen": 112852992 + }, + { + "epoch": 0.03, + "learning_rate": 0.00048777331995987966, + "loss": 3.4624, + "theoretical_loss": 4.693184239306744, + "tokens_seen": 112918528 + }, + { + "epoch": 0.03, + "learning_rate": 0.00048776328986960884, + "loss": 3.411, + "theoretical_loss": 4.692813164112819, + "tokens_seen": 112984064 + }, + { + "epoch": 0.03, + "objective/train/docs_used": 74817, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.8190953731536865, + "objective/train/theoretical_loss": 4.692442364324931, + "objective/train/tokens_used": 133509600, + "theoretical_loss": 4.692442364324931, + "tokens_seen": 113049600 + }, + { + "epoch": 0.03, + "learning_rate": 0.000487753259779338, + "loss": 3.4511, + "theoretical_loss": 4.692442364324931, + "tokens_seen": 113049600 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004877432296890672, + "loss": 3.6126, + "theoretical_loss": 4.692071839579201, + "tokens_seen": 113115136 + }, + { + "epoch": 0.03, + "learning_rate": 0.00048773319959879644, + "loss": 3.2247, + "theoretical_loss": 4.6917015895124425, + "tokens_seen": 113180672 + }, + { + "epoch": 0.03, + "learning_rate": 0.00048772316950852556, + "loss": 3.5138, + "theoretical_loss": 4.691331613762153, + "tokens_seen": 113246208 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004877131394182548, + "loss": 3.4228, + "theoretical_loss": 4.690961911966523, + "tokens_seen": 113311744 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004877031093279839, + "loss": 3.4015, + "theoretical_loss": 4.690592483764427, + "tokens_seen": 113377280 + }, + { + "epoch": 0.03, + "learning_rate": 0.00048769307923771316, + "loss": 3.6467, + "theoretical_loss": 4.690223328795424, + "tokens_seen": 113442816 + }, + { + "epoch": 0.03, + "learning_rate": 0.00048768304914744234, + "loss": 3.2994, + "theoretical_loss": 4.689854446699757, + "tokens_seen": 113508352 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004876730190571715, + "loss": 3.5569, + "theoretical_loss": 4.689485837118347, + "tokens_seen": 113573888 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004876629889669007, + "loss": 3.4775, + "theoretical_loss": 4.689117499692798, + "tokens_seen": 113639424 + }, + { + "epoch": 0.03, + "learning_rate": 0.00048765295887662994, + "loss": 3.558, + "theoretical_loss": 4.688749434065389, + "tokens_seen": 113704960 + }, + { + "epoch": 0.03, + "learning_rate": 0.00048764292878635907, + "loss": 3.4569, + "theoretical_loss": 4.688381639879076, + "tokens_seen": 113770496 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004876328986960883, + "loss": 3.5202, + "theoretical_loss": 4.68801411677749, + "tokens_seen": 113836032 + }, + { + "epoch": 0.03, + "learning_rate": 0.00048762286860581743, + "loss": 3.4741, + "theoretical_loss": 4.687646864404934, + "tokens_seen": 113901568 + }, + { + "epoch": 0.03, + "learning_rate": 0.00048761283851554666, + "loss": 3.2901, + "theoretical_loss": 4.687279882406381, + "tokens_seen": 113967104 + }, + { + "epoch": 0.03, + "learning_rate": 0.00048760280842527584, + "loss": 3.5359, + "theoretical_loss": 4.686913170427477, + "tokens_seen": 114032640 + }, + { + "epoch": 0.03, + "learning_rate": 0.000487592778335005, + "loss": 3.4748, + "theoretical_loss": 4.68654672811453, + "tokens_seen": 114098176 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004875827482447342, + "loss": 3.4518, + "theoretical_loss": 4.68618055511452, + "tokens_seen": 114163712 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004875727181544634, + "loss": 3.4131, + "theoretical_loss": 4.685814651075088, + "tokens_seen": 114229248 + }, + { + "epoch": 0.03, + "learning_rate": 0.00048756268806419257, + "loss": 3.3579, + "theoretical_loss": 4.685449015644537, + "tokens_seen": 114294784 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004875526579739218, + "loss": 3.4673, + "theoretical_loss": 4.685083648471835, + "tokens_seen": 114360320 + }, + { + "epoch": 0.03, + "learning_rate": 0.00048754262788365093, + "loss": 3.5622, + "theoretical_loss": 4.684718549206607, + "tokens_seen": 114425856 + }, + { + "epoch": 0.03, + "learning_rate": 0.00048753259779338017, + "loss": 3.6253, + "theoretical_loss": 4.6843537174991345, + "tokens_seen": 114491392 + }, + { + "epoch": 0.03, + "learning_rate": 0.00048752256770310935, + "loss": 3.2159, + "theoretical_loss": 4.6839891530003595, + "tokens_seen": 114556928 + }, + { + "epoch": 0.03, + "learning_rate": 0.00048751253761283853, + "loss": 3.4459, + "theoretical_loss": 4.683624855361876, + "tokens_seen": 114622464 + }, + { + "epoch": 0.03, + "objective/train/docs_used": 76169, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.472637414932251, + "objective/train/theoretical_loss": 4.68326082423593, + "objective/train/tokens_used": 135148000, + "theoretical_loss": 4.68326082423593, + "tokens_seen": 114688000 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004875025075225677, + "loss": 3.4635, + "theoretical_loss": 4.68326082423593, + "tokens_seen": 114688000 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004874924774322969, + "loss": 3.443, + "theoretical_loss": 4.682897059275422, + "tokens_seen": 114753536 + }, + { + "epoch": 0.03, + "learning_rate": 0.00048748244734202607, + "loss": 3.5193, + "theoretical_loss": 4.682533560133901, + "tokens_seen": 114819072 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004874724172517553, + "loss": 3.3296, + "theoretical_loss": 4.682170326465565, + "tokens_seen": 114884608 + }, + { + "epoch": 0.03, + "learning_rate": 0.00048746238716148443, + "loss": 3.3268, + "theoretical_loss": 4.681807357925257, + "tokens_seen": 114950144 + }, + { + "epoch": 0.03, + "learning_rate": 0.00048745235707121367, + "loss": 3.3407, + "theoretical_loss": 4.681444654168468, + "tokens_seen": 115015680 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004874423269809428, + "loss": 3.1065, + "theoretical_loss": 4.68108221485133, + "tokens_seen": 115081216 + }, + { + "epoch": 0.03, + "learning_rate": 0.00048743229689067203, + "loss": 3.3778, + "theoretical_loss": 4.680720039630617, + "tokens_seen": 115146752 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004874222668004012, + "loss": 3.5193, + "theoretical_loss": 4.680358128163747, + "tokens_seen": 115212288 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004874122367101304, + "loss": 3.4961, + "theoretical_loss": 4.679996480108773, + "tokens_seen": 115277824 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004874022066198596, + "loss": 3.1536, + "theoretical_loss": 4.6796350951243895, + "tokens_seen": 115343360 + }, + { + "epoch": 0.03, + "learning_rate": 0.00048739217652958876, + "loss": 3.3588, + "theoretical_loss": 4.679273972869922, + "tokens_seen": 115408896 + }, + { + "epoch": 0.03, + "learning_rate": 0.00048738214643931794, + "loss": 3.6314, + "theoretical_loss": 4.678913113005333, + "tokens_seen": 115474432 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048737211634904717, + "loss": 3.3236, + "theoretical_loss": 4.6785525151912175, + "tokens_seen": 115539968 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004873620862587763, + "loss": 3.3678, + "theoretical_loss": 4.678192179088802, + "tokens_seen": 115605504 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048735205616850553, + "loss": 3.4886, + "theoretical_loss": 4.6778321043599425, + "tokens_seen": 115671040 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004873420260782347, + "loss": 3.634, + "theoretical_loss": 4.677472290667122, + "tokens_seen": 115736576 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004873319959879639, + "loss": 3.4002, + "theoretical_loss": 4.677112737673453, + "tokens_seen": 115802112 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004873219658976931, + "loss": 3.4607, + "theoretical_loss": 4.676753445042669, + "tokens_seen": 115867648 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048731193580742226, + "loss": 3.3732, + "theoretical_loss": 4.676394412439132, + "tokens_seen": 115933184 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048730190571715144, + "loss": 3.3785, + "theoretical_loss": 4.6760356395278215, + "tokens_seen": 115998720 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004872918756268807, + "loss": 3.3735, + "theoretical_loss": 4.675677125974339, + "tokens_seen": 116064256 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004872818455366098, + "loss": 3.259, + "theoretical_loss": 4.675318871444908, + "tokens_seen": 116129792 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048727181544633904, + "loss": 3.3408, + "theoretical_loss": 4.674960875606366, + "tokens_seen": 116195328 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048726178535606816, + "loss": 3.247, + "theoretical_loss": 4.674603138126168, + "tokens_seen": 116260864 + }, + { + "epoch": 0.04, + "objective/train/docs_used": 76931, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.3002774715423584, + "objective/train/theoretical_loss": 4.674245658672382, + "objective/train/tokens_used": 136786400, + "theoretical_loss": 4.674245658672382, + "tokens_seen": 116326400 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004872517552657974, + "loss": 2.9227, + "theoretical_loss": 4.674245658672382, + "tokens_seen": 116326400 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004872417251755266, + "loss": 3.3998, + "theoretical_loss": 4.673888436913694, + "tokens_seen": 116391936 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048723169508525576, + "loss": 3.59, + "theoretical_loss": 4.673531472519397, + "tokens_seen": 116457472 + }, + { + "epoch": 0.04, + "learning_rate": 0.000487221664994985, + "loss": 3.3691, + "theoretical_loss": 4.673174765159393, + "tokens_seen": 116523008 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004872116349047141, + "loss": 3.2292, + "theoretical_loss": 4.672818314504198, + "tokens_seen": 116588544 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048720160481444336, + "loss": 3.5257, + "theoretical_loss": 4.6724621202249335, + "tokens_seen": 116654080 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048719157472417254, + "loss": 3.5958, + "theoretical_loss": 4.672106181993324, + "tokens_seen": 116719616 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004871815446339017, + "loss": 3.4603, + "theoretical_loss": 4.6717504994817, + "tokens_seen": 116785152 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004871715145436309, + "loss": 3.4717, + "theoretical_loss": 4.671395072362996, + "tokens_seen": 116850688 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048716148445336014, + "loss": 3.5419, + "theoretical_loss": 4.671039900310747, + "tokens_seen": 116916224 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048715145436308927, + "loss": 3.5374, + "theoretical_loss": 4.670684982999088, + "tokens_seen": 116981760 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004871414242728185, + "loss": 3.1384, + "theoretical_loss": 4.670330320102753, + "tokens_seen": 117047296 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048713139418254763, + "loss": 3.6241, + "theoretical_loss": 4.669975911297072, + "tokens_seen": 117112832 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048712136409227686, + "loss": 3.2034, + "theoretical_loss": 4.669621756257971, + "tokens_seen": 117178368 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048711133400200604, + "loss": 3.1406, + "theoretical_loss": 4.669267854661973, + "tokens_seen": 117243904 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004871013039117352, + "loss": 3.5656, + "theoretical_loss": 4.668914206186189, + "tokens_seen": 117309440 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004870912738214644, + "loss": 3.5188, + "theoretical_loss": 4.6685608105083265, + "tokens_seen": 117374976 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004870812437311936, + "loss": 3.4507, + "theoretical_loss": 4.66820766730668, + "tokens_seen": 117440512 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048707121364092277, + "loss": 3.4366, + "theoretical_loss": 4.667854776260132, + "tokens_seen": 117506048 + }, + { + "epoch": 0.04, + "learning_rate": 0.000487061183550652, + "loss": 3.2462, + "theoretical_loss": 4.667502137048155, + "tokens_seen": 117571584 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048705115346038113, + "loss": 3.2731, + "theoretical_loss": 4.667149749350805, + "tokens_seen": 117637120 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048704112337011037, + "loss": 3.5116, + "theoretical_loss": 4.666797612848723, + "tokens_seen": 117702656 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048703109327983955, + "loss": 3.5031, + "theoretical_loss": 4.666445727223134, + "tokens_seen": 117768192 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048702106318956873, + "loss": 3.4129, + "theoretical_loss": 4.666094092155843, + "tokens_seen": 117833728 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004870110330992979, + "loss": 3.4719, + "theoretical_loss": 4.665742707329238, + "tokens_seen": 117899264 + }, + { + "epoch": 0.04, + "objective/train/docs_used": 78354, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.9217355251312256, + "objective/train/theoretical_loss": 4.665391572426282, + "objective/train/tokens_used": 138424800, + "theoretical_loss": 4.665391572426282, + "tokens_seen": 117964800 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004870010030090271, + "loss": 3.5261, + "theoretical_loss": 4.665391572426282, + "tokens_seen": 117964800 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048699097291875627, + "loss": 3.1807, + "theoretical_loss": 4.665040687130518, + "tokens_seen": 118030336 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004869809428284855, + "loss": 3.4102, + "theoretical_loss": 4.664690051126065, + "tokens_seen": 118095872 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048697091273821463, + "loss": 3.4225, + "theoretical_loss": 4.664339664097617, + "tokens_seen": 118161408 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048696088264794387, + "loss": 3.5318, + "theoretical_loss": 4.66398952573044, + "tokens_seen": 118226944 + }, + { + "epoch": 0.04, + "learning_rate": 0.000486950852557673, + "loss": 3.3987, + "theoretical_loss": 4.663639635710373, + "tokens_seen": 118292480 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048694082246740223, + "loss": 3.2866, + "theoretical_loss": 4.663289993723826, + "tokens_seen": 118358016 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004869307923771314, + "loss": 3.5113, + "theoretical_loss": 4.662940599457777, + "tokens_seen": 118423552 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004869207622868606, + "loss": 3.4545, + "theoretical_loss": 4.662591452599774, + "tokens_seen": 118489088 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004869107321965898, + "loss": 3.5142, + "theoretical_loss": 4.662242552837929, + "tokens_seen": 118554624 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048690070210631896, + "loss": 3.4958, + "theoretical_loss": 4.661893899860923, + "tokens_seen": 118620160 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048689067201604814, + "loss": 2.9126, + "theoretical_loss": 4.6615454933579965, + "tokens_seen": 118685696 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048688064192577737, + "loss": 3.3089, + "theoretical_loss": 4.661197333018957, + "tokens_seen": 118751232 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004868706118355065, + "loss": 3.2343, + "theoretical_loss": 4.66084941853417, + "tokens_seen": 118816768 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048686058174523573, + "loss": 3.4087, + "theoretical_loss": 4.6605017495945615, + "tokens_seen": 118882304 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004868505516549649, + "loss": 3.3099, + "theoretical_loss": 4.660154325891618, + "tokens_seen": 118947840 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004868405215646941, + "loss": 3.4114, + "theoretical_loss": 4.659807147117382, + "tokens_seen": 119013376 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004868304914744233, + "loss": 3.308, + "theoretical_loss": 4.6594602129644525, + "tokens_seen": 119078912 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048682046138415246, + "loss": 3.398, + "theoretical_loss": 4.659113523125981, + "tokens_seen": 119144448 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048681043129388164, + "loss": 3.4281, + "theoretical_loss": 4.6587670772956775, + "tokens_seen": 119209984 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004868004012036109, + "loss": 3.44, + "theoretical_loss": 4.658420875167799, + "tokens_seen": 119275520 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048679037111334, + "loss": 3.3893, + "theoretical_loss": 4.658074916437155, + "tokens_seen": 119341056 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048678034102306924, + "loss": 3.498, + "theoretical_loss": 4.657729200799105, + "tokens_seen": 119406592 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048677031093279836, + "loss": 3.6136, + "theoretical_loss": 4.657383727949558, + "tokens_seen": 119472128 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004867602808425276, + "loss": 3.3699, + "theoretical_loss": 4.657038497584967, + "tokens_seen": 119537664 + }, + { + "epoch": 0.04, + "objective/train/docs_used": 78905, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.197535276412964, + "objective/train/theoretical_loss": 4.656693509402331, + "objective/train/tokens_used": 140063200, + "theoretical_loss": 4.656693509402331, + "tokens_seen": 119603200 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004867502507522568, + "loss": 3.3284, + "theoretical_loss": 4.656693509402331, + "tokens_seen": 119603200 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048674022066198596, + "loss": 3.4335, + "theoretical_loss": 4.6563487630991975, + "tokens_seen": 119668736 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048673019057171514, + "loss": 3.3969, + "theoretical_loss": 4.656004258373651, + "tokens_seen": 119734272 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004867201604814443, + "loss": 3.2785, + "theoretical_loss": 4.655659994924323, + "tokens_seen": 119799808 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004867101303911735, + "loss": 3.5232, + "theoretical_loss": 4.655315972450383, + "tokens_seen": 119865344 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048670010030090274, + "loss": 3.4709, + "theoretical_loss": 4.65497219065154, + "tokens_seen": 119930880 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048669007021063187, + "loss": 3.1621, + "theoretical_loss": 4.654628649228041, + "tokens_seen": 119996416 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004866800401203611, + "loss": 3.4993, + "theoretical_loss": 4.654285347880672, + "tokens_seen": 120061952 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004866700100300903, + "loss": 3.2093, + "theoretical_loss": 4.653942286310749, + "tokens_seen": 120127488 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048665997993981947, + "loss": 3.5648, + "theoretical_loss": 4.653599464220129, + "tokens_seen": 120193024 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048664994984954865, + "loss": 3.4329, + "theoretical_loss": 4.653256881311198, + "tokens_seen": 120258560 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048663991975927783, + "loss": 3.679, + "theoretical_loss": 4.6529145372868745, + "tokens_seen": 120324096 + }, + { + "epoch": 0.04, + "learning_rate": 0.000486629889669007, + "loss": 3.5076, + "theoretical_loss": 4.652572431850608, + "tokens_seen": 120389632 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048661985957873624, + "loss": 3.3315, + "theoretical_loss": 4.652230564706377, + "tokens_seen": 120455168 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048660982948846537, + "loss": 3.5002, + "theoretical_loss": 4.651888935558688, + "tokens_seen": 120520704 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004865997993981946, + "loss": 3.5856, + "theoretical_loss": 4.651547544112575, + "tokens_seen": 120586240 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048658976930792373, + "loss": 3.742, + "theoretical_loss": 4.651206390073597, + "tokens_seen": 120651776 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048657973921765297, + "loss": 3.4095, + "theoretical_loss": 4.650865473147837, + "tokens_seen": 120717312 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048656970912738215, + "loss": 3.5222, + "theoretical_loss": 4.650524793041903, + "tokens_seen": 120782848 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048655967903711133, + "loss": 3.5061, + "theoretical_loss": 4.650184349462922, + "tokens_seen": 120848384 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004865496489468405, + "loss": 3.3906, + "theoretical_loss": 4.649844142118544, + "tokens_seen": 120913920 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048653961885656975, + "loss": 3.454, + "theoretical_loss": 4.6495041707169396, + "tokens_seen": 120979456 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004865295887662989, + "loss": 3.3339, + "theoretical_loss": 4.649164434966794, + "tokens_seen": 121044992 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004865195586760281, + "loss": 3.6652, + "theoretical_loss": 4.648824934577313, + "tokens_seen": 121110528 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048650952858575724, + "loss": 3.4645, + "theoretical_loss": 4.648485669258216, + "tokens_seen": 121176064 + }, + { + "epoch": 0.04, + "objective/train/docs_used": 79692, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.9736289978027344, + "objective/train/theoretical_loss": 4.648146638719739, + "objective/train/tokens_used": 141701600, + "theoretical_loss": 4.648146638719739, + "tokens_seen": 121241600 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048649949849548647, + "loss": 3.4528, + "theoretical_loss": 4.648146638719739, + "tokens_seen": 121241600 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048648946840521565, + "loss": 3.4371, + "theoretical_loss": 4.647807842672631, + "tokens_seen": 121307136 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048647943831494483, + "loss": 3.6325, + "theoretical_loss": 4.647469280828153, + "tokens_seen": 121372672 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048646940822467407, + "loss": 3.074, + "theoretical_loss": 4.647130952898077, + "tokens_seen": 121438208 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004864593781344032, + "loss": 3.5242, + "theoretical_loss": 4.646792858594686, + "tokens_seen": 121503744 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048644934804413243, + "loss": 3.6688, + "theoretical_loss": 4.64645499763077, + "tokens_seen": 121569280 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004864393179538616, + "loss": 3.2291, + "theoretical_loss": 4.646117369719629, + "tokens_seen": 121634816 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004864292878635908, + "loss": 3.1516, + "theoretical_loss": 4.645779974575069, + "tokens_seen": 121700352 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048641925777332, + "loss": 3.2044, + "theoretical_loss": 4.6454428119113995, + "tokens_seen": 121765888 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048640922768304916, + "loss": 3.3072, + "theoretical_loss": 4.6451058814434365, + "tokens_seen": 121831424 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048639919759277834, + "loss": 3.2776, + "theoretical_loss": 4.644769182886495, + "tokens_seen": 121896960 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048638916750250757, + "loss": 3.6095, + "theoretical_loss": 4.644432715956399, + "tokens_seen": 121962496 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004863791374122367, + "loss": 3.4414, + "theoretical_loss": 4.644096480369466, + "tokens_seen": 122028032 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048636910732196593, + "loss": 3.1281, + "theoretical_loss": 4.643760475842518, + "tokens_seen": 122093568 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004863590772316951, + "loss": 3.3452, + "theoretical_loss": 4.6434247020928705, + "tokens_seen": 122159104 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004863490471414243, + "loss": 3.5932, + "theoretical_loss": 4.643089158838341, + "tokens_seen": 122224640 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004863390170511535, + "loss": 3.3176, + "theoretical_loss": 4.642753845797243, + "tokens_seen": 122290176 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048632898696088266, + "loss": 3.481, + "theoretical_loss": 4.642418762688379, + "tokens_seen": 122355712 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048631895687061184, + "loss": 3.5405, + "theoretical_loss": 4.642083909231053, + "tokens_seen": 122421248 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004863089267803411, + "loss": 3.6172, + "theoretical_loss": 4.641749285145057, + "tokens_seen": 122486784 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004862988966900702, + "loss": 3.4751, + "theoretical_loss": 4.641414890150675, + "tokens_seen": 122552320 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048628886659979944, + "loss": 3.4076, + "theoretical_loss": 4.641080723968684, + "tokens_seen": 122617856 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048627883650952857, + "loss": 3.4744, + "theoretical_loss": 4.6407467863203475, + "tokens_seen": 122683392 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004862688064192578, + "loss": 3.3313, + "theoretical_loss": 4.640413076927418, + "tokens_seen": 122748928 + }, + { + "epoch": 0.04, + "learning_rate": 0.000486258776328987, + "loss": 3.0625, + "theoretical_loss": 4.6400795955121374, + "tokens_seen": 122814464 + }, + { + "epoch": 0.04, + "objective/train/docs_used": 81051, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.273958683013916, + "objective/train/theoretical_loss": 4.639746341797229, + "objective/train/tokens_used": 143340000, + "theoretical_loss": 4.639746341797229, + "tokens_seen": 122880000 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048624874623871616, + "loss": 3.5055, + "theoretical_loss": 4.639746341797229, + "tokens_seen": 122880000 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048623871614844534, + "loss": 3.4553, + "theoretical_loss": 4.639413315505905, + "tokens_seen": 122945536 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004862286860581745, + "loss": 3.4082, + "theoretical_loss": 4.639080516361861, + "tokens_seen": 123011072 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004862186559679037, + "loss": 3.3843, + "theoretical_loss": 4.638747944089273, + "tokens_seen": 123076608 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048620862587763294, + "loss": 3.3584, + "theoretical_loss": 4.638415598412799, + "tokens_seen": 123142144 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048619859578736207, + "loss": 3.285, + "theoretical_loss": 4.638083479057579, + "tokens_seen": 123207680 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004861885656970913, + "loss": 3.2169, + "theoretical_loss": 4.637751585749234, + "tokens_seen": 123273216 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004861785356068205, + "loss": 3.2718, + "theoretical_loss": 4.6374199182138565, + "tokens_seen": 123338752 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048616850551654967, + "loss": 3.4323, + "theoretical_loss": 4.637088476178025, + "tokens_seen": 123404288 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048615847542627885, + "loss": 3.3324, + "theoretical_loss": 4.636757259368787, + "tokens_seen": 123469824 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048614844533600803, + "loss": 3.6044, + "theoretical_loss": 4.636426267513668, + "tokens_seen": 123535360 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004861384152457372, + "loss": 3.4497, + "theoretical_loss": 4.636095500340669, + "tokens_seen": 123600896 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048612838515546644, + "loss": 3.5109, + "theoretical_loss": 4.635764957578261, + "tokens_seen": 123666432 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048611835506519557, + "loss": 3.2299, + "theoretical_loss": 4.635434638955388, + "tokens_seen": 123731968 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004861083249749248, + "loss": 3.4359, + "theoretical_loss": 4.635104544201465, + "tokens_seen": 123797504 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048609829488465393, + "loss": 3.3737, + "theoretical_loss": 4.634774673046376, + "tokens_seen": 123863040 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048608826479438317, + "loss": 3.2749, + "theoretical_loss": 4.634445025220475, + "tokens_seen": 123928576 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048607823470411235, + "loss": 3.3556, + "theoretical_loss": 4.634115600454582, + "tokens_seen": 123994112 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048606820461384153, + "loss": 3.3388, + "theoretical_loss": 4.633786398479983, + "tokens_seen": 124059648 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004860581745235707, + "loss": 3.1468, + "theoretical_loss": 4.6334574190284314, + "tokens_seen": 124125184 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048604814443329995, + "loss": 3.2246, + "theoretical_loss": 4.633128661832145, + "tokens_seen": 124190720 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004860381143430291, + "loss": 3.5981, + "theoretical_loss": 4.632800126623803, + "tokens_seen": 124256256 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004860280842527583, + "loss": 3.4044, + "theoretical_loss": 4.632471813136547, + "tokens_seen": 124321792 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048601805416248744, + "loss": 3.4309, + "theoretical_loss": 4.632143721103983, + "tokens_seen": 124387328 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048600802407221667, + "loss": 3.4409, + "theoretical_loss": 4.631815850260173, + "tokens_seen": 124452864 + }, + { + "epoch": 0.04, + "objective/train/docs_used": 81846, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.6155059337615967, + "objective/train/theoretical_loss": 4.631488200339643, + "objective/train/tokens_used": 144978400, + "theoretical_loss": 4.631488200339643, + "tokens_seen": 124518400 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048599799398194585, + "loss": 3.4182, + "theoretical_loss": 4.631488200339643, + "tokens_seen": 124518400 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048598796389167503, + "loss": 3.4847, + "theoretical_loss": 4.63116077107737, + "tokens_seen": 124583936 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004859779338014042, + "loss": 3.5333, + "theoretical_loss": 4.630833562208797, + "tokens_seen": 124649472 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004859679037111334, + "loss": 3.2189, + "theoretical_loss": 4.630506573469815, + "tokens_seen": 124715008 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004859578736208626, + "loss": 3.3049, + "theoretical_loss": 4.630179804596775, + "tokens_seen": 124780544 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004859478435305918, + "loss": 3.3721, + "theoretical_loss": 4.629853255326481, + "tokens_seen": 124846080 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048593781344032094, + "loss": 3.4475, + "theoretical_loss": 4.629526925396189, + "tokens_seen": 124911616 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004859277833500502, + "loss": 3.3819, + "theoretical_loss": 4.6292008145436085, + "tokens_seen": 124977152 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004859177532597793, + "loss": 3.4153, + "theoretical_loss": 4.628874922506897, + "tokens_seen": 125042688 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048590772316950854, + "loss": 3.5372, + "theoretical_loss": 4.628549249024666, + "tokens_seen": 125108224 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004858976930792377, + "loss": 3.2878, + "theoretical_loss": 4.628223793835975, + "tokens_seen": 125173760 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004858876629889669, + "loss": 3.1345, + "theoretical_loss": 4.627898556680327, + "tokens_seen": 125239296 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004858776328986961, + "loss": 3.5378, + "theoretical_loss": 4.627573537297678, + "tokens_seen": 125304832 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004858676028084253, + "loss": 3.257, + "theoretical_loss": 4.627248735428427, + "tokens_seen": 125370368 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048585757271815444, + "loss": 3.5364, + "theoretical_loss": 4.6269241508134185, + "tokens_seen": 125435904 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004858475426278837, + "loss": 3.5306, + "theoretical_loss": 4.6265997831939405, + "tokens_seen": 125501440 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004858375125376128, + "loss": 3.4566, + "theoretical_loss": 4.6262756323117245, + "tokens_seen": 125566976 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048582748244734204, + "loss": 3.1179, + "theoretical_loss": 4.625951697908944, + "tokens_seen": 125632512 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004858174523570712, + "loss": 3.3013, + "theoretical_loss": 4.625627979728212, + "tokens_seen": 125698048 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004858074222668004, + "loss": 3.3677, + "theoretical_loss": 4.625304477512584, + "tokens_seen": 125763584 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004857973921765296, + "loss": 3.443, + "theoretical_loss": 4.624981191005554, + "tokens_seen": 125829120 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048578736208625877, + "loss": 3.3865, + "theoretical_loss": 4.624658119951052, + "tokens_seen": 125894656 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048577733199598795, + "loss": 3.4286, + "theoretical_loss": 4.624335264093447, + "tokens_seen": 125960192 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004857673019057172, + "loss": 3.3829, + "theoretical_loss": 4.624012623177544, + "tokens_seen": 126025728 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004857572718154463, + "loss": 3.4995, + "theoretical_loss": 4.623690196948582, + "tokens_seen": 126091264 + }, + { + "epoch": 0.04, + "objective/train/docs_used": 83055, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.784626007080078, + "objective/train/theoretical_loss": 4.623367985152234, + "objective/train/tokens_used": 146616800, + "theoretical_loss": 4.623367985152234, + "tokens_seen": 126156800 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048574724172517554, + "loss": 3.4698, + "theoretical_loss": 4.623367985152234, + "tokens_seen": 126156800 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048573721163490467, + "loss": 3.3665, + "theoretical_loss": 4.623045987534609, + "tokens_seen": 126222336 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004857271815446339, + "loss": 3.5286, + "theoretical_loss": 4.622724203842246, + "tokens_seen": 126287872 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048571715145436314, + "loss": 3.1883, + "theoretical_loss": 4.622402633822114, + "tokens_seen": 126353408 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048570712136409227, + "loss": 3.1845, + "theoretical_loss": 4.622081277221616, + "tokens_seen": 126418944 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004856970912738215, + "loss": 3.3815, + "theoretical_loss": 4.62176013378858, + "tokens_seen": 126484480 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004856870611835507, + "loss": 3.2044, + "theoretical_loss": 4.621439203271267, + "tokens_seen": 126550016 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048567703109327987, + "loss": 3.4265, + "theoretical_loss": 4.621118485418362, + "tokens_seen": 126615552 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048566700100300905, + "loss": 3.2876, + "theoretical_loss": 4.620797979978978, + "tokens_seen": 126681088 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048565697091273823, + "loss": 3.4705, + "theoretical_loss": 4.620477686702651, + "tokens_seen": 126746624 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004856469408224674, + "loss": 3.08, + "theoretical_loss": 4.620157605339347, + "tokens_seen": 126812160 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048563691073219664, + "loss": 3.549, + "theoretical_loss": 4.619837735639452, + "tokens_seen": 126877696 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048562688064192577, + "loss": 3.5377, + "theoretical_loss": 4.619518077353776, + "tokens_seen": 126943232 + }, + { + "epoch": 0.04, + "learning_rate": 0.000485616850551655, + "loss": 3.5239, + "theoretical_loss": 4.619198630233547, + "tokens_seen": 127008768 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048560682046138413, + "loss": 3.402, + "theoretical_loss": 4.6188793940304205, + "tokens_seen": 127074304 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048559679037111337, + "loss": 3.3869, + "theoretical_loss": 4.618560368496466, + "tokens_seen": 127139840 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048558676028084255, + "loss": 3.2437, + "theoretical_loss": 4.618241553384175, + "tokens_seen": 127205376 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048557673019057173, + "loss": 3.5848, + "theoretical_loss": 4.617922948446459, + "tokens_seen": 127270912 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004855667001003009, + "loss": 2.9942, + "theoretical_loss": 4.617604553436642, + "tokens_seen": 127336448 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048555667001003015, + "loss": 3.5606, + "theoretical_loss": 4.617286368108466, + "tokens_seen": 127401984 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004855466399197593, + "loss": 3.4445, + "theoretical_loss": 4.6169683922160925, + "tokens_seen": 127467520 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004855366098294885, + "loss": 3.3326, + "theoretical_loss": 4.616650625514091, + "tokens_seen": 127533056 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048552657973921764, + "loss": 3.4342, + "theoretical_loss": 4.616333067757449, + "tokens_seen": 127598592 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048551654964894687, + "loss": 3.48, + "theoretical_loss": 4.616015718701563, + "tokens_seen": 127664128 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048550651955867605, + "loss": 3.2618, + "theoretical_loss": 4.615698578102245, + "tokens_seen": 127729664 + }, + { + "epoch": 0.04, + "objective/train/docs_used": 83592, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.068225860595703, + "objective/train/theoretical_loss": 4.615381645715717, + "objective/train/tokens_used": 148255200, + "theoretical_loss": 4.615381645715717, + "tokens_seen": 127795200 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048549648946840523, + "loss": 3.1804, + "theoretical_loss": 4.615381645715717, + "tokens_seen": 127795200 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004854864593781344, + "loss": 3.4404, + "theoretical_loss": 4.615064921298608, + "tokens_seen": 127860736 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004854764292878636, + "loss": 3.6451, + "theoretical_loss": 4.61474840460796, + "tokens_seen": 127926272 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004854663991975928, + "loss": 3.3584, + "theoretical_loss": 4.614432095401219, + "tokens_seen": 127991808 + }, + { + "epoch": 0.04, + "learning_rate": 0.000485456369107322, + "loss": 3.5192, + "theoretical_loss": 4.614115993436242, + "tokens_seen": 128057344 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048544633901705114, + "loss": 3.1517, + "theoretical_loss": 4.613800098471291, + "tokens_seen": 128122880 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004854363089267804, + "loss": 3.2304, + "theoretical_loss": 4.613484410265032, + "tokens_seen": 128188416 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004854262788365095, + "loss": 3.5812, + "theoretical_loss": 4.613168928576538, + "tokens_seen": 128253952 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048541624874623874, + "loss": 3.3067, + "theoretical_loss": 4.612853653165283, + "tokens_seen": 128319488 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004854062186559679, + "loss": 3.3353, + "theoretical_loss": 4.612538583791146, + "tokens_seen": 128385024 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004853961885656971, + "loss": 3.6273, + "theoretical_loss": 4.612223720214407, + "tokens_seen": 128450560 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004853861584754263, + "loss": 3.309, + "theoretical_loss": 4.611909062195749, + "tokens_seen": 128516096 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004853761283851555, + "loss": 3.372, + "theoretical_loss": 4.61159460949625, + "tokens_seen": 128581632 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048536609829488464, + "loss": 3.593, + "theoretical_loss": 4.611280361877393, + "tokens_seen": 128647168 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004853560682046139, + "loss": 3.4064, + "theoretical_loss": 4.610966319101056, + "tokens_seen": 128712704 + }, + { + "epoch": 0.04, + "learning_rate": 0.000485346038114343, + "loss": 3.51, + "theoretical_loss": 4.610652480929515, + "tokens_seen": 128778240 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048533600802407224, + "loss": 3.1953, + "theoretical_loss": 4.610338847125445, + "tokens_seen": 128843776 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004853259779338014, + "loss": 3.5004, + "theoretical_loss": 4.610025417451913, + "tokens_seen": 128909312 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004853159478435306, + "loss": 3.3487, + "theoretical_loss": 4.6097121916723856, + "tokens_seen": 128974848 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004853059177532598, + "loss": 3.4453, + "theoretical_loss": 4.609399169550718, + "tokens_seen": 129040384 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048529588766298897, + "loss": 3.2506, + "theoretical_loss": 4.609086350851165, + "tokens_seen": 129105920 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048528585757271815, + "loss": 3.1674, + "theoretical_loss": 4.6087737353383655, + "tokens_seen": 129171456 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004852758274824474, + "loss": 3.4917, + "theoretical_loss": 4.6084613227773605, + "tokens_seen": 129236992 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004852657973921765, + "loss": 3.4098, + "theoretical_loss": 4.608149112933571, + "tokens_seen": 129302528 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048525576730190574, + "loss": 3.4741, + "theoretical_loss": 4.607837105572816, + "tokens_seen": 129368064 + }, + { + "epoch": 0.04, + "objective/train/docs_used": 84814, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.5462865829467773, + "objective/train/theoretical_loss": 4.607525300461299, + "objective/train/tokens_used": 149893600, + "theoretical_loss": 4.607525300461299, + "tokens_seen": 129433600 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048524573721163487, + "loss": 3.4219, + "theoretical_loss": 4.607525300461299, + "tokens_seen": 129433600 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004852357071213641, + "loss": 3.5306, + "theoretical_loss": 4.607213697365613, + "tokens_seen": 129499136 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004852256770310933, + "loss": 3.4483, + "theoretical_loss": 4.606902296052739, + "tokens_seen": 129564672 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048521564694082247, + "loss": 3.4257, + "theoretical_loss": 4.6065910962900425, + "tokens_seen": 129630208 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048520561685055165, + "loss": 3.5188, + "theoretical_loss": 4.606280097845277, + "tokens_seen": 129695744 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004851955867602809, + "loss": 3.5821, + "theoretical_loss": 4.60596930048658, + "tokens_seen": 129761280 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048518555667001, + "loss": 3.4408, + "theoretical_loss": 4.605658703982471, + "tokens_seen": 129826816 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048517552657973925, + "loss": 3.4081, + "theoretical_loss": 4.6053483081018545, + "tokens_seen": 129892352 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004851654964894684, + "loss": 3.3554, + "theoretical_loss": 4.605038112614018, + "tokens_seen": 129957888 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004851554663991976, + "loss": 3.4616, + "theoretical_loss": 4.604728117288631, + "tokens_seen": 130023424 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004851454363089268, + "loss": 3.4282, + "theoretical_loss": 4.604418321895739, + "tokens_seen": 130088960 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048513540621865597, + "loss": 3.3901, + "theoretical_loss": 4.604108726205774, + "tokens_seen": 130154496 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048512537612838515, + "loss": 3.2869, + "theoretical_loss": 4.60379932998954, + "tokens_seen": 130220032 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048511534603811433, + "loss": 3.4003, + "theoretical_loss": 4.6034901330182265, + "tokens_seen": 130285568 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004851053159478435, + "loss": 3.1953, + "theoretical_loss": 4.603181135063394, + "tokens_seen": 130351104 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048509528585757275, + "loss": 3.4428, + "theoretical_loss": 4.6028723358969845, + "tokens_seen": 130416640 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004850852557673019, + "loss": 3.3328, + "theoretical_loss": 4.602563735291312, + "tokens_seen": 130482176 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004850752256770311, + "loss": 3.4588, + "theoretical_loss": 4.602255333019068, + "tokens_seen": 130547712 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048506519558676024, + "loss": 3.2303, + "theoretical_loss": 4.6019471288533165, + "tokens_seen": 130613248 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004850551654964895, + "loss": 3.1379, + "theoretical_loss": 4.601639122567497, + "tokens_seen": 130678784 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048504513540621866, + "loss": 3.4158, + "theoretical_loss": 4.601331313935418, + "tokens_seen": 130744320 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048503510531594784, + "loss": 3.3123, + "theoretical_loss": 4.601023702731264, + "tokens_seen": 130809856 + }, + { + "epoch": 0.04, + "learning_rate": 0.000485025075225677, + "loss": 3.4928, + "theoretical_loss": 4.600716288729587, + "tokens_seen": 130875392 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048501504513540625, + "loss": 3.3954, + "theoretical_loss": 4.600409071705312, + "tokens_seen": 130940928 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004850050150451354, + "loss": 3.3409, + "theoretical_loss": 4.60010205143373, + "tokens_seen": 131006464 + }, + { + "epoch": 0.04, + "objective/train/docs_used": 85560, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.4484026432037354, + "objective/train/theoretical_loss": 4.599795227690505, + "objective/train/tokens_used": 151532000, + "theoretical_loss": 4.599795227690505, + "tokens_seen": 131072000 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004849949849548646, + "loss": 3.3038, + "theoretical_loss": 4.599795227690505, + "tokens_seen": 131072000 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004849849548645938, + "loss": 3.3538, + "theoretical_loss": 4.5994886002516635, + "tokens_seen": 131137536 + }, + { + "epoch": 0.04, + "learning_rate": 0.000484974924774323, + "loss": 3.5226, + "theoretical_loss": 4.599182168893604, + "tokens_seen": 131203072 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004849648946840522, + "loss": 3.4058, + "theoretical_loss": 4.598875933393089, + "tokens_seen": 131268608 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048495486459378134, + "loss": 3.1081, + "theoretical_loss": 4.5985698935272445, + "tokens_seen": 131334144 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004849448345035106, + "loss": 3.3463, + "theoretical_loss": 4.598264049073565, + "tokens_seen": 131399680 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004849348044132397, + "loss": 3.0983, + "theoretical_loss": 4.597958399809908, + "tokens_seen": 131465216 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048492477432296894, + "loss": 3.2027, + "theoretical_loss": 4.59765294551449, + "tokens_seen": 131530752 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004849147442326981, + "loss": 3.5707, + "theoretical_loss": 4.597347685965897, + "tokens_seen": 131596288 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004849047141424273, + "loss": 3.6566, + "theoretical_loss": 4.597042620943069, + "tokens_seen": 131661824 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004848946840521565, + "loss": 3.2833, + "theoretical_loss": 4.596737750225311, + "tokens_seen": 131727360 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004848846539618857, + "loss": 3.3376, + "theoretical_loss": 4.596433073592289, + "tokens_seen": 131792896 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048487462387161484, + "loss": 3.2996, + "theoretical_loss": 4.596128590824026, + "tokens_seen": 131858432 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004848645937813441, + "loss": 3.1223, + "theoretical_loss": 4.595824301700904, + "tokens_seen": 131923968 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004848545636910732, + "loss": 3.4071, + "theoretical_loss": 4.595520206003663, + "tokens_seen": 131989504 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048484453360080244, + "loss": 3.4257, + "theoretical_loss": 4.595216303513399, + "tokens_seen": 132055040 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004848345035105316, + "loss": 3.4109, + "theoretical_loss": 4.594912594011566, + "tokens_seen": 132120576 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004848244734202608, + "loss": 3.3532, + "theoretical_loss": 4.594609077279973, + "tokens_seen": 132186112 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048481444332999, + "loss": 3.4505, + "theoretical_loss": 4.594305753100782, + "tokens_seen": 132251648 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048480441323971917, + "loss": 3.1057, + "theoretical_loss": 4.594002621256511, + "tokens_seen": 132317184 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048479438314944835, + "loss": 3.242, + "theoretical_loss": 4.59369968153003, + "tokens_seen": 132382720 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004847843530591776, + "loss": 3.2813, + "theoretical_loss": 4.593396933704562, + "tokens_seen": 132448256 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004847743229689067, + "loss": 3.2752, + "theoretical_loss": 4.593094377563681, + "tokens_seen": 132513792 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048476429287863594, + "loss": 3.1239, + "theoretical_loss": 4.592792012891314, + "tokens_seen": 132579328 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048475426278836507, + "loss": 3.4682, + "theoretical_loss": 4.592489839471735, + "tokens_seen": 132644864 + }, + { + "epoch": 0.04, + "objective/train/docs_used": 86181, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.1618175506591797, + "objective/train/theoretical_loss": 4.592187857089571, + "objective/train/tokens_used": 153170400, + "theoretical_loss": 4.592187857089571, + "tokens_seen": 132710400 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004847442326980943, + "loss": 3.2079, + "theoretical_loss": 4.592187857089571, + "tokens_seen": 132710400 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004847342026078235, + "loss": 3.4512, + "theoretical_loss": 4.591886065529795, + "tokens_seen": 132775936 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048472417251755267, + "loss": 3.2247, + "theoretical_loss": 4.591584464577728, + "tokens_seen": 132841472 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048471414242728185, + "loss": 3.269, + "theoretical_loss": 4.591283054019041, + "tokens_seen": 132907008 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004847041123370111, + "loss": 3.4997, + "theoretical_loss": 4.5909818336397485, + "tokens_seen": 132972544 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004846940822467402, + "loss": 3.3062, + "theoretical_loss": 4.590680803226213, + "tokens_seen": 133038080 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048468405215646945, + "loss": 3.5196, + "theoretical_loss": 4.590379962565141, + "tokens_seen": 133103616 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004846740220661986, + "loss": 3.2837, + "theoretical_loss": 4.590079311443583, + "tokens_seen": 133169152 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004846639919759278, + "loss": 3.1525, + "theoretical_loss": 4.589778849648934, + "tokens_seen": 133234688 + }, + { + "epoch": 0.04, + "learning_rate": 0.000484653961885657, + "loss": 3.4237, + "theoretical_loss": 4.589478576968932, + "tokens_seen": 133300224 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048464393179538617, + "loss": 3.2379, + "theoretical_loss": 4.589178493191655, + "tokens_seen": 133365760 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048463390170511535, + "loss": 3.2568, + "theoretical_loss": 4.588878598105527, + "tokens_seen": 133431296 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048462387161484453, + "loss": 3.3572, + "theoretical_loss": 4.588578891499308, + "tokens_seen": 133496832 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004846138415245737, + "loss": 3.3589, + "theoretical_loss": 4.588279373162101, + "tokens_seen": 133562368 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048460381143430295, + "loss": 3.4065, + "theoretical_loss": 4.587980042883347, + "tokens_seen": 133627904 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004845937813440321, + "loss": 3.1927, + "theoretical_loss": 4.587680900452824, + "tokens_seen": 133693440 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004845837512537613, + "loss": 3.7051, + "theoretical_loss": 4.587381945660653, + "tokens_seen": 133758976 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048457372116349044, + "loss": 3.3832, + "theoretical_loss": 4.587083178297288, + "tokens_seen": 133824512 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004845636910732197, + "loss": 3.3373, + "theoretical_loss": 4.5867845981535185, + "tokens_seen": 133890048 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048455366098294886, + "loss": 3.1244, + "theoretical_loss": 4.586486205020474, + "tokens_seen": 133955584 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048454363089267804, + "loss": 3.2697, + "theoretical_loss": 4.586187998689616, + "tokens_seen": 134021120 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004845336008024072, + "loss": 3.2758, + "theoretical_loss": 4.585889978952741, + "tokens_seen": 134086656 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048452357071213645, + "loss": 3.2439, + "theoretical_loss": 4.58559214560198, + "tokens_seen": 134152192 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004845135406218656, + "loss": 3.3953, + "theoretical_loss": 4.585294498429796, + "tokens_seen": 134217728 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004845035105315948, + "loss": 3.3078, + "theoretical_loss": 4.584997037228986, + "tokens_seen": 134283264 + }, + { + "epoch": 0.04, + "objective/train/docs_used": 87535, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.1161417961120605, + "objective/train/theoretical_loss": 4.584699761792674, + "objective/train/tokens_used": 154808800, + "theoretical_loss": 4.584699761792674, + "tokens_seen": 134348800 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048449348044132394, + "loss": 3.1776, + "theoretical_loss": 4.584699761792674, + "tokens_seen": 134348800 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004844834503510532, + "loss": 3.4011, + "theoretical_loss": 4.5844026719143205, + "tokens_seen": 134414336 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048447342026078236, + "loss": 3.6531, + "theoretical_loss": 4.5841057673877135, + "tokens_seen": 134479872 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048446339017051154, + "loss": 3.1186, + "theoretical_loss": 4.5838090480069695, + "tokens_seen": 134545408 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004844533600802407, + "loss": 3.1638, + "theoretical_loss": 4.5835125135665375, + "tokens_seen": 134610944 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004844433299899699, + "loss": 3.6398, + "theoretical_loss": 4.583216163861191, + "tokens_seen": 134676480 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004844332998996991, + "loss": 3.5227, + "theoretical_loss": 4.58291999868603, + "tokens_seen": 134742016 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004844232698094283, + "loss": 3.5111, + "theoretical_loss": 4.582624017836489, + "tokens_seen": 134807552 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048441323971915745, + "loss": 3.3788, + "theoretical_loss": 4.582328221108318, + "tokens_seen": 134873088 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004844032096288867, + "loss": 3.2425, + "theoretical_loss": 4.5820326082976, + "tokens_seen": 134938624 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048439317953861586, + "loss": 3.3977, + "theoretical_loss": 4.581737179200739, + "tokens_seen": 135004160 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048438314944834504, + "loss": 3.1764, + "theoretical_loss": 4.581441933614466, + "tokens_seen": 135069696 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004843731193580742, + "loss": 3.4576, + "theoretical_loss": 4.581146871335832, + "tokens_seen": 135135232 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004843630892678034, + "loss": 3.3952, + "theoretical_loss": 4.580851992162214, + "tokens_seen": 135200768 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004843530591775326, + "loss": 3.2809, + "theoretical_loss": 4.5805572958913086, + "tokens_seen": 135266304 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004843430290872618, + "loss": 3.4331, + "theoretical_loss": 4.580262782321135, + "tokens_seen": 135331840 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048433299899699095, + "loss": 3.5156, + "theoretical_loss": 4.579968451250032, + "tokens_seen": 135397376 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004843229689067202, + "loss": 3.3807, + "theoretical_loss": 4.579674302476661, + "tokens_seen": 135462912 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004843129388164493, + "loss": 3.3503, + "theoretical_loss": 4.579380335800001, + "tokens_seen": 135528448 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048430290872617855, + "loss": 3.2947, + "theoretical_loss": 4.579086551019348, + "tokens_seen": 135593984 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048429287863590773, + "loss": 3.1548, + "theoretical_loss": 4.5787929479343195, + "tokens_seen": 135659520 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004842828485456369, + "loss": 3.1051, + "theoretical_loss": 4.578499526344848, + "tokens_seen": 135725056 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004842728184553661, + "loss": 3.358, + "theoretical_loss": 4.578206286051184, + "tokens_seen": 135790592 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048426278836509527, + "loss": 3.2007, + "theoretical_loss": 4.5779132268538945, + "tokens_seen": 135856128 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048425275827482445, + "loss": 3.4396, + "theoretical_loss": 4.577620348553859, + "tokens_seen": 135921664 + }, + { + "epoch": 0.04, + "objective/train/docs_used": 88189, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.043414831161499, + "objective/train/theoretical_loss": 4.577327650952276, + "objective/train/tokens_used": 156447200, + "theoretical_loss": 4.577327650952276, + "tokens_seen": 135987200 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004842427281845537, + "loss": 3.2112, + "theoretical_loss": 4.577327650952276, + "tokens_seen": 135987200 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048423269809428287, + "loss": 3.1292, + "theoretical_loss": 4.5770351338506545, + "tokens_seen": 136052736 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048422266800401205, + "loss": 2.9997, + "theoretical_loss": 4.57674279705082, + "tokens_seen": 136118272 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004842126379137413, + "loss": 3.6417, + "theoretical_loss": 4.57645064035491, + "tokens_seen": 136183808 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004842026078234704, + "loss": 3.2719, + "theoretical_loss": 4.576158663565371, + "tokens_seen": 136249344 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048419257773319965, + "loss": 3.2378, + "theoretical_loss": 4.575866866484967, + "tokens_seen": 136314880 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004841825476429288, + "loss": 3.2435, + "theoretical_loss": 4.575575248916767, + "tokens_seen": 136380416 + }, + { + "epoch": 0.04, + "learning_rate": 0.000484172517552658, + "loss": 3.0393, + "theoretical_loss": 4.575283810664155, + "tokens_seen": 136445952 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004841624874623872, + "loss": 3.2731, + "theoretical_loss": 4.574992551530822, + "tokens_seen": 136511488 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048415245737211637, + "loss": 3.1887, + "theoretical_loss": 4.574701471320768, + "tokens_seen": 136577024 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048414242728184555, + "loss": 3.3261, + "theoretical_loss": 4.574410569838304, + "tokens_seen": 136642560 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048413239719157473, + "loss": 3.3891, + "theoretical_loss": 4.574119846888045, + "tokens_seen": 136708096 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004841223671013039, + "loss": 3.2844, + "theoretical_loss": 4.573829302274915, + "tokens_seen": 136773632 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048411233701103315, + "loss": 3.2397, + "theoretical_loss": 4.573538935804146, + "tokens_seen": 136839168 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004841023069207623, + "loss": 3.6331, + "theoretical_loss": 4.573248747281273, + "tokens_seen": 136904704 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004840922768304915, + "loss": 3.1251, + "theoretical_loss": 4.5729587365121365, + "tokens_seen": 136970240 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048408224674022064, + "loss": 3.1943, + "theoretical_loss": 4.572668903302886, + "tokens_seen": 137035776 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004840722166499499, + "loss": 3.28, + "theoretical_loss": 4.572379247459969, + "tokens_seen": 137101312 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048406218655967906, + "loss": 3.4938, + "theoretical_loss": 4.57208976879014, + "tokens_seen": 137166848 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048405215646940824, + "loss": 3.4793, + "theoretical_loss": 4.571800467100456, + "tokens_seen": 137232384 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004840421263791374, + "loss": 3.1596, + "theoretical_loss": 4.5715113421982725, + "tokens_seen": 137297920 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048403209628886665, + "loss": 3.3151, + "theoretical_loss": 4.571222393891253, + "tokens_seen": 137363456 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004840220661985958, + "loss": 3.6099, + "theoretical_loss": 4.570933621987356, + "tokens_seen": 137428992 + }, + { + "epoch": 0.04, + "learning_rate": 0.000484012036108325, + "loss": 3.5888, + "theoretical_loss": 4.570645026294844, + "tokens_seen": 137494528 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048400200601805414, + "loss": 3.2192, + "theoretical_loss": 4.570356606622278, + "tokens_seen": 137560064 + }, + { + "epoch": 0.04, + "objective/train/docs_used": 88907, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.109947681427002, + "objective/train/theoretical_loss": 4.570068362778516, + "objective/train/tokens_used": 158085600, + "theoretical_loss": 4.570068362778516, + "tokens_seen": 137625600 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004839919759277834, + "loss": 3.316, + "theoretical_loss": 4.570068362778516, + "tokens_seen": 137625600 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048398194583751256, + "loss": 3.5263, + "theoretical_loss": 4.569780294572718, + "tokens_seen": 137691136 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048397191574724174, + "loss": 3.385, + "theoretical_loss": 4.569492401814339, + "tokens_seen": 137756672 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004839618856569709, + "loss": 3.2054, + "theoretical_loss": 4.569204684313133, + "tokens_seen": 137822208 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004839518555667001, + "loss": 3.3325, + "theoretical_loss": 4.568917141879149, + "tokens_seen": 137887744 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004839418254764293, + "loss": 2.9677, + "theoretical_loss": 4.568629774322736, + "tokens_seen": 137953280 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004839317953861585, + "loss": 3.2, + "theoretical_loss": 4.568342581454532, + "tokens_seen": 138018816 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048392176529588765, + "loss": 3.2137, + "theoretical_loss": 4.568055563085476, + "tokens_seen": 138084352 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004839117352056169, + "loss": 3.3768, + "theoretical_loss": 4.567768719026797, + "tokens_seen": 138149888 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048390170511534606, + "loss": 3.4504, + "theoretical_loss": 4.567482049090019, + "tokens_seen": 138215424 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048389167502507524, + "loss": 3.378, + "theoretical_loss": 4.567195553086961, + "tokens_seen": 138280960 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004838816449348044, + "loss": 3.2308, + "theoretical_loss": 4.566909230829729, + "tokens_seen": 138346496 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004838716148445336, + "loss": 3.2836, + "theoretical_loss": 4.566623082130729, + "tokens_seen": 138412032 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004838615847542628, + "loss": 3.4326, + "theoretical_loss": 4.566337106802651, + "tokens_seen": 138477568 + }, + { + "epoch": 0.04, + "learning_rate": 0.000483851554663992, + "loss": 3.1526, + "theoretical_loss": 4.56605130465848, + "tokens_seen": 138543104 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048384152457372115, + "loss": 3.4851, + "theoretical_loss": 4.565765675511487, + "tokens_seen": 138608640 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004838314944834504, + "loss": 3.5763, + "theoretical_loss": 4.565480219175237, + "tokens_seen": 138674176 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004838214643931795, + "loss": 3.3947, + "theoretical_loss": 4.56519493546358, + "tokens_seen": 138739712 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048381143430290875, + "loss": 3.5138, + "theoretical_loss": 4.56490982419066, + "tokens_seen": 138805248 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048380140421263793, + "loss": 3.5022, + "theoretical_loss": 4.564624885170902, + "tokens_seen": 138870784 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004837913741223671, + "loss": 3.2728, + "theoretical_loss": 4.564340118219022, + "tokens_seen": 138936320 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004837813440320963, + "loss": 3.4237, + "theoretical_loss": 4.56405552315002, + "tokens_seen": 139001856 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048377131394182547, + "loss": 3.3895, + "theoretical_loss": 4.563771099779187, + "tokens_seen": 139067392 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048376128385155465, + "loss": 3.5882, + "theoretical_loss": 4.563486847922093, + "tokens_seen": 139132928 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004837512537612839, + "loss": 3.4316, + "theoretical_loss": 4.563202767394597, + "tokens_seen": 139198464 + }, + { + "epoch": 0.04, + "objective/train/docs_used": 90119, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.974191427230835, + "objective/train/theoretical_loss": 4.562918858012843, + "objective/train/tokens_used": 159724000, + "theoretical_loss": 4.562918858012843, + "tokens_seen": 139264000 + }, + { + "epoch": 0.04, + "learning_rate": 0.000483741223671013, + "loss": 3.6345, + "theoretical_loss": 4.562918858012843, + "tokens_seen": 139264000 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048373119358074225, + "loss": 3.205, + "theoretical_loss": 4.562635119593255, + "tokens_seen": 139329536 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048372116349047143, + "loss": 3.2882, + "theoretical_loss": 4.562351551952542, + "tokens_seen": 139395072 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004837111334002006, + "loss": 3.5052, + "theoretical_loss": 4.5620681549076965, + "tokens_seen": 139460608 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004837011033099298, + "loss": 3.4682, + "theoretical_loss": 4.561784928275992, + "tokens_seen": 139526144 + }, + { + "epoch": 0.04, + "learning_rate": 0.000483691073219659, + "loss": 3.2171, + "theoretical_loss": 4.561501871874984, + "tokens_seen": 139591680 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048368104312938816, + "loss": 3.2232, + "theoretical_loss": 4.561218985522507, + "tokens_seen": 139657216 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004836710130391174, + "loss": 3.6344, + "theoretical_loss": 4.560936269036679, + "tokens_seen": 139722752 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004836609829488465, + "loss": 3.3014, + "theoretical_loss": 4.560653722235895, + "tokens_seen": 139788288 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048365095285857575, + "loss": 3.4618, + "theoretical_loss": 4.560371344938831, + "tokens_seen": 139853824 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004836409227683049, + "loss": 3.2854, + "theoretical_loss": 4.560089136964439, + "tokens_seen": 139919360 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004836308926780341, + "loss": 3.1137, + "theoretical_loss": 4.559807098131953, + "tokens_seen": 139984896 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004836208625877633, + "loss": 3.1085, + "theoretical_loss": 4.559525228260882, + "tokens_seen": 140050432 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004836108324974925, + "loss": 3.2936, + "theoretical_loss": 4.559243527171011, + "tokens_seen": 140115968 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048360080240722166, + "loss": 3.1881, + "theoretical_loss": 4.558961994682403, + "tokens_seen": 140181504 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048359077231695084, + "loss": 3.2675, + "theoretical_loss": 4.558680630615397, + "tokens_seen": 140247040 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048358074222668, + "loss": 3.411, + "theoretical_loss": 4.558399434790607, + "tokens_seen": 140312576 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048357071213640926, + "loss": 3.2842, + "theoretical_loss": 4.558118407028921, + "tokens_seen": 140378112 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004835606820461384, + "loss": 3.6055, + "theoretical_loss": 4.557837547151502, + "tokens_seen": 140443648 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004835506519558676, + "loss": 3.2576, + "theoretical_loss": 4.557556854979786, + "tokens_seen": 140509184 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004835406218655968, + "loss": 3.3391, + "theoretical_loss": 4.5572763303354815, + "tokens_seen": 140574720 + }, + { + "epoch": 0.04, + "learning_rate": 0.000483530591775326, + "loss": 3.3281, + "theoretical_loss": 4.556995973040574, + "tokens_seen": 140640256 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048352056168505516, + "loss": 3.262, + "theoretical_loss": 4.556715782917314, + "tokens_seen": 140705792 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048351053159478434, + "loss": 3.3591, + "theoretical_loss": 4.556435759788229, + "tokens_seen": 140771328 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004835005015045135, + "loss": 3.3146, + "theoretical_loss": 4.556155903476114, + "tokens_seen": 140836864 + }, + { + "epoch": 0.04, + "objective/train/docs_used": 90792, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.841050863265991, + "objective/train/theoretical_loss": 4.555876213804037, + "objective/train/tokens_used": 161362400, + "theoretical_loss": 4.555876213804037, + "tokens_seen": 140902400 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048349047141424276, + "loss": 3.3738, + "theoretical_loss": 4.555876213804037, + "tokens_seen": 140902400 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048348044132397194, + "loss": 3.5298, + "theoretical_loss": 4.555596690595333, + "tokens_seen": 140967936 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004834704112337011, + "loss": 3.2319, + "theoretical_loss": 4.555317333673611, + "tokens_seen": 141033472 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004834603811434303, + "loss": 3.2189, + "theoretical_loss": 4.555038142862742, + "tokens_seen": 141099008 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004834503510531595, + "loss": 3.2627, + "theoretical_loss": 4.5547591179868725, + "tokens_seen": 141164544 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004834403209628887, + "loss": 3.2293, + "theoretical_loss": 4.554480258870409, + "tokens_seen": 141230080 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048343029087261785, + "loss": 3.4886, + "theoretical_loss": 4.554201565338033, + "tokens_seen": 141295616 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004834202607823471, + "loss": 3.4271, + "theoretical_loss": 4.5539230372146875, + "tokens_seen": 141361152 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048341023069207626, + "loss": 3.3072, + "theoretical_loss": 4.553644674325584, + "tokens_seen": 141426688 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048340020060180544, + "loss": 3.4699, + "theoretical_loss": 4.553366476496198, + "tokens_seen": 141492224 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004833901705115346, + "loss": 3.448, + "theoretical_loss": 4.553088443552269, + "tokens_seen": 141557760 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004833801404212638, + "loss": 3.1839, + "theoretical_loss": 4.552810575319806, + "tokens_seen": 141623296 + }, + { + "epoch": 0.04, + "learning_rate": 0.000483370110330993, + "loss": 3.2458, + "theoretical_loss": 4.552532871625077, + "tokens_seen": 141688832 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004833600802407222, + "loss": 3.2432, + "theoretical_loss": 4.5522553322946155, + "tokens_seen": 141754368 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048335005015045135, + "loss": 3.4625, + "theoretical_loss": 4.551977957155217, + "tokens_seen": 141819904 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004833400200601806, + "loss": 3.4216, + "theoretical_loss": 4.5517007460339425, + "tokens_seen": 141885440 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004833299899699097, + "loss": 3.357, + "theoretical_loss": 4.551423698758111, + "tokens_seen": 141950976 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048331995987963895, + "loss": 3.2079, + "theoretical_loss": 4.551146815155304, + "tokens_seen": 142016512 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048330992978936813, + "loss": 3.0932, + "theoretical_loss": 4.550870095053366, + "tokens_seen": 142082048 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004832998996990973, + "loss": 3.6194, + "theoretical_loss": 4.550593538280398, + "tokens_seen": 142147584 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004832898696088265, + "loss": 3.1281, + "theoretical_loss": 4.550317144664766, + "tokens_seen": 142213120 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048327983951855567, + "loss": 3.3763, + "theoretical_loss": 4.55004091403509, + "tokens_seen": 142278656 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048326980942828485, + "loss": 3.4224, + "theoretical_loss": 4.5497648462202545, + "tokens_seen": 142344192 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004832597793380141, + "loss": 3.1911, + "theoretical_loss": 4.549488941049397, + "tokens_seen": 142409728 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004832497492477432, + "loss": 3.5291, + "theoretical_loss": 4.549213198351914, + "tokens_seen": 142475264 + }, + { + "epoch": 0.04, + "objective/train/docs_used": 92175, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.857307195663452, + "objective/train/theoretical_loss": 4.548937617957463, + "objective/train/tokens_used": 163000800, + "theoretical_loss": 4.548937617957463, + "tokens_seen": 142540800 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048323971915747245, + "loss": 3.2511, + "theoretical_loss": 4.548937617957463, + "tokens_seen": 142540800 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048322968906720163, + "loss": 3.3592, + "theoretical_loss": 4.548662199695954, + "tokens_seen": 142606336 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004832196589769308, + "loss": 3.3668, + "theoretical_loss": 4.548386943397556, + "tokens_seen": 142671872 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048320962888666, + "loss": 3.4362, + "theoretical_loss": 4.548111848892693, + "tokens_seen": 142737408 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004831995987963892, + "loss": 3.2594, + "theoretical_loss": 4.547836916012042, + "tokens_seen": 142802944 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048318956870611836, + "loss": 3.159, + "theoretical_loss": 4.547562144586539, + "tokens_seen": 142868480 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004831795386158476, + "loss": 3.6512, + "theoretical_loss": 4.547287534447372, + "tokens_seen": 142934016 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004831695085255767, + "loss": 3.3008, + "theoretical_loss": 4.5470130854259825, + "tokens_seen": 142999552 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048315947843530595, + "loss": 3.3839, + "theoretical_loss": 4.546738797354065, + "tokens_seen": 143065088 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004831494483450351, + "loss": 3.4761, + "theoretical_loss": 4.546464670063569, + "tokens_seen": 143130624 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004831394182547643, + "loss": 3.2573, + "theoretical_loss": 4.546190703386695, + "tokens_seen": 143196160 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004831293881644935, + "loss": 3.3416, + "theoretical_loss": 4.545916897155894, + "tokens_seen": 143261696 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004831193580742227, + "loss": 3.2611, + "theoretical_loss": 4.54564325120387, + "tokens_seen": 143327232 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048310932798395186, + "loss": 3.5618, + "theoretical_loss": 4.545369765363578, + "tokens_seen": 143392768 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048309929789368104, + "loss": 3.3966, + "theoretical_loss": 4.545096439468223, + "tokens_seen": 143458304 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004830892678034102, + "loss": 3.4235, + "theoretical_loss": 4.544823273351257, + "tokens_seen": 143523840 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048307923771313946, + "loss": 3.1933, + "theoretical_loss": 4.544550266846388, + "tokens_seen": 143589376 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004830692076228686, + "loss": 3.6045, + "theoretical_loss": 4.544277419787566, + "tokens_seen": 143654912 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004830591775325978, + "loss": 3.4117, + "theoretical_loss": 4.544004732008993, + "tokens_seen": 143720448 + }, + { + "epoch": 0.04, + "learning_rate": 0.000483049147442327, + "loss": 3.3116, + "theoretical_loss": 4.543732203345119, + "tokens_seen": 143785984 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004830391173520562, + "loss": 3.3471, + "theoretical_loss": 4.543459833630639, + "tokens_seen": 143851520 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048302908726178536, + "loss": 3.3008, + "theoretical_loss": 4.543187622700497, + "tokens_seen": 143917056 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048301905717151454, + "loss": 3.1974, + "theoretical_loss": 4.542915570389884, + "tokens_seen": 143982592 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004830090270812437, + "loss": 3.4365, + "theoretical_loss": 4.542643676534234, + "tokens_seen": 144048128 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048299899699097296, + "loss": 3.42, + "theoretical_loss": 4.542371940969231, + "tokens_seen": 144113664 + }, + { + "epoch": 0.04, + "objective/train/docs_used": 92798, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.8370907306671143, + "objective/train/theoretical_loss": 4.542100363530799, + "objective/train/tokens_used": 164639200, + "theoretical_loss": 4.542100363530799, + "tokens_seen": 144179200 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004829889669007021, + "loss": 3.2519, + "theoretical_loss": 4.542100363530799, + "tokens_seen": 144179200 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004829789368104313, + "loss": 3.469, + "theoretical_loss": 4.54182894405511, + "tokens_seen": 144244736 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048296890672016045, + "loss": 3.3804, + "theoretical_loss": 4.5415576823785795, + "tokens_seen": 144310272 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004829588766298897, + "loss": 3.227, + "theoretical_loss": 4.541286578337866, + "tokens_seen": 144375808 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048294884653961887, + "loss": 3.2238, + "theoretical_loss": 4.541015631769872, + "tokens_seen": 144441344 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048293881644934805, + "loss": 3.1567, + "theoretical_loss": 4.5407448425117405, + "tokens_seen": 144506880 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048292878635907723, + "loss": 3.282, + "theoretical_loss": 4.540474210400859, + "tokens_seen": 144572416 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048291875626880646, + "loss": 3.4719, + "theoretical_loss": 4.540203735274855, + "tokens_seen": 144637952 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004829087261785356, + "loss": 3.3559, + "theoretical_loss": 4.5399334169716, + "tokens_seen": 144703488 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004828986960882648, + "loss": 3.2419, + "theoretical_loss": 4.539663255329202, + "tokens_seen": 144769024 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048288866599799395, + "loss": 3.2905, + "theoretical_loss": 4.539393250186015, + "tokens_seen": 144834560 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004828786359077232, + "loss": 3.2132, + "theoretical_loss": 4.539123401380625, + "tokens_seen": 144900096 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048286860581745237, + "loss": 3.3788, + "theoretical_loss": 4.538853708751866, + "tokens_seen": 144965632 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048285857572718155, + "loss": 3.5326, + "theoretical_loss": 4.538584172138804, + "tokens_seen": 145031168 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048284854563691073, + "loss": 3.5429, + "theoretical_loss": 4.538314791380748, + "tokens_seen": 145096704 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004828385155466399, + "loss": 3.5515, + "theoretical_loss": 4.538045566317242, + "tokens_seen": 145162240 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004828284854563691, + "loss": 3.3134, + "theoretical_loss": 4.537776496788071, + "tokens_seen": 145227776 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048281845536609833, + "loss": 3.5195, + "theoretical_loss": 4.537507582633253, + "tokens_seen": 145293312 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048280842527582746, + "loss": 3.3029, + "theoretical_loss": 4.537238823693045, + "tokens_seen": 145358848 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004827983951855567, + "loss": 3.224, + "theoretical_loss": 4.536970219807939, + "tokens_seen": 145424384 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004827883650952858, + "loss": 3.3656, + "theoretical_loss": 4.536701770818665, + "tokens_seen": 145489920 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048277833500501505, + "loss": 3.3768, + "theoretical_loss": 4.536433476566185, + "tokens_seen": 145555456 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048276830491474423, + "loss": 3.2194, + "theoretical_loss": 4.536165336891699, + "tokens_seen": 145620992 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004827582748244734, + "loss": 3.4202, + "theoretical_loss": 4.535897351636638, + "tokens_seen": 145686528 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004827482447342026, + "loss": 3.5616, + "theoretical_loss": 4.53562952064267, + "tokens_seen": 145752064 + }, + { + "epoch": 0.04, + "objective/train/docs_used": 94036, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.990285873413086, + "objective/train/theoretical_loss": 4.535361843751696, + "objective/train/tokens_used": 166277600, + "theoretical_loss": 4.535361843751696, + "tokens_seen": 145817600 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048273821464393183, + "loss": 3.1237, + "theoretical_loss": 4.535361843751696, + "tokens_seen": 145817600 + }, + { + "epoch": 0.04, + "learning_rate": 0.000482728184553661, + "loss": 3.397, + "theoretical_loss": 4.535094320805847, + "tokens_seen": 145883136 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004827181544633902, + "loss": 3.2256, + "theoretical_loss": 4.534826951647489, + "tokens_seen": 145948672 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004827081243731194, + "loss": 3.3973, + "theoretical_loss": 4.5345597361192205, + "tokens_seen": 146014208 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048269809428284856, + "loss": 3.282, + "theoretical_loss": 4.53429267406387, + "tokens_seen": 146079744 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004826880641925778, + "loss": 3.2167, + "theoretical_loss": 4.5340257653244995, + "tokens_seen": 146145280 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004826780341023069, + "loss": 3.4357, + "theoretical_loss": 4.5337590097444, + "tokens_seen": 146210816 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048266800401203615, + "loss": 3.2758, + "theoretical_loss": 4.533492407167093, + "tokens_seen": 146276352 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004826579739217653, + "loss": 3.3079, + "theoretical_loss": 4.53322595743633, + "tokens_seen": 146341888 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004826479438314945, + "loss": 3.1206, + "theoretical_loss": 4.5329596603960916, + "tokens_seen": 146407424 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004826379137412237, + "loss": 3.2727, + "theoretical_loss": 4.53269351589059, + "tokens_seen": 146472960 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004826278836509529, + "loss": 3.2763, + "theoretical_loss": 4.532427523764261, + "tokens_seen": 146538496 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048261785356068206, + "loss": 3.3849, + "theoretical_loss": 4.532161683861773, + "tokens_seen": 146604032 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048260782347041124, + "loss": 3.1596, + "theoretical_loss": 4.5318959960280205, + "tokens_seen": 146669568 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004825977933801404, + "loss": 3.2396, + "theoretical_loss": 4.531630460108125, + "tokens_seen": 146735104 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048258776328986966, + "loss": 3.2984, + "theoretical_loss": 4.531365075947434, + "tokens_seen": 146800640 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004825777331995988, + "loss": 3.4815, + "theoretical_loss": 4.531099843391524, + "tokens_seen": 146866176 + }, + { + "epoch": 0.04, + "learning_rate": 0.000482567703109328, + "loss": 3.3054, + "theoretical_loss": 4.5308347622861955, + "tokens_seen": 146931712 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004825576730190572, + "loss": 3.2911, + "theoretical_loss": 4.5305698324774735, + "tokens_seen": 146997248 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004825476429287864, + "loss": 3.6255, + "theoretical_loss": 4.530305053811611, + "tokens_seen": 147062784 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048253761283851556, + "loss": 3.4052, + "theoretical_loss": 4.530040426135084, + "tokens_seen": 147128320 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048252758274824474, + "loss": 3.496, + "theoretical_loss": 4.529775949294593, + "tokens_seen": 147193856 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004825175526579739, + "loss": 3.5377, + "theoretical_loss": 4.529511623137061, + "tokens_seen": 147259392 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048250752256770316, + "loss": 3.3151, + "theoretical_loss": 4.529247447509637, + "tokens_seen": 147324928 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004824974924774323, + "loss": 3.3374, + "theoretical_loss": 4.528983422259691, + "tokens_seen": 147390464 + }, + { + "epoch": 0.04, + "objective/train/docs_used": 94726, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.953251361846924, + "objective/train/theoretical_loss": 4.528719547234816, + "objective/train/tokens_used": 167916000, + "theoretical_loss": 4.528719547234816, + "tokens_seen": 147456000 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004824874623871615, + "loss": 3.4406, + "theoretical_loss": 4.528719547234816, + "tokens_seen": 147456000 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048247743229689065, + "loss": 3.6595, + "theoretical_loss": 4.528455822282828, + "tokens_seen": 147521536 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004824674022066199, + "loss": 3.3066, + "theoretical_loss": 4.528192247251763, + "tokens_seen": 147587072 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048245737211634907, + "loss": 3.2948, + "theoretical_loss": 4.52792882198988, + "tokens_seen": 147652608 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048244734202607825, + "loss": 3.2822, + "theoretical_loss": 4.527665546345656, + "tokens_seen": 147718144 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048243731193580743, + "loss": 2.9503, + "theoretical_loss": 4.5274024201677925, + "tokens_seen": 147783680 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048242728184553666, + "loss": 3.3107, + "theoretical_loss": 4.527139443305209, + "tokens_seen": 147849216 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004824172517552658, + "loss": 3.5266, + "theoretical_loss": 4.526876615607042, + "tokens_seen": 147914752 + }, + { + "epoch": 0.04, + "learning_rate": 0.000482407221664995, + "loss": 3.3327, + "theoretical_loss": 4.526613936922654, + "tokens_seen": 147980288 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048239719157472415, + "loss": 3.3261, + "theoretical_loss": 4.526351407101618, + "tokens_seen": 148045824 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004823871614844534, + "loss": 3.4825, + "theoretical_loss": 4.526089025993732, + "tokens_seen": 148111360 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048237713139418257, + "loss": 3.0878, + "theoretical_loss": 4.525826793449008, + "tokens_seen": 148176896 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048236710130391175, + "loss": 3.2488, + "theoretical_loss": 4.525564709317678, + "tokens_seen": 148242432 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048235707121364093, + "loss": 3.4734, + "theoretical_loss": 4.525302773450187, + "tokens_seen": 148307968 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004823470411233701, + "loss": 3.4456, + "theoretical_loss": 4.525040985697203, + "tokens_seen": 148373504 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004823370110330993, + "loss": 3.4067, + "theoretical_loss": 4.524779345909604, + "tokens_seen": 148439040 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048232698094282853, + "loss": 3.0241, + "theoretical_loss": 4.524517853938489, + "tokens_seen": 148504576 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048231695085255766, + "loss": 3.303, + "theoretical_loss": 4.524256509635169, + "tokens_seen": 148570112 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004823069207622869, + "loss": 3.4487, + "theoretical_loss": 4.523995312851174, + "tokens_seen": 148635648 + }, + { + "epoch": 0.05, + "learning_rate": 0.000482296890672016, + "loss": 3.2333, + "theoretical_loss": 4.523734263438241, + "tokens_seen": 148701184 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048228686058174525, + "loss": 3.3882, + "theoretical_loss": 4.52347336124833, + "tokens_seen": 148766720 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048227683049147443, + "loss": 3.3968, + "theoretical_loss": 4.52321260613361, + "tokens_seen": 148832256 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004822668004012036, + "loss": 3.1745, + "theoretical_loss": 4.522951997946466, + "tokens_seen": 148897792 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004822567703109328, + "loss": 3.208, + "theoretical_loss": 4.522691536539492, + "tokens_seen": 148963328 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048224674022066203, + "loss": 3.5049, + "theoretical_loss": 4.522431221765498, + "tokens_seen": 149028864 + }, + { + "epoch": 0.05, + "objective/train/docs_used": 95869, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.491877794265747, + "objective/train/theoretical_loss": 4.522171053477507, + "objective/train/tokens_used": 169554400, + "theoretical_loss": 4.522171053477507, + "tokens_seen": 149094400 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048223671013039116, + "loss": 3.3234, + "theoretical_loss": 4.522171053477507, + "tokens_seen": 149094400 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004822266800401204, + "loss": 3.3719, + "theoretical_loss": 4.5219110315287505, + "tokens_seen": 149159936 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004822166499498495, + "loss": 3.3828, + "theoretical_loss": 4.521651155772675, + "tokens_seen": 149225472 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048220661985957876, + "loss": 3.3359, + "theoretical_loss": 4.521391426062934, + "tokens_seen": 149291008 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048219658976930794, + "loss": 3.3976, + "theoretical_loss": 4.521131842253396, + "tokens_seen": 149356544 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004821865596790371, + "loss": 3.3314, + "theoretical_loss": 4.520872404198139, + "tokens_seen": 149422080 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004821765295887663, + "loss": 3.4846, + "theoretical_loss": 4.520613111751445, + "tokens_seen": 149487616 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004821664994984955, + "loss": 3.2052, + "theoretical_loss": 4.520353964767814, + "tokens_seen": 149553152 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048215646940822466, + "loss": 3.2258, + "theoretical_loss": 4.5200949631019505, + "tokens_seen": 149618688 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004821464393179539, + "loss": 3.1412, + "theoretical_loss": 4.519836106608768, + "tokens_seen": 149684224 + }, + { + "epoch": 0.05, + "learning_rate": 0.000482136409227683, + "loss": 3.4078, + "theoretical_loss": 4.519577395143388, + "tokens_seen": 149749760 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048212637913741226, + "loss": 3.1593, + "theoretical_loss": 4.519318828561142, + "tokens_seen": 149815296 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004821163490471414, + "loss": 3.1689, + "theoretical_loss": 4.519060406717565, + "tokens_seen": 149880832 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004821063189568706, + "loss": 3.1324, + "theoretical_loss": 4.518802129468405, + "tokens_seen": 149946368 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004820962888665998, + "loss": 3.354, + "theoretical_loss": 4.51854399666961, + "tokens_seen": 150011904 + }, + { + "epoch": 0.05, + "learning_rate": 0.000482086258776329, + "loss": 3.3396, + "theoretical_loss": 4.518286008177341, + "tokens_seen": 150077440 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048207622868605816, + "loss": 3.3702, + "theoretical_loss": 4.51802816384796, + "tokens_seen": 150142976 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004820661985957874, + "loss": 3.4021, + "theoretical_loss": 4.517770463538038, + "tokens_seen": 150208512 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048205616850551653, + "loss": 3.4426, + "theoretical_loss": 4.517512907104347, + "tokens_seen": 150274048 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048204613841524576, + "loss": 3.3319, + "theoretical_loss": 4.517255494403868, + "tokens_seen": 150339584 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004820361083249749, + "loss": 3.1712, + "theoretical_loss": 4.516998225293785, + "tokens_seen": 150405120 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004820260782347041, + "loss": 3.294, + "theoretical_loss": 4.516741099631485, + "tokens_seen": 150470656 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004820160481444333, + "loss": 3.0, + "theoretical_loss": 4.51648411727456, + "tokens_seen": 150536192 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004820060180541625, + "loss": 3.4023, + "theoretical_loss": 4.5162272780808035, + "tokens_seen": 150601728 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048199598796389167, + "loss": 3.0665, + "theoretical_loss": 4.515970581908216, + "tokens_seen": 150667264 + }, + { + "epoch": 0.05, + "objective/train/docs_used": 96442, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.4400134086608887, + "objective/train/theoretical_loss": 4.515714028614996, + "objective/train/tokens_used": 171192800, + "theoretical_loss": 4.515714028614996, + "tokens_seen": 150732800 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048198595787362085, + "loss": 3.3154, + "theoretical_loss": 4.515714028614996, + "tokens_seen": 150732800 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004819759277833501, + "loss": 3.3389, + "theoretical_loss": 4.515457618059546, + "tokens_seen": 150798336 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048196589769307927, + "loss": 3.3009, + "theoretical_loss": 4.515201350100471, + "tokens_seen": 150863872 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048195586760280845, + "loss": 3.1341, + "theoretical_loss": 4.514945224596577, + "tokens_seen": 150929408 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048194583751253763, + "loss": 3.4125, + "theoretical_loss": 4.5146892414068684, + "tokens_seen": 150994944 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048193580742226686, + "loss": 3.1004, + "theoretical_loss": 4.514433400390554, + "tokens_seen": 151060480 + }, + { + "epoch": 0.05, + "learning_rate": 0.000481925777331996, + "loss": 3.4489, + "theoretical_loss": 4.514177701407042, + "tokens_seen": 151126016 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004819157472417252, + "loss": 3.4083, + "theoretical_loss": 4.51392214431594, + "tokens_seen": 151191552 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048190571715145435, + "loss": 3.2392, + "theoretical_loss": 4.513666728977054, + "tokens_seen": 151257088 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004818956870611836, + "loss": 3.3398, + "theoretical_loss": 4.51341145525039, + "tokens_seen": 151322624 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048188565697091277, + "loss": 3.6117, + "theoretical_loss": 4.513156322996155, + "tokens_seen": 151388160 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048187562688064195, + "loss": 3.1927, + "theoretical_loss": 4.512901332074751, + "tokens_seen": 151453696 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048186559679037113, + "loss": 3.3057, + "theoretical_loss": 4.5126464823467805, + "tokens_seen": 151519232 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004818555667001003, + "loss": 3.2035, + "theoretical_loss": 4.512391773673042, + "tokens_seen": 151584768 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004818455366098295, + "loss": 3.1402, + "theoretical_loss": 4.5121372059145335, + "tokens_seen": 151650304 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048183550651955873, + "loss": 3.3522, + "theoretical_loss": 4.511882778932447, + "tokens_seen": 151715840 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048182547642928786, + "loss": 3.239, + "theoretical_loss": 4.511628492588174, + "tokens_seen": 151781376 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004818154463390171, + "loss": 3.3342, + "theoretical_loss": 4.5113743467433, + "tokens_seen": 151846912 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004818054162487462, + "loss": 3.2241, + "theoretical_loss": 4.511120341259608, + "tokens_seen": 151912448 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048179538615847545, + "loss": 3.2438, + "theoretical_loss": 4.510866475999077, + "tokens_seen": 151977984 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048178535606820463, + "loss": 3.1373, + "theoretical_loss": 4.510612750823878, + "tokens_seen": 152043520 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004817753259779338, + "loss": 3.2093, + "theoretical_loss": 4.5103591655963795, + "tokens_seen": 152109056 + }, + { + "epoch": 0.05, + "learning_rate": 0.000481765295887663, + "loss": 3.4628, + "theoretical_loss": 4.510105720179144, + "tokens_seen": 152174592 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048175526579739223, + "loss": 3.1515, + "theoretical_loss": 4.5098524144349295, + "tokens_seen": 152240128 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048174523570712136, + "loss": 3.3996, + "theoretical_loss": 4.509599248226683, + "tokens_seen": 152305664 + }, + { + "epoch": 0.05, + "objective/train/docs_used": 97778, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.846565008163452, + "objective/train/theoretical_loss": 4.509346221417552, + "objective/train/tokens_used": 172831200, + "theoretical_loss": 4.509346221417552, + "tokens_seen": 152371200 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004817352056168506, + "loss": 3.3255, + "theoretical_loss": 4.509346221417552, + "tokens_seen": 152371200 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004817251755265797, + "loss": 3.4219, + "theoretical_loss": 4.509093333870869, + "tokens_seen": 152436736 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048171514543630896, + "loss": 3.393, + "theoretical_loss": 4.508840585450166, + "tokens_seen": 152502272 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048170511534603814, + "loss": 3.1983, + "theoretical_loss": 4.508587976019164, + "tokens_seen": 152567808 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004816950852557673, + "loss": 3.3573, + "theoretical_loss": 4.508335505441774, + "tokens_seen": 152633344 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004816850551654965, + "loss": 3.3479, + "theoretical_loss": 4.508083173582105, + "tokens_seen": 152698880 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004816750250752257, + "loss": 3.5365, + "theoretical_loss": 4.507830980304451, + "tokens_seen": 152764416 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048166499498495486, + "loss": 3.165, + "theoretical_loss": 4.5075789254733, + "tokens_seen": 152829952 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004816549648946841, + "loss": 3.2017, + "theoretical_loss": 4.507327008953329, + "tokens_seen": 152895488 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004816449348044132, + "loss": 3.4427, + "theoretical_loss": 4.507075230609407, + "tokens_seen": 152961024 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048163490471414246, + "loss": 3.5607, + "theoretical_loss": 4.506823590306591, + "tokens_seen": 153026560 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004816248746238716, + "loss": 3.1581, + "theoretical_loss": 4.506572087910127, + "tokens_seen": 153092096 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004816148445336008, + "loss": 3.1972, + "theoretical_loss": 4.506320723285455, + "tokens_seen": 153157632 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048160481444333, + "loss": 3.3458, + "theoretical_loss": 4.506069496298198, + "tokens_seen": 153223168 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004815947843530592, + "loss": 3.1806, + "theoretical_loss": 4.5058184068141705, + "tokens_seen": 153288704 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048158475426278837, + "loss": 3.411, + "theoretical_loss": 4.505567454699373, + "tokens_seen": 153354240 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004815747241725176, + "loss": 3.5175, + "theoretical_loss": 4.505316639819997, + "tokens_seen": 153419776 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048156469408224673, + "loss": 3.5944, + "theoretical_loss": 4.505065962042418, + "tokens_seen": 153485312 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048155466399197596, + "loss": 3.398, + "theoretical_loss": 4.504815421233202, + "tokens_seen": 153550848 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004815446339017051, + "loss": 3.2123, + "theoretical_loss": 4.504565017259097, + "tokens_seen": 153616384 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004815346038114343, + "loss": 3.208, + "theoretical_loss": 4.504314749987044, + "tokens_seen": 153681920 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004815245737211635, + "loss": 3.3597, + "theoretical_loss": 4.504064619284163, + "tokens_seen": 153747456 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004815145436308927, + "loss": 3.5044, + "theoretical_loss": 4.503814625017766, + "tokens_seen": 153812992 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048150451354062187, + "loss": 3.0857, + "theoretical_loss": 4.5035647670553445, + "tokens_seen": 153878528 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048149448345035105, + "loss": 3.1861, + "theoretical_loss": 4.503315045264581, + "tokens_seen": 153944064 + }, + { + "epoch": 0.05, + "objective/train/docs_used": 98099, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.38057017326355, + "objective/train/theoretical_loss": 4.503065459513339, + "objective/train/tokens_used": 174469600, + "theoretical_loss": 4.503065459513339, + "tokens_seen": 154009600 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048148445336008023, + "loss": 3.4343, + "theoretical_loss": 4.503065459513339, + "tokens_seen": 154009600 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048147442326980947, + "loss": 3.3424, + "theoretical_loss": 4.502816009669665, + "tokens_seen": 154075136 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004814643931795386, + "loss": 3.3995, + "theoretical_loss": 4.502566695601795, + "tokens_seen": 154140672 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048145436308926783, + "loss": 3.2388, + "theoretical_loss": 4.502317517178142, + "tokens_seen": 154206208 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048144433299899696, + "loss": 2.9615, + "theoretical_loss": 4.502068474267309, + "tokens_seen": 154271744 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004814343029087262, + "loss": 3.2668, + "theoretical_loss": 4.501819566738076, + "tokens_seen": 154337280 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048142427281845537, + "loss": 3.3564, + "theoretical_loss": 4.501570794459411, + "tokens_seen": 154402816 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048141424272818455, + "loss": 3.016, + "theoretical_loss": 4.501322157300461, + "tokens_seen": 154468352 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048140421263791373, + "loss": 3.3948, + "theoretical_loss": 4.501073655130554, + "tokens_seen": 154533888 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048139418254764297, + "loss": 3.4663, + "theoretical_loss": 4.500825287819205, + "tokens_seen": 154599424 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004813841524573721, + "loss": 3.2626, + "theoretical_loss": 4.500577055236104, + "tokens_seen": 154664960 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048137412236710133, + "loss": 3.364, + "theoretical_loss": 4.500328957251128, + "tokens_seen": 154730496 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048136409227683046, + "loss": 3.4725, + "theoretical_loss": 4.500080993734329, + "tokens_seen": 154796032 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004813540621865597, + "loss": 3.3051, + "theoretical_loss": 4.499833164555944, + "tokens_seen": 154861568 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004813440320962889, + "loss": 3.2788, + "theoretical_loss": 4.499585469586387, + "tokens_seen": 154927104 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048133400200601806, + "loss": 3.596, + "theoretical_loss": 4.499337908696255, + "tokens_seen": 154992640 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048132397191574724, + "loss": 3.2666, + "theoretical_loss": 4.499090481756321, + "tokens_seen": 155058176 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004813139418254764, + "loss": 3.1353, + "theoretical_loss": 4.498843188637538, + "tokens_seen": 155123712 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004813039117352056, + "loss": 3.336, + "theoretical_loss": 4.498596029211041, + "tokens_seen": 155189248 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048129388164493483, + "loss": 3.2455, + "theoretical_loss": 4.498349003348137, + "tokens_seen": 155254784 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048128385155466396, + "loss": 3.2869, + "theoretical_loss": 4.4981021109203185, + "tokens_seen": 155320320 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004812738214643932, + "loss": 3.5069, + "theoretical_loss": 4.49785535179925, + "tokens_seen": 155385856 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004812637913741223, + "loss": 3.5582, + "theoretical_loss": 4.497608725856776, + "tokens_seen": 155451392 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048125376128385156, + "loss": 3.219, + "theoretical_loss": 4.497362232964919, + "tokens_seen": 155516928 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048124373119358074, + "loss": 3.2262, + "theoretical_loss": 4.497115872995876, + "tokens_seen": 155582464 + }, + { + "epoch": 0.05, + "objective/train/docs_used": 99496, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.3245973587036133, + "objective/train/theoretical_loss": 4.496869645822022, + "objective/train/tokens_used": 176108000, + "theoretical_loss": 4.496869645822022, + "tokens_seen": 155648000 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004812337011033099, + "loss": 2.9805, + "theoretical_loss": 4.496869645822022, + "tokens_seen": 155648000 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048122367101303916, + "loss": 3.3748, + "theoretical_loss": 4.496623551315908, + "tokens_seen": 155713536 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048121364092276834, + "loss": 3.2504, + "theoretical_loss": 4.496377589350261, + "tokens_seen": 155779072 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004812036108324975, + "loss": 3.3977, + "theoretical_loss": 4.496131759797984, + "tokens_seen": 155844608 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004811935807422267, + "loss": 3.4156, + "theoretical_loss": 4.495886062532153, + "tokens_seen": 155910144 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004811835506519559, + "loss": 3.1635, + "theoretical_loss": 4.495640497426023, + "tokens_seen": 155975680 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048117352056168506, + "loss": 3.4067, + "theoretical_loss": 4.495395064353019, + "tokens_seen": 156041216 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004811634904714143, + "loss": 3.1671, + "theoretical_loss": 4.4951497631867445, + "tokens_seen": 156106752 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004811534603811434, + "loss": 3.5499, + "theoretical_loss": 4.494904593800973, + "tokens_seen": 156172288 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048114343029087266, + "loss": 3.4013, + "theoretical_loss": 4.4946595560696565, + "tokens_seen": 156237824 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004811334002006018, + "loss": 3.4994, + "theoretical_loss": 4.494414649866915, + "tokens_seen": 156303360 + }, + { + "epoch": 0.05, + "learning_rate": 0.000481123370110331, + "loss": 3.3378, + "theoretical_loss": 4.494169875067046, + "tokens_seen": 156368896 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004811133400200602, + "loss": 3.388, + "theoretical_loss": 4.493925231544516, + "tokens_seen": 156434432 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004811033099297894, + "loss": 3.0508, + "theoretical_loss": 4.493680719173968, + "tokens_seen": 156499968 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048109327983951857, + "loss": 3.1507, + "theoretical_loss": 4.4934363378302145, + "tokens_seen": 156565504 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004810832497492478, + "loss": 3.1549, + "theoretical_loss": 4.493192087388239, + "tokens_seen": 156631040 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048107321965897693, + "loss": 2.9902, + "theoretical_loss": 4.4929479677232, + "tokens_seen": 156696576 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048106318956870616, + "loss": 3.3567, + "theoretical_loss": 4.4927039787104235, + "tokens_seen": 156762112 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004810531594784353, + "loss": 3.3827, + "theoretical_loss": 4.4924601202254095, + "tokens_seen": 156827648 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004810431293881645, + "loss": 3.2089, + "theoretical_loss": 4.492216392143826, + "tokens_seen": 156893184 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004810330992978937, + "loss": 3.2874, + "theoretical_loss": 4.491972794341514, + "tokens_seen": 156958720 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004810230692076229, + "loss": 3.3188, + "theoretical_loss": 4.49172932669448, + "tokens_seen": 157024256 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048101303911735207, + "loss": 3.3073, + "theoretical_loss": 4.491485989078906, + "tokens_seen": 157089792 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048100300902708125, + "loss": 3.5011, + "theoretical_loss": 4.491242781371138, + "tokens_seen": 157155328 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048099297893681043, + "loss": 3.4143, + "theoretical_loss": 4.490999703447697, + "tokens_seen": 157220864 + }, + { + "epoch": 0.05, + "objective/train/docs_used": 100000, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.5053956508636475, + "objective/train/theoretical_loss": 4.4907567551852665, + "objective/train/tokens_used": 177746400, + "theoretical_loss": 4.4907567551852665, + "tokens_seen": 157286400 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048098294884653967, + "loss": 3.1921, + "theoretical_loss": 4.4907567551852665, + "tokens_seen": 157286400 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004809729187562688, + "loss": 3.4296, + "theoretical_loss": 4.490513936460702, + "tokens_seen": 157351936 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048096288866599803, + "loss": 3.4721, + "theoretical_loss": 4.490271247151027, + "tokens_seen": 157417472 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048095285857572716, + "loss": 3.4052, + "theoretical_loss": 4.490028687133432, + "tokens_seen": 157483008 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004809428284854564, + "loss": 3.3674, + "theoretical_loss": 4.489786256285276, + "tokens_seen": 157548544 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048093279839518557, + "loss": 3.3074, + "theoretical_loss": 4.489543954484084, + "tokens_seen": 157614080 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048092276830491475, + "loss": 3.4224, + "theoretical_loss": 4.489301781607551, + "tokens_seen": 157679616 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048091273821464393, + "loss": 3.4244, + "theoretical_loss": 4.489059737533534, + "tokens_seen": 157745152 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048090270812437317, + "loss": 3.2276, + "theoretical_loss": 4.48881782214006, + "tokens_seen": 157810688 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004808926780341023, + "loss": 3.4517, + "theoretical_loss": 4.48857603530532, + "tokens_seen": 157876224 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048088264794383153, + "loss": 3.248, + "theoretical_loss": 4.488334376907673, + "tokens_seen": 157941760 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048087261785356066, + "loss": 3.466, + "theoretical_loss": 4.4880928468256425, + "tokens_seen": 158007296 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004808625877632899, + "loss": 3.1882, + "theoretical_loss": 4.487851444937916, + "tokens_seen": 158072832 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004808525576730191, + "loss": 3.1961, + "theoretical_loss": 4.487610171123347, + "tokens_seen": 158138368 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048084252758274826, + "loss": 3.456, + "theoretical_loss": 4.487369025260954, + "tokens_seen": 158203904 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048083249749247744, + "loss": 3.5022, + "theoretical_loss": 4.48712800722992, + "tokens_seen": 158269440 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004808224674022066, + "loss": 3.4946, + "theoretical_loss": 4.48688711690959, + "tokens_seen": 158334976 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004808124373119358, + "loss": 3.2507, + "theoretical_loss": 4.486646354179475, + "tokens_seen": 158400512 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048080240722166503, + "loss": 3.2019, + "theoretical_loss": 4.48640571891925, + "tokens_seen": 158466048 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048079237713139416, + "loss": 3.3399, + "theoretical_loss": 4.48616521100875, + "tokens_seen": 158531584 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004807823470411234, + "loss": 3.2544, + "theoretical_loss": 4.485924830327974, + "tokens_seen": 158597120 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004807723169508525, + "loss": 3.2142, + "theoretical_loss": 4.485684576757087, + "tokens_seen": 158662656 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048076228686058176, + "loss": 3.5072, + "theoretical_loss": 4.485444450176413, + "tokens_seen": 158728192 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048075225677031094, + "loss": 3.4206, + "theoretical_loss": 4.485204450466437, + "tokens_seen": 158793728 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004807422266800401, + "loss": 3.2192, + "theoretical_loss": 4.484964577507808, + "tokens_seen": 158859264 + }, + { + "epoch": 0.05, + "objective/train/docs_used": 101476, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.3228378295898438, + "objective/train/theoretical_loss": 4.484724831181337, + "objective/train/tokens_used": 179384800, + "theoretical_loss": 4.484724831181337, + "tokens_seen": 158924800 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004807321965897693, + "loss": 3.2896, + "theoretical_loss": 4.484724831181337, + "tokens_seen": 158924800 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048072216649949854, + "loss": 3.5084, + "theoretical_loss": 4.4844852113679945, + "tokens_seen": 158990336 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048071213640922766, + "loss": 3.4679, + "theoretical_loss": 4.484245717948913, + "tokens_seen": 159055872 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004807021063189569, + "loss": 3.2778, + "theoretical_loss": 4.484006350805385, + "tokens_seen": 159121408 + }, + { + "epoch": 0.05, + "learning_rate": 0.000480692076228686, + "loss": 3.2778, + "theoretical_loss": 4.483767109818862, + "tokens_seen": 159186944 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048068204613841526, + "loss": 3.5164, + "theoretical_loss": 4.483527994870958, + "tokens_seen": 159252480 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048067201604814444, + "loss": 3.2239, + "theoretical_loss": 4.483289005843445, + "tokens_seen": 159318016 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004806619859578736, + "loss": 3.3276, + "theoretical_loss": 4.483050142618255, + "tokens_seen": 159383552 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004806519558676028, + "loss": 3.1697, + "theoretical_loss": 4.482811405077482, + "tokens_seen": 159449088 + }, + { + "epoch": 0.05, + "learning_rate": 0.000480641925777332, + "loss": 3.1915, + "theoretical_loss": 4.482572793103373, + "tokens_seen": 159514624 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048063189568706117, + "loss": 3.4525, + "theoretical_loss": 4.482334306578339, + "tokens_seen": 159580160 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004806218655967904, + "loss": 3.4491, + "theoretical_loss": 4.482095945384946, + "tokens_seen": 159645696 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048061183550651953, + "loss": 3.5396, + "theoretical_loss": 4.481857709405919, + "tokens_seen": 159711232 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048060180541624877, + "loss": 3.7235, + "theoretical_loss": 4.4816195985241425, + "tokens_seen": 159776768 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048059177532597795, + "loss": 3.3959, + "theoretical_loss": 4.481381612622657, + "tokens_seen": 159842304 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048058174523570713, + "loss": 3.4112, + "theoretical_loss": 4.481143751584659, + "tokens_seen": 159907840 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004805717151454363, + "loss": 3.0964, + "theoretical_loss": 4.480906015293505, + "tokens_seen": 159973376 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004805616850551655, + "loss": 3.3323, + "theoretical_loss": 4.480668403632706, + "tokens_seen": 160038912 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048055165496489467, + "loss": 3.0926, + "theoretical_loss": 4.480430916485929, + "tokens_seen": 160104448 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004805416248746239, + "loss": 3.7539, + "theoretical_loss": 4.480193553736999, + "tokens_seen": 160169984 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048053159478435303, + "loss": 3.3289, + "theoretical_loss": 4.479956315269897, + "tokens_seen": 160235520 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048052156469408227, + "loss": 3.0114, + "theoretical_loss": 4.479719200968757, + "tokens_seen": 160301056 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004805115346038114, + "loss": 3.3172, + "theoretical_loss": 4.479482210717871, + "tokens_seen": 160366592 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048050150451354063, + "loss": 3.2286, + "theoretical_loss": 4.479245344401685, + "tokens_seen": 160432128 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004804914744232698, + "loss": 3.24, + "theoretical_loss": 4.479008601904798, + "tokens_seen": 160497664 + }, + { + "epoch": 0.05, + "objective/train/docs_used": 102054, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.287539005279541, + "objective/train/theoretical_loss": 4.478771983111967, + "objective/train/tokens_used": 181023200, + "theoretical_loss": 4.478771983111967, + "tokens_seen": 160563200 + }, + { + "epoch": 0.05, + "learning_rate": 0.000480481444332999, + "loss": 3.2253, + "theoretical_loss": 4.478771983111967, + "tokens_seen": 160563200 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048047141424272823, + "loss": 3.3772, + "theoretical_loss": 4.478535487908101, + "tokens_seen": 160628736 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048046138415245736, + "loss": 3.3127, + "theoretical_loss": 4.478299116178265, + "tokens_seen": 160694272 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004804513540621866, + "loss": 3.2784, + "theoretical_loss": 4.478062867807674, + "tokens_seen": 160759808 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048044132397191577, + "loss": 3.1882, + "theoretical_loss": 4.4778267426817, + "tokens_seen": 160825344 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048043129388164495, + "loss": 3.1845, + "theoretical_loss": 4.477590740685867, + "tokens_seen": 160890880 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048042126379137413, + "loss": 3.1551, + "theoretical_loss": 4.47735486170585, + "tokens_seen": 160956416 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048041123370110337, + "loss": 3.3968, + "theoretical_loss": 4.47711910562748, + "tokens_seen": 161021952 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004804012036108325, + "loss": 3.1993, + "theoretical_loss": 4.4768834723367394, + "tokens_seen": 161087488 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048039117352056173, + "loss": 3.2284, + "theoretical_loss": 4.4766479617197605, + "tokens_seen": 161153024 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048038114343029086, + "loss": 3.06, + "theoretical_loss": 4.476412573662829, + "tokens_seen": 161218560 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004803711133400201, + "loss": 3.5181, + "theoretical_loss": 4.4761773080523835, + "tokens_seen": 161284096 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004803610832497493, + "loss": 3.311, + "theoretical_loss": 4.475942164775013, + "tokens_seen": 161349632 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048035105315947846, + "loss": 3.3443, + "theoretical_loss": 4.475707143717455, + "tokens_seen": 161415168 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048034102306920764, + "loss": 3.3287, + "theoretical_loss": 4.475472244766601, + "tokens_seen": 161480704 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004803309929789368, + "loss": 3.2144, + "theoretical_loss": 4.475237467809492, + "tokens_seen": 161546240 + }, + { + "epoch": 0.05, + "learning_rate": 0.000480320962888666, + "loss": 3.0972, + "theoretical_loss": 4.47500281273332, + "tokens_seen": 161611776 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048031093279839523, + "loss": 3.2474, + "theoretical_loss": 4.474768279425424, + "tokens_seen": 161677312 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048030090270812436, + "loss": 3.4611, + "theoretical_loss": 4.474533867773299, + "tokens_seen": 161742848 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004802908726178536, + "loss": 2.9781, + "theoretical_loss": 4.474299577664581, + "tokens_seen": 161808384 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004802808425275827, + "loss": 3.3853, + "theoretical_loss": 4.474065408987063, + "tokens_seen": 161873920 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048027081243731196, + "loss": 3.0433, + "theoretical_loss": 4.473831361628682, + "tokens_seen": 161939456 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048026078234704114, + "loss": 3.5184, + "theoretical_loss": 4.473597435477526, + "tokens_seen": 162004992 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004802507522567703, + "loss": 3.2771, + "theoretical_loss": 4.473363630421831, + "tokens_seen": 162070528 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004802407221664995, + "loss": 3.264, + "theoretical_loss": 4.473129946349982, + "tokens_seen": 162136064 + }, + { + "epoch": 0.05, + "objective/train/docs_used": 102718, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.609177350997925, + "objective/train/theoretical_loss": 4.472896383150508, + "objective/train/tokens_used": 182661600, + "theoretical_loss": 4.472896383150508, + "tokens_seen": 162201600 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048023069207622874, + "loss": 3.4304, + "theoretical_loss": 4.472896383150508, + "tokens_seen": 162201600 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048022066198595786, + "loss": 3.088, + "theoretical_loss": 4.472662940712091, + "tokens_seen": 162267136 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004802106318956871, + "loss": 3.0931, + "theoretical_loss": 4.472429618923558, + "tokens_seen": 162332672 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048020060180541623, + "loss": 3.0317, + "theoretical_loss": 4.472196417673883, + "tokens_seen": 162398208 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048019057171514546, + "loss": 3.4279, + "theoretical_loss": 4.471963336852187, + "tokens_seen": 162463744 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048018054162487464, + "loss": 3.2777, + "theoretical_loss": 4.471730376347738, + "tokens_seen": 162529280 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004801705115346038, + "loss": 3.1922, + "theoretical_loss": 4.4714975360499505, + "tokens_seen": 162594816 + }, + { + "epoch": 0.05, + "learning_rate": 0.000480160481444333, + "loss": 3.5322, + "theoretical_loss": 4.471264815848384, + "tokens_seen": 162660352 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004801504513540622, + "loss": 3.1, + "theoretical_loss": 4.471032215632746, + "tokens_seen": 162725888 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048014042126379137, + "loss": 3.1249, + "theoretical_loss": 4.470799735292889, + "tokens_seen": 162791424 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004801303911735206, + "loss": 3.3883, + "theoretical_loss": 4.470567374718808, + "tokens_seen": 162856960 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048012036108324973, + "loss": 3.1361, + "theoretical_loss": 4.470335133800649, + "tokens_seen": 162922496 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048011033099297897, + "loss": 3.2812, + "theoretical_loss": 4.470103012428696, + "tokens_seen": 162988032 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048010030090270815, + "loss": 3.3892, + "theoretical_loss": 4.469871010493383, + "tokens_seen": 163053568 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048009027081243733, + "loss": 3.2024, + "theoretical_loss": 4.469639127885287, + "tokens_seen": 163119104 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004800802407221665, + "loss": 3.3988, + "theoretical_loss": 4.4694073644951295, + "tokens_seen": 163184640 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004800702106318957, + "loss": 3.2923, + "theoretical_loss": 4.469175720213771, + "tokens_seen": 163250176 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048006018054162487, + "loss": 3.1748, + "theoretical_loss": 4.468944194932225, + "tokens_seen": 163315712 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004800501504513541, + "loss": 3.2072, + "theoretical_loss": 4.468712788541639, + "tokens_seen": 163381248 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048004012036108323, + "loss": 3.4557, + "theoretical_loss": 4.46848150093331, + "tokens_seen": 163446784 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048003009027081247, + "loss": 3.0576, + "theoretical_loss": 4.468250331998676, + "tokens_seen": 163512320 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004800200601805416, + "loss": 3.3139, + "theoretical_loss": 4.468019281629316, + "tokens_seen": 163577856 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048001003009027083, + "loss": 3.3134, + "theoretical_loss": 4.467788349716955, + "tokens_seen": 163643392 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048, + "loss": 3.1948, + "theoretical_loss": 4.467557536153457, + "tokens_seen": 163708928 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004799899699097292, + "loss": 3.1366, + "theoretical_loss": 4.467326840830829, + "tokens_seen": 163774464 + }, + { + "debugging/Self-BLEU-5": 0.4072555680918346, + "debugging/distinct-1-grams": 0.8137531241311615, + "debugging/distinct-2-grams": 0.9866498101723051, + "debugging/entropy-1-grams": 5.4466195562752855, + "debugging/entropy-2-grams": 6.104764435286928, + "debugging/length": 544.5, + "debugging/num_segments": 6, + "debugging/score": 0.002729943707102512, + "debugging/score_std": 0.0025596098515178914, + "epoch": 0.05, + "objective/train/docs_used": 104069, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.122515916824341, + "objective/train/theoretical_loss": 4.467096263641219, + "objective/train/tokens_used": 184300000, + "theoretical_loss": 4.467096263641219, + "tokens_seen": 163840000 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004799799398194584, + "loss": 3.364, + "theoretical_loss": 4.467096263641219, + "tokens_seen": 163840000 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047996990972918756, + "loss": 3.254, + "theoretical_loss": 4.466865804476919, + "tokens_seen": 163905536 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047995987963891674, + "loss": 3.4626, + "theoretical_loss": 4.466635463230359, + "tokens_seen": 163971072 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047994984954864597, + "loss": 3.0926, + "theoretical_loss": 4.466405239794113, + "tokens_seen": 164036608 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004799398194583751, + "loss": 3.1514, + "theoretical_loss": 4.466175134060894, + "tokens_seen": 164102144 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047992978936810433, + "loss": 3.351, + "theoretical_loss": 4.465945145923554, + "tokens_seen": 164167680 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004799197592778335, + "loss": 3.0217, + "theoretical_loss": 4.4657152752750875, + "tokens_seen": 164233216 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004799097291875627, + "loss": 3.3081, + "theoretical_loss": 4.465485522008629, + "tokens_seen": 164298752 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004798996990972919, + "loss": 3.0959, + "theoretical_loss": 4.465255886017452, + "tokens_seen": 164364288 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047988966900702106, + "loss": 3.1881, + "theoretical_loss": 4.465026367194971, + "tokens_seen": 164429824 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047987963891675024, + "loss": 3.3868, + "theoretical_loss": 4.464796965434738, + "tokens_seen": 164495360 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004798696088264795, + "loss": 3.1284, + "theoretical_loss": 4.464567680630443, + "tokens_seen": 164560896 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004798595787362086, + "loss": 3.2482, + "theoretical_loss": 4.464338512675919, + "tokens_seen": 164626432 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047984954864593784, + "loss": 3.0729, + "theoretical_loss": 4.464109461465133, + "tokens_seen": 164691968 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047983951855566696, + "loss": 3.2284, + "theoretical_loss": 4.4638805268921935, + "tokens_seen": 164757504 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004798294884653962, + "loss": 3.1582, + "theoretical_loss": 4.463651708851346, + "tokens_seen": 164823040 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004798194583751254, + "loss": 3.1417, + "theoretical_loss": 4.463423007236974, + "tokens_seen": 164888576 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047980942828485456, + "loss": 3.2099, + "theoretical_loss": 4.4631944219436, + "tokens_seen": 164954112 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047979939819458374, + "loss": 3.5012, + "theoretical_loss": 4.462965952865879, + "tokens_seen": 165019648 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004797893681043129, + "loss": 3.3506, + "theoretical_loss": 4.46273759989861, + "tokens_seen": 165085184 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004797793380140421, + "loss": 3.1468, + "theoretical_loss": 4.462509362936723, + "tokens_seen": 165150720 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047976930792377134, + "loss": 3.278, + "theoretical_loss": 4.46228124187529, + "tokens_seen": 165216256 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047975927783350047, + "loss": 3.4265, + "theoretical_loss": 4.462053236609516, + "tokens_seen": 165281792 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004797492477432297, + "loss": 3.3982, + "theoretical_loss": 4.461825347034742, + "tokens_seen": 165347328 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004797392176529589, + "loss": 3.3485, + "theoretical_loss": 4.461597573046449, + "tokens_seen": 165412864 + }, + { + "epoch": 0.05, + "objective/train/docs_used": 105300, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.0064985752105713, + "objective/train/theoretical_loss": 4.461369914540247, + "objective/train/tokens_used": 185938400, + "theoretical_loss": 4.461369914540247, + "tokens_seen": 165478400 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047972918756268807, + "loss": 3.0503, + "theoretical_loss": 4.461369914540247, + "tokens_seen": 165478400 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004797191574724173, + "loss": 3.2915, + "theoretical_loss": 4.4611423714118885, + "tokens_seen": 165543936 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047970912738214643, + "loss": 3.3318, + "theoretical_loss": 4.460914943557256, + "tokens_seen": 165609472 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047969909729187566, + "loss": 3.5956, + "theoretical_loss": 4.460687630872371, + "tokens_seen": 165675008 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047968906720160484, + "loss": 3.2215, + "theoretical_loss": 4.46046043325339, + "tokens_seen": 165740544 + }, + { + "epoch": 0.05, + "learning_rate": 0.000479679037111334, + "loss": 3.2531, + "theoretical_loss": 4.460233350596599, + "tokens_seen": 165806080 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004796690070210632, + "loss": 3.19, + "theoretical_loss": 4.460006382798425, + "tokens_seen": 165871616 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004796589769307924, + "loss": 3.2943, + "theoretical_loss": 4.459779529755423, + "tokens_seen": 165937152 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047964894684052157, + "loss": 3.2706, + "theoretical_loss": 4.459552791364288, + "tokens_seen": 166002688 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004796389167502508, + "loss": 3.3049, + "theoretical_loss": 4.459326167521844, + "tokens_seen": 166068224 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047962888665997993, + "loss": 3.2977, + "theoretical_loss": 4.4590996581250515, + "tokens_seen": 166133760 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047961885656970917, + "loss": 3.2425, + "theoretical_loss": 4.458873263071002, + "tokens_seen": 166199296 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047960882647943835, + "loss": 3.0658, + "theoretical_loss": 4.458646982256921, + "tokens_seen": 166264832 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047959879638916753, + "loss": 3.3019, + "theoretical_loss": 4.458420815580169, + "tokens_seen": 166330368 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004795887662988967, + "loss": 3.5914, + "theoretical_loss": 4.458194762938234, + "tokens_seen": 166395904 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004795787362086259, + "loss": 3.5516, + "theoretical_loss": 4.457968824228743, + "tokens_seen": 166461440 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047956870611835507, + "loss": 3.0776, + "theoretical_loss": 4.457742999349449, + "tokens_seen": 166526976 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004795586760280843, + "loss": 3.3158, + "theoretical_loss": 4.4575172881982414, + "tokens_seen": 166592512 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047954864593781343, + "loss": 2.9814, + "theoretical_loss": 4.457291690673139, + "tokens_seen": 166658048 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047953861584754267, + "loss": 3.1817, + "theoretical_loss": 4.457066206672291, + "tokens_seen": 166723584 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004795285857572718, + "loss": 3.125, + "theoretical_loss": 4.456840836093983, + "tokens_seen": 166789120 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047951855566700103, + "loss": 3.3363, + "theoretical_loss": 4.456615578836625, + "tokens_seen": 166854656 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004795085255767302, + "loss": 2.9642, + "theoretical_loss": 4.456390434798762, + "tokens_seen": 166920192 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004794984954864594, + "loss": 3.0932, + "theoretical_loss": 4.45616540387907, + "tokens_seen": 166985728 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004794884653961886, + "loss": 3.4142, + "theoretical_loss": 4.4559404859763525, + "tokens_seen": 167051264 + }, + { + "epoch": 0.05, + "objective/train/docs_used": 106148, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.7205657958984375, + "objective/train/theoretical_loss": 4.455715680989545, + "objective/train/tokens_used": 187576800, + "theoretical_loss": 4.455715680989545, + "tokens_seen": 167116800 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047947843530591776, + "loss": 3.3954, + "theoretical_loss": 4.455715680989545, + "tokens_seen": 167116800 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047946840521564694, + "loss": 3.3232, + "theoretical_loss": 4.455490988817713, + "tokens_seen": 167182336 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047945837512537617, + "loss": 3.3872, + "theoretical_loss": 4.4552664093600525, + "tokens_seen": 167247872 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004794483450351053, + "loss": 3.3941, + "theoretical_loss": 4.455041942515887, + "tokens_seen": 167313408 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047943831494483453, + "loss": 3.2813, + "theoretical_loss": 4.454817588184669, + "tokens_seen": 167378944 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004794282848545637, + "loss": 3.1614, + "theoretical_loss": 4.454593346265984, + "tokens_seen": 167444480 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004794182547642929, + "loss": 3.148, + "theoretical_loss": 4.454369216659542, + "tokens_seen": 167510016 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004794082246740221, + "loss": 3.3197, + "theoretical_loss": 4.454145199265183, + "tokens_seen": 167575552 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047939819458375126, + "loss": 3.0525, + "theoretical_loss": 4.453921293982877, + "tokens_seen": 167641088 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047938816449348044, + "loss": 3.0177, + "theoretical_loss": 4.453697500712722, + "tokens_seen": 167706624 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004793781344032097, + "loss": 3.3305, + "theoretical_loss": 4.453473819354942, + "tokens_seen": 167772160 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004793681043129388, + "loss": 3.2661, + "theoretical_loss": 4.453250249809889, + "tokens_seen": 167837696 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047935807422266804, + "loss": 3.4484, + "theoretical_loss": 4.453026791978045, + "tokens_seen": 167903232 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047934804413239716, + "loss": 3.2061, + "theoretical_loss": 4.4528034457600185, + "tokens_seen": 167968768 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004793380140421264, + "loss": 3.2846, + "theoretical_loss": 4.452580211056542, + "tokens_seen": 168034304 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004793279839518556, + "loss": 3.2659, + "theoretical_loss": 4.452357087768481, + "tokens_seen": 168099840 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047931795386158476, + "loss": 3.3937, + "theoretical_loss": 4.45213407579682, + "tokens_seen": 168165376 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047930792377131394, + "loss": 3.5216, + "theoretical_loss": 4.451911175042679, + "tokens_seen": 168230912 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004792978936810431, + "loss": 3.2824, + "theoretical_loss": 4.451688385407296, + "tokens_seen": 168296448 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004792878635907723, + "loss": 3.2781, + "theoretical_loss": 4.451465706792041, + "tokens_seen": 168361984 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047927783350050154, + "loss": 3.2492, + "theoretical_loss": 4.4512431390984055, + "tokens_seen": 168427520 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047926780341023067, + "loss": 3.0039, + "theoretical_loss": 4.451020682228011, + "tokens_seen": 168493056 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004792577733199599, + "loss": 3.3954, + "theoretical_loss": 4.450798336082601, + "tokens_seen": 168558592 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004792477432296891, + "loss": 3.4103, + "theoretical_loss": 4.450576100564046, + "tokens_seen": 168624128 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047923771313941827, + "loss": 3.3796, + "theoretical_loss": 4.450353975574341, + "tokens_seen": 168689664 + }, + { + "epoch": 0.05, + "objective/train/docs_used": 106755, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.610914707183838, + "objective/train/theoretical_loss": 4.450131961015606, + "objective/train/tokens_used": 189215200, + "theoretical_loss": 4.450131961015606, + "tokens_seen": 168755200 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047922768304914745, + "loss": 3.0115, + "theoretical_loss": 4.450131961015606, + "tokens_seen": 168755200 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047921765295887663, + "loss": 3.2394, + "theoretical_loss": 4.449910056790086, + "tokens_seen": 168820736 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004792076228686058, + "loss": 3.0145, + "theoretical_loss": 4.44968826280015, + "tokens_seen": 168886272 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047919759277833504, + "loss": 3.1414, + "theoretical_loss": 4.4494665789482895, + "tokens_seen": 168951808 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047918756268806417, + "loss": 3.4367, + "theoretical_loss": 4.449245005137125, + "tokens_seen": 169017344 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004791775325977934, + "loss": 2.9607, + "theoretical_loss": 4.449023541269395, + "tokens_seen": 169082880 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047916750250752253, + "loss": 3.3094, + "theoretical_loss": 4.448802187247966, + "tokens_seen": 169148416 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047915747241725177, + "loss": 3.2629, + "theoretical_loss": 4.448580942975825, + "tokens_seen": 169213952 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047914744232698095, + "loss": 3.2376, + "theoretical_loss": 4.448359808356084, + "tokens_seen": 169279488 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047913741223671013, + "loss": 3.2962, + "theoretical_loss": 4.448138783291979, + "tokens_seen": 169345024 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004791273821464393, + "loss": 3.192, + "theoretical_loss": 4.447917867686863, + "tokens_seen": 169410560 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047911735205616855, + "loss": 3.626, + "theoretical_loss": 4.44769706144422, + "tokens_seen": 169476096 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004791073219658977, + "loss": 3.3122, + "theoretical_loss": 4.44747636446765, + "tokens_seen": 169541632 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004790972918756269, + "loss": 3.4109, + "theoretical_loss": 4.447255776660878, + "tokens_seen": 169607168 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047908726178535604, + "loss": 3.3341, + "theoretical_loss": 4.44703529792775, + "tokens_seen": 169672704 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047907723169508527, + "loss": 3.4113, + "theoretical_loss": 4.446814928172234, + "tokens_seen": 169738240 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047906720160481445, + "loss": 3.1274, + "theoretical_loss": 4.446594667298421, + "tokens_seen": 169803776 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047905717151454363, + "loss": 3.3019, + "theoretical_loss": 4.446374515210521, + "tokens_seen": 169869312 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004790471414242728, + "loss": 3.2222, + "theoretical_loss": 4.446154471812866, + "tokens_seen": 169934848 + }, + { + "epoch": 0.05, + "learning_rate": 0.000479037111334002, + "loss": 3.2455, + "theoretical_loss": 4.445934537009911, + "tokens_seen": 170000384 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004790270812437312, + "loss": 3.296, + "theoretical_loss": 4.445714710706228, + "tokens_seen": 170065920 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004790170511534604, + "loss": 3.3642, + "theoretical_loss": 4.445494992806513, + "tokens_seen": 170131456 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047900702106318954, + "loss": 3.3999, + "theoretical_loss": 4.44527538321558, + "tokens_seen": 170196992 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004789969909729188, + "loss": 3.3887, + "theoretical_loss": 4.445055881838365, + "tokens_seen": 170262528 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004789869608826479, + "loss": 3.4913, + "theoretical_loss": 4.444836488579924, + "tokens_seen": 170328064 + }, + { + "epoch": 0.05, + "objective/train/docs_used": 107830, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.48449444770813, + "objective/train/theoretical_loss": 4.44461720334543, + "objective/train/tokens_used": 190853600, + "theoretical_loss": 4.44461720334543, + "tokens_seen": 170393600 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047897693079237714, + "loss": 3.2715, + "theoretical_loss": 4.44461720334543, + "tokens_seen": 170393600 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047896690070210637, + "loss": 3.0477, + "theoretical_loss": 4.444398026040179, + "tokens_seen": 170459136 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004789568706118355, + "loss": 3.2522, + "theoretical_loss": 4.444178956569585, + "tokens_seen": 170524672 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047894684052156473, + "loss": 3.4596, + "theoretical_loss": 4.443959994839181, + "tokens_seen": 170590208 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004789368104312939, + "loss": 2.9501, + "theoretical_loss": 4.44374114075462, + "tokens_seen": 170655744 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004789267803410231, + "loss": 3.3983, + "theoretical_loss": 4.443522394221671, + "tokens_seen": 170721280 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004789167502507523, + "loss": 3.2128, + "theoretical_loss": 4.443303755146225, + "tokens_seen": 170786816 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047890672016048146, + "loss": 3.0624, + "theoretical_loss": 4.443085223434291, + "tokens_seen": 170852352 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047889669007021064, + "loss": 3.2411, + "theoretical_loss": 4.442866798991993, + "tokens_seen": 170917888 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004788866599799399, + "loss": 3.3814, + "theoretical_loss": 4.442648481725577, + "tokens_seen": 170983424 + }, + { + "epoch": 0.05, + "learning_rate": 0.000478876629889669, + "loss": 3.2555, + "theoretical_loss": 4.442430271541404, + "tokens_seen": 171048960 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047886659979939824, + "loss": 3.2421, + "theoretical_loss": 4.442212168345956, + "tokens_seen": 171114496 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047885656970912736, + "loss": 3.0656, + "theoretical_loss": 4.4419941720458285, + "tokens_seen": 171180032 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004788465396188566, + "loss": 3.2846, + "theoretical_loss": 4.441776282547736, + "tokens_seen": 171245568 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004788365095285858, + "loss": 3.3535, + "theoretical_loss": 4.441558499758511, + "tokens_seen": 171311104 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047882647943831496, + "loss": 3.0785, + "theoretical_loss": 4.441340823585101, + "tokens_seen": 171376640 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047881644934804414, + "loss": 3.2276, + "theoretical_loss": 4.441123253934572, + "tokens_seen": 171442176 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004788064192577733, + "loss": 3.3016, + "theoretical_loss": 4.440905790714105, + "tokens_seen": 171507712 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004787963891675025, + "loss": 3.2406, + "theoretical_loss": 4.440688433830999, + "tokens_seen": 171573248 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047878635907723174, + "loss": 3.2669, + "theoretical_loss": 4.440471183192667, + "tokens_seen": 171638784 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047877632898696087, + "loss": 3.0818, + "theoretical_loss": 4.440254038706639, + "tokens_seen": 171704320 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004787662988966901, + "loss": 3.406, + "theoretical_loss": 4.440037000280561, + "tokens_seen": 171769856 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004787562688064193, + "loss": 3.3441, + "theoretical_loss": 4.439820067822195, + "tokens_seen": 171835392 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047874623871614847, + "loss": 3.009, + "theoretical_loss": 4.439603241239416, + "tokens_seen": 171900928 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047873620862587765, + "loss": 3.2221, + "theoretical_loss": 4.439386520440218, + "tokens_seen": 171966464 + }, + { + "epoch": 0.05, + "objective/train/docs_used": 108471, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.1990857124328613, + "objective/train/theoretical_loss": 4.439169905332706, + "objective/train/tokens_used": 192492000, + "theoretical_loss": 4.439169905332706, + "tokens_seen": 172032000 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047872617853560683, + "loss": 3.2003, + "theoretical_loss": 4.439169905332706, + "tokens_seen": 172032000 + }, + { + "epoch": 0.05, + "learning_rate": 0.000478716148445336, + "loss": 3.1812, + "theoretical_loss": 4.438953395825102, + "tokens_seen": 172097536 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047870611835506524, + "loss": 3.2771, + "theoretical_loss": 4.438736991825744, + "tokens_seen": 172163072 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047869608826479437, + "loss": 3.0257, + "theoretical_loss": 4.438520693243079, + "tokens_seen": 172228608 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004786860581745236, + "loss": 3.2079, + "theoretical_loss": 4.4383044999856756, + "tokens_seen": 172294144 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047867602808425273, + "loss": 3.1905, + "theoretical_loss": 4.438088411962211, + "tokens_seen": 172359680 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047866599799398197, + "loss": 2.9904, + "theoretical_loss": 4.437872429081477, + "tokens_seen": 172425216 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047865596790371115, + "loss": 2.9984, + "theoretical_loss": 4.437656551252381, + "tokens_seen": 172490752 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047864593781344033, + "loss": 3.1714, + "theoretical_loss": 4.4374407783839445, + "tokens_seen": 172556288 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004786359077231695, + "loss": 3.277, + "theoretical_loss": 4.437225110385297, + "tokens_seen": 172621824 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047862587763289875, + "loss": 3.2842, + "theoretical_loss": 4.4370095471656885, + "tokens_seen": 172687360 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004786158475426279, + "loss": 3.334, + "theoretical_loss": 4.436794088634477, + "tokens_seen": 172752896 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004786058174523571, + "loss": 3.0771, + "theoretical_loss": 4.4365787347011345, + "tokens_seen": 172818432 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047859578736208624, + "loss": 3.17, + "theoretical_loss": 4.436363485275246, + "tokens_seen": 172883968 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047858575727181547, + "loss": 3.086, + "theoretical_loss": 4.436148340266508, + "tokens_seen": 172949504 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047857572718154465, + "loss": 3.2349, + "theoretical_loss": 4.435933299584729, + "tokens_seen": 173015040 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047856569709127383, + "loss": 3.2119, + "theoretical_loss": 4.4357183631398325, + "tokens_seen": 173080576 + }, + { + "epoch": 0.05, + "learning_rate": 0.000478555667001003, + "loss": 3.1316, + "theoretical_loss": 4.435503530841849, + "tokens_seen": 173146112 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004785456369107322, + "loss": 3.2008, + "theoretical_loss": 4.435288802600926, + "tokens_seen": 173211648 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004785356068204614, + "loss": 3.4882, + "theoretical_loss": 4.4350741783273175, + "tokens_seen": 173277184 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004785255767301906, + "loss": 3.3088, + "theoretical_loss": 4.434859657931392, + "tokens_seen": 173342720 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047851554663991974, + "loss": 3.1826, + "theoretical_loss": 4.434645241323629, + "tokens_seen": 173408256 + }, + { + "epoch": 0.05, + "learning_rate": 0.000478505516549649, + "loss": 3.2865, + "theoretical_loss": 4.434430928414617, + "tokens_seen": 173473792 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004784954864593781, + "loss": 3.1275, + "theoretical_loss": 4.434216719115057, + "tokens_seen": 173539328 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047848545636910734, + "loss": 3.5013, + "theoretical_loss": 4.43400261333576, + "tokens_seen": 173604864 + }, + { + "epoch": 0.05, + "objective/train/docs_used": 109645, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.2769198417663574, + "objective/train/theoretical_loss": 4.433788610987646, + "objective/train/tokens_used": 194130400, + "theoretical_loss": 4.433788610987646, + "tokens_seen": 173670400 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004784754262788365, + "loss": 3.274, + "theoretical_loss": 4.433788610987646, + "tokens_seen": 173670400 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004784653961885657, + "loss": 3.2798, + "theoretical_loss": 4.433574711981749, + "tokens_seen": 173735936 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004784553660982949, + "loss": 3.2293, + "theoretical_loss": 4.433360916229209, + "tokens_seen": 173801472 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004784453360080241, + "loss": 3.1387, + "theoretical_loss": 4.433147223641278, + "tokens_seen": 173867008 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047843530591775324, + "loss": 3.5462, + "theoretical_loss": 4.432933634129318, + "tokens_seen": 173932544 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004784252758274825, + "loss": 3.3114, + "theoretical_loss": 4.4327201476047975, + "tokens_seen": 173998080 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004784152457372116, + "loss": 3.4395, + "theoretical_loss": 4.432506763979299, + "tokens_seen": 174063616 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047840521564694084, + "loss": 3.2343, + "theoretical_loss": 4.432293483164512, + "tokens_seen": 174129152 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047839518555667, + "loss": 3.1148, + "theoretical_loss": 4.432080305072233, + "tokens_seen": 174194688 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004783851554663992, + "loss": 3.2719, + "theoretical_loss": 4.43186722961437, + "tokens_seen": 174260224 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004783751253761284, + "loss": 3.1626, + "theoretical_loss": 4.431654256702938, + "tokens_seen": 174325760 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047836509528585756, + "loss": 3.1065, + "theoretical_loss": 4.431441386250063, + "tokens_seen": 174391296 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047835506519558675, + "loss": 2.8464, + "theoretical_loss": 4.4312286181679745, + "tokens_seen": 174456832 + }, + { + "epoch": 0.05, + "learning_rate": 0.000478345035105316, + "loss": 3.4557, + "theoretical_loss": 4.431015952369016, + "tokens_seen": 174522368 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004783350050150451, + "loss": 3.1439, + "theoretical_loss": 4.430803388765636, + "tokens_seen": 174587904 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047832497492477434, + "loss": 3.281, + "theoretical_loss": 4.430590927270388, + "tokens_seen": 174653440 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047831494483450347, + "loss": 3.1745, + "theoretical_loss": 4.430378567795938, + "tokens_seen": 174718976 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004783049147442327, + "loss": 3.443, + "theoretical_loss": 4.430166310255057, + "tokens_seen": 174784512 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004782948846539619, + "loss": 3.1798, + "theoretical_loss": 4.429954154560624, + "tokens_seen": 174850048 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047828485456369107, + "loss": 3.1181, + "theoretical_loss": 4.429742100625624, + "tokens_seen": 174915584 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047827482447342025, + "loss": 3.2796, + "theoretical_loss": 4.429530148363151, + "tokens_seen": 174981120 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004782647943831495, + "loss": 3.3092, + "theoretical_loss": 4.429318297686402, + "tokens_seen": 175046656 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004782547642928786, + "loss": 3.305, + "theoretical_loss": 4.429106548508685, + "tokens_seen": 175112192 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047824473420260785, + "loss": 3.1845, + "theoretical_loss": 4.428894900743411, + "tokens_seen": 175177728 + }, + { + "epoch": 0.05, + "learning_rate": 0.000478234704112337, + "loss": 3.2327, + "theoretical_loss": 4.428683354304098, + "tokens_seen": 175243264 + }, + { + "epoch": 0.05, + "objective/train/docs_used": 110166, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.4295928478240967, + "objective/train/theoretical_loss": 4.428471909104372, + "objective/train/tokens_used": 195768800, + "theoretical_loss": 4.428471909104372, + "tokens_seen": 175308800 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004782246740220662, + "loss": 3.2399, + "theoretical_loss": 4.428471909104372, + "tokens_seen": 175308800 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047821464393179544, + "loss": 3.4346, + "theoretical_loss": 4.428260565057964, + "tokens_seen": 175374336 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047820461384152457, + "loss": 3.3069, + "theoretical_loss": 4.428049322078708, + "tokens_seen": 175439872 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004781945837512538, + "loss": 3.1985, + "theoretical_loss": 4.427838180080547, + "tokens_seen": 175505408 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047818455366098293, + "loss": 3.4471, + "theoretical_loss": 4.4276271389775275, + "tokens_seen": 175570944 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047817452357071217, + "loss": 3.3491, + "theoretical_loss": 4.427416198683803, + "tokens_seen": 175636480 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047816449348044135, + "loss": 3.0362, + "theoretical_loss": 4.427205359113629, + "tokens_seen": 175702016 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047815446339017053, + "loss": 3.208, + "theoretical_loss": 4.42699462018137, + "tokens_seen": 175767552 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004781444332998997, + "loss": 3.5011, + "theoretical_loss": 4.42678398180149, + "tokens_seen": 175833088 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047813440320962895, + "loss": 3.3444, + "theoretical_loss": 4.426573443888563, + "tokens_seen": 175898624 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004781243731193581, + "loss": 3.079, + "theoretical_loss": 4.426363006357263, + "tokens_seen": 175964160 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004781143430290873, + "loss": 3.2537, + "theoretical_loss": 4.426152669122374, + "tokens_seen": 176029696 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047810431293881644, + "loss": 3.2052, + "theoretical_loss": 4.425942432098774, + "tokens_seen": 176095232 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047809428284854567, + "loss": 3.2307, + "theoretical_loss": 4.425732295201455, + "tokens_seen": 176160768 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047808425275827485, + "loss": 3.2647, + "theoretical_loss": 4.425522258345508, + "tokens_seen": 176226304 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047807422266800403, + "loss": 3.2675, + "theoretical_loss": 4.425312321446127, + "tokens_seen": 176291840 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004780641925777332, + "loss": 3.0024, + "theoretical_loss": 4.425102484418613, + "tokens_seen": 176357376 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004780541624874624, + "loss": 3.3125, + "theoretical_loss": 4.424892747178365, + "tokens_seen": 176422912 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004780441323971916, + "loss": 3.1194, + "theoretical_loss": 4.42468310964089, + "tokens_seen": 176488448 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004780341023069208, + "loss": 3.0572, + "theoretical_loss": 4.424473571721794, + "tokens_seen": 176553984 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047802407221664994, + "loss": 3.2069, + "theoretical_loss": 4.42426413333679, + "tokens_seen": 176619520 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004780140421263792, + "loss": 3.3892, + "theoretical_loss": 4.424054794401689, + "tokens_seen": 176685056 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004780040120361083, + "loss": 3.3894, + "theoretical_loss": 4.423845554832406, + "tokens_seen": 176750592 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047799398194583754, + "loss": 3.104, + "theoretical_loss": 4.42363641454496, + "tokens_seen": 176816128 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004779839518555667, + "loss": 3.3822, + "theoretical_loss": 4.423427373455471, + "tokens_seen": 176881664 + }, + { + "epoch": 0.05, + "objective/train/docs_used": 110915, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.2131428718566895, + "objective/train/theoretical_loss": 4.42321843148016, + "objective/train/tokens_used": 197407200, + "theoretical_loss": 4.42321843148016, + "tokens_seen": 176947200 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004779739217652959, + "loss": 3.1635, + "theoretical_loss": 4.42321843148016, + "tokens_seen": 176947200 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004779638916750251, + "loss": 3.1166, + "theoretical_loss": 4.423009588535351, + "tokens_seen": 177012736 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004779538615847543, + "loss": 3.3268, + "theoretical_loss": 4.422800844537466, + "tokens_seen": 177078272 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047794383149448344, + "loss": 3.2876, + "theoretical_loss": 4.422592199403036, + "tokens_seen": 177143808 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004779338014042127, + "loss": 3.3092, + "theoretical_loss": 4.422383653048685, + "tokens_seen": 177209344 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004779237713139418, + "loss": 3.1588, + "theoretical_loss": 4.422175205391145, + "tokens_seen": 177274880 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047791374122367104, + "loss": 3.1771, + "theoretical_loss": 4.421966856347243, + "tokens_seen": 177340416 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004779037111334002, + "loss": 3.091, + "theoretical_loss": 4.421758605833912, + "tokens_seen": 177405952 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004778936810431294, + "loss": 3.1262, + "theoretical_loss": 4.421550453768181, + "tokens_seen": 177471488 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004778836509528586, + "loss": 3.3769, + "theoretical_loss": 4.421342400067183, + "tokens_seen": 177537024 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047787362086258776, + "loss": 3.1324, + "theoretical_loss": 4.42113444464815, + "tokens_seen": 177602560 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047786359077231695, + "loss": 3.0876, + "theoretical_loss": 4.420926587428411, + "tokens_seen": 177668096 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004778535606820462, + "loss": 3.3605, + "theoretical_loss": 4.420718828325403, + "tokens_seen": 177733632 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004778435305917753, + "loss": 3.2724, + "theoretical_loss": 4.420511167256656, + "tokens_seen": 177799168 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047783350050150454, + "loss": 3.0219, + "theoretical_loss": 4.4203036041398, + "tokens_seen": 177864704 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047782347041123367, + "loss": 3.3263, + "theoretical_loss": 4.420096138892568, + "tokens_seen": 177930240 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004778134403209629, + "loss": 3.1365, + "theoretical_loss": 4.419888771432789, + "tokens_seen": 177995776 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004778034102306921, + "loss": 3.1645, + "theoretical_loss": 4.419681501678395, + "tokens_seen": 178061312 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047779338014042127, + "loss": 3.0607, + "theoretical_loss": 4.419474329547413, + "tokens_seen": 178126848 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047778335005015045, + "loss": 3.2835, + "theoretical_loss": 4.419267254957971, + "tokens_seen": 178192384 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004777733199598797, + "loss": 3.4579, + "theoretical_loss": 4.419060277828295, + "tokens_seen": 178257920 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004777632898696088, + "loss": 3.4154, + "theoretical_loss": 4.41885339807671, + "tokens_seen": 178323456 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047775325977933805, + "loss": 3.4246, + "theoretical_loss": 4.4186466156216415, + "tokens_seen": 178388992 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004777432296890672, + "loss": 3.167, + "theoretical_loss": 4.418439930381609, + "tokens_seen": 178454528 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004777331995987964, + "loss": 3.2456, + "theoretical_loss": 4.418233342275233, + "tokens_seen": 178520064 + }, + { + "epoch": 0.05, + "objective/train/docs_used": 112413, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.0976438522338867, + "objective/train/theoretical_loss": 4.418026851221231, + "objective/train/tokens_used": 199045600, + "theoretical_loss": 4.418026851221231, + "tokens_seen": 178585600 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004777231695085256, + "loss": 3.0861, + "theoretical_loss": 4.418026851221231, + "tokens_seen": 178585600 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047771313941825477, + "loss": 3.0485, + "theoretical_loss": 4.4178204571384185, + "tokens_seen": 178651136 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047770310932798395, + "loss": 3.3325, + "theoretical_loss": 4.41761415994571, + "tokens_seen": 178716672 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047769307923771313, + "loss": 3.1803, + "theoretical_loss": 4.417407959562116, + "tokens_seen": 178782208 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004776830491474423, + "loss": 3.0034, + "theoretical_loss": 4.417201855906742, + "tokens_seen": 178847744 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047767301905717155, + "loss": 3.1023, + "theoretical_loss": 4.416995848898797, + "tokens_seen": 178913280 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004776629889669007, + "loss": 3.2336, + "theoretical_loss": 4.4167899384575815, + "tokens_seen": 178978816 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004776529588766299, + "loss": 3.3796, + "theoretical_loss": 4.416584124502495, + "tokens_seen": 179044352 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047764292878635904, + "loss": 3.1261, + "theoretical_loss": 4.416378406953033, + "tokens_seen": 179109888 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004776328986960883, + "loss": 3.112, + "theoretical_loss": 4.41617278572879, + "tokens_seen": 179175424 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047762286860581746, + "loss": 3.2963, + "theoretical_loss": 4.4159672607494524, + "tokens_seen": 179240960 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047761283851554664, + "loss": 3.1597, + "theoretical_loss": 4.415761831934808, + "tokens_seen": 179306496 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004776028084252758, + "loss": 3.0727, + "theoretical_loss": 4.415556499204737, + "tokens_seen": 179372032 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047759277833500505, + "loss": 3.0216, + "theoretical_loss": 4.415351262479216, + "tokens_seen": 179437568 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004775827482447342, + "loss": 3.2077, + "theoretical_loss": 4.415146121678321, + "tokens_seen": 179503104 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004775727181544634, + "loss": 3.193, + "theoretical_loss": 4.414941076722219, + "tokens_seen": 179568640 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047756268806419254, + "loss": 3.4055, + "theoretical_loss": 4.4147361275311745, + "tokens_seen": 179634176 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004775526579739218, + "loss": 3.2131, + "theoretical_loss": 4.414531274025548, + "tokens_seen": 179699712 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047754262788365096, + "loss": 3.276, + "theoretical_loss": 4.414326516125795, + "tokens_seen": 179765248 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047753259779338014, + "loss": 3.2095, + "theoretical_loss": 4.414121853752466, + "tokens_seen": 179830784 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004775225677031093, + "loss": 3.0624, + "theoretical_loss": 4.413917286826205, + "tokens_seen": 179896320 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004775125376128385, + "loss": 3.2954, + "theoretical_loss": 4.413712815267752, + "tokens_seen": 179961856 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004775025075225677, + "loss": 3.2637, + "theoretical_loss": 4.413508438997944, + "tokens_seen": 180027392 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004774924774322969, + "loss": 3.1727, + "theoretical_loss": 4.4133041579377075, + "tokens_seen": 180092928 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047748244734202605, + "loss": 3.212, + "theoretical_loss": 4.413099972008068, + "tokens_seen": 180158464 + }, + { + "epoch": 0.05, + "objective/train/docs_used": 112982, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.1012556552886963, + "objective/train/theoretical_loss": 4.412895881130142, + "objective/train/tokens_used": 200684000, + "theoretical_loss": 4.412895881130142, + "tokens_seen": 180224000 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004774724172517553, + "loss": 3.2505, + "theoretical_loss": 4.412895881130142, + "tokens_seen": 180224000 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004774623871614845, + "loss": 3.447, + "theoretical_loss": 4.412691885225141, + "tokens_seen": 180289536 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047745235707121364, + "loss": 3.353, + "theoretical_loss": 4.412487984214373, + "tokens_seen": 180355072 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004774423269809429, + "loss": 3.1408, + "theoretical_loss": 4.412284178019235, + "tokens_seen": 180420608 + }, + { + "epoch": 0.05, + "learning_rate": 0.000477432296890672, + "loss": 3.3407, + "theoretical_loss": 4.412080466561221, + "tokens_seen": 180486144 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047742226680040124, + "loss": 3.0194, + "theoretical_loss": 4.411876849761917, + "tokens_seen": 180551680 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004774122367101304, + "loss": 3.4386, + "theoretical_loss": 4.411673327543005, + "tokens_seen": 180617216 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004774022066198596, + "loss": 3.1405, + "theoretical_loss": 4.4114698998262565, + "tokens_seen": 180682752 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004773921765295888, + "loss": 3.026, + "theoretical_loss": 4.411266566533539, + "tokens_seen": 180748288 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047738214643931797, + "loss": 3.3335, + "theoretical_loss": 4.41106332758681, + "tokens_seen": 180813824 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047737211634904715, + "loss": 3.2686, + "theoretical_loss": 4.41086018290812, + "tokens_seen": 180879360 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004773620862587764, + "loss": 3.2655, + "theoretical_loss": 4.410657132419617, + "tokens_seen": 180944896 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004773520561685055, + "loss": 3.1595, + "theoretical_loss": 4.410454176043537, + "tokens_seen": 181010432 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047734202607823474, + "loss": 3.321, + "theoretical_loss": 4.410251313702208, + "tokens_seen": 181075968 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047733199598796387, + "loss": 3.3804, + "theoretical_loss": 4.410048545318052, + "tokens_seen": 181141504 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004773219658976931, + "loss": 2.9908, + "theoretical_loss": 4.409845870813582, + "tokens_seen": 181207040 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004773119358074223, + "loss": 3.1721, + "theoretical_loss": 4.409643290111404, + "tokens_seen": 181272576 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047730190571715147, + "loss": 3.1854, + "theoretical_loss": 4.409440803134215, + "tokens_seen": 181338112 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047729187562688065, + "loss": 3.2051, + "theoretical_loss": 4.409238409804804, + "tokens_seen": 181403648 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004772818455366099, + "loss": 3.3612, + "theoretical_loss": 4.409036110046051, + "tokens_seen": 181469184 + }, + { + "epoch": 0.06, + "learning_rate": 0.000477271815446339, + "loss": 3.1355, + "theoretical_loss": 4.408833903780926, + "tokens_seen": 181534720 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047726178535606825, + "loss": 3.2812, + "theoretical_loss": 4.408631790932494, + "tokens_seen": 181600256 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004772517552657974, + "loss": 3.0538, + "theoretical_loss": 4.408429771423909, + "tokens_seen": 181665792 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004772417251755266, + "loss": 3.1945, + "theoretical_loss": 4.408227845178414, + "tokens_seen": 181731328 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004772316950852558, + "loss": 3.296, + "theoretical_loss": 4.408026012119344, + "tokens_seen": 181796864 + }, + { + "epoch": 0.06, + "objective/train/docs_used": 114216, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.0965561866760254, + "objective/train/theoretical_loss": 4.407824272170128, + "objective/train/tokens_used": 202322400, + "theoretical_loss": 4.407824272170128, + "tokens_seen": 181862400 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047722166499498497, + "loss": 3.1195, + "theoretical_loss": 4.407824272170128, + "tokens_seen": 181862400 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047721163490471415, + "loss": 3.2961, + "theoretical_loss": 4.407622625254279, + "tokens_seen": 181927936 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047720160481444333, + "loss": 3.3368, + "theoretical_loss": 4.407421071295406, + "tokens_seen": 181993472 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004771915747241725, + "loss": 3.3405, + "theoretical_loss": 4.407219610217206, + "tokens_seen": 182059008 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047718154463390175, + "loss": 3.2996, + "theoretical_loss": 4.407018241943467, + "tokens_seen": 182124544 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004771715145436309, + "loss": 3.1271, + "theoretical_loss": 4.406816966398064, + "tokens_seen": 182190080 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004771614844533601, + "loss": 3.2203, + "theoretical_loss": 4.406615783504965, + "tokens_seen": 182255616 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047715145436308924, + "loss": 3.0705, + "theoretical_loss": 4.4064146931882275, + "tokens_seen": 182321152 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004771414242728185, + "loss": 3.1355, + "theoretical_loss": 4.406213695371996, + "tokens_seen": 182386688 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047713139418254766, + "loss": 3.1314, + "theoretical_loss": 4.406012789980506, + "tokens_seen": 182452224 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047712136409227684, + "loss": 3.0828, + "theoretical_loss": 4.405811976938084, + "tokens_seen": 182517760 + }, + { + "epoch": 0.06, + "learning_rate": 0.000477111334002006, + "loss": 3.2309, + "theoretical_loss": 4.405611256169143, + "tokens_seen": 182583296 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047710130391173525, + "loss": 3.149, + "theoretical_loss": 4.405410627598185, + "tokens_seen": 182648832 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004770912738214644, + "loss": 3.1255, + "theoretical_loss": 4.405210091149802, + "tokens_seen": 182714368 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004770812437311936, + "loss": 3.4633, + "theoretical_loss": 4.405009646748674, + "tokens_seen": 182779904 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047707121364092274, + "loss": 3.4309, + "theoretical_loss": 4.404809294319572, + "tokens_seen": 182845440 + }, + { + "epoch": 0.06, + "learning_rate": 0.000477061183550652, + "loss": 3.3709, + "theoretical_loss": 4.40460903378735, + "tokens_seen": 182910976 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047705115346038116, + "loss": 3.351, + "theoretical_loss": 4.404408865076955, + "tokens_seen": 182976512 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047704112337011034, + "loss": 3.0974, + "theoretical_loss": 4.404208788113422, + "tokens_seen": 183042048 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004770310932798395, + "loss": 3.3235, + "theoretical_loss": 4.404008802821871, + "tokens_seen": 183107584 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004770210631895687, + "loss": 3.2266, + "theoretical_loss": 4.4038089091275125, + "tokens_seen": 183173120 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004770110330992979, + "loss": 3.2177, + "theoretical_loss": 4.403609106955645, + "tokens_seen": 183238656 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004770010030090271, + "loss": 3.2219, + "theoretical_loss": 4.403409396231651, + "tokens_seen": 183304192 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047699097291875625, + "loss": 3.2755, + "theoretical_loss": 4.403209776881004, + "tokens_seen": 183369728 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004769809428284855, + "loss": 3.3628, + "theoretical_loss": 4.403010248829265, + "tokens_seen": 183435264 + }, + { + "epoch": 0.06, + "objective/train/docs_used": 114775, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.1911120414733887, + "objective/train/theoretical_loss": 4.4028108120020795, + "objective/train/tokens_used": 203960800, + "theoretical_loss": 4.4028108120020795, + "tokens_seen": 183500800 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047697091273821466, + "loss": 3.3216, + "theoretical_loss": 4.4028108120020795, + "tokens_seen": 183500800 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047696088264794384, + "loss": 2.9694, + "theoretical_loss": 4.402611466325182, + "tokens_seen": 183566336 + }, + { + "epoch": 0.06, + "learning_rate": 0.000476950852557673, + "loss": 2.9904, + "theoretical_loss": 4.4024122117243945, + "tokens_seen": 183631872 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004769408224674022, + "loss": 3.4241, + "theoretical_loss": 4.402213048125624, + "tokens_seen": 183697408 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004769307923771314, + "loss": 3.4352, + "theoretical_loss": 4.4020139754548655, + "tokens_seen": 183762944 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004769207622868606, + "loss": 3.2402, + "theoretical_loss": 4.401814993638199, + "tokens_seen": 183828480 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047691073219658975, + "loss": 3.3919, + "theoretical_loss": 4.4016161026017935, + "tokens_seen": 183894016 + }, + { + "epoch": 0.06, + "learning_rate": 0.000476900702106319, + "loss": 3.3524, + "theoretical_loss": 4.401417302271902, + "tokens_seen": 183959552 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004768906720160481, + "loss": 3.1893, + "theoretical_loss": 4.401218592574865, + "tokens_seen": 184025088 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047688064192577735, + "loss": 3.2633, + "theoretical_loss": 4.401019973437108, + "tokens_seen": 184090624 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047687061183550653, + "loss": 3.1084, + "theoretical_loss": 4.400821444785143, + "tokens_seen": 184156160 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004768605817452357, + "loss": 3.211, + "theoretical_loss": 4.400623006545567, + "tokens_seen": 184221696 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004768505516549649, + "loss": 3.1031, + "theoretical_loss": 4.400424658645065, + "tokens_seen": 184287232 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047684052156469407, + "loss": 3.2263, + "theoretical_loss": 4.400226401010404, + "tokens_seen": 184352768 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047683049147442325, + "loss": 3.4032, + "theoretical_loss": 4.40002823356844, + "tokens_seen": 184418304 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004768204613841525, + "loss": 3.3498, + "theoretical_loss": 4.39983015624611, + "tokens_seen": 184483840 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004768104312938816, + "loss": 3.2921, + "theoretical_loss": 4.39963216897044, + "tokens_seen": 184549376 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047680040120361085, + "loss": 3.4203, + "theoretical_loss": 4.3994342716685395, + "tokens_seen": 184614912 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047679037111334003, + "loss": 3.2533, + "theoretical_loss": 4.399236464267602, + "tokens_seen": 184680448 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004767803410230692, + "loss": 3.1343, + "theoretical_loss": 4.399038746694908, + "tokens_seen": 184745984 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004767703109327984, + "loss": 3.2046, + "theoretical_loss": 4.398841118877819, + "tokens_seen": 184811520 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004767602808425276, + "loss": 3.1554, + "theoretical_loss": 4.398643580743785, + "tokens_seen": 184877056 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047675025075225676, + "loss": 3.2564, + "theoretical_loss": 4.398446132220338, + "tokens_seen": 184942592 + }, + { + "epoch": 0.06, + "learning_rate": 0.000476740220661986, + "loss": 3.2765, + "theoretical_loss": 4.3982487732350934, + "tokens_seen": 185008128 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004767301905717151, + "loss": 3.4265, + "theoretical_loss": 4.398051503715753, + "tokens_seen": 185073664 + }, + { + "epoch": 0.06, + "objective/train/docs_used": 115939, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.440326452255249, + "objective/train/theoretical_loss": 4.397854323590102, + "objective/train/tokens_used": 205599200, + "theoretical_loss": 4.397854323590102, + "tokens_seen": 185139200 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047672016048144435, + "loss": 3.0732, + "theoretical_loss": 4.397854323590102, + "tokens_seen": 185139200 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047671013039117353, + "loss": 3.2298, + "theoretical_loss": 4.397657232786008, + "tokens_seen": 185204736 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004767001003009027, + "loss": 3.3746, + "theoretical_loss": 4.397460231231424, + "tokens_seen": 185270272 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047669007021063195, + "loss": 3.2388, + "theoretical_loss": 4.397263318854384, + "tokens_seen": 185335808 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004766800401203611, + "loss": 3.1408, + "theoretical_loss": 4.39706649558301, + "tokens_seen": 185401344 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004766700100300903, + "loss": 2.9736, + "theoretical_loss": 4.396869761345503, + "tokens_seen": 185466880 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047665997993981944, + "loss": 3.093, + "theoretical_loss": 4.396673116070147, + "tokens_seen": 185532416 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004766499498495487, + "loss": 3.2755, + "theoretical_loss": 4.396476559685315, + "tokens_seen": 185597952 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047663991975927786, + "loss": 3.1636, + "theoretical_loss": 4.396280092119455, + "tokens_seen": 185663488 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047662988966900704, + "loss": 3.1822, + "theoretical_loss": 4.3960837133011035, + "tokens_seen": 185729024 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004766198595787362, + "loss": 3.3732, + "theoretical_loss": 4.395887423158877, + "tokens_seen": 185794560 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047660982948846545, + "loss": 3.2677, + "theoretical_loss": 4.395691221621476, + "tokens_seen": 185860096 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004765997993981946, + "loss": 3.0931, + "theoretical_loss": 4.395495108617682, + "tokens_seen": 185925632 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004765897693079238, + "loss": 3.1729, + "theoretical_loss": 4.39529908407636, + "tokens_seen": 185991168 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047657973921765294, + "loss": 3.4346, + "theoretical_loss": 4.3951031479264575, + "tokens_seen": 186056704 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004765697091273822, + "loss": 3.1241, + "theoretical_loss": 4.394907300097002, + "tokens_seen": 186122240 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047655967903711136, + "loss": 3.3395, + "theoretical_loss": 4.394711540517106, + "tokens_seen": 186187776 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047654964894684054, + "loss": 3.2064, + "theoretical_loss": 4.39451586911596, + "tokens_seen": 186253312 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004765396188565697, + "loss": 3.1737, + "theoretical_loss": 4.39432028582284, + "tokens_seen": 186318848 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004765295887662989, + "loss": 3.0046, + "theoretical_loss": 4.394124790567101, + "tokens_seen": 186384384 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004765195586760281, + "loss": 3.0175, + "theoretical_loss": 4.3939293832781825, + "tokens_seen": 186449920 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004765095285857573, + "loss": 2.9164, + "theoretical_loss": 4.393734063885599, + "tokens_seen": 186515456 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047649949849548645, + "loss": 3.2775, + "theoretical_loss": 4.3935388323189555, + "tokens_seen": 186580992 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004764894684052157, + "loss": 3.224, + "theoretical_loss": 4.39334368850793, + "tokens_seen": 186646528 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047647943831494486, + "loss": 2.8372, + "theoretical_loss": 4.3931486323822835, + "tokens_seen": 186712064 + }, + { + "epoch": 0.06, + "objective/train/docs_used": 116562, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.249227285385132, + "objective/train/theoretical_loss": 4.392953663871862, + "objective/train/tokens_used": 207237600, + "theoretical_loss": 4.392953663871862, + "tokens_seen": 186777600 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047646940822467404, + "loss": 3.2635, + "theoretical_loss": 4.392953663871862, + "tokens_seen": 186777600 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004764593781344032, + "loss": 3.2692, + "theoretical_loss": 4.392758782906586, + "tokens_seen": 186843136 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004764493480441324, + "loss": 3.3263, + "theoretical_loss": 4.392563989416462, + "tokens_seen": 186908672 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004764393179538616, + "loss": 3.2587, + "theoretical_loss": 4.392369283331574, + "tokens_seen": 186974208 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004764292878635908, + "loss": 3.0956, + "theoretical_loss": 4.392174664582085, + "tokens_seen": 187039744 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047641925777331995, + "loss": 3.4258, + "theoretical_loss": 4.391980133098244, + "tokens_seen": 187105280 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004764092276830492, + "loss": 3.2159, + "theoretical_loss": 4.391785688810373, + "tokens_seen": 187170816 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004763991975927783, + "loss": 3.1703, + "theoretical_loss": 4.391591331648879, + "tokens_seen": 187236352 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047638916750250755, + "loss": 3.004, + "theoretical_loss": 4.391397061544247, + "tokens_seen": 187301888 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047637913741223673, + "loss": 3.1715, + "theoretical_loss": 4.391202878427042, + "tokens_seen": 187367424 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004763691073219659, + "loss": 3.3668, + "theoretical_loss": 4.3910087822279085, + "tokens_seen": 187432960 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004763590772316951, + "loss": 3.3547, + "theoretical_loss": 4.390814772877571, + "tokens_seen": 187498496 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047634904714142427, + "loss": 3.3076, + "theoretical_loss": 4.390620850306832, + "tokens_seen": 187564032 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047633901705115345, + "loss": 3.1025, + "theoretical_loss": 4.390427014446575, + "tokens_seen": 187629568 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004763289869608827, + "loss": 3.2145, + "theoretical_loss": 4.390233265227764, + "tokens_seen": 187695104 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004763189568706118, + "loss": 3.1065, + "theoretical_loss": 4.390039602581437, + "tokens_seen": 187760640 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047630892678034105, + "loss": 3.3582, + "theoretical_loss": 4.389846026438715, + "tokens_seen": 187826176 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047629889669007023, + "loss": 3.3055, + "theoretical_loss": 4.3896525367307975, + "tokens_seen": 187891712 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004762888665997994, + "loss": 3.2101, + "theoretical_loss": 4.389459133388962, + "tokens_seen": 187957248 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004762788365095286, + "loss": 3.365, + "theoretical_loss": 4.3892658163445635, + "tokens_seen": 188022784 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004762688064192578, + "loss": 3.2138, + "theoretical_loss": 4.389072585529037, + "tokens_seen": 188088320 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047625877632898696, + "loss": 3.2676, + "theoretical_loss": 4.388879440873897, + "tokens_seen": 188153856 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004762487462387162, + "loss": 3.0752, + "theoretical_loss": 4.388686382310732, + "tokens_seen": 188219392 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004762387161484453, + "loss": 3.1453, + "theoretical_loss": 4.388493409771213, + "tokens_seen": 188284928 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047622868605817455, + "loss": 3.2416, + "theoretical_loss": 4.388300523187087, + "tokens_seen": 188350464 + }, + { + "epoch": 0.06, + "objective/train/docs_used": 117858, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.125227212905884, + "objective/train/theoretical_loss": 4.3881077224901786, + "objective/train/tokens_used": 208876000, + "theoretical_loss": 4.3881077224901786, + "tokens_seen": 188416000 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004762186559679037, + "loss": 3.012, + "theoretical_loss": 4.3881077224901786, + "tokens_seen": 188416000 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004762086258776329, + "loss": 3.0011, + "theoretical_loss": 4.38791500761239, + "tokens_seen": 188481536 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004761985957873621, + "loss": 2.9857, + "theoretical_loss": 4.387722378485703, + "tokens_seen": 188547072 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004761885656970913, + "loss": 3.3563, + "theoretical_loss": 4.3875298350421765, + "tokens_seen": 188612608 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047617853560682046, + "loss": 3.0966, + "theoretical_loss": 4.387337377213943, + "tokens_seen": 188678144 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047616850551654964, + "loss": 3.1881, + "theoretical_loss": 4.387145004933218, + "tokens_seen": 188743680 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004761584754262788, + "loss": 3.3411, + "theoretical_loss": 4.38695271813229, + "tokens_seen": 188809216 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047614844533600806, + "loss": 3.2266, + "theoretical_loss": 4.386760516743526, + "tokens_seen": 188874752 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004761384152457372, + "loss": 3.1784, + "theoretical_loss": 4.38656840069937, + "tokens_seen": 188940288 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004761283851554664, + "loss": 3.1781, + "theoretical_loss": 4.386376369932344, + "tokens_seen": 189005824 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004761183550651956, + "loss": 3.2914, + "theoretical_loss": 4.386184424375044, + "tokens_seen": 189071360 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004761083249749248, + "loss": 3.2668, + "theoretical_loss": 4.385992563960145, + "tokens_seen": 189136896 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047609829488465396, + "loss": 3.3815, + "theoretical_loss": 4.385800788620397, + "tokens_seen": 189202432 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047608826479438314, + "loss": 3.2324, + "theoretical_loss": 4.385609098288628, + "tokens_seen": 189267968 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004760782347041123, + "loss": 3.2619, + "theoretical_loss": 4.385417492897741, + "tokens_seen": 189333504 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047606820461384156, + "loss": 3.0182, + "theoretical_loss": 4.385225972380715, + "tokens_seen": 189399040 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004760581745235707, + "loss": 3.2542, + "theoretical_loss": 4.385034536670606, + "tokens_seen": 189464576 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004760481444332999, + "loss": 3.2078, + "theoretical_loss": 4.384843185700544, + "tokens_seen": 189530112 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047603811434302905, + "loss": 3.167, + "theoretical_loss": 4.384651919403739, + "tokens_seen": 189595648 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004760280842527583, + "loss": 3.4475, + "theoretical_loss": 4.384460737713471, + "tokens_seen": 189661184 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047601805416248746, + "loss": 3.2957, + "theoretical_loss": 4.384269640563101, + "tokens_seen": 189726720 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047600802407221665, + "loss": 3.3133, + "theoretical_loss": 4.384078627886062, + "tokens_seen": 189792256 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004759979939819458, + "loss": 3.0252, + "theoretical_loss": 4.383887699615863, + "tokens_seen": 189857792 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047598796389167506, + "loss": 3.3256, + "theoretical_loss": 4.38369685568609, + "tokens_seen": 189923328 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004759779338014042, + "loss": 3.2009, + "theoretical_loss": 4.383506096030401, + "tokens_seen": 189988864 + }, + { + "epoch": 0.06, + "objective/train/docs_used": 118609, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.1224095821380615, + "objective/train/theoretical_loss": 4.383315420582533, + "objective/train/tokens_used": 210514400, + "theoretical_loss": 4.383315420582533, + "tokens_seen": 190054400 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004759679037111334, + "loss": 3.396, + "theoretical_loss": 4.383315420582533, + "tokens_seen": 190054400 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004759578736208626, + "loss": 2.8923, + "theoretical_loss": 4.383124829276294, + "tokens_seen": 190119936 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004759478435305918, + "loss": 3.4653, + "theoretical_loss": 4.38293432204557, + "tokens_seen": 190185472 + }, + { + "epoch": 0.06, + "learning_rate": 0.000475937813440321, + "loss": 3.0714, + "theoretical_loss": 4.382743898824321, + "tokens_seen": 190251008 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047592778335005015, + "loss": 3.2187, + "theoretical_loss": 4.3825535595465785, + "tokens_seen": 190316544 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004759177532597794, + "loss": 3.1661, + "theoretical_loss": 4.382363304146453, + "tokens_seen": 190382080 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004759077231695085, + "loss": 3.0922, + "theoretical_loss": 4.382173132558126, + "tokens_seen": 190447616 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047589769307923775, + "loss": 3.0571, + "theoretical_loss": 4.381983044715856, + "tokens_seen": 190513152 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047588766298896693, + "loss": 3.2283, + "theoretical_loss": 4.381793040553973, + "tokens_seen": 190578688 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004758776328986961, + "loss": 3.398, + "theoretical_loss": 4.381603120006883, + "tokens_seen": 190644224 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004758676028084253, + "loss": 3.0587, + "theoretical_loss": 4.381413283009065, + "tokens_seen": 190709760 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047585757271815447, + "loss": 3.3559, + "theoretical_loss": 4.381223529495073, + "tokens_seen": 190775296 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047584754262788365, + "loss": 3.1721, + "theoretical_loss": 4.381033859399532, + "tokens_seen": 190840832 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004758375125376129, + "loss": 3.2333, + "theoretical_loss": 4.380844272657145, + "tokens_seen": 190906368 + }, + { + "epoch": 0.06, + "learning_rate": 0.000475827482447342, + "loss": 3.2712, + "theoretical_loss": 4.380654769202683, + "tokens_seen": 190971904 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047581745235707125, + "loss": 2.9861, + "theoretical_loss": 4.380465348970995, + "tokens_seen": 191037440 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047580742226680043, + "loss": 3.2341, + "theoretical_loss": 4.380276011897003, + "tokens_seen": 191102976 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004757973921765296, + "loss": 3.5546, + "theoretical_loss": 4.380086757915698, + "tokens_seen": 191168512 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004757873620862588, + "loss": 3.2386, + "theoretical_loss": 4.379897586962148, + "tokens_seen": 191234048 + }, + { + "epoch": 0.06, + "learning_rate": 0.000475777331995988, + "loss": 3.1928, + "theoretical_loss": 4.379708498971494, + "tokens_seen": 191299584 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047576730190571716, + "loss": 3.245, + "theoretical_loss": 4.379519493878948, + "tokens_seen": 191365120 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004757572718154464, + "loss": 3.0325, + "theoretical_loss": 4.379330571619795, + "tokens_seen": 191430656 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004757472417251755, + "loss": 3.0047, + "theoretical_loss": 4.379141732129394, + "tokens_seen": 191496192 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047573721163490475, + "loss": 3.4233, + "theoretical_loss": 4.378952975343175, + "tokens_seen": 191561728 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004757271815446339, + "loss": 3.2114, + "theoretical_loss": 4.378764301196642, + "tokens_seen": 191627264 + }, + { + "epoch": 0.06, + "objective/train/docs_used": 119339, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.2042396068573, + "objective/train/theoretical_loss": 4.37857570962537, + "objective/train/tokens_used": 212152800, + "theoretical_loss": 4.37857570962537, + "tokens_seen": 191692800 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004757171514543631, + "loss": 3.0125, + "theoretical_loss": 4.37857570962537, + "tokens_seen": 191692800 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004757071213640923, + "loss": 3.1076, + "theoretical_loss": 4.378387200565006, + "tokens_seen": 191758336 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004756970912738215, + "loss": 3.2585, + "theoretical_loss": 4.378198773951272, + "tokens_seen": 191823872 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047568706118355066, + "loss": 3.1924, + "theoretical_loss": 4.378010429719957, + "tokens_seen": 191889408 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047567703109327984, + "loss": 3.1931, + "theoretical_loss": 4.377822167806928, + "tokens_seen": 191954944 + }, + { + "epoch": 0.06, + "learning_rate": 0.000475667001003009, + "loss": 3.2357, + "theoretical_loss": 4.377633988148117, + "tokens_seen": 192020480 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047565697091273826, + "loss": 3.2306, + "theoretical_loss": 4.377445890679534, + "tokens_seen": 192086016 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004756469408224674, + "loss": 3.0877, + "theoretical_loss": 4.377257875337257, + "tokens_seen": 192151552 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004756369107321966, + "loss": 3.1574, + "theoretical_loss": 4.377069942057436, + "tokens_seen": 192217088 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004756268806419258, + "loss": 3.1049, + "theoretical_loss": 4.376882090776293, + "tokens_seen": 192282624 + }, + { + "epoch": 0.06, + "learning_rate": 0.000475616850551655, + "loss": 3.4829, + "theoretical_loss": 4.376694321430121, + "tokens_seen": 192348160 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047560682046138416, + "loss": 3.1526, + "theoretical_loss": 4.376506633955286, + "tokens_seen": 192413696 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047559679037111334, + "loss": 3.2882, + "theoretical_loss": 4.376319028288219, + "tokens_seen": 192479232 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004755867602808425, + "loss": 3.5306, + "theoretical_loss": 4.37613150436543, + "tokens_seen": 192544768 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047557673019057176, + "loss": 3.1131, + "theoretical_loss": 4.375944062123496, + "tokens_seen": 192610304 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004755667001003009, + "loss": 3.1642, + "theoretical_loss": 4.375756701499063, + "tokens_seen": 192675840 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004755566700100301, + "loss": 3.2798, + "theoretical_loss": 4.3755694224288515, + "tokens_seen": 192741376 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047554663991975925, + "loss": 3.2001, + "theoretical_loss": 4.375382224849648, + "tokens_seen": 192806912 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004755366098294885, + "loss": 3.0408, + "theoretical_loss": 4.375195108698316, + "tokens_seen": 192872448 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047552657973921766, + "loss": 3.09, + "theoretical_loss": 4.375008073911781, + "tokens_seen": 192937984 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047551654964894685, + "loss": 3.0579, + "theoretical_loss": 4.374821120427047, + "tokens_seen": 193003520 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047550651955867603, + "loss": 3.5089, + "theoretical_loss": 4.374634248181182, + "tokens_seen": 193069056 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047549648946840526, + "loss": 3.3129, + "theoretical_loss": 4.3744474571113265, + "tokens_seen": 193134592 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004754864593781344, + "loss": 3.0337, + "theoretical_loss": 4.374260747154692, + "tokens_seen": 193200128 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004754764292878636, + "loss": 3.1201, + "theoretical_loss": 4.374074118248559, + "tokens_seen": 193265664 + }, + { + "epoch": 0.06, + "objective/train/docs_used": 120055, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.348609209060669, + "objective/train/theoretical_loss": 4.373887570330275, + "objective/train/tokens_used": 213791200, + "theoretical_loss": 4.373887570330275, + "tokens_seen": 193331200 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047546639919759275, + "loss": 3.3769, + "theoretical_loss": 4.373887570330275, + "tokens_seen": 193331200 + }, + { + "epoch": 0.06, + "learning_rate": 0.000475456369107322, + "loss": 3.2671, + "theoretical_loss": 4.373701103337263, + "tokens_seen": 193396736 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047544633901705117, + "loss": 3.1433, + "theoretical_loss": 4.373514717207009, + "tokens_seen": 193462272 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047543630892678035, + "loss": 3.1673, + "theoretical_loss": 4.373328411877073, + "tokens_seen": 193527808 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047542627883650953, + "loss": 3.1922, + "theoretical_loss": 4.373142187285083, + "tokens_seen": 193593344 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004754162487462387, + "loss": 3.2094, + "theoretical_loss": 4.372956043368736, + "tokens_seen": 193658880 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004754062186559679, + "loss": 3.3244, + "theoretical_loss": 4.372769980065797, + "tokens_seen": 193724416 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047539618856569713, + "loss": 3.3183, + "theoretical_loss": 4.372583997314104, + "tokens_seen": 193789952 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047538615847542625, + "loss": 3.3394, + "theoretical_loss": 4.372398095051559, + "tokens_seen": 193855488 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004753761283851555, + "loss": 3.28, + "theoretical_loss": 4.372212273216136, + "tokens_seen": 193921024 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004753660982948846, + "loss": 3.2422, + "theoretical_loss": 4.372026531745877, + "tokens_seen": 193986560 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047535606820461385, + "loss": 3.368, + "theoretical_loss": 4.371840870578891, + "tokens_seen": 194052096 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047534603811434303, + "loss": 3.3347, + "theoretical_loss": 4.37165528965336, + "tokens_seen": 194117632 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004753360080240722, + "loss": 3.3782, + "theoretical_loss": 4.371469788907529, + "tokens_seen": 194183168 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004753259779338014, + "loss": 3.2224, + "theoretical_loss": 4.371284368279714, + "tokens_seen": 194248704 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047531594784353063, + "loss": 3.1302, + "theoretical_loss": 4.3710990277083, + "tokens_seen": 194314240 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047530591775325976, + "loss": 3.161, + "theoretical_loss": 4.3709137671317375, + "tokens_seen": 194379776 + }, + { + "epoch": 0.06, + "learning_rate": 0.000475295887662989, + "loss": 3.2653, + "theoretical_loss": 4.37072858648855, + "tokens_seen": 194445312 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004752858575727181, + "loss": 3.1084, + "theoretical_loss": 4.370543485717322, + "tokens_seen": 194510848 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047527582748244736, + "loss": 3.2944, + "theoretical_loss": 4.370358464756713, + "tokens_seen": 194576384 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047526579739217654, + "loss": 3.1782, + "theoretical_loss": 4.370173523545443, + "tokens_seen": 194641920 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004752557673019057, + "loss": 3.2559, + "theoretical_loss": 4.3699886620223065, + "tokens_seen": 194707456 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004752457372116349, + "loss": 3.3592, + "theoretical_loss": 4.369803880126162, + "tokens_seen": 194772992 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004752357071213641, + "loss": 3.0414, + "theoretical_loss": 4.3696191777959354, + "tokens_seen": 194838528 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047522567703109326, + "loss": 3.061, + "theoretical_loss": 4.369434554970621, + "tokens_seen": 194904064 + }, + { + "epoch": 0.06, + "objective/train/docs_used": 121265, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.2084505558013916, + "objective/train/theoretical_loss": 4.369250011589279, + "objective/train/tokens_used": 215429600, + "theoretical_loss": 4.369250011589279, + "tokens_seen": 194969600 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004752156469408225, + "loss": 3.0417, + "theoretical_loss": 4.369250011589279, + "tokens_seen": 194969600 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004752056168505517, + "loss": 3.5044, + "theoretical_loss": 4.369065547591038, + "tokens_seen": 195035136 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047519558676028086, + "loss": 3.1708, + "theoretical_loss": 4.368881162915095, + "tokens_seen": 195100672 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047518555667001004, + "loss": 3.3253, + "theoretical_loss": 4.36869685750071, + "tokens_seen": 195166208 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004751755265797392, + "loss": 3.3375, + "theoretical_loss": 4.3685126312872145, + "tokens_seen": 195231744 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047516549648946846, + "loss": 3.0336, + "theoretical_loss": 4.368328484214002, + "tokens_seen": 195297280 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004751554663991976, + "loss": 3.3291, + "theoretical_loss": 4.368144416220538, + "tokens_seen": 195362816 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004751454363089268, + "loss": 3.3172, + "theoretical_loss": 4.3679604272463495, + "tokens_seen": 195428352 + }, + { + "epoch": 0.06, + "learning_rate": 0.000475135406218656, + "loss": 3.2975, + "theoretical_loss": 4.367776517231033, + "tokens_seen": 195493888 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004751253761283852, + "loss": 3.511, + "theoretical_loss": 4.367592686114252, + "tokens_seen": 195559424 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047511534603811436, + "loss": 3.1165, + "theoretical_loss": 4.367408933835733, + "tokens_seen": 195624960 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047510531594784354, + "loss": 3.3132, + "theoretical_loss": 4.367225260335272, + "tokens_seen": 195690496 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004750952858575727, + "loss": 3.2948, + "theoretical_loss": 4.36704166555273, + "tokens_seen": 195756032 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047508525576730196, + "loss": 3.2906, + "theoretical_loss": 4.366858149428032, + "tokens_seen": 195821568 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004750752256770311, + "loss": 3.089, + "theoretical_loss": 4.366674711901173, + "tokens_seen": 195887104 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004750651955867603, + "loss": 3.1609, + "theoretical_loss": 4.366491352912211, + "tokens_seen": 195952640 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047505516549648945, + "loss": 3.2893, + "theoretical_loss": 4.366308072401271, + "tokens_seen": 196018176 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004750451354062187, + "loss": 3.2118, + "theoretical_loss": 4.366124870308541, + "tokens_seen": 196083712 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047503510531594787, + "loss": 3.0859, + "theoretical_loss": 4.365941746574278, + "tokens_seen": 196149248 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047502507522567705, + "loss": 3.4212, + "theoretical_loss": 4.3657587011388035, + "tokens_seen": 196214784 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047501504513540623, + "loss": 3.0441, + "theoretical_loss": 4.365575733942503, + "tokens_seen": 196280320 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047500501504513546, + "loss": 3.3144, + "theoretical_loss": 4.365392844925829, + "tokens_seen": 196345856 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004749949849548646, + "loss": 3.2886, + "theoretical_loss": 4.365210034029298, + "tokens_seen": 196411392 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004749849548645938, + "loss": 3.201, + "theoretical_loss": 4.365027301193491, + "tokens_seen": 196476928 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047497492477432295, + "loss": 3.021, + "theoretical_loss": 4.364844646359056, + "tokens_seen": 196542464 + }, + { + "epoch": 0.06, + "objective/train/docs_used": 121711, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.4339725971221924, + "objective/train/theoretical_loss": 4.364662069466704, + "objective/train/tokens_used": 217068000, + "theoretical_loss": 4.364662069466704, + "tokens_seen": 196608000 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004749648946840522, + "loss": 3.2262, + "theoretical_loss": 4.364662069466704, + "tokens_seen": 196608000 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047495486459378137, + "loss": 3.3542, + "theoretical_loss": 4.364479570457213, + "tokens_seen": 196673536 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047494483450351055, + "loss": 3.2435, + "theoretical_loss": 4.364297149271423, + "tokens_seen": 196739072 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047493480441323973, + "loss": 3.0955, + "theoretical_loss": 4.3641148058502415, + "tokens_seen": 196804608 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004749247743229689, + "loss": 3.0556, + "theoretical_loss": 4.363932540134638, + "tokens_seen": 196870144 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004749147442326981, + "loss": 3.4229, + "theoretical_loss": 4.363750352065647, + "tokens_seen": 196935680 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047490471414242733, + "loss": 3.2229, + "theoretical_loss": 4.363568241584368, + "tokens_seen": 197001216 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047489468405215646, + "loss": 3.3384, + "theoretical_loss": 4.363386208631966, + "tokens_seen": 197066752 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004748846539618857, + "loss": 3.302, + "theoretical_loss": 4.363204253149667, + "tokens_seen": 197132288 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004748746238716148, + "loss": 2.9662, + "theoretical_loss": 4.3630223750787644, + "tokens_seen": 197197824 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047486459378134405, + "loss": 3.4634, + "theoretical_loss": 4.362840574360612, + "tokens_seen": 197263360 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047485456369107323, + "loss": 3.1746, + "theoretical_loss": 4.362658850936631, + "tokens_seen": 197328896 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004748445336008024, + "loss": 3.1391, + "theoretical_loss": 4.362477204748305, + "tokens_seen": 197394432 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004748345035105316, + "loss": 3.4169, + "theoretical_loss": 4.362295635737179, + "tokens_seen": 197459968 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047482447342026083, + "loss": 3.2826, + "theoretical_loss": 4.362114143844867, + "tokens_seen": 197525504 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047481444332998996, + "loss": 3.2621, + "theoretical_loss": 4.3619327290130405, + "tokens_seen": 197591040 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004748044132397192, + "loss": 3.4411, + "theoretical_loss": 4.3617513911834385, + "tokens_seen": 197656576 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004747943831494483, + "loss": 3.4049, + "theoretical_loss": 4.361570130297863, + "tokens_seen": 197722112 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047478435305917756, + "loss": 3.4119, + "theoretical_loss": 4.3613889462981765, + "tokens_seen": 197787648 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047477432296890674, + "loss": 3.1474, + "theoretical_loss": 4.361207839126308, + "tokens_seen": 197853184 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004747642928786359, + "loss": 3.3924, + "theoretical_loss": 4.361026808724247, + "tokens_seen": 197918720 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004747542627883651, + "loss": 3.1355, + "theoretical_loss": 4.360845855034049, + "tokens_seen": 197984256 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004747442326980943, + "loss": 2.969, + "theoretical_loss": 4.360664977997828, + "tokens_seen": 198049792 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047473420260782346, + "loss": 3.4454, + "theoretical_loss": 4.360484177557766, + "tokens_seen": 198115328 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004747241725175527, + "loss": 3.1571, + "theoretical_loss": 4.360303453656103, + "tokens_seen": 198180864 + }, + { + "epoch": 0.06, + "objective/train/docs_used": 122248, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.2993173599243164, + "objective/train/theoretical_loss": 4.360122806235145, + "objective/train/tokens_used": 218706400, + "theoretical_loss": 4.360122806235145, + "tokens_seen": 198246400 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004747141424272818, + "loss": 3.2691, + "theoretical_loss": 4.360122806235145, + "tokens_seen": 198246400 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047470411233701106, + "loss": 3.3734, + "theoretical_loss": 4.359942235237257, + "tokens_seen": 198311936 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004746940822467402, + "loss": 3.0638, + "theoretical_loss": 4.359761740604871, + "tokens_seen": 198377472 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004746840521564694, + "loss": 3.1199, + "theoretical_loss": 4.359581322280479, + "tokens_seen": 198443008 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004746740220661986, + "loss": 3.3615, + "theoretical_loss": 4.359400980206634, + "tokens_seen": 198508544 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004746639919759278, + "loss": 3.3912, + "theoretical_loss": 4.359220714325954, + "tokens_seen": 198574080 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047465396188565696, + "loss": 3.0547, + "theoretical_loss": 4.359040524581116, + "tokens_seen": 198639616 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004746439317953862, + "loss": 3.0155, + "theoretical_loss": 4.358860410914861, + "tokens_seen": 198705152 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004746339017051153, + "loss": 3.121, + "theoretical_loss": 4.358680373269993, + "tokens_seen": 198770688 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047462387161484456, + "loss": 3.0596, + "theoretical_loss": 4.358500411589375, + "tokens_seen": 198836224 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004746138415245737, + "loss": 3.0925, + "theoretical_loss": 4.358320525815934, + "tokens_seen": 198901760 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004746038114343029, + "loss": 3.1877, + "theoretical_loss": 4.358140715892658, + "tokens_seen": 198967296 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004745937813440321, + "loss": 3.2906, + "theoretical_loss": 4.357960981762595, + "tokens_seen": 199032832 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004745837512537613, + "loss": 3.4539, + "theoretical_loss": 4.357781323368857, + "tokens_seen": 199098368 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047457372116349047, + "loss": 3.0839, + "theoretical_loss": 4.357601740654617, + "tokens_seen": 199163904 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047456369107321965, + "loss": 3.3382, + "theoretical_loss": 4.357422233563106, + "tokens_seen": 199229440 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047455366098294883, + "loss": 3.1615, + "theoretical_loss": 4.357242802037623, + "tokens_seen": 199294976 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047454363089267807, + "loss": 3.1274, + "theoretical_loss": 4.35706344602152, + "tokens_seen": 199360512 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004745336008024072, + "loss": 3.1434, + "theoretical_loss": 4.356884165458217, + "tokens_seen": 199426048 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047452357071213643, + "loss": 3.1239, + "theoretical_loss": 4.356704960291191, + "tokens_seen": 199491584 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047451354062186555, + "loss": 3.2724, + "theoretical_loss": 4.35652583046398, + "tokens_seen": 199557120 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004745035105315948, + "loss": 3.1847, + "theoretical_loss": 4.356346775920185, + "tokens_seen": 199622656 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047449348044132397, + "loss": 3.2332, + "theoretical_loss": 4.356167796603467, + "tokens_seen": 199688192 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047448345035105315, + "loss": 3.1852, + "theoretical_loss": 4.355988892457546, + "tokens_seen": 199753728 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004744734202607824, + "loss": 2.9517, + "theoretical_loss": 4.355810063426204, + "tokens_seen": 199819264 + }, + { + "epoch": 0.06, + "objective/train/docs_used": 123553, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.2373132705688477, + "objective/train/theoretical_loss": 4.355631309453283, + "objective/train/tokens_used": 220344800, + "theoretical_loss": 4.355631309453283, + "tokens_seen": 199884800 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047446339017051157, + "loss": 3.1159, + "theoretical_loss": 4.355631309453283, + "tokens_seen": 199884800 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047445336008024075, + "loss": 3.1916, + "theoretical_loss": 4.355452630482685, + "tokens_seen": 199950336 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047444332998996993, + "loss": 3.1241, + "theoretical_loss": 4.355274026458375, + "tokens_seen": 200015872 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004744332998996991, + "loss": 3.1847, + "theoretical_loss": 4.355095497324373, + "tokens_seen": 200081408 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004744232698094283, + "loss": 3.076, + "theoretical_loss": 4.354917043024765, + "tokens_seen": 200146944 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047441323971915753, + "loss": 3.3778, + "theoretical_loss": 4.354738663503692, + "tokens_seen": 200212480 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047440320962888666, + "loss": 3.0268, + "theoretical_loss": 4.354560358705358, + "tokens_seen": 200278016 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004743931795386159, + "loss": 3.2272, + "theoretical_loss": 4.354382128574027, + "tokens_seen": 200343552 + }, + { + "epoch": 0.06, + "learning_rate": 0.000474383149448345, + "loss": 2.9804, + "theoretical_loss": 4.35420397305402, + "tokens_seen": 200409088 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047437311935807425, + "loss": 3.3283, + "theoretical_loss": 4.35402589208972, + "tokens_seen": 200474624 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047436308926780343, + "loss": 3.0955, + "theoretical_loss": 4.353847885625571, + "tokens_seen": 200540160 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004743530591775326, + "loss": 3.1665, + "theoretical_loss": 4.353669953606072, + "tokens_seen": 200605696 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004743430290872618, + "loss": 3.3236, + "theoretical_loss": 4.353492095975787, + "tokens_seen": 200671232 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047433299899699103, + "loss": 3.453, + "theoretical_loss": 4.353314312679333, + "tokens_seen": 200736768 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047432296890672016, + "loss": 3.1258, + "theoretical_loss": 4.353136603661392, + "tokens_seen": 200802304 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004743129388164494, + "loss": 3.3115, + "theoretical_loss": 4.352958968866704, + "tokens_seen": 200867840 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004743029087261785, + "loss": 3.3055, + "theoretical_loss": 4.352781408240065, + "tokens_seen": 200933376 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047429287863590776, + "loss": 3.2874, + "theoretical_loss": 4.352603921726334, + "tokens_seen": 200998912 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047428284854563694, + "loss": 3.3695, + "theoretical_loss": 4.352426509270425, + "tokens_seen": 201064448 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004742728184553661, + "loss": 3.2942, + "theoretical_loss": 4.352249170817315, + "tokens_seen": 201129984 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004742627883650953, + "loss": 3.5675, + "theoretical_loss": 4.352071906312037, + "tokens_seen": 201195520 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004742527582748245, + "loss": 3.3367, + "theoretical_loss": 4.351894715699684, + "tokens_seen": 201261056 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047424272818455366, + "loss": 3.2465, + "theoretical_loss": 4.351717598925406, + "tokens_seen": 201326592 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004742326980942829, + "loss": 3.3587, + "theoretical_loss": 4.351540555934414, + "tokens_seen": 201392128 + }, + { + "epoch": 0.06, + "learning_rate": 0.000474222668004012, + "loss": 3.3693, + "theoretical_loss": 4.351363586671976, + "tokens_seen": 201457664 + }, + { + "epoch": 0.06, + "objective/train/docs_used": 124253, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.022651433944702, + "objective/train/theoretical_loss": 4.351186691083417, + "objective/train/tokens_used": 221983200, + "theoretical_loss": 4.351186691083417, + "tokens_seen": 201523200 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047421263791374126, + "loss": 3.2881, + "theoretical_loss": 4.351186691083417, + "tokens_seen": 201523200 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004742026078234704, + "loss": 3.3651, + "theoretical_loss": 4.351009869114124, + "tokens_seen": 201588736 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004741925777331996, + "loss": 3.2988, + "theoretical_loss": 4.350833120709539, + "tokens_seen": 201654272 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004741825476429288, + "loss": 3.044, + "theoretical_loss": 4.350656445815164, + "tokens_seen": 201719808 + }, + { + "epoch": 0.06, + "learning_rate": 0.000474172517552658, + "loss": 3.0445, + "theoretical_loss": 4.350479844376557, + "tokens_seen": 201785344 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047416248746238716, + "loss": 3.1738, + "theoretical_loss": 4.350303316339337, + "tokens_seen": 201850880 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004741524573721164, + "loss": 3.1892, + "theoretical_loss": 4.350126861649178, + "tokens_seen": 201916416 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004741424272818455, + "loss": 3.162, + "theoretical_loss": 4.349950480251813, + "tokens_seen": 201981952 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047413239719157476, + "loss": 3.3348, + "theoretical_loss": 4.349774172093033, + "tokens_seen": 202047488 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004741223671013039, + "loss": 3.215, + "theoretical_loss": 4.349597937118687, + "tokens_seen": 202113024 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004741123370110331, + "loss": 3.2444, + "theoretical_loss": 4.3494217752746795, + "tokens_seen": 202178560 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004741023069207623, + "loss": 3.1566, + "theoretical_loss": 4.349245686506976, + "tokens_seen": 202244096 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004740922768304915, + "loss": 3.2119, + "theoretical_loss": 4.349069670761597, + "tokens_seen": 202309632 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047408224674022067, + "loss": 3.0822, + "theoretical_loss": 4.348893727984619, + "tokens_seen": 202375168 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047407221664994985, + "loss": 3.1091, + "theoretical_loss": 4.348717858122178, + "tokens_seen": 202440704 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047406218655967903, + "loss": 3.2199, + "theoretical_loss": 4.348542061120469, + "tokens_seen": 202506240 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047405215646940827, + "loss": 3.0202, + "theoretical_loss": 4.348366336925739, + "tokens_seen": 202571776 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004740421263791374, + "loss": 3.1683, + "theoretical_loss": 4.3481906854842975, + "tokens_seen": 202637312 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047403209628886663, + "loss": 3.1505, + "theoretical_loss": 4.348015106742507, + "tokens_seen": 202702848 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047402206619859575, + "loss": 3.2487, + "theoretical_loss": 4.347839600646786, + "tokens_seen": 202768384 + }, + { + "epoch": 0.06, + "learning_rate": 0.000474012036108325, + "loss": 3.1066, + "theoretical_loss": 4.347664167143615, + "tokens_seen": 202833920 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047400200601805417, + "loss": 3.4479, + "theoretical_loss": 4.347488806179528, + "tokens_seen": 202899456 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047399197592778335, + "loss": 3.3547, + "theoretical_loss": 4.347313517701114, + "tokens_seen": 202964992 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047398194583751253, + "loss": 3.3868, + "theoretical_loss": 4.347138301655021, + "tokens_seen": 203030528 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047397191574724177, + "loss": 3.1474, + "theoretical_loss": 4.346963157987954, + "tokens_seen": 203096064 + }, + { + "epoch": 0.06, + "objective/train/docs_used": 125838, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.3391339778900146, + "objective/train/theoretical_loss": 4.346788086646671, + "objective/train/tokens_used": 223621600, + "theoretical_loss": 4.346788086646671, + "tokens_seen": 203161600 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004739618856569709, + "loss": 3.1335, + "theoretical_loss": 4.346788086646671, + "tokens_seen": 203161600 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047395185556670013, + "loss": 3.3795, + "theoretical_loss": 4.346613087577991, + "tokens_seen": 203227136 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047394182547642926, + "loss": 3.3084, + "theoretical_loss": 4.346438160728785, + "tokens_seen": 203292672 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004739317953861585, + "loss": 3.2814, + "theoretical_loss": 4.346263306045983, + "tokens_seen": 203358208 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004739217652958877, + "loss": 3.2386, + "theoretical_loss": 4.346088523476569, + "tokens_seen": 203423744 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047391173520561686, + "loss": 3.3004, + "theoretical_loss": 4.345913812967584, + "tokens_seen": 203489280 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047390170511534604, + "loss": 3.1661, + "theoretical_loss": 4.345739174466127, + "tokens_seen": 203554816 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004738916750250752, + "loss": 3.0407, + "theoretical_loss": 4.345564607919348, + "tokens_seen": 203620352 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004738816449348044, + "loss": 3.218, + "theoretical_loss": 4.3453901132744575, + "tokens_seen": 203685888 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047387161484453363, + "loss": 3.2029, + "theoretical_loss": 4.345215690478719, + "tokens_seen": 203751424 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047386158475426276, + "loss": 3.1354, + "theoretical_loss": 4.345041339479453, + "tokens_seen": 203816960 + }, + { + "epoch": 0.06, + "learning_rate": 0.000473851554663992, + "loss": 3.1557, + "theoretical_loss": 4.3448670602240345, + "tokens_seen": 203882496 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004738415245737211, + "loss": 3.2834, + "theoretical_loss": 4.344692852659895, + "tokens_seen": 203948032 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047383149448345036, + "loss": 3.3518, + "theoretical_loss": 4.34451871673452, + "tokens_seen": 204013568 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047382146439317954, + "loss": 3.2446, + "theoretical_loss": 4.344344652395451, + "tokens_seen": 204079104 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004738114343029087, + "loss": 3.1596, + "theoretical_loss": 4.3441706595902865, + "tokens_seen": 204144640 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004738014042126379, + "loss": 3.1622, + "theoretical_loss": 4.343996738266677, + "tokens_seen": 204210176 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047379137412236714, + "loss": 3.264, + "theoretical_loss": 4.343822888372331, + "tokens_seen": 204275712 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047378134403209626, + "loss": 3.0935, + "theoretical_loss": 4.343649109855009, + "tokens_seen": 204341248 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004737713139418255, + "loss": 3.1181, + "theoretical_loss": 4.343475402662529, + "tokens_seen": 204406784 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004737612838515546, + "loss": 3.2375, + "theoretical_loss": 4.343301766742763, + "tokens_seen": 204472320 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047375125376128386, + "loss": 3.0604, + "theoretical_loss": 4.343128202043638, + "tokens_seen": 204537856 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047374122367101304, + "loss": 3.3545, + "theoretical_loss": 4.342954708513136, + "tokens_seen": 204603392 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004737311935807422, + "loss": 3.0508, + "theoretical_loss": 4.342781286099291, + "tokens_seen": 204668928 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047372116349047146, + "loss": 3.0236, + "theoretical_loss": 4.3426079347501965, + "tokens_seen": 204734464 + }, + { + "epoch": 0.06, + "objective/train/docs_used": 126453, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.832430362701416, + "objective/train/theoretical_loss": 4.342434654413995, + "objective/train/tokens_used": 225260000, + "theoretical_loss": 4.342434654413995, + "tokens_seen": 204800000 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004737111334002006, + "loss": 3.1363, + "theoretical_loss": 4.342434654413995, + "tokens_seen": 204800000 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004737011033099298, + "loss": 2.9929, + "theoretical_loss": 4.342261445038888, + "tokens_seen": 204865536 + }, + { + "epoch": 0.06, + "learning_rate": 0.000473691073219659, + "loss": 3.1121, + "theoretical_loss": 4.342088306573128, + "tokens_seen": 204931072 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004736810431293882, + "loss": 3.4405, + "theoretical_loss": 4.341915238965026, + "tokens_seen": 204996608 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047367101303911736, + "loss": 3.299, + "theoretical_loss": 4.34174224216294, + "tokens_seen": 205062144 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004736609829488466, + "loss": 3.2396, + "theoretical_loss": 4.34156931611529, + "tokens_seen": 205127680 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004736509528585757, + "loss": 3.1534, + "theoretical_loss": 4.341396460770547, + "tokens_seen": 205193216 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047364092276830496, + "loss": 3.0966, + "theoretical_loss": 4.341223676077232, + "tokens_seen": 205258752 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004736308926780341, + "loss": 3.3243, + "theoretical_loss": 4.341050961983926, + "tokens_seen": 205324288 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004736208625877633, + "loss": 3.3229, + "theoretical_loss": 4.340878318439261, + "tokens_seen": 205389824 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004736108324974925, + "loss": 3.1547, + "theoretical_loss": 4.340705745391922, + "tokens_seen": 205455360 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004736008024072217, + "loss": 3.2286, + "theoretical_loss": 4.3405332427906504, + "tokens_seen": 205520896 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047359077231695087, + "loss": 3.0644, + "theoretical_loss": 4.340360810584238, + "tokens_seen": 205586432 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047358074222668005, + "loss": 3.4655, + "theoretical_loss": 4.340188448721532, + "tokens_seen": 205651968 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047357071213640923, + "loss": 3.3134, + "theoretical_loss": 4.3400161571514335, + "tokens_seen": 205717504 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047356068204613847, + "loss": 2.9307, + "theoretical_loss": 4.339843935822895, + "tokens_seen": 205783040 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004735506519558676, + "loss": 3.2801, + "theoretical_loss": 4.339671784684923, + "tokens_seen": 205848576 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047354062186559683, + "loss": 3.3474, + "theoretical_loss": 4.339499703686579, + "tokens_seen": 205914112 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047353059177532595, + "loss": 3.4003, + "theoretical_loss": 4.339327692776977, + "tokens_seen": 205979648 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004735205616850552, + "loss": 3.0436, + "theoretical_loss": 4.339155751905282, + "tokens_seen": 206045184 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047351053159478437, + "loss": 3.1985, + "theoretical_loss": 4.338983881020713, + "tokens_seen": 206110720 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047350050150451355, + "loss": 3.2646, + "theoretical_loss": 4.338812080072545, + "tokens_seen": 206176256 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047349047141424273, + "loss": 3.0178, + "theoretical_loss": 4.338640349010101, + "tokens_seen": 206241792 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047348044132397197, + "loss": 3.2123, + "theoretical_loss": 4.3384686877827585, + "tokens_seen": 206307328 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004734704112337011, + "loss": 2.8933, + "theoretical_loss": 4.338297096339951, + "tokens_seen": 206372864 + }, + { + "epoch": 0.06, + "objective/train/docs_used": 127848, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.208740472793579, + "objective/train/theoretical_loss": 4.33812557463116, + "objective/train/tokens_used": 226898400, + "theoretical_loss": 4.33812557463116, + "tokens_seen": 206438400 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047346038114343033, + "loss": 3.1921, + "theoretical_loss": 4.33812557463116, + "tokens_seen": 206438400 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047345035105315946, + "loss": 3.1779, + "theoretical_loss": 4.3379541226059235, + "tokens_seen": 206503936 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004734403209628887, + "loss": 3.5187, + "theoretical_loss": 4.337782740213827, + "tokens_seen": 206569472 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004734302908726179, + "loss": 3.1696, + "theoretical_loss": 4.337611427404514, + "tokens_seen": 206635008 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047342026078234706, + "loss": 3.2575, + "theoretical_loss": 4.337440184127679, + "tokens_seen": 206700544 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047341023069207624, + "loss": 2.896, + "theoretical_loss": 4.337269010333065, + "tokens_seen": 206766080 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004734002006018054, + "loss": 3.1782, + "theoretical_loss": 4.337097905970471, + "tokens_seen": 206831616 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004733901705115346, + "loss": 3.2665, + "theoretical_loss": 4.336926870989748, + "tokens_seen": 206897152 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047338014042126383, + "loss": 3.2986, + "theoretical_loss": 4.336755905340797, + "tokens_seen": 206962688 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047337011033099296, + "loss": 3.2157, + "theoretical_loss": 4.336585008973573, + "tokens_seen": 207028224 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004733600802407222, + "loss": 3.3271, + "theoretical_loss": 4.336414181838082, + "tokens_seen": 207093760 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004733500501504513, + "loss": 3.2844, + "theoretical_loss": 4.336243423884382, + "tokens_seen": 207159296 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047334002006018056, + "loss": 3.2186, + "theoretical_loss": 4.336072735062583, + "tokens_seen": 207224832 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047332998996990974, + "loss": 2.9173, + "theoretical_loss": 4.335902115322847, + "tokens_seen": 207290368 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004733199598796389, + "loss": 3.3235, + "theoretical_loss": 4.335731564615387, + "tokens_seen": 207355904 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004733099297893681, + "loss": 3.3614, + "theoretical_loss": 4.335561082890468, + "tokens_seen": 207421440 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047329989969909734, + "loss": 3.2793, + "theoretical_loss": 4.335390670098407, + "tokens_seen": 207486976 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047328986960882646, + "loss": 2.9718, + "theoretical_loss": 4.335220326189571, + "tokens_seen": 207552512 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004732798395185557, + "loss": 3.0992, + "theoretical_loss": 4.335050051114379, + "tokens_seen": 207618048 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004732698094282848, + "loss": 3.2734, + "theoretical_loss": 4.334879844823304, + "tokens_seen": 207683584 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047325977933801406, + "loss": 3.2116, + "theoretical_loss": 4.334709707266865, + "tokens_seen": 207749120 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047324974924774324, + "loss": 3.0666, + "theoretical_loss": 4.334539638395636, + "tokens_seen": 207814656 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004732397191574724, + "loss": 3.1246, + "theoretical_loss": 4.334369638160242, + "tokens_seen": 207880192 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004732296890672016, + "loss": 3.2958, + "theoretical_loss": 4.334199706511358, + "tokens_seen": 207945728 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004732196589769308, + "loss": 2.988, + "theoretical_loss": 4.334029843399709, + "tokens_seen": 208011264 + }, + { + "epoch": 0.06, + "objective/train/docs_used": 128338, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.3175644874572754, + "objective/train/theoretical_loss": 4.333860048776074, + "objective/train/tokens_used": 228536800, + "theoretical_loss": 4.333860048776074, + "tokens_seen": 208076800 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047320962888665997, + "loss": 3.3092, + "theoretical_loss": 4.333860048776074, + "tokens_seen": 208076800 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004731995987963892, + "loss": 3.121, + "theoretical_loss": 4.33369032259128, + "tokens_seen": 208142336 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047318956870611833, + "loss": 3.3423, + "theoretical_loss": 4.333520664796206, + "tokens_seen": 208207872 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047317953861584757, + "loss": 3.2551, + "theoretical_loss": 4.33335107534178, + "tokens_seen": 208273408 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047316950852557675, + "loss": 3.1864, + "theoretical_loss": 4.333181554178985, + "tokens_seen": 208338944 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047315947843530593, + "loss": 3.0798, + "theoretical_loss": 4.3330121012588485, + "tokens_seen": 208404480 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004731494483450351, + "loss": 3.009, + "theoretical_loss": 4.332842716532454, + "tokens_seen": 208470016 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004731394182547643, + "loss": 3.1507, + "theoretical_loss": 4.332673399950932, + "tokens_seen": 208535552 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047312938816449347, + "loss": 3.0901, + "theoretical_loss": 4.332504151465464, + "tokens_seen": 208601088 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004731193580742227, + "loss": 3.3096, + "theoretical_loss": 4.332334971027284, + "tokens_seen": 208666624 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047310932798395183, + "loss": 3.0521, + "theoretical_loss": 4.332165858587672, + "tokens_seen": 208732160 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047309929789368107, + "loss": 2.8736, + "theoretical_loss": 4.331996814097963, + "tokens_seen": 208797696 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004730892678034102, + "loss": 3.355, + "theoretical_loss": 4.331827837509538, + "tokens_seen": 208863232 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047307923771313943, + "loss": 3.3136, + "theoretical_loss": 4.331658928773831, + "tokens_seen": 208928768 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004730692076228686, + "loss": 3.1275, + "theoretical_loss": 4.331490087842324, + "tokens_seen": 208994304 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004730591775325978, + "loss": 3.2808, + "theoretical_loss": 4.33132131466655, + "tokens_seen": 209059840 + }, + { + "epoch": 0.06, + "learning_rate": 0.000473049147442327, + "loss": 3.3499, + "theoretical_loss": 4.3311526091980905, + "tokens_seen": 209125376 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047303911735205615, + "loss": 3.2705, + "theoretical_loss": 4.330983971388578, + "tokens_seen": 209190912 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047302908726178534, + "loss": 3.2789, + "theoretical_loss": 4.330815401189695, + "tokens_seen": 209256448 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047301905717151457, + "loss": 3.2926, + "theoretical_loss": 4.330646898553173, + "tokens_seen": 209321984 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004730090270812437, + "loss": 3.3805, + "theoretical_loss": 4.330478463430792, + "tokens_seen": 209387520 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047299899699097293, + "loss": 3.3599, + "theoretical_loss": 4.330310095774383, + "tokens_seen": 209453056 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004729889669007021, + "loss": 3.0193, + "theoretical_loss": 4.330141795535828, + "tokens_seen": 209518592 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004729789368104313, + "loss": 3.2521, + "theoretical_loss": 4.329973562667053, + "tokens_seen": 209584128 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047296890672016053, + "loss": 3.0979, + "theoretical_loss": 4.3298053971200385, + "tokens_seen": 209649664 + }, + { + "epoch": 0.06, + "objective/train/docs_used": 129483, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.659421443939209, + "objective/train/theoretical_loss": 4.329637298846812, + "objective/train/tokens_used": 230175200, + "theoretical_loss": 4.329637298846812, + "tokens_seen": 209715200 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047295887662988966, + "loss": 3.4368, + "theoretical_loss": 4.329637298846812, + "tokens_seen": 209715200 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004729488465396189, + "loss": 3.0696, + "theoretical_loss": 4.329469267799451, + "tokens_seen": 209780736 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004729388164493481, + "loss": 3.2288, + "theoretical_loss": 4.32930130393008, + "tokens_seen": 209846272 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047292878635907726, + "loss": 3.3223, + "theoretical_loss": 4.329133407190876, + "tokens_seen": 209911808 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047291875626880644, + "loss": 3.2795, + "theoretical_loss": 4.3289655775340625, + "tokens_seen": 209977344 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004729087261785356, + "loss": 3.3969, + "theoretical_loss": 4.328797814911912, + "tokens_seen": 210042880 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004728986960882648, + "loss": 2.9174, + "theoretical_loss": 4.328630119276747, + "tokens_seen": 210108416 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047288866599799403, + "loss": 3.1225, + "theoretical_loss": 4.328462490580938, + "tokens_seen": 210173952 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047287863590772316, + "loss": 3.0563, + "theoretical_loss": 4.328294928776903, + "tokens_seen": 210239488 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004728686058174524, + "loss": 3.1847, + "theoretical_loss": 4.328127433817112, + "tokens_seen": 210305024 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004728585757271815, + "loss": 3.23, + "theoretical_loss": 4.327960005654081, + "tokens_seen": 210370560 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047284854563691076, + "loss": 3.149, + "theoretical_loss": 4.327792644240374, + "tokens_seen": 210436096 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047283851554663994, + "loss": 3.1957, + "theoretical_loss": 4.327625349528605, + "tokens_seen": 210501632 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004728284854563691, + "loss": 3.1135, + "theoretical_loss": 4.327458121471436, + "tokens_seen": 210567168 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004728184553660983, + "loss": 3.2036, + "theoretical_loss": 4.3272909600215765, + "tokens_seen": 210632704 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047280842527582754, + "loss": 3.1921, + "theoretical_loss": 4.327123865131786, + "tokens_seen": 210698240 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047279839518555666, + "loss": 3.3045, + "theoretical_loss": 4.326956836754871, + "tokens_seen": 210763776 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004727883650952859, + "loss": 3.3457, + "theoretical_loss": 4.326789874843685, + "tokens_seen": 210829312 + }, + { + "epoch": 0.06, + "learning_rate": 0.000472778335005015, + "loss": 3.3425, + "theoretical_loss": 4.326622979351132, + "tokens_seen": 210894848 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047276830491474426, + "loss": 3.3856, + "theoretical_loss": 4.326456150230163, + "tokens_seen": 210960384 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047275827482447344, + "loss": 3.3666, + "theoretical_loss": 4.326289387433776, + "tokens_seen": 211025920 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004727482447342026, + "loss": 2.9358, + "theoretical_loss": 4.326122690915017, + "tokens_seen": 211091456 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004727382146439318, + "loss": 3.1626, + "theoretical_loss": 4.325956060626982, + "tokens_seen": 211156992 + }, + { + "epoch": 0.06, + "learning_rate": 0.000472728184553661, + "loss": 3.1366, + "theoretical_loss": 4.325789496522812, + "tokens_seen": 211222528 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047271815446339017, + "loss": 3.2883, + "theoretical_loss": 4.325622998555697, + "tokens_seen": 211288064 + }, + { + "epoch": 0.06, + "objective/train/docs_used": 130263, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.5621697902679443, + "objective/train/theoretical_loss": 4.3254565666788745, + "objective/train/tokens_used": 231813600, + "theoretical_loss": 4.3254565666788745, + "tokens_seen": 211353600 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004727081243731194, + "loss": 3.4905, + "theoretical_loss": 4.3254565666788745, + "tokens_seen": 211353600 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047269809428284853, + "loss": 3.3763, + "theoretical_loss": 4.325290200845629, + "tokens_seen": 211419136 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047268806419257777, + "loss": 3.2924, + "theoretical_loss": 4.3251239010092934, + "tokens_seen": 211484672 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047267803410230695, + "loss": 3.3776, + "theoretical_loss": 4.324957667123249, + "tokens_seen": 211550208 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047266800401203613, + "loss": 3.3272, + "theoretical_loss": 4.32479149914092, + "tokens_seen": 211615744 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004726579739217653, + "loss": 3.2763, + "theoretical_loss": 4.324625397015783, + "tokens_seen": 211681280 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004726479438314945, + "loss": 3.1934, + "theoretical_loss": 4.3244593607013595, + "tokens_seen": 211746816 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047263791374122367, + "loss": 3.2791, + "theoretical_loss": 4.324293390151218, + "tokens_seen": 211812352 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004726278836509529, + "loss": 3.2578, + "theoretical_loss": 4.324127485318975, + "tokens_seen": 211877888 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047261785356068203, + "loss": 3.0399, + "theoretical_loss": 4.323961646158294, + "tokens_seen": 211943424 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047260782347041127, + "loss": 3.2574, + "theoretical_loss": 4.323795872622884, + "tokens_seen": 212008960 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004725977933801404, + "loss": 3.2161, + "theoretical_loss": 4.323630164666502, + "tokens_seen": 212074496 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047258776328986963, + "loss": 3.1183, + "theoretical_loss": 4.323464522242954, + "tokens_seen": 212140032 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004725777331995988, + "loss": 3.1137, + "theoretical_loss": 4.323298945306089, + "tokens_seen": 212205568 + }, + { + "epoch": 0.06, + "learning_rate": 0.000472567703109328, + "loss": 3.0619, + "theoretical_loss": 4.3231334338098035, + "tokens_seen": 212271104 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004725576730190572, + "loss": 3.4629, + "theoretical_loss": 4.322967987708043, + "tokens_seen": 212336640 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047254764292878636, + "loss": 3.179, + "theoretical_loss": 4.322802606954798, + "tokens_seen": 212402176 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047253761283851554, + "loss": 3.3582, + "theoretical_loss": 4.322637291504106, + "tokens_seen": 212467712 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047252758274824477, + "loss": 3.4527, + "theoretical_loss": 4.32247204131005, + "tokens_seen": 212533248 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004725175526579739, + "loss": 3.4012, + "theoretical_loss": 4.322306856326761, + "tokens_seen": 212598784 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047250752256770313, + "loss": 3.1334, + "theoretical_loss": 4.322141736508415, + "tokens_seen": 212664320 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004724974924774323, + "loss": 3.1282, + "theoretical_loss": 4.321976681809236, + "tokens_seen": 212729856 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004724874623871615, + "loss": 3.3332, + "theoretical_loss": 4.321811692183491, + "tokens_seen": 212795392 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004724774322968907, + "loss": 3.2601, + "theoretical_loss": 4.321646767585497, + "tokens_seen": 212860928 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047246740220661986, + "loss": 3.4236, + "theoretical_loss": 4.3214819079696145, + "tokens_seen": 212926464 + }, + { + "epoch": 0.06, + "objective/train/docs_used": 131630, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.21213960647583, + "objective/train/theoretical_loss": 4.321317113290252, + "objective/train/tokens_used": 233452000, + "theoretical_loss": 4.321317113290252, + "tokens_seen": 212992000 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047245737211634904, + "loss": 2.8933, + "theoretical_loss": 4.321317113290252, + "tokens_seen": 212992000 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004724473420260783, + "loss": 3.0931, + "theoretical_loss": 4.321152383501863, + "tokens_seen": 213057536 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004724373119358074, + "loss": 3.1421, + "theoretical_loss": 4.320987718558945, + "tokens_seen": 213123072 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047242728184553664, + "loss": 3.178, + "theoretical_loss": 4.320823118416046, + "tokens_seen": 213188608 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047241725175526576, + "loss": 3.1124, + "theoretical_loss": 4.320658583027755, + "tokens_seen": 213254144 + }, + { + "epoch": 0.06, + "learning_rate": 0.000472407221664995, + "loss": 3.1339, + "theoretical_loss": 4.32049411234871, + "tokens_seen": 213319680 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004723971915747242, + "loss": 2.7465, + "theoretical_loss": 4.3203297063335935, + "tokens_seen": 213385216 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047238716148445336, + "loss": 3.2522, + "theoretical_loss": 4.320165364937134, + "tokens_seen": 213450752 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047237713139418254, + "loss": 3.225, + "theoretical_loss": 4.320001088114105, + "tokens_seen": 213516288 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004723671013039117, + "loss": 3.0445, + "theoretical_loss": 4.319836875819325, + "tokens_seen": 213581824 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004723570712136409, + "loss": 3.3149, + "theoretical_loss": 4.31967272800766, + "tokens_seen": 213647360 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047234704112337014, + "loss": 3.1508, + "theoretical_loss": 4.319508644634021, + "tokens_seen": 213712896 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047233701103309927, + "loss": 3.1067, + "theoretical_loss": 4.319344625653361, + "tokens_seen": 213778432 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004723269809428285, + "loss": 3.1235, + "theoretical_loss": 4.319180671020684, + "tokens_seen": 213843968 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004723169508525577, + "loss": 3.3633, + "theoretical_loss": 4.319016780691033, + "tokens_seen": 213909504 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047230692076228686, + "loss": 3.247, + "theoretical_loss": 4.318852954619501, + "tokens_seen": 213975040 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047229689067201605, + "loss": 3.1027, + "theoretical_loss": 4.318689192761225, + "tokens_seen": 214040576 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004722868605817452, + "loss": 3.1884, + "theoretical_loss": 4.318525495071385, + "tokens_seen": 214106112 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004722768304914744, + "loss": 3.2689, + "theoretical_loss": 4.318361861505207, + "tokens_seen": 214171648 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047226680040120364, + "loss": 3.0063, + "theoretical_loss": 4.318198292017964, + "tokens_seen": 214237184 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047225677031093277, + "loss": 3.2365, + "theoretical_loss": 4.318034786564971, + "tokens_seen": 214302720 + }, + { + "epoch": 0.06, + "learning_rate": 0.000472246740220662, + "loss": 3.2503, + "theoretical_loss": 4.31787134510159, + "tokens_seen": 214368256 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047223671013039113, + "loss": 3.296, + "theoretical_loss": 4.3177079675832255, + "tokens_seen": 214433792 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047222668004012037, + "loss": 3.2151, + "theoretical_loss": 4.317544653965329, + "tokens_seen": 214499328 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004722166499498496, + "loss": 3.2008, + "theoretical_loss": 4.3173814042033944, + "tokens_seen": 214564864 + }, + { + "epoch": 0.07, + "objective/train/docs_used": 132228, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.3430302143096924, + "objective/train/theoretical_loss": 4.317218218252963, + "objective/train/tokens_used": 235090400, + "theoretical_loss": 4.317218218252963, + "tokens_seen": 214630400 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047220661985957873, + "loss": 3.1547, + "theoretical_loss": 4.317218218252963, + "tokens_seen": 214630400 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047219658976930797, + "loss": 2.8774, + "theoretical_loss": 4.317055096069618, + "tokens_seen": 214695936 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047218655967903715, + "loss": 3.0886, + "theoretical_loss": 4.316892037608987, + "tokens_seen": 214761472 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047217652958876633, + "loss": 3.2875, + "theoretical_loss": 4.316729042826745, + "tokens_seen": 214827008 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004721664994984955, + "loss": 3.0392, + "theoretical_loss": 4.316566111678609, + "tokens_seen": 214892544 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004721564694082247, + "loss": 3.2117, + "theoretical_loss": 4.316403244120339, + "tokens_seen": 214958080 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047214643931795387, + "loss": 3.3941, + "theoretical_loss": 4.3162404401077445, + "tokens_seen": 215023616 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004721364092276831, + "loss": 3.1674, + "theoretical_loss": 4.316077699596671, + "tokens_seen": 215089152 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047212637913741223, + "loss": 3.1696, + "theoretical_loss": 4.315915022543016, + "tokens_seen": 215154688 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047211634904714147, + "loss": 3.1937, + "theoretical_loss": 4.315752408902716, + "tokens_seen": 215220224 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004721063189568706, + "loss": 3.142, + "theoretical_loss": 4.315589858631755, + "tokens_seen": 215285760 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047209628886659983, + "loss": 3.2296, + "theoretical_loss": 4.315427371686157, + "tokens_seen": 215351296 + }, + { + "epoch": 0.07, + "learning_rate": 0.000472086258776329, + "loss": 3.1915, + "theoretical_loss": 4.315264948021994, + "tokens_seen": 215416832 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004720762286860582, + "loss": 3.3199, + "theoretical_loss": 4.315102587595379, + "tokens_seen": 215482368 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004720661985957874, + "loss": 3.3334, + "theoretical_loss": 4.31494029036247, + "tokens_seen": 215547904 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047205616850551656, + "loss": 3.2662, + "theoretical_loss": 4.314778056279468, + "tokens_seen": 215613440 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047204613841524574, + "loss": 3.1994, + "theoretical_loss": 4.314615885302619, + "tokens_seen": 215678976 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047203610832497497, + "loss": 2.8855, + "theoretical_loss": 4.314453777388209, + "tokens_seen": 215744512 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004720260782347041, + "loss": 3.2391, + "theoretical_loss": 4.314291732492573, + "tokens_seen": 215810048 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047201604814443333, + "loss": 3.2191, + "theoretical_loss": 4.314129750572087, + "tokens_seen": 215875584 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004720060180541625, + "loss": 3.2209, + "theoretical_loss": 4.3139678315831675, + "tokens_seen": 215941120 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004719959879638917, + "loss": 3.4565, + "theoretical_loss": 4.313805975482278, + "tokens_seen": 216006656 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004719859578736209, + "loss": 3.2291, + "theoretical_loss": 4.313644182225926, + "tokens_seen": 216072192 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047197592778335006, + "loss": 3.0005, + "theoretical_loss": 4.313482451770659, + "tokens_seen": 216137728 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047196589769307924, + "loss": 3.3178, + "theoretical_loss": 4.313320784073069, + "tokens_seen": 216203264 + }, + { + "epoch": 0.07, + "objective/train/docs_used": 133529, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.712810516357422, + "objective/train/theoretical_loss": 4.3131591790897925, + "objective/train/tokens_used": 236728800, + "theoretical_loss": 4.3131591790897925, + "tokens_seen": 216268800 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004719558676028085, + "loss": 3.4463, + "theoretical_loss": 4.3131591790897925, + "tokens_seen": 216268800 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004719458375125376, + "loss": 3.1367, + "theoretical_loss": 4.3129976367775065, + "tokens_seen": 216334336 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047193580742226684, + "loss": 3.2712, + "theoretical_loss": 4.312836157092934, + "tokens_seen": 216399872 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047192577733199596, + "loss": 2.9203, + "theoretical_loss": 4.312674739992839, + "tokens_seen": 216465408 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004719157472417252, + "loss": 3.1237, + "theoretical_loss": 4.31251338543403, + "tokens_seen": 216530944 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004719057171514544, + "loss": 3.0423, + "theoretical_loss": 4.312352093373354, + "tokens_seen": 216596480 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047189568706118356, + "loss": 3.1801, + "theoretical_loss": 4.312190863767708, + "tokens_seen": 216662016 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047188565697091274, + "loss": 3.2321, + "theoretical_loss": 4.312029696574027, + "tokens_seen": 216727552 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004718756268806419, + "loss": 3.4227, + "theoretical_loss": 4.311868591749287, + "tokens_seen": 216793088 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004718655967903711, + "loss": 3.218, + "theoretical_loss": 4.311707549250514, + "tokens_seen": 216858624 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047185556670010034, + "loss": 2.9338, + "theoretical_loss": 4.311546569034767, + "tokens_seen": 216924160 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047184553660982947, + "loss": 3.0379, + "theoretical_loss": 4.311385651059155, + "tokens_seen": 216989696 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004718355065195587, + "loss": 3.4089, + "theoretical_loss": 4.311224795280825, + "tokens_seen": 217055232 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004718254764292879, + "loss": 3.0412, + "theoretical_loss": 4.3110640016569715, + "tokens_seen": 217120768 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047181544633901706, + "loss": 3.3065, + "theoretical_loss": 4.310903270144825, + "tokens_seen": 217186304 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047180541624874625, + "loss": 3.2586, + "theoretical_loss": 4.310742600701664, + "tokens_seen": 217251840 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004717953861584754, + "loss": 3.2692, + "theoretical_loss": 4.310581993284805, + "tokens_seen": 217317376 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004717853560682046, + "loss": 3.1189, + "theoretical_loss": 4.310421447851609, + "tokens_seen": 217382912 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047177532597793384, + "loss": 3.3316, + "theoretical_loss": 4.310260964359479, + "tokens_seen": 217448448 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047176529588766297, + "loss": 2.9637, + "theoretical_loss": 4.310100542765858, + "tokens_seen": 217513984 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004717552657973922, + "loss": 3.3276, + "theoretical_loss": 4.309940183028236, + "tokens_seen": 217579520 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047174523570712133, + "loss": 3.1227, + "theoretical_loss": 4.309779885104139, + "tokens_seen": 217645056 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047173520561685057, + "loss": 3.161, + "theoretical_loss": 4.309619648951139, + "tokens_seen": 217710592 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047172517552657975, + "loss": 3.3967, + "theoretical_loss": 4.3094594745268475, + "tokens_seen": 217776128 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047171514543630893, + "loss": 3.0518, + "theoretical_loss": 4.30929936178892, + "tokens_seen": 217841664 + }, + { + "epoch": 0.07, + "objective/train/docs_used": 134228, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.4435603618621826, + "objective/train/theoretical_loss": 4.309139310695053, + "objective/train/tokens_used": 238367200, + "theoretical_loss": 4.309139310695053, + "tokens_seen": 217907200 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004717051153460381, + "loss": 3.1092, + "theoretical_loss": 4.309139310695053, + "tokens_seen": 217907200 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047169508525576735, + "loss": 3.2399, + "theoretical_loss": 4.308979321202983, + "tokens_seen": 217972736 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004716850551654965, + "loss": 3.3268, + "theoretical_loss": 4.308819393270491, + "tokens_seen": 218038272 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004716750250752257, + "loss": 3.0694, + "theoretical_loss": 4.308659526855396, + "tokens_seen": 218103808 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047166499498495484, + "loss": 3.3126, + "theoretical_loss": 4.308499721915563, + "tokens_seen": 218169344 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047165496489468407, + "loss": 3.0966, + "theoretical_loss": 4.308339978408897, + "tokens_seen": 218234880 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047164493480441325, + "loss": 3.2024, + "theoretical_loss": 4.308180296293341, + "tokens_seen": 218300416 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047163490471414243, + "loss": 3.0746, + "theoretical_loss": 4.308020675526883, + "tokens_seen": 218365952 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004716248746238716, + "loss": 3.4115, + "theoretical_loss": 4.307861116067554, + "tokens_seen": 218431488 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004716148445336008, + "loss": 3.0598, + "theoretical_loss": 4.30770161787342, + "tokens_seen": 218497024 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047160481444333, + "loss": 3.1639, + "theoretical_loss": 4.307542180902594, + "tokens_seen": 218562560 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004715947843530592, + "loss": 3.3503, + "theoretical_loss": 4.307382805113228, + "tokens_seen": 218628096 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047158475426278834, + "loss": 3.0688, + "theoretical_loss": 4.307223490463516, + "tokens_seen": 218693632 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004715747241725176, + "loss": 3.1234, + "theoretical_loss": 4.307064236911692, + "tokens_seen": 218759168 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004715646940822467, + "loss": 3.3338, + "theoretical_loss": 4.30690504441603, + "tokens_seen": 218824704 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047155466399197594, + "loss": 3.085, + "theoretical_loss": 4.306745912934849, + "tokens_seen": 218890240 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004715446339017051, + "loss": 3.1988, + "theoretical_loss": 4.306586842426504, + "tokens_seen": 218955776 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004715346038114343, + "loss": 3.236, + "theoretical_loss": 4.306427832849394, + "tokens_seen": 219021312 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004715245737211635, + "loss": 3.2309, + "theoretical_loss": 4.306268884161959, + "tokens_seen": 219086848 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004715145436308927, + "loss": 3.3585, + "theoretical_loss": 4.306109996322679, + "tokens_seen": 219152384 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047150451354062184, + "loss": 3.0296, + "theoretical_loss": 4.305951169290073, + "tokens_seen": 219217920 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004714944834503511, + "loss": 3.2751, + "theoretical_loss": 4.305792403022703, + "tokens_seen": 219283456 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004714844533600802, + "loss": 3.0347, + "theoretical_loss": 4.305633697479171, + "tokens_seen": 219348992 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047147442326980944, + "loss": 3.2133, + "theoretical_loss": 4.305475052618119, + "tokens_seen": 219414528 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004714643931795387, + "loss": 3.2636, + "theoretical_loss": 4.30531646839823, + "tokens_seen": 219480064 + }, + { + "epoch": 0.07, + "objective/train/docs_used": 135676, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.7242591381073, + "objective/train/theoretical_loss": 4.305157944778228, + "objective/train/tokens_used": 240005600, + "theoretical_loss": 4.305157944778228, + "tokens_seen": 219545600 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004714543630892678, + "loss": 3.033, + "theoretical_loss": 4.305157944778228, + "tokens_seen": 219545600 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047144433299899704, + "loss": 3.2922, + "theoretical_loss": 4.304999481716876, + "tokens_seen": 219611136 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047143430290872616, + "loss": 3.3129, + "theoretical_loss": 4.304841079172979, + "tokens_seen": 219676672 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004714242728184554, + "loss": 3.1466, + "theoretical_loss": 4.30468273710538, + "tokens_seen": 219742208 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004714142427281846, + "loss": 3.2923, + "theoretical_loss": 4.304524455472965, + "tokens_seen": 219807744 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047140421263791376, + "loss": 3.1803, + "theoretical_loss": 4.304366234234659, + "tokens_seen": 219873280 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047139418254764294, + "loss": 3.0837, + "theoretical_loss": 4.304208073349426, + "tokens_seen": 219938816 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004713841524573721, + "loss": 3.1949, + "theoretical_loss": 4.304049972776271, + "tokens_seen": 220004352 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004713741223671013, + "loss": 3.2699, + "theoretical_loss": 4.30389193247424, + "tokens_seen": 220069888 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047136409227683054, + "loss": 3.0824, + "theoretical_loss": 4.303733952402419, + "tokens_seen": 220135424 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047135406218655967, + "loss": 3.3143, + "theoretical_loss": 4.303576032519931, + "tokens_seen": 220200960 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004713440320962889, + "loss": 3.2935, + "theoretical_loss": 4.303418172785943, + "tokens_seen": 220266496 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004713340020060181, + "loss": 3.4022, + "theoretical_loss": 4.303260373159659, + "tokens_seen": 220332032 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047132397191574726, + "loss": 3.2172, + "theoretical_loss": 4.303102633600322, + "tokens_seen": 220397568 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047131394182547645, + "loss": 3.2858, + "theoretical_loss": 4.30294495406722, + "tokens_seen": 220463104 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047130391173520563, + "loss": 3.3566, + "theoretical_loss": 4.3027873345196745, + "tokens_seen": 220528640 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004712938816449348, + "loss": 3.4721, + "theoretical_loss": 4.302629774917049, + "tokens_seen": 220594176 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047128385155466404, + "loss": 3.1524, + "theoretical_loss": 4.302472275218748, + "tokens_seen": 220659712 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047127382146439317, + "loss": 3.1871, + "theoretical_loss": 4.302314835384214, + "tokens_seen": 220725248 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004712637913741224, + "loss": 3.2289, + "theoretical_loss": 4.30215745537293, + "tokens_seen": 220790784 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047125376128385153, + "loss": 3.1151, + "theoretical_loss": 4.302000135144416, + "tokens_seen": 220856320 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047124373119358077, + "loss": 3.2239, + "theoretical_loss": 4.301842874658235, + "tokens_seen": 220921856 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047123370110330995, + "loss": 3.262, + "theoretical_loss": 4.301685673873987, + "tokens_seen": 220987392 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047122367101303913, + "loss": 3.3815, + "theoretical_loss": 4.301528532751312, + "tokens_seen": 221052928 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004712136409227683, + "loss": 3.3069, + "theoretical_loss": 4.301371451249888, + "tokens_seen": 221118464 + }, + { + "epoch": 0.07, + "objective/train/docs_used": 136371, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.688603639602661, + "objective/train/theoretical_loss": 4.301214429329433, + "objective/train/tokens_used": 241644000, + "theoretical_loss": 4.301214429329433, + "tokens_seen": 221184000 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047120361083249755, + "loss": 3.1277, + "theoretical_loss": 4.301214429329433, + "tokens_seen": 221184000 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004711935807422267, + "loss": 3.3198, + "theoretical_loss": 4.301057466949707, + "tokens_seen": 221249536 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004711835506519559, + "loss": 3.1314, + "theoretical_loss": 4.300900564070504, + "tokens_seen": 221315072 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047117352056168504, + "loss": 3.1535, + "theoretical_loss": 4.30074372065166, + "tokens_seen": 221380608 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047116349047141427, + "loss": 3.1082, + "theoretical_loss": 4.300586936653049, + "tokens_seen": 221446144 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047115346038114345, + "loss": 3.1216, + "theoretical_loss": 4.300430212034587, + "tokens_seen": 221511680 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047114343029087263, + "loss": 3.1814, + "theoretical_loss": 4.300273546756223, + "tokens_seen": 221577216 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004711334002006018, + "loss": 3.1909, + "theoretical_loss": 4.300116940777951, + "tokens_seen": 221642752 + }, + { + "epoch": 0.07, + "learning_rate": 0.000471123370110331, + "loss": 3.1137, + "theoretical_loss": 4.299960394059799, + "tokens_seen": 221708288 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004711133400200602, + "loss": 3.4328, + "theoretical_loss": 4.299803906561835, + "tokens_seen": 221773824 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004711033099297894, + "loss": 3.2642, + "theoretical_loss": 4.29964747824417, + "tokens_seen": 221839360 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047109327983951854, + "loss": 2.96, + "theoretical_loss": 4.299491109066947, + "tokens_seen": 221904896 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004710832497492478, + "loss": 3.3055, + "theoretical_loss": 4.299334798990351, + "tokens_seen": 221970432 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004710732196589769, + "loss": 3.2613, + "theoretical_loss": 4.2991785479746065, + "tokens_seen": 222035968 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047106318956870614, + "loss": 3.0836, + "theoretical_loss": 4.299022355979974, + "tokens_seen": 222101504 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004710531594784353, + "loss": 3.1318, + "theoretical_loss": 4.298866222966755, + "tokens_seen": 222167040 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004710431293881645, + "loss": 3.1645, + "theoretical_loss": 4.298710148895286, + "tokens_seen": 222232576 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004710330992978937, + "loss": 3.1871, + "theoretical_loss": 4.298554133725946, + "tokens_seen": 222298112 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004710230692076229, + "loss": 3.1399, + "theoretical_loss": 4.298398177419149, + "tokens_seen": 222363648 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047101303911735204, + "loss": 3.1229, + "theoretical_loss": 4.298242279935349, + "tokens_seen": 222429184 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004710030090270813, + "loss": 3.0416, + "theoretical_loss": 4.2980864412350375, + "tokens_seen": 222494720 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004709929789368104, + "loss": 3.0125, + "theoretical_loss": 4.297930661278745, + "tokens_seen": 222560256 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047098294884653964, + "loss": 3.3504, + "theoretical_loss": 4.297774940027038, + "tokens_seen": 222625792 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004709729187562688, + "loss": 3.1926, + "theoretical_loss": 4.297619277440523, + "tokens_seen": 222691328 + }, + { + "epoch": 0.07, + "learning_rate": 0.000470962888665998, + "loss": 3.1824, + "theoretical_loss": 4.297463673479846, + "tokens_seen": 222756864 + }, + { + "epoch": 0.07, + "objective/train/docs_used": 137650, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.839879035949707, + "objective/train/theoretical_loss": 4.297308128105687, + "objective/train/tokens_used": 243282400, + "theoretical_loss": 4.297308128105687, + "tokens_seen": 222822400 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004709528585757272, + "loss": 2.9774, + "theoretical_loss": 4.297308128105687, + "tokens_seen": 222822400 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047094282848545636, + "loss": 3.065, + "theoretical_loss": 4.297152641278767, + "tokens_seen": 222887936 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047093279839518555, + "loss": 3.2057, + "theoretical_loss": 4.296997212959842, + "tokens_seen": 222953472 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004709227683049148, + "loss": 3.4341, + "theoretical_loss": 4.296841843109711, + "tokens_seen": 223019008 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004709127382146439, + "loss": 3.2099, + "theoretical_loss": 4.296686531689204, + "tokens_seen": 223084544 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047090270812437314, + "loss": 3.2885, + "theoretical_loss": 4.296531278659193, + "tokens_seen": 223150080 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047089267803410227, + "loss": 2.9526, + "theoretical_loss": 4.296376083980589, + "tokens_seen": 223215616 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004708826479438315, + "loss": 3.2951, + "theoretical_loss": 4.296220947614337, + "tokens_seen": 223281152 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004708726178535607, + "loss": 3.4286, + "theoretical_loss": 4.296065869521421, + "tokens_seen": 223346688 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047086258776328987, + "loss": 3.2966, + "theoretical_loss": 4.295910849662862, + "tokens_seen": 223412224 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047085255767301905, + "loss": 3.0904, + "theoretical_loss": 4.2957558879997215, + "tokens_seen": 223477760 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004708425275827483, + "loss": 3.0458, + "theoretical_loss": 4.295600984493093, + "tokens_seen": 223543296 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004708324974924774, + "loss": 3.2588, + "theoretical_loss": 4.295446139104112, + "tokens_seen": 223608832 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047082246740220665, + "loss": 3.14, + "theoretical_loss": 4.295291351793951, + "tokens_seen": 223674368 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004708124373119358, + "loss": 3.1552, + "theoretical_loss": 4.295136622523817, + "tokens_seen": 223739904 + }, + { + "epoch": 0.07, + "learning_rate": 0.000470802407221665, + "loss": 3.1462, + "theoretical_loss": 4.294981951254956, + "tokens_seen": 223805440 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004707923771313942, + "loss": 3.2273, + "theoretical_loss": 4.294827337948651, + "tokens_seen": 223870976 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047078234704112337, + "loss": 3.3522, + "theoretical_loss": 4.294672782566224, + "tokens_seen": 223936512 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047077231695085255, + "loss": 3.2264, + "theoretical_loss": 4.29451828506903, + "tokens_seen": 224002048 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047076228686058173, + "loss": 3.0935, + "theoretical_loss": 4.294363845418465, + "tokens_seen": 224067584 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004707522567703109, + "loss": 3.1355, + "theoretical_loss": 4.29420946357596, + "tokens_seen": 224133120 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047074222668004015, + "loss": 3.2008, + "theoretical_loss": 4.294055139502985, + "tokens_seen": 224198656 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004707321965897693, + "loss": 3.2019, + "theoretical_loss": 4.293900873161043, + "tokens_seen": 224264192 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004707221664994985, + "loss": 3.1207, + "theoretical_loss": 4.293746664511678, + "tokens_seen": 224329728 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047071213640922775, + "loss": 3.2845, + "theoretical_loss": 4.293592513516469, + "tokens_seen": 224395264 + }, + { + "epoch": 0.07, + "objective/train/docs_used": 138458, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.224550724029541, + "objective/train/theoretical_loss": 4.293438420137031, + "objective/train/tokens_used": 244920800, + "theoretical_loss": 4.293438420137031, + "tokens_seen": 224460800 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004707021063189569, + "loss": 3.3098, + "theoretical_loss": 4.293438420137031, + "tokens_seen": 224460800 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004706920762286861, + "loss": 3.0378, + "theoretical_loss": 4.293284384335017, + "tokens_seen": 224526336 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047068204613841524, + "loss": 3.1923, + "theoretical_loss": 4.293130406072118, + "tokens_seen": 224591872 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047067201604814447, + "loss": 3.0745, + "theoretical_loss": 4.292976485310057, + "tokens_seen": 224657408 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047066198595787365, + "loss": 2.9871, + "theoretical_loss": 4.2928226220106005, + "tokens_seen": 224722944 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047065195586760283, + "loss": 3.2895, + "theoretical_loss": 4.292668816135545, + "tokens_seen": 224788480 + }, + { + "epoch": 0.07, + "learning_rate": 0.000470641925777332, + "loss": 3.1025, + "theoretical_loss": 4.292515067646727, + "tokens_seen": 224854016 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004706318956870612, + "loss": 3.1386, + "theoretical_loss": 4.29236137650602, + "tokens_seen": 224919552 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004706218655967904, + "loss": 3.3601, + "theoretical_loss": 4.2922077426753305, + "tokens_seen": 224985088 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004706118355065196, + "loss": 3.1949, + "theoretical_loss": 4.292054166116605, + "tokens_seen": 225050624 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047060180541624874, + "loss": 3.3203, + "theoretical_loss": 4.291900646791825, + "tokens_seen": 225116160 + }, + { + "epoch": 0.07, + "learning_rate": 0.000470591775325978, + "loss": 3.3352, + "theoretical_loss": 4.2917471846630075, + "tokens_seen": 225181696 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004705817452357071, + "loss": 3.2554, + "theoretical_loss": 4.291593779692207, + "tokens_seen": 225247232 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047057171514543634, + "loss": 3.4009, + "theoretical_loss": 4.291440431841513, + "tokens_seen": 225312768 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004705616850551655, + "loss": 3.174, + "theoretical_loss": 4.291287141073053, + "tokens_seen": 225378304 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004705516549648947, + "loss": 3.1361, + "theoretical_loss": 4.291133907348989, + "tokens_seen": 225443840 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004705416248746239, + "loss": 3.2254, + "theoretical_loss": 4.29098073063152, + "tokens_seen": 225509376 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004705315947843531, + "loss": 3.0334, + "theoretical_loss": 4.29082761088288, + "tokens_seen": 225574912 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047052156469408224, + "loss": 3.1637, + "theoretical_loss": 4.290674548065338, + "tokens_seen": 225640448 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004705115346038115, + "loss": 3.2989, + "theoretical_loss": 4.290521542141203, + "tokens_seen": 225705984 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004705015045135406, + "loss": 3.0939, + "theoretical_loss": 4.290368593072817, + "tokens_seen": 225771520 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047049147442326984, + "loss": 3.182, + "theoretical_loss": 4.290215700822556, + "tokens_seen": 225837056 + }, + { + "epoch": 0.07, + "learning_rate": 0.000470481444332999, + "loss": 3.3886, + "theoretical_loss": 4.290062865352837, + "tokens_seen": 225902592 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004704714142427282, + "loss": 2.8248, + "theoretical_loss": 4.289910086626108, + "tokens_seen": 225968128 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004704613841524574, + "loss": 3.3256, + "theoretical_loss": 4.289757364604855, + "tokens_seen": 226033664 + }, + { + "epoch": 0.07, + "objective/train/docs_used": 139641, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.3089590072631836, + "objective/train/theoretical_loss": 4.2896046992515995, + "objective/train/tokens_used": 246559200, + "theoretical_loss": 4.2896046992515995, + "tokens_seen": 226099200 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047045135406218656, + "loss": 3.1431, + "theoretical_loss": 4.2896046992515995, + "tokens_seen": 226099200 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047044132397191575, + "loss": 2.8636, + "theoretical_loss": 4.289452090528897, + "tokens_seen": 226164736 + }, + { + "epoch": 0.07, + "learning_rate": 0.000470431293881645, + "loss": 3.0843, + "theoretical_loss": 4.289299538399341, + "tokens_seen": 226230272 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004704212637913741, + "loss": 3.1064, + "theoretical_loss": 4.28914704282556, + "tokens_seen": 226295808 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047041123370110334, + "loss": 3.4158, + "theoretical_loss": 4.288994603770215, + "tokens_seen": 226361344 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047040120361083247, + "loss": 3.0657, + "theoretical_loss": 4.288842221196007, + "tokens_seen": 226426880 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004703911735205617, + "loss": 3.0784, + "theoretical_loss": 4.28868989506567, + "tokens_seen": 226492416 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004703811434302909, + "loss": 3.2676, + "theoretical_loss": 4.288537625341974, + "tokens_seen": 226557952 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047037111334002007, + "loss": 3.1534, + "theoretical_loss": 4.288385411987722, + "tokens_seen": 226623488 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047036108324974925, + "loss": 3.2291, + "theoretical_loss": 4.288233254965755, + "tokens_seen": 226689024 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004703510531594785, + "loss": 3.031, + "theoretical_loss": 4.2880811542389505, + "tokens_seen": 226754560 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004703410230692076, + "loss": 3.3898, + "theoretical_loss": 4.287929109770217, + "tokens_seen": 226820096 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047033099297893685, + "loss": 3.2323, + "theoretical_loss": 4.287777121522501, + "tokens_seen": 226885632 + }, + { + "epoch": 0.07, + "learning_rate": 0.000470320962888666, + "loss": 3.3985, + "theoretical_loss": 4.287625189458781, + "tokens_seen": 226951168 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004703109327983952, + "loss": 3.2782, + "theoretical_loss": 4.287473313542077, + "tokens_seen": 227016704 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004703009027081244, + "loss": 3.2877, + "theoretical_loss": 4.287321493735438, + "tokens_seen": 227082240 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047029087261785357, + "loss": 3.37, + "theoretical_loss": 4.287169730001949, + "tokens_seen": 227147776 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047028084252758275, + "loss": 3.2498, + "theoretical_loss": 4.287018022304733, + "tokens_seen": 227213312 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047027081243731193, + "loss": 3.2654, + "theoretical_loss": 4.286866370606943, + "tokens_seen": 227278848 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004702607823470411, + "loss": 3.2819, + "theoretical_loss": 4.286714774871772, + "tokens_seen": 227344384 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047025075225677035, + "loss": 3.1087, + "theoretical_loss": 4.286563235062444, + "tokens_seen": 227409920 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004702407221664995, + "loss": 3.0908, + "theoretical_loss": 4.28641175114222, + "tokens_seen": 227475456 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004702306920762287, + "loss": 3.2981, + "theoretical_loss": 4.286260323074394, + "tokens_seen": 227540992 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047022066198595784, + "loss": 3.1672, + "theoretical_loss": 4.286108950822296, + "tokens_seen": 227606528 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004702106318956871, + "loss": 3.1738, + "theoretical_loss": 4.285957634349289, + "tokens_seen": 227672064 + }, + { + "epoch": 0.07, + "objective/train/docs_used": 140376, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.3152148723602295, + "objective/train/theoretical_loss": 4.285806373618774, + "objective/train/tokens_used": 248197600, + "theoretical_loss": 4.285806373618774, + "tokens_seen": 227737600 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047020060180541626, + "loss": 3.0675, + "theoretical_loss": 4.285806373618774, + "tokens_seen": 227737600 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047019057171514544, + "loss": 3.388, + "theoretical_loss": 4.285655168594182, + "tokens_seen": 227803136 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004701805416248746, + "loss": 3.1444, + "theoretical_loss": 4.285504019238982, + "tokens_seen": 227868672 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047017051153460385, + "loss": 3.258, + "theoretical_loss": 4.285352925516676, + "tokens_seen": 227934208 + }, + { + "epoch": 0.07, + "learning_rate": 0.000470160481444333, + "loss": 3.1719, + "theoretical_loss": 4.2852018873908, + "tokens_seen": 227999744 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004701504513540622, + "loss": 3.2246, + "theoretical_loss": 4.285050904824925, + "tokens_seen": 228065280 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047014042126379134, + "loss": 3.3831, + "theoretical_loss": 4.284899977782658, + "tokens_seen": 228130816 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004701303911735206, + "loss": 3.2275, + "theoretical_loss": 4.284749106227636, + "tokens_seen": 228196352 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047012036108324976, + "loss": 3.1204, + "theoretical_loss": 4.284598290123535, + "tokens_seen": 228261888 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047011033099297894, + "loss": 3.288, + "theoretical_loss": 4.284447529434061, + "tokens_seen": 228327424 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004701003009027081, + "loss": 3.0952, + "theoretical_loss": 4.284296824122959, + "tokens_seen": 228392960 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004700902708124373, + "loss": 3.3164, + "theoretical_loss": 4.284146174154003, + "tokens_seen": 228458496 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004700802407221665, + "loss": 3.0692, + "theoretical_loss": 4.283995579491004, + "tokens_seen": 228524032 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004700702106318957, + "loss": 3.252, + "theoretical_loss": 4.283845040097807, + "tokens_seen": 228589568 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047006018054162484, + "loss": 3.1247, + "theoretical_loss": 4.28369455593829, + "tokens_seen": 228655104 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004700501504513541, + "loss": 3.1848, + "theoretical_loss": 4.2835441269763646, + "tokens_seen": 228720640 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047004012036108326, + "loss": 3.0464, + "theoretical_loss": 4.283393753175979, + "tokens_seen": 228786176 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047003009027081244, + "loss": 3.104, + "theoretical_loss": 4.283243434501112, + "tokens_seen": 228851712 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004700200601805416, + "loss": 3.2732, + "theoretical_loss": 4.283093170915778, + "tokens_seen": 228917248 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004700100300902708, + "loss": 3.297, + "theoretical_loss": 4.282942962384023, + "tokens_seen": 228982784 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047, + "loss": 3.1767, + "theoretical_loss": 4.282792808869932, + "tokens_seen": 229048320 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004699899699097292, + "loss": 3.1809, + "theoretical_loss": 4.282642710337618, + "tokens_seen": 229113856 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046997993981945835, + "loss": 3.0385, + "theoretical_loss": 4.28249266675123, + "tokens_seen": 229179392 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004699699097291876, + "loss": 3.3402, + "theoretical_loss": 4.282342678074951, + "tokens_seen": 229244928 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046995987963891676, + "loss": 3.0843, + "theoretical_loss": 4.2821927442729955, + "tokens_seen": 229310464 + }, + { + "epoch": 0.07, + "objective/train/docs_used": 141693, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.2135519981384277, + "objective/train/theoretical_loss": 4.282042865309616, + "objective/train/tokens_used": 249836000, + "theoretical_loss": 4.282042865309616, + "tokens_seen": 229376000 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046994984954864595, + "loss": 3.2484, + "theoretical_loss": 4.282042865309616, + "tokens_seen": 229376000 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004699398194583752, + "loss": 3.2872, + "theoretical_loss": 4.281893041149093, + "tokens_seen": 229441536 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004699297893681043, + "loss": 3.2066, + "theoretical_loss": 4.2817432717557455, + "tokens_seen": 229507072 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046991975927783354, + "loss": 3.3088, + "theoretical_loss": 4.28159355709392, + "tokens_seen": 229572608 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046990972918756267, + "loss": 3.0353, + "theoretical_loss": 4.281443897128004, + "tokens_seen": 229638144 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004698996990972919, + "loss": 3.391, + "theoretical_loss": 4.2812942918224115, + "tokens_seen": 229703680 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004698896690070211, + "loss": 3.2215, + "theoretical_loss": 4.281144741141593, + "tokens_seen": 229769216 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046987963891675027, + "loss": 3.547, + "theoretical_loss": 4.280995245050032, + "tokens_seen": 229834752 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046986960882647945, + "loss": 3.0622, + "theoretical_loss": 4.2808458035122445, + "tokens_seen": 229900288 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004698595787362087, + "loss": 3.001, + "theoretical_loss": 4.2806964164927805, + "tokens_seen": 229965824 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004698495486459378, + "loss": 3.5159, + "theoretical_loss": 4.280547083956224, + "tokens_seen": 230031360 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046983951855566705, + "loss": 3.0416, + "theoretical_loss": 4.280397805867188, + "tokens_seen": 230096896 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004698294884653962, + "loss": 3.0223, + "theoretical_loss": 4.280248582190324, + "tokens_seen": 230162432 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004698194583751254, + "loss": 3.3188, + "theoretical_loss": 4.280099412890312, + "tokens_seen": 230227968 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004698094282848546, + "loss": 3.3351, + "theoretical_loss": 4.279950297931869, + "tokens_seen": 230293504 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046979939819458377, + "loss": 3.2325, + "theoretical_loss": 4.27980123727974, + "tokens_seen": 230359040 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046978936810431295, + "loss": 3.002, + "theoretical_loss": 4.279652230898709, + "tokens_seen": 230424576 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046977933801404213, + "loss": 3.0345, + "theoretical_loss": 4.279503278753586, + "tokens_seen": 230490112 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004697693079237713, + "loss": 2.9771, + "theoretical_loss": 4.27935438080922, + "tokens_seen": 230555648 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046975927783350055, + "loss": 3.3242, + "theoretical_loss": 4.27920553703049, + "tokens_seen": 230621184 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004697492477432297, + "loss": 3.5207, + "theoretical_loss": 4.279056747382306, + "tokens_seen": 230686720 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004697392176529589, + "loss": 3.1005, + "theoretical_loss": 4.278908011829613, + "tokens_seen": 230752256 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046972918756268804, + "loss": 3.341, + "theoretical_loss": 4.27875933033739, + "tokens_seen": 230817792 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004697191574724173, + "loss": 3.0874, + "theoretical_loss": 4.278610702870646, + "tokens_seen": 230883328 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046970912738214646, + "loss": 3.0862, + "theoretical_loss": 4.278462129394423, + "tokens_seen": 230948864 + }, + { + "epoch": 0.07, + "objective/train/docs_used": 142407, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.8267900943756104, + "objective/train/theoretical_loss": 4.278313609873795, + "objective/train/tokens_used": 251474400, + "theoretical_loss": 4.278313609873795, + "tokens_seen": 231014400 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046969909729187564, + "loss": 3.025, + "theoretical_loss": 4.278313609873795, + "tokens_seen": 231014400 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004696890672016048, + "loss": 3.0411, + "theoretical_loss": 4.278165144273871, + "tokens_seen": 231079936 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046967903711133405, + "loss": 3.044, + "theoretical_loss": 4.27801673255979, + "tokens_seen": 231145472 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004696690070210632, + "loss": 3.113, + "theoretical_loss": 4.277868374696725, + "tokens_seen": 231211008 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004696589769307924, + "loss": 2.9389, + "theoretical_loss": 4.277720070649879, + "tokens_seen": 231276544 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046964894684052154, + "loss": 2.9033, + "theoretical_loss": 4.277571820384491, + "tokens_seen": 231342080 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004696389167502508, + "loss": 3.1625, + "theoretical_loss": 4.277423623865829, + "tokens_seen": 231407616 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046962888665997996, + "loss": 3.0961, + "theoretical_loss": 4.277275481059195, + "tokens_seen": 231473152 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046961885656970914, + "loss": 2.9911, + "theoretical_loss": 4.2771273919299215, + "tokens_seen": 231538688 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004696088264794383, + "loss": 3.0094, + "theoretical_loss": 4.276979356443377, + "tokens_seen": 231604224 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004695987963891675, + "loss": 3.0933, + "theoretical_loss": 4.276831374564957, + "tokens_seen": 231669760 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004695887662988967, + "loss": 3.0499, + "theoretical_loss": 4.276683446260093, + "tokens_seen": 231735296 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004695787362086259, + "loss": 3.0474, + "theoretical_loss": 4.276535571494247, + "tokens_seen": 231800832 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046956870611835505, + "loss": 3.066, + "theoretical_loss": 4.276387750232913, + "tokens_seen": 231866368 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004695586760280843, + "loss": 3.178, + "theoretical_loss": 4.276239982441617, + "tokens_seen": 231931904 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046954864593781346, + "loss": 3.1004, + "theoretical_loss": 4.276092268085918, + "tokens_seen": 231997440 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046953861584754264, + "loss": 2.9467, + "theoretical_loss": 4.275944607131406, + "tokens_seen": 232062976 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004695285857572718, + "loss": 3.2609, + "theoretical_loss": 4.275796999543703, + "tokens_seen": 232128512 + }, + { + "epoch": 0.07, + "learning_rate": 0.000469518555667001, + "loss": 3.2627, + "theoretical_loss": 4.275649445288461, + "tokens_seen": 232194048 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004695085255767302, + "loss": 3.211, + "theoretical_loss": 4.275501944331367, + "tokens_seen": 232259584 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004694984954864594, + "loss": 2.7646, + "theoretical_loss": 4.275354496638139, + "tokens_seen": 232325120 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046948846539618855, + "loss": 3.2126, + "theoretical_loss": 4.275207102174525, + "tokens_seen": 232390656 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004694784353059178, + "loss": 3.257, + "theoretical_loss": 4.275059760906305, + "tokens_seen": 232456192 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004694684052156469, + "loss": 2.8919, + "theoretical_loss": 4.2749124727992935, + "tokens_seen": 232521728 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046945837512537615, + "loss": 3.059, + "theoretical_loss": 4.274765237819333, + "tokens_seen": 232587264 + }, + { + "epoch": 0.07, + "objective/train/docs_used": 143093, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.1605868339538574, + "objective/train/theoretical_loss": 4.274618055932298, + "objective/train/tokens_used": 253112800, + "theoretical_loss": 4.274618055932298, + "tokens_seen": 232652800 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004694483450351053, + "loss": 3.2542, + "theoretical_loss": 4.274618055932298, + "tokens_seen": 232652800 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004694383149448345, + "loss": 3.2164, + "theoretical_loss": 4.2744709271040975, + "tokens_seen": 232718336 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004694282848545637, + "loss": 3.0795, + "theoretical_loss": 4.27432385130067, + "tokens_seen": 232783872 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046941825476429287, + "loss": 3.1837, + "theoretical_loss": 4.274176828487984, + "tokens_seen": 232849408 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046940822467402205, + "loss": 2.9049, + "theoretical_loss": 4.2740298586320415, + "tokens_seen": 232914944 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004693981945837513, + "loss": 3.3365, + "theoretical_loss": 4.273882941698876, + "tokens_seen": 232980480 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004693881644934804, + "loss": 2.8665, + "theoretical_loss": 4.27373607765455, + "tokens_seen": 233046016 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046937813440320965, + "loss": 3.2377, + "theoretical_loss": 4.2735892664651605, + "tokens_seen": 233111552 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046936810431293883, + "loss": 2.9248, + "theoretical_loss": 4.273442508096833, + "tokens_seen": 233177088 + }, + { + "epoch": 0.07, + "learning_rate": 0.000469358074222668, + "loss": 3.0984, + "theoretical_loss": 4.273295802515726, + "tokens_seen": 233242624 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004693480441323972, + "loss": 3.0799, + "theoretical_loss": 4.273149149688028, + "tokens_seen": 233308160 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004693380140421264, + "loss": 3.2695, + "theoretical_loss": 4.27300254957996, + "tokens_seen": 233373696 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046932798395185555, + "loss": 3.2082, + "theoretical_loss": 4.272856002157772, + "tokens_seen": 233439232 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004693179538615848, + "loss": 3.2353, + "theoretical_loss": 4.272709507387748, + "tokens_seen": 233504768 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004693079237713139, + "loss": 2.8927, + "theoretical_loss": 4.2725630652362, + "tokens_seen": 233570304 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046929789368104315, + "loss": 2.929, + "theoretical_loss": 4.272416675669473, + "tokens_seen": 233635840 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004692878635907723, + "loss": 3.2172, + "theoretical_loss": 4.272270338653942, + "tokens_seen": 233701376 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004692778335005015, + "loss": 3.1903, + "theoretical_loss": 4.272124054156014, + "tokens_seen": 233766912 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004692678034102307, + "loss": 3.39, + "theoretical_loss": 4.271977822142125, + "tokens_seen": 233832448 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004692577733199599, + "loss": 3.1742, + "theoretical_loss": 4.271831642578745, + "tokens_seen": 233897984 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046924774322968906, + "loss": 3.0853, + "theoretical_loss": 4.27168551543237, + "tokens_seen": 233963520 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046923771313941824, + "loss": 3.1935, + "theoretical_loss": 4.271539440669532, + "tokens_seen": 234029056 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004692276830491474, + "loss": 3.0012, + "theoretical_loss": 4.27139341825679, + "tokens_seen": 234094592 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046921765295887666, + "loss": 3.2946, + "theoretical_loss": 4.271247448160736, + "tokens_seen": 234160128 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046920762286860584, + "loss": 3.4353, + "theoretical_loss": 4.27110153034799, + "tokens_seen": 234225664 + }, + { + "epoch": 0.07, + "objective/train/docs_used": 144323, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.2330448627471924, + "objective/train/theoretical_loss": 4.270955664785207, + "objective/train/tokens_used": 254751200, + "theoretical_loss": 4.270955664785207, + "tokens_seen": 234291200 + }, + { + "epoch": 0.07, + "learning_rate": 0.000469197592778335, + "loss": 3.3123, + "theoretical_loss": 4.270955664785207, + "tokens_seen": 234291200 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046918756268806425, + "loss": 3.2566, + "theoretical_loss": 4.2708098514390676, + "tokens_seen": 234356736 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004691775325977934, + "loss": 3.1794, + "theoretical_loss": 4.270664090276286, + "tokens_seen": 234422272 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004691675025075226, + "loss": 3.2189, + "theoretical_loss": 4.2705183812636065, + "tokens_seen": 234487808 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046915747241725174, + "loss": 3.531, + "theoretical_loss": 4.270372724367803, + "tokens_seen": 234553344 + }, + { + "epoch": 0.07, + "learning_rate": 0.000469147442326981, + "loss": 3.2723, + "theoretical_loss": 4.270227119555681, + "tokens_seen": 234618880 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046913741223671016, + "loss": 3.1009, + "theoretical_loss": 4.270081566794076, + "tokens_seen": 234684416 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046912738214643934, + "loss": 3.1586, + "theoretical_loss": 4.269936066049852, + "tokens_seen": 234749952 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004691173520561685, + "loss": 3.1741, + "theoretical_loss": 4.269790617289907, + "tokens_seen": 234815488 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004691073219658977, + "loss": 3.3225, + "theoretical_loss": 4.269645220481166, + "tokens_seen": 234881024 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004690972918756269, + "loss": 3.0885, + "theoretical_loss": 4.269499875590587, + "tokens_seen": 234946560 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004690872617853561, + "loss": 3.1221, + "theoretical_loss": 4.269354582585156, + "tokens_seen": 235012096 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046907723169508525, + "loss": 3.2268, + "theoretical_loss": 4.269209341431889, + "tokens_seen": 235077632 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004690672016048145, + "loss": 3.0364, + "theoretical_loss": 4.269064152097835, + "tokens_seen": 235143168 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046905717151454366, + "loss": 3.0581, + "theoretical_loss": 4.26891901455007, + "tokens_seen": 235208704 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046904714142427284, + "loss": 3.0874, + "theoretical_loss": 4.268773928755701, + "tokens_seen": 235274240 + }, + { + "epoch": 0.07, + "learning_rate": 0.000469037111334002, + "loss": 3.2326, + "theoretical_loss": 4.268628894681868, + "tokens_seen": 235339776 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004690270812437312, + "loss": 3.285, + "theoretical_loss": 4.268483912295735, + "tokens_seen": 235405312 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004690170511534604, + "loss": 3.1795, + "theoretical_loss": 4.268338981564502, + "tokens_seen": 235470848 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004690070210631896, + "loss": 3.2583, + "theoretical_loss": 4.268194102455395, + "tokens_seen": 235536384 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046899699097291875, + "loss": 3.1932, + "theoretical_loss": 4.26804927493567, + "tokens_seen": 235601920 + }, + { + "epoch": 0.07, + "learning_rate": 0.000468986960882648, + "loss": 3.0061, + "theoretical_loss": 4.267904498972618, + "tokens_seen": 235667456 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004689769307923771, + "loss": 2.9983, + "theoretical_loss": 4.267759774533552, + "tokens_seen": 235732992 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046896690070210635, + "loss": 3.0886, + "theoretical_loss": 4.267615101585821, + "tokens_seen": 235798528 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046895687061183553, + "loss": 3.2664, + "theoretical_loss": 4.267470480096801, + "tokens_seen": 235864064 + }, + { + "epoch": 0.07, + "objective/train/docs_used": 145084, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.3232319355010986, + "objective/train/theoretical_loss": 4.267325910033897, + "objective/train/tokens_used": 256389600, + "theoretical_loss": 4.267325910033897, + "tokens_seen": 235929600 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004689468405215647, + "loss": 3.3695, + "theoretical_loss": 4.267325910033897, + "tokens_seen": 235929600 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004689368104312939, + "loss": 3.1766, + "theoretical_loss": 4.267181391364547, + "tokens_seen": 235995136 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046892678034102307, + "loss": 2.9129, + "theoretical_loss": 4.267036924056215, + "tokens_seen": 236060672 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046891675025075225, + "loss": 2.9832, + "theoretical_loss": 4.266892508076397, + "tokens_seen": 236126208 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004689067201604815, + "loss": 3.1616, + "theoretical_loss": 4.266748143392617, + "tokens_seen": 236191744 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004688966900702106, + "loss": 3.0588, + "theoretical_loss": 4.26660382997243, + "tokens_seen": 236257280 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046888665997993985, + "loss": 3.2491, + "theoretical_loss": 4.26645956778342, + "tokens_seen": 236322816 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046887662988966903, + "loss": 3.2823, + "theoretical_loss": 4.2663153567932, + "tokens_seen": 236388352 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004688665997993982, + "loss": 3.0464, + "theoretical_loss": 4.266171196969412, + "tokens_seen": 236453888 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004688565697091274, + "loss": 2.9732, + "theoretical_loss": 4.2660270882797295, + "tokens_seen": 236519424 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004688465396188566, + "loss": 3.1662, + "theoretical_loss": 4.265883030691853, + "tokens_seen": 236584960 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046883650952858575, + "loss": 3.1374, + "theoretical_loss": 4.265739024173515, + "tokens_seen": 236650496 + }, + { + "epoch": 0.07, + "learning_rate": 0.000468826479438315, + "loss": 2.993, + "theoretical_loss": 4.265595068692473, + "tokens_seen": 236716032 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004688164493480441, + "loss": 2.9253, + "theoretical_loss": 4.26545116421652, + "tokens_seen": 236781568 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046880641925777335, + "loss": 3.135, + "theoretical_loss": 4.265307310713471, + "tokens_seen": 236847104 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004687963891675025, + "loss": 3.2528, + "theoretical_loss": 4.2651635081511765, + "tokens_seen": 236912640 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004687863590772317, + "loss": 2.8376, + "theoretical_loss": 4.265019756497512, + "tokens_seen": 236978176 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004687763289869609, + "loss": 2.8369, + "theoretical_loss": 4.264876055720386, + "tokens_seen": 237043712 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004687662988966901, + "loss": 3.0845, + "theoretical_loss": 4.264732405787731, + "tokens_seen": 237109248 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046875626880641926, + "loss": 3.067, + "theoretical_loss": 4.264588806667513, + "tokens_seen": 237174784 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046874623871614844, + "loss": 2.996, + "theoretical_loss": 4.264445258327724, + "tokens_seen": 237240320 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004687362086258776, + "loss": 3.0437, + "theoretical_loss": 4.264301760736389, + "tokens_seen": 237305856 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046872617853560686, + "loss": 2.9054, + "theoretical_loss": 4.264158313861557, + "tokens_seen": 237371392 + }, + { + "epoch": 0.07, + "learning_rate": 0.000468716148445336, + "loss": 3.1007, + "theoretical_loss": 4.264014917671309, + "tokens_seen": 237436928 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004687061183550652, + "loss": 2.9186, + "theoretical_loss": 4.2638715721337554, + "tokens_seen": 237502464 + }, + { + "epoch": 0.07, + "objective/train/docs_used": 146518, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.3827011585235596, + "objective/train/theoretical_loss": 4.263728277217032, + "objective/train/tokens_used": 258028000, + "theoretical_loss": 4.263728277217032, + "tokens_seen": 237568000 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004686960882647944, + "loss": 3.0815, + "theoretical_loss": 4.263728277217032, + "tokens_seen": 237568000 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004686860581745236, + "loss": 3.2321, + "theoretical_loss": 4.263585032889306, + "tokens_seen": 237633536 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046867602808425276, + "loss": 3.3322, + "theoretical_loss": 4.263441839118776, + "tokens_seen": 237699072 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046866599799398194, + "loss": 2.8462, + "theoretical_loss": 4.2632986958736625, + "tokens_seen": 237764608 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004686559679037111, + "loss": 3.2385, + "theoretical_loss": 4.263155603122221, + "tokens_seen": 237830144 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046864593781344036, + "loss": 3.1947, + "theoretical_loss": 4.263012560832733, + "tokens_seen": 237895680 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004686359077231695, + "loss": 3.1607, + "theoretical_loss": 4.262869568973508, + "tokens_seen": 237961216 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004686258776328987, + "loss": 3.1302, + "theoretical_loss": 4.262726627512886, + "tokens_seen": 238026752 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046861584754262785, + "loss": 3.2301, + "theoretical_loss": 4.262583736419234, + "tokens_seen": 238092288 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004686058174523571, + "loss": 3.1809, + "theoretical_loss": 4.26244089566095, + "tokens_seen": 238157824 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046859578736208626, + "loss": 3.2716, + "theoretical_loss": 4.262298105206456, + "tokens_seen": 238223360 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046858575727181545, + "loss": 3.1308, + "theoretical_loss": 4.262155365024207, + "tokens_seen": 238288896 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004685757271815446, + "loss": 3.0112, + "theoretical_loss": 4.262012675082685, + "tokens_seen": 238354432 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004685656970912738, + "loss": 3.2504, + "theoretical_loss": 4.261870035350399, + "tokens_seen": 238419968 + }, + { + "epoch": 0.07, + "learning_rate": 0.000468555667001003, + "loss": 2.8631, + "theoretical_loss": 4.261727445795888, + "tokens_seen": 238485504 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004685456369107322, + "loss": 3.3798, + "theoretical_loss": 4.26158490638772, + "tokens_seen": 238551040 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046853560682046135, + "loss": 3.1364, + "theoretical_loss": 4.261442417094488, + "tokens_seen": 238616576 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004685255767301906, + "loss": 2.9032, + "theoretical_loss": 4.261299977884816, + "tokens_seen": 238682112 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046851554663991977, + "loss": 3.2458, + "theoretical_loss": 4.2611575887273565, + "tokens_seen": 238747648 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046850551654964895, + "loss": 3.2734, + "theoretical_loss": 4.261015249590789, + "tokens_seen": 238813184 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046849548645937813, + "loss": 3.2059, + "theoretical_loss": 4.260872960443822, + "tokens_seen": 238878720 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004684854563691073, + "loss": 2.9761, + "theoretical_loss": 4.260730721255191, + "tokens_seen": 238944256 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004684754262788365, + "loss": 3.2039, + "theoretical_loss": 4.260588531993662, + "tokens_seen": 239009792 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046846539618856573, + "loss": 3.1275, + "theoretical_loss": 4.260446392628026, + "tokens_seen": 239075328 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004684553660982949, + "loss": 3.3135, + "theoretical_loss": 4.2603043031271035, + "tokens_seen": 239140864 + }, + { + "epoch": 0.07, + "objective/train/docs_used": 147039, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.874567747116089, + "objective/train/theoretical_loss": 4.260162263459744, + "objective/train/tokens_used": 259666400, + "theoretical_loss": 4.260162263459744, + "tokens_seen": 239206400 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004684453360080241, + "loss": 3.0995, + "theoretical_loss": 4.260162263459744, + "tokens_seen": 239206400 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046843530591775327, + "loss": 3.2374, + "theoretical_loss": 4.260020273594824, + "tokens_seen": 239271936 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046842527582748245, + "loss": 3.3045, + "theoretical_loss": 4.259878333501247, + "tokens_seen": 239337472 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004684152457372117, + "loss": 3.0821, + "theoretical_loss": 4.259736443147946, + "tokens_seen": 239403008 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004684052156469408, + "loss": 3.253, + "theoretical_loss": 4.259594602503881, + "tokens_seen": 239468544 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046839518555667005, + "loss": 3.0982, + "theoretical_loss": 4.259452811538041, + "tokens_seen": 239534080 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046838515546639923, + "loss": 3.1884, + "theoretical_loss": 4.259311070219441, + "tokens_seen": 239599616 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004683751253761284, + "loss": 3.0644, + "theoretical_loss": 4.259169378517125, + "tokens_seen": 239665152 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004683650952858576, + "loss": 3.0827, + "theoretical_loss": 4.259027736400165, + "tokens_seen": 239730688 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004683550651955868, + "loss": 3.2492, + "theoretical_loss": 4.258886143837661, + "tokens_seen": 239796224 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046834503510531595, + "loss": 3.2114, + "theoretical_loss": 4.258744600798739, + "tokens_seen": 239861760 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004683350050150452, + "loss": 3.3475, + "theoretical_loss": 4.2586031072525525, + "tokens_seen": 239927296 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004683249749247743, + "loss": 3.2166, + "theoretical_loss": 4.258461663168285, + "tokens_seen": 239992832 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046831494483450355, + "loss": 2.9147, + "theoretical_loss": 4.258320268515147, + "tokens_seen": 240058368 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004683049147442327, + "loss": 3.0942, + "theoretical_loss": 4.258178923262376, + "tokens_seen": 240123904 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004682948846539619, + "loss": 3.4146, + "theoretical_loss": 4.258037627379235, + "tokens_seen": 240189440 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004682848545636911, + "loss": 3.2589, + "theoretical_loss": 4.257896380835018, + "tokens_seen": 240254976 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004682748244734203, + "loss": 3.0668, + "theoretical_loss": 4.257755183599045, + "tokens_seen": 240320512 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046826479438314946, + "loss": 3.1518, + "theoretical_loss": 4.257614035640662, + "tokens_seen": 240386048 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046825476429287864, + "loss": 3.0437, + "theoretical_loss": 4.257472936929246, + "tokens_seen": 240451584 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004682447342026078, + "loss": 3.3181, + "theoretical_loss": 4.257331887434198, + "tokens_seen": 240517120 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046823470411233706, + "loss": 3.2362, + "theoretical_loss": 4.257190887124946, + "tokens_seen": 240582656 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004682246740220662, + "loss": 3.2266, + "theoretical_loss": 4.25704993597095, + "tokens_seen": 240648192 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004682146439317954, + "loss": 3.1797, + "theoretical_loss": 4.256909033941691, + "tokens_seen": 240713728 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004682046138415246, + "loss": 3.2339, + "theoretical_loss": 4.256768181006683, + "tokens_seen": 240779264 + }, + { + "epoch": 0.07, + "objective/train/docs_used": 148118, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.263885736465454, + "objective/train/theoretical_loss": 4.2566273771354615, + "objective/train/tokens_used": 261304800, + "theoretical_loss": 4.2566273771354615, + "tokens_seen": 240844800 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004681945837512538, + "loss": 3.0414, + "theoretical_loss": 4.2566273771354615, + "tokens_seen": 240844800 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046818455366098296, + "loss": 3.2383, + "theoretical_loss": 4.256486622297595, + "tokens_seen": 240910336 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046817452357071214, + "loss": 3.089, + "theoretical_loss": 4.256345916462674, + "tokens_seen": 240975872 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004681644934804413, + "loss": 3.1002, + "theoretical_loss": 4.256205259600321, + "tokens_seen": 241041408 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046815446339017056, + "loss": 3.2227, + "theoretical_loss": 4.256064651680182, + "tokens_seen": 241106944 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004681444332998997, + "loss": 3.4244, + "theoretical_loss": 4.255924092671931, + "tokens_seen": 241172480 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004681344032096289, + "loss": 3.312, + "theoretical_loss": 4.255783582545269, + "tokens_seen": 241238016 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046812437311935805, + "loss": 3.2017, + "theoretical_loss": 4.255643121269924, + "tokens_seen": 241303552 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004681143430290873, + "loss": 3.322, + "theoretical_loss": 4.255502708815651, + "tokens_seen": 241369088 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046810431293881646, + "loss": 3.2484, + "theoretical_loss": 4.255362345152234, + "tokens_seen": 241434624 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046809428284854565, + "loss": 3.0736, + "theoretical_loss": 4.255222030249479, + "tokens_seen": 241500160 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004680842527582748, + "loss": 3.2653, + "theoretical_loss": 4.255081764077224, + "tokens_seen": 241565696 + }, + { + "epoch": 0.07, + "learning_rate": 0.000468074222668004, + "loss": 3.3819, + "theoretical_loss": 4.25494154660533, + "tokens_seen": 241631232 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004680641925777332, + "loss": 3.1568, + "theoretical_loss": 4.254801377803689, + "tokens_seen": 241696768 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004680541624874624, + "loss": 3.1751, + "theoretical_loss": 4.254661257642215, + "tokens_seen": 241762304 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046804413239719155, + "loss": 3.1643, + "theoretical_loss": 4.254521186090852, + "tokens_seen": 241827840 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004680341023069208, + "loss": 3.3101, + "theoretical_loss": 4.254381163119568, + "tokens_seen": 241893376 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046802407221664997, + "loss": 3.4466, + "theoretical_loss": 4.254241188698361, + "tokens_seen": 241958912 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046801404212637915, + "loss": 3.0985, + "theoretical_loss": 4.2541012627972545, + "tokens_seen": 242024448 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046800401203610833, + "loss": 3.1687, + "theoretical_loss": 4.2539613853862965, + "tokens_seen": 242089984 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004679939819458375, + "loss": 3.3695, + "theoretical_loss": 4.253821556435565, + "tokens_seen": 242155520 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004679839518555667, + "loss": 3.0407, + "theoretical_loss": 4.253681775915161, + "tokens_seen": 242221056 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046797392176529593, + "loss": 3.1967, + "theoretical_loss": 4.253542043795215, + "tokens_seen": 242286592 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046796389167502505, + "loss": 3.1599, + "theoretical_loss": 4.253402360045882, + "tokens_seen": 242352128 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004679538615847543, + "loss": 3.1065, + "theoretical_loss": 4.253262724637346, + "tokens_seen": 242417664 + }, + { + "epoch": 0.07, + "objective/train/docs_used": 148826, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.2221624851226807, + "objective/train/theoretical_loss": 4.253123137539814, + "objective/train/tokens_used": 262943200, + "theoretical_loss": 4.253123137539814, + "tokens_seen": 242483200 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004679438314944834, + "loss": 3.3545, + "theoretical_loss": 4.253123137539814, + "tokens_seen": 242483200 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046793380140421265, + "loss": 2.8025, + "theoretical_loss": 4.252983598723521, + "tokens_seen": 242548736 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046792377131394183, + "loss": 3.2439, + "theoretical_loss": 4.25284410815873, + "tokens_seen": 242614272 + }, + { + "epoch": 0.07, + "learning_rate": 0.000467913741223671, + "loss": 3.1771, + "theoretical_loss": 4.2527046658157275, + "tokens_seen": 242679808 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004679037111334002, + "loss": 3.1684, + "theoretical_loss": 4.252565271664828, + "tokens_seen": 242745344 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046789368104312943, + "loss": 3.2102, + "theoretical_loss": 4.252425925676373, + "tokens_seen": 242810880 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046788365095285856, + "loss": 3.1045, + "theoretical_loss": 4.252286627820727, + "tokens_seen": 242876416 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004678736208625878, + "loss": 3.1938, + "theoretical_loss": 4.252147378068285, + "tokens_seen": 242941952 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004678635907723169, + "loss": 3.1913, + "theoretical_loss": 4.252008176389465, + "tokens_seen": 243007488 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046785356068204616, + "loss": 3.1493, + "theoretical_loss": 4.251869022754712, + "tokens_seen": 243073024 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046784353059177534, + "loss": 3.0924, + "theoretical_loss": 4.251729917134498, + "tokens_seen": 243138560 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004678335005015045, + "loss": 2.9893, + "theoretical_loss": 4.251590859499322, + "tokens_seen": 243204096 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004678234704112337, + "loss": 3.1228, + "theoretical_loss": 4.251451849819704, + "tokens_seen": 243269632 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004678134403209629, + "loss": 3.0794, + "theoretical_loss": 4.251312888066197, + "tokens_seen": 243335168 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046780341023069206, + "loss": 2.9701, + "theoretical_loss": 4.251173974209375, + "tokens_seen": 243400704 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004677933801404213, + "loss": 3.0512, + "theoretical_loss": 4.251035108219839, + "tokens_seen": 243466240 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004677833500501504, + "loss": 2.9104, + "theoretical_loss": 4.250896290068218, + "tokens_seen": 243531776 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046777331995987966, + "loss": 2.9683, + "theoretical_loss": 4.250757519725165, + "tokens_seen": 243597312 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004677632898696088, + "loss": 3.0521, + "theoretical_loss": 4.25061879716136, + "tokens_seen": 243662848 + }, + { + "epoch": 0.07, + "learning_rate": 0.000467753259779338, + "loss": 3.1174, + "theoretical_loss": 4.250480122347507, + "tokens_seen": 243728384 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004677432296890672, + "loss": 3.0046, + "theoretical_loss": 4.250341495254337, + "tokens_seen": 243793920 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004677331995987964, + "loss": 3.2529, + "theoretical_loss": 4.250202915852608, + "tokens_seen": 243859456 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046772316950852556, + "loss": 3.1652, + "theoretical_loss": 4.250064384113102, + "tokens_seen": 243924992 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004677131394182548, + "loss": 3.063, + "theoretical_loss": 4.249925900006627, + "tokens_seen": 243990528 + }, + { + "epoch": 0.07, + "learning_rate": 0.000467703109327984, + "loss": 3.0702, + "theoretical_loss": 4.249787463504019, + "tokens_seen": 244056064 + }, + { + "epoch": 0.07, + "objective/train/docs_used": 150000, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.974370241165161, + "objective/train/theoretical_loss": 4.249649074576134, + "objective/train/tokens_used": 264581600, + "theoretical_loss": 4.249649074576134, + "tokens_seen": 244121600 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046769307923771316, + "loss": 3.0065, + "theoretical_loss": 4.249649074576134, + "tokens_seen": 244121600 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046768304914744234, + "loss": 3.2049, + "theoretical_loss": 4.249510733193862, + "tokens_seen": 244187136 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004676730190571715, + "loss": 3.2831, + "theoretical_loss": 4.249372439328111, + "tokens_seen": 244252672 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046766298896690076, + "loss": 3.0317, + "theoretical_loss": 4.249234192949818, + "tokens_seen": 244318208 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004676529588766299, + "loss": 2.8645, + "theoretical_loss": 4.249095994029947, + "tokens_seen": 244383744 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004676429287863591, + "loss": 3.2092, + "theoretical_loss": 4.248957842539484, + "tokens_seen": 244449280 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046763289869608825, + "loss": 3.1265, + "theoretical_loss": 4.248819738449442, + "tokens_seen": 244514816 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004676228686058175, + "loss": 3.1373, + "theoretical_loss": 4.2486816817308615, + "tokens_seen": 244580352 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046761283851554666, + "loss": 3.1034, + "theoretical_loss": 4.248543672354805, + "tokens_seen": 244645888 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046760280842527585, + "loss": 3.07, + "theoretical_loss": 4.248405710292364, + "tokens_seen": 244711424 + }, + { + "epoch": 0.07, + "learning_rate": 0.000467592778335005, + "loss": 3.2771, + "theoretical_loss": 4.248267795514652, + "tokens_seen": 244776960 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004675827482447342, + "loss": 2.9297, + "theoretical_loss": 4.248129927992808, + "tokens_seen": 244842496 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004675727181544634, + "loss": 3.3286, + "theoretical_loss": 4.247992107698002, + "tokens_seen": 244908032 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004675626880641926, + "loss": 3.114, + "theoretical_loss": 4.247854334601421, + "tokens_seen": 244973568 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046755265797392175, + "loss": 3.1593, + "theoretical_loss": 4.247716608674283, + "tokens_seen": 245039104 + }, + { + "epoch": 0.07, + "learning_rate": 0.000467542627883651, + "loss": 3.2549, + "theoretical_loss": 4.247578929887829, + "tokens_seen": 245104640 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046753259779338017, + "loss": 3.1528, + "theoretical_loss": 4.247441298213326, + "tokens_seen": 245170176 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046752256770310935, + "loss": 3.184, + "theoretical_loss": 4.247303713622067, + "tokens_seen": 245235712 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046751253761283853, + "loss": 2.9793, + "theoretical_loss": 4.247166176085367, + "tokens_seen": 245301248 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004675025075225677, + "loss": 3.1446, + "theoretical_loss": 4.247028685574569, + "tokens_seen": 245366784 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004674924774322969, + "loss": 3.0552, + "theoretical_loss": 4.246891242061041, + "tokens_seen": 245432320 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046748244734202613, + "loss": 2.9006, + "theoretical_loss": 4.246753845516174, + "tokens_seen": 245497856 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046747241725175525, + "loss": 3.1904, + "theoretical_loss": 4.246616495911388, + "tokens_seen": 245563392 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004674623871614845, + "loss": 3.3241, + "theoretical_loss": 4.246479193218123, + "tokens_seen": 245628928 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004674523570712136, + "loss": 3.0002, + "theoretical_loss": 4.246341937407848, + "tokens_seen": 245694464 + }, + { + "epoch": 0.07, + "objective/train/docs_used": 150657, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.440032958984375, + "objective/train/theoretical_loss": 4.246204728452055, + "objective/train/tokens_used": 266220000, + "theoretical_loss": 4.246204728452055, + "tokens_seen": 245760000 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046744232698094285, + "loss": 3.3179, + "theoretical_loss": 4.246204728452055, + "tokens_seen": 245760000 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046743229689067203, + "loss": 3.273, + "theoretical_loss": 4.246067566322259, + "tokens_seen": 245825536 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004674222668004012, + "loss": 3.3121, + "theoretical_loss": 4.245930450990007, + "tokens_seen": 245891072 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004674122367101304, + "loss": 2.9493, + "theoretical_loss": 4.245793382426861, + "tokens_seen": 245956608 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046740220661985963, + "loss": 2.9085, + "theoretical_loss": 4.245656360604417, + "tokens_seen": 246022144 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046739217652958876, + "loss": 3.1549, + "theoretical_loss": 4.24551938549429, + "tokens_seen": 246087680 + }, + { + "epoch": 0.07, + "learning_rate": 0.000467382146439318, + "loss": 3.1751, + "theoretical_loss": 4.2453824570681205, + "tokens_seen": 246153216 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004673721163490471, + "loss": 3.0445, + "theoretical_loss": 4.245245575297577, + "tokens_seen": 246218752 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046736208625877636, + "loss": 3.0221, + "theoretical_loss": 4.2451087401543495, + "tokens_seen": 246284288 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046735205616850554, + "loss": 3.0738, + "theoretical_loss": 4.244971951610154, + "tokens_seen": 246349824 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004673420260782347, + "loss": 3.2211, + "theoretical_loss": 4.24483520963673, + "tokens_seen": 246415360 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004673319959879639, + "loss": 3.1129, + "theoretical_loss": 4.244698514205844, + "tokens_seen": 246480896 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004673219658976931, + "loss": 3.1848, + "theoretical_loss": 4.244561865289285, + "tokens_seen": 246546432 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046731193580742226, + "loss": 3.2135, + "theoretical_loss": 4.244425262858867, + "tokens_seen": 246611968 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004673019057171515, + "loss": 3.2466, + "theoretical_loss": 4.2442887068864295, + "tokens_seen": 246677504 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004672918756268806, + "loss": 3.2217, + "theoretical_loss": 4.244152197343835, + "tokens_seen": 246743040 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046728184553660986, + "loss": 2.9363, + "theoretical_loss": 4.244015734202973, + "tokens_seen": 246808576 + }, + { + "epoch": 0.07, + "learning_rate": 0.000467271815446339, + "loss": 3.3383, + "theoretical_loss": 4.243879317435755, + "tokens_seen": 246874112 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004672617853560682, + "loss": 3.111, + "theoretical_loss": 4.243742947014117, + "tokens_seen": 246939648 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004672517552657974, + "loss": 3.1101, + "theoretical_loss": 4.243606622910021, + "tokens_seen": 247005184 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004672417251755266, + "loss": 3.0126, + "theoretical_loss": 4.243470345095453, + "tokens_seen": 247070720 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046723169508525576, + "loss": 3.0905, + "theoretical_loss": 4.2433341135424225, + "tokens_seen": 247136256 + }, + { + "epoch": 0.07, + "learning_rate": 0.000467221664994985, + "loss": 2.9421, + "theoretical_loss": 4.243197928222964, + "tokens_seen": 247201792 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004672116349047141, + "loss": 3.3044, + "theoretical_loss": 4.243061789109136, + "tokens_seen": 247267328 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046720160481444336, + "loss": 3.2194, + "theoretical_loss": 4.242925696173021, + "tokens_seen": 247332864 + }, + { + "epoch": 0.07, + "objective/train/docs_used": 151862, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.3030314445495605, + "objective/train/theoretical_loss": 4.2427896493867285, + "objective/train/tokens_used": 267858400, + "theoretical_loss": 4.2427896493867285, + "tokens_seen": 247398400 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004671915747241725, + "loss": 3.0573, + "theoretical_loss": 4.2427896493867285, + "tokens_seen": 247398400 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004671815446339017, + "loss": 3.0589, + "theoretical_loss": 4.242653648722387, + "tokens_seen": 247463936 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004671715145436309, + "loss": 2.8884, + "theoretical_loss": 4.242517694152154, + "tokens_seen": 247529472 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004671614844533601, + "loss": 3.3057, + "theoretical_loss": 4.24238178564821, + "tokens_seen": 247595008 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046715145436308927, + "loss": 3.2249, + "theoretical_loss": 4.242245923182756, + "tokens_seen": 247660544 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046714142427281845, + "loss": 3.2008, + "theoretical_loss": 4.242110106728022, + "tokens_seen": 247726080 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046713139418254763, + "loss": 3.1185, + "theoretical_loss": 4.241974336256261, + "tokens_seen": 247791616 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046712136409227686, + "loss": 3.1252, + "theoretical_loss": 4.241838611739748, + "tokens_seen": 247857152 + }, + { + "epoch": 0.08, + "learning_rate": 0.000467111334002006, + "loss": 3.3753, + "theoretical_loss": 4.241702933150783, + "tokens_seen": 247922688 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004671013039117352, + "loss": 3.0951, + "theoretical_loss": 4.241567300461693, + "tokens_seen": 247988224 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046709127382146435, + "loss": 3.1228, + "theoretical_loss": 4.241431713644823, + "tokens_seen": 248053760 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004670812437311936, + "loss": 3.2219, + "theoretical_loss": 4.241296172672547, + "tokens_seen": 248119296 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046707121364092277, + "loss": 3.1535, + "theoretical_loss": 4.24116067751726, + "tokens_seen": 248184832 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046706118355065195, + "loss": 3.2456, + "theoretical_loss": 4.241025228151383, + "tokens_seen": 248250368 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046705115346038113, + "loss": 3.0877, + "theoretical_loss": 4.24088982454736, + "tokens_seen": 248315904 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046704112337011037, + "loss": 2.9972, + "theoretical_loss": 4.240754466677659, + "tokens_seen": 248381440 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004670310932798395, + "loss": 3.167, + "theoretical_loss": 4.240619154514771, + "tokens_seen": 248446976 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046702106318956873, + "loss": 3.0545, + "theoretical_loss": 4.240483888031212, + "tokens_seen": 248512512 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046701103309929786, + "loss": 2.9543, + "theoretical_loss": 4.240348667199521, + "tokens_seen": 248578048 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004670010030090271, + "loss": 3.0375, + "theoretical_loss": 4.240213491992261, + "tokens_seen": 248643584 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004669909729187563, + "loss": 3.4415, + "theoretical_loss": 4.240078362382019, + "tokens_seen": 248709120 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046698094282848545, + "loss": 3.146, + "theoretical_loss": 4.239943278341404, + "tokens_seen": 248774656 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046697091273821464, + "loss": 3.185, + "theoretical_loss": 4.239808239843052, + "tokens_seen": 248840192 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004669608826479438, + "loss": 3.1956, + "theoretical_loss": 4.239673246859619, + "tokens_seen": 248905728 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046695085255767305, + "loss": 3.0067, + "theoretical_loss": 4.239538299363788, + "tokens_seen": 248971264 + }, + { + "epoch": 0.08, + "objective/train/docs_used": 152583, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.055074691772461, + "objective/train/theoretical_loss": 4.239403397328261, + "objective/train/tokens_used": 269496800, + "theoretical_loss": 4.239403397328261, + "tokens_seen": 249036800 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046694082246740223, + "loss": 3.1003, + "theoretical_loss": 4.239403397328261, + "tokens_seen": 249036800 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004669307923771314, + "loss": 3.1628, + "theoretical_loss": 4.239268540725769, + "tokens_seen": 249102336 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004669207622868606, + "loss": 3.1076, + "theoretical_loss": 4.239133729529064, + "tokens_seen": 249167872 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046691073219658983, + "loss": 3.179, + "theoretical_loss": 4.2389989637109196, + "tokens_seen": 249233408 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046690070210631896, + "loss": 3.0111, + "theoretical_loss": 4.2388642432441355, + "tokens_seen": 249298944 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004668906720160482, + "loss": 3.1068, + "theoretical_loss": 4.238729568101535, + "tokens_seen": 249364480 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004668806419257773, + "loss": 3.3072, + "theoretical_loss": 4.238594938255963, + "tokens_seen": 249430016 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046687061183550656, + "loss": 2.9278, + "theoretical_loss": 4.2384603536802885, + "tokens_seen": 249495552 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046686058174523574, + "loss": 3.0771, + "theoretical_loss": 4.238325814347404, + "tokens_seen": 249561088 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004668505516549649, + "loss": 3.338, + "theoretical_loss": 4.238191320230227, + "tokens_seen": 249626624 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004668405215646941, + "loss": 3.0884, + "theoretical_loss": 4.238056871301695, + "tokens_seen": 249692160 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004668304914744233, + "loss": 3.1595, + "theoretical_loss": 4.237922467534771, + "tokens_seen": 249757696 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046682046138415246, + "loss": 3.4463, + "theoretical_loss": 4.237788108902441, + "tokens_seen": 249823232 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004668104312938817, + "loss": 3.4304, + "theoretical_loss": 4.237653795377714, + "tokens_seen": 249888768 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004668004012036108, + "loss": 3.2941, + "theoretical_loss": 4.237519526933622, + "tokens_seen": 249954304 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046679037111334006, + "loss": 2.903, + "theoretical_loss": 4.2373853035432205, + "tokens_seen": 250019840 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004667803410230692, + "loss": 3.174, + "theoretical_loss": 4.237251125179588, + "tokens_seen": 250085376 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004667703109327984, + "loss": 2.9676, + "theoretical_loss": 4.237116991815826, + "tokens_seen": 250150912 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004667602808425276, + "loss": 3.2225, + "theoretical_loss": 4.23698290342506, + "tokens_seen": 250216448 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004667502507522568, + "loss": 3.0144, + "theoretical_loss": 4.236848859980437, + "tokens_seen": 250281984 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046674022066198596, + "loss": 3.0452, + "theoretical_loss": 4.23671486145513, + "tokens_seen": 250347520 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004667301905717152, + "loss": 3.3803, + "theoretical_loss": 4.236580907822331, + "tokens_seen": 250413056 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004667201604814443, + "loss": 2.8512, + "theoretical_loss": 4.236446999055257, + "tokens_seen": 250478592 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046671013039117356, + "loss": 3.2175, + "theoretical_loss": 4.2363131351271495, + "tokens_seen": 250544128 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004667001003009027, + "loss": 3.1343, + "theoretical_loss": 4.2361793160112695, + "tokens_seen": 250609664 + }, + { + "epoch": 0.08, + "objective/train/docs_used": 153242, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.0893537998199463, + "objective/train/theoretical_loss": 4.236045541680905, + "objective/train/tokens_used": 271135200, + "theoretical_loss": 4.236045541680905, + "tokens_seen": 250675200 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004666900702106319, + "loss": 3.1103, + "theoretical_loss": 4.236045541680905, + "tokens_seen": 250675200 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004666800401203611, + "loss": 2.9528, + "theoretical_loss": 4.235911812109363, + "tokens_seen": 250740736 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004666700100300903, + "loss": 3.105, + "theoretical_loss": 4.235778127269976, + "tokens_seen": 250806272 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046665997993981947, + "loss": 3.0719, + "theoretical_loss": 4.235644487136098, + "tokens_seen": 250871808 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046664994984954865, + "loss": 2.934, + "theoretical_loss": 4.235510891681108, + "tokens_seen": 250937344 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046663991975927783, + "loss": 3.2493, + "theoretical_loss": 4.235377340878404, + "tokens_seen": 251002880 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046662988966900706, + "loss": 3.1748, + "theoretical_loss": 4.23524383470141, + "tokens_seen": 251068416 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004666198595787362, + "loss": 3.3832, + "theoretical_loss": 4.235110373123572, + "tokens_seen": 251133952 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046660982948846543, + "loss": 3.035, + "theoretical_loss": 4.2349769561183574, + "tokens_seen": 251199488 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046659979939819455, + "loss": 2.9441, + "theoretical_loss": 4.2348435836592575, + "tokens_seen": 251265024 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004665897693079238, + "loss": 3.1246, + "theoretical_loss": 4.234710255719786, + "tokens_seen": 251330560 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046657973921765297, + "loss": 3.0054, + "theoretical_loss": 4.234576972273481, + "tokens_seen": 251396096 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046656970912738215, + "loss": 3.2966, + "theoretical_loss": 4.234443733293899, + "tokens_seen": 251461632 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046655967903711133, + "loss": 3.1876, + "theoretical_loss": 4.234310538754624, + "tokens_seen": 251527168 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046654964894684057, + "loss": 3.2616, + "theoretical_loss": 4.2341773886292575, + "tokens_seen": 251592704 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004665396188565697, + "loss": 3.4585, + "theoretical_loss": 4.234044282891429, + "tokens_seen": 251658240 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046652958876629893, + "loss": 3.176, + "theoretical_loss": 4.233911221514787, + "tokens_seen": 251723776 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046651955867602806, + "loss": 3.4007, + "theoretical_loss": 4.233778204473002, + "tokens_seen": 251789312 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004665095285857573, + "loss": 3.423, + "theoretical_loss": 4.23364523173977, + "tokens_seen": 251854848 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004664994984954865, + "loss": 2.9969, + "theoretical_loss": 4.233512303288807, + "tokens_seen": 251920384 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046648946840521565, + "loss": 3.142, + "theoretical_loss": 4.233379419093851, + "tokens_seen": 251985920 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046647943831494484, + "loss": 2.9608, + "theoretical_loss": 4.233246579128666, + "tokens_seen": 252051456 + }, + { + "epoch": 0.08, + "learning_rate": 0.000466469408224674, + "loss": 3.2141, + "theoretical_loss": 4.233113783367033, + "tokens_seen": 252116992 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004664593781344032, + "loss": 3.3402, + "theoretical_loss": 4.232981031782761, + "tokens_seen": 252182528 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046644934804413243, + "loss": 3.2154, + "theoretical_loss": 4.232848324349677, + "tokens_seen": 252248064 + }, + { + "epoch": 0.08, + "objective/train/docs_used": 154565, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.148193597793579, + "objective/train/theoretical_loss": 4.232715661041632, + "objective/train/tokens_used": 272773600, + "theoretical_loss": 4.232715661041632, + "tokens_seen": 252313600 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046643931795386156, + "loss": 3.4049, + "theoretical_loss": 4.232715661041632, + "tokens_seen": 252313600 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004664292878635908, + "loss": 3.334, + "theoretical_loss": 4.232583041832499, + "tokens_seen": 252379136 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004664192577733199, + "loss": 3.0422, + "theoretical_loss": 4.232450466696174, + "tokens_seen": 252444672 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046640922768304916, + "loss": 3.2218, + "theoretical_loss": 4.2323179356065745, + "tokens_seen": 252510208 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046639919759277834, + "loss": 2.9353, + "theoretical_loss": 4.23218544853764, + "tokens_seen": 252575744 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004663891675025075, + "loss": 3.339, + "theoretical_loss": 4.232053005463333, + "tokens_seen": 252641280 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004663791374122367, + "loss": 3.13, + "theoretical_loss": 4.231920606357638, + "tokens_seen": 252706816 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046636910732196594, + "loss": 3.108, + "theoretical_loss": 4.231788251194559, + "tokens_seen": 252772352 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046635907723169506, + "loss": 3.2639, + "theoretical_loss": 4.231655939948127, + "tokens_seen": 252837888 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004663490471414243, + "loss": 3.06, + "theoretical_loss": 4.231523672592392, + "tokens_seen": 252903424 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004663390170511534, + "loss": 3.0639, + "theoretical_loss": 4.231391449101425, + "tokens_seen": 252968960 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046632898696088266, + "loss": 3.1918, + "theoretical_loss": 4.231259269449322, + "tokens_seen": 253034496 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046631895687061184, + "loss": 3.1834, + "theoretical_loss": 4.231127133610198, + "tokens_seen": 253100032 + }, + { + "epoch": 0.08, + "learning_rate": 0.000466308926780341, + "loss": 3.0298, + "theoretical_loss": 4.230995041558194, + "tokens_seen": 253165568 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004662988966900702, + "loss": 3.167, + "theoretical_loss": 4.230862993267468, + "tokens_seen": 253231104 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004662888665997994, + "loss": 3.0906, + "theoretical_loss": 4.230730988712205, + "tokens_seen": 253296640 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046627883650952857, + "loss": 2.9818, + "theoretical_loss": 4.230599027866606, + "tokens_seen": 253362176 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004662688064192578, + "loss": 3.1265, + "theoretical_loss": 4.2304671107048994, + "tokens_seen": 253427712 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046625877632898693, + "loss": 2.9768, + "theoretical_loss": 4.2303352372013325, + "tokens_seen": 253493248 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046624874623871616, + "loss": 3.0033, + "theoretical_loss": 4.230203407330176, + "tokens_seen": 253558784 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046623871614844535, + "loss": 3.037, + "theoretical_loss": 4.230071621065721, + "tokens_seen": 253624320 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004662286860581745, + "loss": 3.1625, + "theoretical_loss": 4.2299398783822815, + "tokens_seen": 253689856 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004662186559679037, + "loss": 3.0437, + "theoretical_loss": 4.229808179254192, + "tokens_seen": 253755392 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004662086258776329, + "loss": 3.2906, + "theoretical_loss": 4.22967652365581, + "tokens_seen": 253820928 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004661985957873621, + "loss": 3.1545, + "theoretical_loss": 4.229544911561513, + "tokens_seen": 253886464 + }, + { + "epoch": 0.08, + "objective/train/docs_used": 156106, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.288883924484253, + "objective/train/theoretical_loss": 4.229413342945703, + "objective/train/tokens_used": 274412000, + "theoretical_loss": 4.229413342945703, + "tokens_seen": 253952000 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004661885656970913, + "loss": 3.3003, + "theoretical_loss": 4.229413342945703, + "tokens_seen": 253952000 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004661785356068205, + "loss": 3.1244, + "theoretical_loss": 4.229281817782801, + "tokens_seen": 254017536 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046616850551654967, + "loss": 3.1214, + "theoretical_loss": 4.229150336047251, + "tokens_seen": 254083072 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046615847542627885, + "loss": 3.289, + "theoretical_loss": 4.229018897713519, + "tokens_seen": 254148608 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046614844533600803, + "loss": 3.1937, + "theoretical_loss": 4.22888750275609, + "tokens_seen": 254214144 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046613841524573727, + "loss": 3.3532, + "theoretical_loss": 4.228756151149475, + "tokens_seen": 254279680 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004661283851554664, + "loss": 3.1176, + "theoretical_loss": 4.228624842868202, + "tokens_seen": 254345216 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046611835506519563, + "loss": 3.0796, + "theoretical_loss": 4.228493577886824, + "tokens_seen": 254410752 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046610832497492475, + "loss": 2.9175, + "theoretical_loss": 4.228362356179913, + "tokens_seen": 254476288 + }, + { + "epoch": 0.08, + "learning_rate": 0.000466098294884654, + "loss": 3.1493, + "theoretical_loss": 4.228231177722063, + "tokens_seen": 254541824 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046608826479438317, + "loss": 3.2473, + "theoretical_loss": 4.228100042487892, + "tokens_seen": 254607360 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046607823470411235, + "loss": 3.354, + "theoretical_loss": 4.227968950452035, + "tokens_seen": 254672896 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046606820461384153, + "loss": 3.0903, + "theoretical_loss": 4.227837901589153, + "tokens_seen": 254738432 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046605817452357077, + "loss": 3.0287, + "theoretical_loss": 4.227706895873924, + "tokens_seen": 254803968 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004660481444332999, + "loss": 3.1269, + "theoretical_loss": 4.227575933281051, + "tokens_seen": 254869504 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046603811434302913, + "loss": 3.1888, + "theoretical_loss": 4.227445013785257, + "tokens_seen": 254935040 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046602808425275826, + "loss": 3.1169, + "theoretical_loss": 4.227314137361285, + "tokens_seen": 255000576 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004660180541624875, + "loss": 3.2877, + "theoretical_loss": 4.227183303983901, + "tokens_seen": 255066112 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004660080240722167, + "loss": 3.1152, + "theoretical_loss": 4.227052513627893, + "tokens_seen": 255131648 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046599799398194586, + "loss": 3.1075, + "theoretical_loss": 4.226921766268067, + "tokens_seen": 255197184 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046598796389167504, + "loss": 3.0752, + "theoretical_loss": 4.226791061879253, + "tokens_seen": 255262720 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004659779338014042, + "loss": 3.03, + "theoretical_loss": 4.226660400436302, + "tokens_seen": 255328256 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004659679037111334, + "loss": 3.1293, + "theoretical_loss": 4.226529781914084, + "tokens_seen": 255393792 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046595787362086263, + "loss": 3.1227, + "theoretical_loss": 4.226399206287493, + "tokens_seen": 255459328 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046594784353059176, + "loss": 2.9702, + "theoretical_loss": 4.226268673531442, + "tokens_seen": 255524864 + }, + { + "epoch": 0.08, + "objective/train/docs_used": 156630, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.5435431003570557, + "objective/train/theoretical_loss": 4.226138183620867, + "objective/train/tokens_used": 276050400, + "theoretical_loss": 4.226138183620867, + "tokens_seen": 255590400 + }, + { + "epoch": 0.08, + "learning_rate": 0.000465937813440321, + "loss": 2.9812, + "theoretical_loss": 4.226138183620867, + "tokens_seen": 255590400 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004659277833500501, + "loss": 3.0519, + "theoretical_loss": 4.226007736530723, + "tokens_seen": 255655936 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046591775325977936, + "loss": 3.1841, + "theoretical_loss": 4.225877332235987, + "tokens_seen": 255721472 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046590772316950854, + "loss": 2.9659, + "theoretical_loss": 4.225746970711657, + "tokens_seen": 255787008 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004658976930792377, + "loss": 3.3536, + "theoretical_loss": 4.225616651932753, + "tokens_seen": 255852544 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004658876629889669, + "loss": 3.392, + "theoretical_loss": 4.225486375874315, + "tokens_seen": 255918080 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046587763289869614, + "loss": 3.0656, + "theoretical_loss": 4.225356142511402, + "tokens_seen": 255983616 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046586760280842526, + "loss": 2.8899, + "theoretical_loss": 4.225225951819099, + "tokens_seen": 256049152 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004658575727181545, + "loss": 3.2571, + "theoretical_loss": 4.225095803772507, + "tokens_seen": 256114688 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004658475426278836, + "loss": 3.0785, + "theoretical_loss": 4.22496569834675, + "tokens_seen": 256180224 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046583751253761286, + "loss": 3.145, + "theoretical_loss": 4.224835635516973, + "tokens_seen": 256245760 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046582748244734204, + "loss": 2.8934, + "theoretical_loss": 4.224705615258341, + "tokens_seen": 256311296 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004658174523570712, + "loss": 2.9095, + "theoretical_loss": 4.224575637546041, + "tokens_seen": 256376832 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004658074222668004, + "loss": 2.8229, + "theoretical_loss": 4.224445702355279, + "tokens_seen": 256442368 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004657973921765296, + "loss": 3.0357, + "theoretical_loss": 4.2243158096612845, + "tokens_seen": 256507904 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046578736208625877, + "loss": 2.8497, + "theoretical_loss": 4.224185959439305, + "tokens_seen": 256573440 + }, + { + "epoch": 0.08, + "learning_rate": 0.000465777331995988, + "loss": 3.1126, + "theoretical_loss": 4.22405615166461, + "tokens_seen": 256638976 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046576730190571713, + "loss": 3.1401, + "theoretical_loss": 4.22392638631249, + "tokens_seen": 256704512 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046575727181544636, + "loss": 3.1382, + "theoretical_loss": 4.223796663358255, + "tokens_seen": 256770048 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046574724172517555, + "loss": 3.2955, + "theoretical_loss": 4.223666982777237, + "tokens_seen": 256835584 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004657372116349047, + "loss": 3.3054, + "theoretical_loss": 4.223537344544788, + "tokens_seen": 256901120 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004657271815446339, + "loss": 3.2315, + "theoretical_loss": 4.223407748636282, + "tokens_seen": 256966656 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004657171514543631, + "loss": 3.0709, + "theoretical_loss": 4.22327819502711, + "tokens_seen": 257032192 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046570712136409227, + "loss": 2.8788, + "theoretical_loss": 4.223148683692687, + "tokens_seen": 257097728 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004656970912738215, + "loss": 3.1017, + "theoretical_loss": 4.223019214608446, + "tokens_seen": 257163264 + }, + { + "epoch": 0.08, + "objective/train/docs_used": 157258, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.3907642364501953, + "objective/train/theoretical_loss": 4.222889787749845, + "objective/train/tokens_used": 277688800, + "theoretical_loss": 4.222889787749845, + "tokens_seen": 257228800 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046568706118355063, + "loss": 2.9536, + "theoretical_loss": 4.222889787749845, + "tokens_seen": 257228800 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046567703109327987, + "loss": 3.3622, + "theoretical_loss": 4.222760403092358, + "tokens_seen": 257294336 + }, + { + "epoch": 0.08, + "learning_rate": 0.000465667001003009, + "loss": 3.2016, + "theoretical_loss": 4.22263106061148, + "tokens_seen": 257359872 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046565697091273823, + "loss": 3.1998, + "theoretical_loss": 4.222501760282729, + "tokens_seen": 257425408 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004656469408224674, + "loss": 3.1238, + "theoretical_loss": 4.22237250208164, + "tokens_seen": 257490944 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004656369107321966, + "loss": 3.1857, + "theoretical_loss": 4.222243285983772, + "tokens_seen": 257556480 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004656268806419258, + "loss": 2.8578, + "theoretical_loss": 4.222114111964703, + "tokens_seen": 257622016 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046561685055165495, + "loss": 3.0813, + "theoretical_loss": 4.221984980000029, + "tokens_seen": 257687552 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046560682046138414, + "loss": 3.0411, + "theoretical_loss": 4.2218558900653695, + "tokens_seen": 257753088 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046559679037111337, + "loss": 2.8988, + "theoretical_loss": 4.221726842136364, + "tokens_seen": 257818624 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004655867602808425, + "loss": 3.0962, + "theoretical_loss": 4.2215978361886695, + "tokens_seen": 257884160 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046557673019057173, + "loss": 2.8258, + "theoretical_loss": 4.221468872197967, + "tokens_seen": 257949696 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004655667001003009, + "loss": 3.1102, + "theoretical_loss": 4.221339950139956, + "tokens_seen": 258015232 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004655566700100301, + "loss": 3.158, + "theoretical_loss": 4.221211069990357, + "tokens_seen": 258080768 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004655466399197593, + "loss": 2.988, + "theoretical_loss": 4.221082231724908, + "tokens_seen": 258146304 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046553660982948846, + "loss": 3.1691, + "theoretical_loss": 4.22095343531937, + "tokens_seen": 258211840 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046552657973921764, + "loss": 3.2189, + "theoretical_loss": 4.220824680749525, + "tokens_seen": 258277376 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004655165496489469, + "loss": 2.9362, + "theoretical_loss": 4.220695967991171, + "tokens_seen": 258342912 + }, + { + "epoch": 0.08, + "learning_rate": 0.000465506519558676, + "loss": 3.1545, + "theoretical_loss": 4.220567297020131, + "tokens_seen": 258408448 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046549648946840524, + "loss": 3.0796, + "theoretical_loss": 4.220438667812244, + "tokens_seen": 258473984 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046548645937813436, + "loss": 3.0365, + "theoretical_loss": 4.220310080343373, + "tokens_seen": 258539520 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004654764292878636, + "loss": 3.1576, + "theoretical_loss": 4.220181534589398, + "tokens_seen": 258605056 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004654663991975928, + "loss": 3.2543, + "theoretical_loss": 4.22005303052622, + "tokens_seen": 258670592 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046545636910732196, + "loss": 3.1193, + "theoretical_loss": 4.219924568129759, + "tokens_seen": 258736128 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004654463390170512, + "loss": 3.0812, + "theoretical_loss": 4.219796147375957, + "tokens_seen": 258801664 + }, + { + "epoch": 0.08, + "objective/train/docs_used": 158510, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.648303270339966, + "objective/train/theoretical_loss": 4.219667768240775, + "objective/train/tokens_used": 279327200, + "theoretical_loss": 4.219667768240775, + "tokens_seen": 258867200 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004654363089267803, + "loss": 3.1929, + "theoretical_loss": 4.219667768240775, + "tokens_seen": 258867200 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046542627883650956, + "loss": 3.1791, + "theoretical_loss": 4.219539430700195, + "tokens_seen": 258932736 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046541624874623874, + "loss": 2.9421, + "theoretical_loss": 4.2194111347302155, + "tokens_seen": 258998272 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004654062186559679, + "loss": 3.2183, + "theoretical_loss": 4.219282880306859, + "tokens_seen": 259063808 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004653961885656971, + "loss": 3.193, + "theoretical_loss": 4.219154667406166, + "tokens_seen": 259129344 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046538615847542634, + "loss": 3.1217, + "theoretical_loss": 4.219026496004198, + "tokens_seen": 259194880 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046537612838515546, + "loss": 3.0638, + "theoretical_loss": 4.218898366077035, + "tokens_seen": 259260416 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004653660982948847, + "loss": 3.2555, + "theoretical_loss": 4.218770277600775, + "tokens_seen": 259325952 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004653560682046138, + "loss": 3.0011, + "theoretical_loss": 4.218642230551541, + "tokens_seen": 259391488 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046534603811434306, + "loss": 3.1561, + "theoretical_loss": 4.218514224905472, + "tokens_seen": 259457024 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046533600802407224, + "loss": 3.1305, + "theoretical_loss": 4.218386260638727, + "tokens_seen": 259522560 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004653259779338014, + "loss": 3.2608, + "theoretical_loss": 4.2182583377274865, + "tokens_seen": 259588096 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004653159478435306, + "loss": 3.1983, + "theoretical_loss": 4.218130456147948, + "tokens_seen": 259653632 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004653059177532598, + "loss": 3.2366, + "theoretical_loss": 4.218002615876332, + "tokens_seen": 259719168 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046529588766298897, + "loss": 3.0201, + "theoretical_loss": 4.217874816888877, + "tokens_seen": 259784704 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004652858575727182, + "loss": 3.0065, + "theoretical_loss": 4.217747059161839, + "tokens_seen": 259850240 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046527582748244733, + "loss": 3.2433, + "theoretical_loss": 4.217619342671498, + "tokens_seen": 259915776 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046526579739217656, + "loss": 2.998, + "theoretical_loss": 4.2174916673941505, + "tokens_seen": 259981312 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046525576730190575, + "loss": 3.1428, + "theoretical_loss": 4.217364033306113, + "tokens_seen": 260046848 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004652457372116349, + "loss": 2.996, + "theoretical_loss": 4.217236440383724, + "tokens_seen": 260112384 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004652357071213641, + "loss": 3.228, + "theoretical_loss": 4.217108888603337, + "tokens_seen": 260177920 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004652256770310933, + "loss": 3.1325, + "theoretical_loss": 4.21698137794133, + "tokens_seen": 260243456 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046521564694082247, + "loss": 2.895, + "theoretical_loss": 4.216853908374097, + "tokens_seen": 260308992 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004652056168505517, + "loss": 3.0931, + "theoretical_loss": 4.216726479878052, + "tokens_seen": 260374528 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046519558676028083, + "loss": 2.9594, + "theoretical_loss": 4.216599092429631, + "tokens_seen": 260440064 + }, + { + "epoch": 0.08, + "objective/train/docs_used": 159098, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.1803290843963623, + "objective/train/theoretical_loss": 4.216471746005286, + "objective/train/tokens_used": 280965600, + "theoretical_loss": 4.216471746005286, + "tokens_seen": 260505600 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046518555667001007, + "loss": 2.967, + "theoretical_loss": 4.216471746005286, + "tokens_seen": 260505600 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004651755265797392, + "loss": 3.1332, + "theoretical_loss": 4.216344440581491, + "tokens_seen": 260571136 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046516549648946843, + "loss": 3.1521, + "theoretical_loss": 4.2162171761347365, + "tokens_seen": 260636672 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004651554663991976, + "loss": 3.0712, + "theoretical_loss": 4.2160899526415365, + "tokens_seen": 260702208 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004651454363089268, + "loss": 3.0385, + "theoretical_loss": 4.215962770078422, + "tokens_seen": 260767744 + }, + { + "epoch": 0.08, + "learning_rate": 0.000465135406218656, + "loss": 3.163, + "theoretical_loss": 4.215835628421942, + "tokens_seen": 260833280 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046512537612838515, + "loss": 3.1195, + "theoretical_loss": 4.215708527648667, + "tokens_seen": 260898816 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046511534603811434, + "loss": 3.0471, + "theoretical_loss": 4.215581467735187, + "tokens_seen": 260964352 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046510531594784357, + "loss": 2.956, + "theoretical_loss": 4.215454448658109, + "tokens_seen": 261029888 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004650952858575727, + "loss": 3.3698, + "theoretical_loss": 4.215327470394062, + "tokens_seen": 261095424 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046508525576730193, + "loss": 3.2779, + "theoretical_loss": 4.215200532919691, + "tokens_seen": 261160960 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004650752256770311, + "loss": 3.2967, + "theoretical_loss": 4.215073636211664, + "tokens_seen": 261226496 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004650651955867603, + "loss": 3.2427, + "theoretical_loss": 4.214946780246666, + "tokens_seen": 261292032 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004650551654964895, + "loss": 2.8436, + "theoretical_loss": 4.214819965001401, + "tokens_seen": 261357568 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046504513540621866, + "loss": 2.9477, + "theoretical_loss": 4.214693190452593, + "tokens_seen": 261423104 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046503510531594784, + "loss": 3.2566, + "theoretical_loss": 4.214566456576984, + "tokens_seen": 261488640 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004650250752256771, + "loss": 3.1072, + "theoretical_loss": 4.214439763351336, + "tokens_seen": 261554176 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004650150451354062, + "loss": 3.1733, + "theoretical_loss": 4.214313110752431, + "tokens_seen": 261619712 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046500501504513544, + "loss": 3.0154, + "theoretical_loss": 4.214186498757069, + "tokens_seen": 261685248 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046499498495486456, + "loss": 3.0535, + "theoretical_loss": 4.214059927342068, + "tokens_seen": 261750784 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004649849548645938, + "loss": 3.1657, + "theoretical_loss": 4.213933396484267, + "tokens_seen": 261816320 + }, + { + "epoch": 0.08, + "learning_rate": 0.000464974924774323, + "loss": 2.9417, + "theoretical_loss": 4.213806906160523, + "tokens_seen": 261881856 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046496489468405216, + "loss": 3.3274, + "theoretical_loss": 4.213680456347712, + "tokens_seen": 261947392 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046495486459378134, + "loss": 3.1786, + "theoretical_loss": 4.213554047022729, + "tokens_seen": 262012928 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004649448345035105, + "loss": 3.3239, + "theoretical_loss": 4.213427678162489, + "tokens_seen": 262078464 + }, + { + "epoch": 0.08, + "objective/train/docs_used": 160106, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.018378973007202, + "objective/train/theoretical_loss": 4.213301349743924, + "objective/train/tokens_used": 282604000, + "theoretical_loss": 4.213301349743924, + "tokens_seen": 262144000 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004649348044132397, + "loss": 3.1955, + "theoretical_loss": 4.213301349743924, + "tokens_seen": 262144000 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046492477432296894, + "loss": 3.0967, + "theoretical_loss": 4.2131750617439865, + "tokens_seen": 262209536 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046491474423269807, + "loss": 3.2456, + "theoretical_loss": 4.213048814139647, + "tokens_seen": 262275072 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004649047141424273, + "loss": 3.2926, + "theoretical_loss": 4.212922606907895, + "tokens_seen": 262340608 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004648946840521565, + "loss": 3.282, + "theoretical_loss": 4.21279644002574, + "tokens_seen": 262406144 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046488465396188566, + "loss": 3.227, + "theoretical_loss": 4.212670313470209, + "tokens_seen": 262471680 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046487462387161485, + "loss": 3.0664, + "theoretical_loss": 4.212544227218347, + "tokens_seen": 262537216 + }, + { + "epoch": 0.08, + "learning_rate": 0.000464864593781344, + "loss": 2.9033, + "theoretical_loss": 4.21241818124722, + "tokens_seen": 262602752 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004648545636910732, + "loss": 3.3065, + "theoretical_loss": 4.212292175533912, + "tokens_seen": 262668288 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046484453360080244, + "loss": 3.1799, + "theoretical_loss": 4.212166210055526, + "tokens_seen": 262733824 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046483450351053157, + "loss": 3.3237, + "theoretical_loss": 4.212040284789181, + "tokens_seen": 262799360 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004648244734202608, + "loss": 3.0471, + "theoretical_loss": 4.211914399712019, + "tokens_seen": 262864896 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046481444332998993, + "loss": 2.94, + "theoretical_loss": 4.211788554801198, + "tokens_seen": 262930432 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046480441323971917, + "loss": 3.058, + "theoretical_loss": 4.211662750033895, + "tokens_seen": 262995968 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046479438314944835, + "loss": 2.8963, + "theoretical_loss": 4.211536985387307, + "tokens_seen": 263061504 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046478435305917753, + "loss": 3.1982, + "theoretical_loss": 4.211411260838647, + "tokens_seen": 263127040 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004647743229689067, + "loss": 2.9601, + "theoretical_loss": 4.2112855763651496, + "tokens_seen": 263192576 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046476429287863595, + "loss": 3.0841, + "theoretical_loss": 4.211159931944065, + "tokens_seen": 263258112 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004647542627883651, + "loss": 3.0865, + "theoretical_loss": 4.211034327552666, + "tokens_seen": 263323648 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004647442326980943, + "loss": 3.0591, + "theoretical_loss": 4.210908763168239, + "tokens_seen": 263389184 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046473420260782344, + "loss": 3.0858, + "theoretical_loss": 4.210783238768093, + "tokens_seen": 263454720 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046472417251755267, + "loss": 3.1614, + "theoretical_loss": 4.210657754329553, + "tokens_seen": 263520256 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004647141424272819, + "loss": 3.1009, + "theoretical_loss": 4.210532309829965, + "tokens_seen": 263585792 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046470411233701103, + "loss": 2.9607, + "theoretical_loss": 4.21040690524669, + "tokens_seen": 263651328 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046469408224674027, + "loss": 3.1981, + "theoretical_loss": 4.21028154055711, + "tokens_seen": 263716864 + }, + { + "epoch": 0.08, + "objective/train/docs_used": 160649, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.922471284866333, + "objective/train/theoretical_loss": 4.2101562157386265, + "objective/train/tokens_used": 284242400, + "theoretical_loss": 4.2101562157386265, + "tokens_seen": 263782400 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004646840521564694, + "loss": 2.9811, + "theoretical_loss": 4.2101562157386265, + "tokens_seen": 263782400 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046467402206619863, + "loss": 3.1279, + "theoretical_loss": 4.210030930768655, + "tokens_seen": 263847936 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004646639919759278, + "loss": 3.0295, + "theoretical_loss": 4.2099056856246335, + "tokens_seen": 263913472 + }, + { + "epoch": 0.08, + "learning_rate": 0.000464653961885657, + "loss": 3.0133, + "theoretical_loss": 4.209780480284017, + "tokens_seen": 263979008 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004646439317953862, + "loss": 3.2384, + "theoretical_loss": 4.209655314724279, + "tokens_seen": 264044544 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046463390170511535, + "loss": 3.0111, + "theoretical_loss": 4.209530188922911, + "tokens_seen": 264110080 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046462387161484454, + "loss": 3.1056, + "theoretical_loss": 4.209405102857422, + "tokens_seen": 264175616 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046461384152457377, + "loss": 2.9345, + "theoretical_loss": 4.209280056505342, + "tokens_seen": 264241152 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004646038114343029, + "loss": 3.0779, + "theoretical_loss": 4.209155049844217, + "tokens_seen": 264306688 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046459378134403213, + "loss": 3.0116, + "theoretical_loss": 4.209030082851612, + "tokens_seen": 264372224 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004645837512537613, + "loss": 3.0935, + "theoretical_loss": 4.208905155505109, + "tokens_seen": 264437760 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004645737211634905, + "loss": 3.126, + "theoretical_loss": 4.20878026778231, + "tokens_seen": 264503296 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004645636910732197, + "loss": 3.2886, + "theoretical_loss": 4.208655419660834, + "tokens_seen": 264568832 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046455366098294886, + "loss": 3.0061, + "theoretical_loss": 4.208530611118321, + "tokens_seen": 264634368 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046454363089267804, + "loss": 3.0988, + "theoretical_loss": 4.208405842132423, + "tokens_seen": 264699904 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004645336008024073, + "loss": 2.7914, + "theoretical_loss": 4.208281112680817, + "tokens_seen": 264765440 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004645235707121364, + "loss": 3.2416, + "theoretical_loss": 4.208156422741195, + "tokens_seen": 264830976 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046451354062186564, + "loss": 2.8958, + "theoretical_loss": 4.208031772291265, + "tokens_seen": 264896512 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046450351053159476, + "loss": 3.0539, + "theoretical_loss": 4.207907161308757, + "tokens_seen": 264962048 + }, + { + "epoch": 0.08, + "learning_rate": 0.000464493480441324, + "loss": 3.1805, + "theoretical_loss": 4.2077825897714165, + "tokens_seen": 265027584 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004644834503510532, + "loss": 2.8898, + "theoretical_loss": 4.207658057657008, + "tokens_seen": 265093120 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046447342026078236, + "loss": 3.2754, + "theoretical_loss": 4.207533564943316, + "tokens_seen": 265158656 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046446339017051154, + "loss": 3.1402, + "theoretical_loss": 4.207409111608138, + "tokens_seen": 265224192 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004644533600802407, + "loss": 3.3612, + "theoretical_loss": 4.2072846976292935, + "tokens_seen": 265289728 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004644433299899699, + "loss": 3.1182, + "theoretical_loss": 4.2071603229846195, + "tokens_seen": 265355264 + }, + { + "epoch": 0.08, + "objective/train/docs_used": 161970, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.8406832218170166, + "objective/train/theoretical_loss": 4.20703598765197, + "objective/train/tokens_used": 285880800, + "theoretical_loss": 4.20703598765197, + "tokens_seen": 265420800 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046443329989969914, + "loss": 2.9082, + "theoretical_loss": 4.20703598765197, + "tokens_seen": 265420800 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046442326980942827, + "loss": 3.0459, + "theoretical_loss": 4.206911691609217, + "tokens_seen": 265486336 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004644132397191575, + "loss": 3.3701, + "theoretical_loss": 4.206787434834251, + "tokens_seen": 265551872 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004644032096288867, + "loss": 2.9945, + "theoretical_loss": 4.20666321730498, + "tokens_seen": 265617408 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046439317953861586, + "loss": 3.1087, + "theoretical_loss": 4.206539038999329, + "tokens_seen": 265682944 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046438314944834505, + "loss": 3.0794, + "theoretical_loss": 4.206414899895244, + "tokens_seen": 265748480 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004643731193580742, + "loss": 2.9984, + "theoretical_loss": 4.206290799970685, + "tokens_seen": 265814016 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004643630892678034, + "loss": 3.401, + "theoretical_loss": 4.206166739203632, + "tokens_seen": 265879552 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046435305917753264, + "loss": 2.8809, + "theoretical_loss": 4.206042717572082, + "tokens_seen": 265945088 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046434302908726177, + "loss": 2.9364, + "theoretical_loss": 4.20591873505405, + "tokens_seen": 266010624 + }, + { + "epoch": 0.08, + "learning_rate": 0.000464332998996991, + "loss": 2.9368, + "theoretical_loss": 4.20579479162757, + "tokens_seen": 266076160 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046432296890672013, + "loss": 3.3266, + "theoretical_loss": 4.205670887270691, + "tokens_seen": 266141696 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046431293881644937, + "loss": 2.9977, + "theoretical_loss": 4.205547021961482, + "tokens_seen": 266207232 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046430290872617855, + "loss": 2.9025, + "theoretical_loss": 4.205423195678029, + "tokens_seen": 266272768 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046429287863590773, + "loss": 3.1234, + "theoretical_loss": 4.205299408398435, + "tokens_seen": 266338304 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004642828485456369, + "loss": 2.9499, + "theoretical_loss": 4.2051756601008226, + "tokens_seen": 266403840 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046427281845536615, + "loss": 3.0119, + "theoretical_loss": 4.20505195076333, + "tokens_seen": 266469376 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004642627883650953, + "loss": 3.1889, + "theoretical_loss": 4.204928280364115, + "tokens_seen": 266534912 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004642527582748245, + "loss": 3.1874, + "theoretical_loss": 4.20480464888135, + "tokens_seen": 266600448 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046424272818455364, + "loss": 3.1772, + "theoretical_loss": 4.204681056293228, + "tokens_seen": 266665984 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046423269809428287, + "loss": 3.1051, + "theoretical_loss": 4.204557502577957, + "tokens_seen": 266731520 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046422266800401205, + "loss": 2.7565, + "theoretical_loss": 4.204433987713767, + "tokens_seen": 266797056 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046421263791374123, + "loss": 3.079, + "theoretical_loss": 4.2043105116789, + "tokens_seen": 266862592 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004642026078234704, + "loss": 2.9333, + "theoretical_loss": 4.204187074451617, + "tokens_seen": 266928128 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004641925777331996, + "loss": 3.0724, + "theoretical_loss": 4.204063676010202, + "tokens_seen": 266993664 + }, + { + "epoch": 0.08, + "objective/train/docs_used": 162606, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.9326999187469482, + "objective/train/theoretical_loss": 4.203940316332948, + "objective/train/tokens_used": 287519200, + "theoretical_loss": 4.203940316332948, + "tokens_seen": 267059200 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004641825476429288, + "loss": 2.8403, + "theoretical_loss": 4.203940316332948, + "tokens_seen": 267059200 + }, + { + "epoch": 0.08, + "learning_rate": 0.000464172517552658, + "loss": 2.9159, + "theoretical_loss": 4.203816995398171, + "tokens_seen": 267124736 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046416248746238714, + "loss": 2.9578, + "theoretical_loss": 4.203693713184203, + "tokens_seen": 267190272 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004641524573721164, + "loss": 3.0943, + "theoretical_loss": 4.203570469669392, + "tokens_seen": 267255808 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004641424272818455, + "loss": 2.8963, + "theoretical_loss": 4.203447264832107, + "tokens_seen": 267321344 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046413239719157474, + "loss": 3.3, + "theoretical_loss": 4.203324098650731, + "tokens_seen": 267386880 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004641223671013039, + "loss": 3.0447, + "theoretical_loss": 4.203200971103666, + "tokens_seen": 267452416 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004641123370110331, + "loss": 3.0596, + "theoretical_loss": 4.20307788216933, + "tokens_seen": 267517952 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004641023069207623, + "loss": 2.9888, + "theoretical_loss": 4.202954831826159, + "tokens_seen": 267583488 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004640922768304915, + "loss": 2.9303, + "theoretical_loss": 4.202831820052609, + "tokens_seen": 267649024 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046408224674022064, + "loss": 3.077, + "theoretical_loss": 4.202708846827148, + "tokens_seen": 267714560 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004640722166499499, + "loss": 3.1159, + "theoretical_loss": 4.202585912128266, + "tokens_seen": 267780096 + }, + { + "epoch": 0.08, + "learning_rate": 0.000464062186559679, + "loss": 3.2216, + "theoretical_loss": 4.202463015934468, + "tokens_seen": 267845632 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046405215646940824, + "loss": 2.976, + "theoretical_loss": 4.202340158224277, + "tokens_seen": 267911168 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004640421263791374, + "loss": 2.9674, + "theoretical_loss": 4.202217338976231, + "tokens_seen": 267976704 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004640320962888666, + "loss": 3.3398, + "theoretical_loss": 4.2020945581688895, + "tokens_seen": 268042240 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004640220661985958, + "loss": 3.1878, + "theoretical_loss": 4.201971815780826, + "tokens_seen": 268107776 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046401203610832496, + "loss": 3.1188, + "theoretical_loss": 4.201849111790631, + "tokens_seen": 268173312 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046400200601805414, + "loss": 3.0391, + "theoretical_loss": 4.201726446176915, + "tokens_seen": 268238848 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004639919759277834, + "loss": 3.0081, + "theoretical_loss": 4.201603818918302, + "tokens_seen": 268304384 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004639819458375125, + "loss": 3.0651, + "theoretical_loss": 4.201481229993435, + "tokens_seen": 268369920 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046397191574724174, + "loss": 3.0006, + "theoretical_loss": 4.201358679380976, + "tokens_seen": 268435456 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004639618856569709, + "loss": 3.0677, + "theoretical_loss": 4.201236167059601, + "tokens_seen": 268500992 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004639518555667001, + "loss": 2.8345, + "theoretical_loss": 4.201113693008002, + "tokens_seen": 268566528 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046394182547642934, + "loss": 2.7628, + "theoretical_loss": 4.200991257204894, + "tokens_seen": 268632064 + }, + { + "epoch": 0.08, + "objective/train/docs_used": 163909, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.3210434913635254, + "objective/train/theoretical_loss": 4.2008688596290025, + "objective/train/tokens_used": 289157600, + "theoretical_loss": 4.2008688596290025, + "tokens_seen": 268697600 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046393179538615847, + "loss": 3.1274, + "theoretical_loss": 4.2008688596290025, + "tokens_seen": 268697600 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004639217652958877, + "loss": 3.2468, + "theoretical_loss": 4.200746500259073, + "tokens_seen": 268763136 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004639117352056169, + "loss": 3.0926, + "theoretical_loss": 4.200624179073869, + "tokens_seen": 268828672 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046390170511534606, + "loss": 3.3197, + "theoretical_loss": 4.2005018960521685, + "tokens_seen": 268894208 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046389167502507525, + "loss": 2.8972, + "theoretical_loss": 4.200379651172769, + "tokens_seen": 268959744 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004638816449348044, + "loss": 3.1323, + "theoretical_loss": 4.200257444414483, + "tokens_seen": 269025280 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004638716148445336, + "loss": 3.0084, + "theoretical_loss": 4.200135275756139, + "tokens_seen": 269090816 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046386158475426284, + "loss": 3.0795, + "theoretical_loss": 4.200013145176587, + "tokens_seen": 269156352 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046385155466399197, + "loss": 3.0963, + "theoretical_loss": 4.199891052654689, + "tokens_seen": 269221888 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004638415245737212, + "loss": 2.9818, + "theoretical_loss": 4.199768998169326, + "tokens_seen": 269287424 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046383149448345033, + "loss": 3.1024, + "theoretical_loss": 4.199646981699395, + "tokens_seen": 269352960 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046382146439317957, + "loss": 3.1726, + "theoretical_loss": 4.199525003223812, + "tokens_seen": 269418496 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046381143430290875, + "loss": 3.2413, + "theoretical_loss": 4.199403062721506, + "tokens_seen": 269484032 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046380140421263793, + "loss": 3.0264, + "theoretical_loss": 4.199281160171427, + "tokens_seen": 269549568 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004637913741223671, + "loss": 2.9991, + "theoretical_loss": 4.1991592955525405, + "tokens_seen": 269615104 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046378134403209635, + "loss": 3.3086, + "theoretical_loss": 4.199037468843825, + "tokens_seen": 269680640 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004637713139418255, + "loss": 2.956, + "theoretical_loss": 4.198915680024282, + "tokens_seen": 269746176 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004637612838515547, + "loss": 3.0576, + "theoretical_loss": 4.198793929072925, + "tokens_seen": 269811712 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046375125376128384, + "loss": 3.1268, + "theoretical_loss": 4.198672215968785, + "tokens_seen": 269877248 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046374122367101307, + "loss": 3.2694, + "theoretical_loss": 4.198550540690912, + "tokens_seen": 269942784 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046373119358074225, + "loss": 2.9604, + "theoretical_loss": 4.198428903218371, + "tokens_seen": 270008320 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046372116349047143, + "loss": 2.8685, + "theoretical_loss": 4.198307303530243, + "tokens_seen": 270073856 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004637111334002006, + "loss": 3.0788, + "theoretical_loss": 4.198185741605628, + "tokens_seen": 270139392 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004637011033099298, + "loss": 3.2208, + "theoretical_loss": 4.19806421742364, + "tokens_seen": 270204928 + }, + { + "epoch": 0.08, + "learning_rate": 0.000463691073219659, + "loss": 3.2454, + "theoretical_loss": 4.197942730963412, + "tokens_seen": 270270464 + }, + { + "epoch": 0.08, + "objective/train/docs_used": 164570, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.4892289638519287, + "objective/train/theoretical_loss": 4.19782128220409, + "objective/train/tokens_used": 290796000, + "theoretical_loss": 4.19782128220409, + "tokens_seen": 270336000 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004636810431293882, + "loss": 3.0646, + "theoretical_loss": 4.19782128220409, + "tokens_seen": 270336000 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046367101303911734, + "loss": 3.1432, + "theoretical_loss": 4.19769987112484, + "tokens_seen": 270401536 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004636609829488466, + "loss": 3.134, + "theoretical_loss": 4.1975784977048445, + "tokens_seen": 270467072 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004636509528585757, + "loss": 3.199, + "theoretical_loss": 4.1974571619233, + "tokens_seen": 270532608 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046364092276830494, + "loss": 3.1501, + "theoretical_loss": 4.197335863759422, + "tokens_seen": 270598144 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004636308926780341, + "loss": 3.1634, + "theoretical_loss": 4.1972146031924416, + "tokens_seen": 270663680 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004636208625877633, + "loss": 3.032, + "theoretical_loss": 4.197093380201606, + "tokens_seen": 270729216 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004636108324974925, + "loss": 3.1938, + "theoretical_loss": 4.196972194766179, + "tokens_seen": 270794752 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004636008024072217, + "loss": 3.1349, + "theoretical_loss": 4.196851046865442, + "tokens_seen": 270860288 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046359077231695084, + "loss": 3.0471, + "theoretical_loss": 4.1967299364786905, + "tokens_seen": 270925824 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004635807422266801, + "loss": 3.2991, + "theoretical_loss": 4.196608863585239, + "tokens_seen": 270991360 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004635707121364092, + "loss": 3.2406, + "theoretical_loss": 4.1964878281644165, + "tokens_seen": 271056896 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046356068204613844, + "loss": 3.0728, + "theoretical_loss": 4.19636683019557, + "tokens_seen": 271122432 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004635506519558676, + "loss": 2.9971, + "theoretical_loss": 4.196245869658061, + "tokens_seen": 271187968 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004635406218655968, + "loss": 3.2962, + "theoretical_loss": 4.1961249465312696, + "tokens_seen": 271253504 + }, + { + "epoch": 0.08, + "learning_rate": 0.000463530591775326, + "loss": 3.2454, + "theoretical_loss": 4.196004060794589, + "tokens_seen": 271319040 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046352056168505516, + "loss": 3.1115, + "theoretical_loss": 4.195883212427433, + "tokens_seen": 271384576 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046351053159478434, + "loss": 3.0641, + "theoretical_loss": 4.195762401409229, + "tokens_seen": 271450112 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004635005015045136, + "loss": 3.0171, + "theoretical_loss": 4.19564162771942, + "tokens_seen": 271515648 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004634904714142427, + "loss": 3.1574, + "theoretical_loss": 4.195520891337466, + "tokens_seen": 271581184 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046348044132397194, + "loss": 3.3199, + "theoretical_loss": 4.195400192242845, + "tokens_seen": 271646720 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046347041123370107, + "loss": 3.0698, + "theoretical_loss": 4.19527953041505, + "tokens_seen": 271712256 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004634603811434303, + "loss": 2.9288, + "theoretical_loss": 4.19515890583359, + "tokens_seen": 271777792 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004634503510531595, + "loss": 2.9584, + "theoretical_loss": 4.195038318477989, + "tokens_seen": 271843328 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046344032096288867, + "loss": 3.0864, + "theoretical_loss": 4.194917768327789, + "tokens_seen": 271908864 + }, + { + "epoch": 0.08, + "objective/train/docs_used": 165961, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.39284610748291, + "objective/train/theoretical_loss": 4.194797255362549, + "objective/train/tokens_used": 292434400, + "theoretical_loss": 4.194797255362549, + "tokens_seen": 271974400 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046343029087261785, + "loss": 3.0759, + "theoretical_loss": 4.194797255362549, + "tokens_seen": 271974400 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004634202607823471, + "loss": 3.1504, + "theoretical_loss": 4.194676779561841, + "tokens_seen": 272039936 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004634102306920762, + "loss": 3.2928, + "theoretical_loss": 4.194556340905256, + "tokens_seen": 272105472 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046340020060180545, + "loss": 3.2224, + "theoretical_loss": 4.194435939372401, + "tokens_seen": 272171008 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046339017051153457, + "loss": 3.2887, + "theoretical_loss": 4.194315574942896, + "tokens_seen": 272236544 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004633801404212638, + "loss": 3.0728, + "theoretical_loss": 4.194195247596381, + "tokens_seen": 272302080 + }, + { + "epoch": 0.08, + "learning_rate": 0.000463370110330993, + "loss": 2.966, + "theoretical_loss": 4.19407495731251, + "tokens_seen": 272367616 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046336008024072217, + "loss": 3.1345, + "theoretical_loss": 4.193954704070952, + "tokens_seen": 272433152 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046335005015045135, + "loss": 2.8613, + "theoretical_loss": 4.193834487851396, + "tokens_seen": 272498688 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046334002006018053, + "loss": 2.9628, + "theoretical_loss": 4.193714308633542, + "tokens_seen": 272564224 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004633299899699097, + "loss": 3.0543, + "theoretical_loss": 4.1935941663971095, + "tokens_seen": 272629760 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046331995987963895, + "loss": 3.2005, + "theoretical_loss": 4.193474061121833, + "tokens_seen": 272695296 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004633099297893681, + "loss": 3.0841, + "theoretical_loss": 4.193353992787463, + "tokens_seen": 272760832 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004632998996990973, + "loss": 3.1344, + "theoretical_loss": 4.193233961373766, + "tokens_seen": 272826368 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046328986960882644, + "loss": 3.0634, + "theoretical_loss": 4.1931139668605235, + "tokens_seen": 272891904 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004632798395185557, + "loss": 3.3775, + "theoretical_loss": 4.192994009227535, + "tokens_seen": 272957440 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046326980942828485, + "loss": 2.8788, + "theoretical_loss": 4.192874088454613, + "tokens_seen": 273022976 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046325977933801404, + "loss": 3.0913, + "theoretical_loss": 4.19275420452159, + "tokens_seen": 273088512 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004632497492477432, + "loss": 3.0962, + "theoretical_loss": 4.192634357408309, + "tokens_seen": 273154048 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046323971915747245, + "loss": 3.0276, + "theoretical_loss": 4.192514547094634, + "tokens_seen": 273219584 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004632296890672016, + "loss": 3.0959, + "theoretical_loss": 4.192394773560441, + "tokens_seen": 273285120 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004632196589769308, + "loss": 3.1401, + "theoretical_loss": 4.192275036785625, + "tokens_seen": 273350656 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046320962888666, + "loss": 2.936, + "theoretical_loss": 4.192155336750094, + "tokens_seen": 273416192 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004631995987963892, + "loss": 3.082, + "theoretical_loss": 4.192035673433773, + "tokens_seen": 273481728 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004631895687061184, + "loss": 3.1713, + "theoretical_loss": 4.191916046816605, + "tokens_seen": 273547264 + }, + { + "epoch": 0.08, + "objective/train/docs_used": 166474, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.4257972240448, + "objective/train/theoretical_loss": 4.191796456878544, + "objective/train/tokens_used": 294072800, + "theoretical_loss": 4.191796456878544, + "tokens_seen": 273612800 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046317953861584754, + "loss": 3.0705, + "theoretical_loss": 4.191796456878544, + "tokens_seen": 273612800 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004631695085255768, + "loss": 3.0263, + "theoretical_loss": 4.191676903599563, + "tokens_seen": 273678336 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004631594784353059, + "loss": 3.0322, + "theoretical_loss": 4.191557386959651, + "tokens_seen": 273743872 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046314944834503514, + "loss": 3.107, + "theoretical_loss": 4.191437906938811, + "tokens_seen": 273809408 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004631394182547643, + "loss": 3.212, + "theoretical_loss": 4.191318463517062, + "tokens_seen": 273874944 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004631293881644935, + "loss": 3.1908, + "theoretical_loss": 4.19119905667444, + "tokens_seen": 273940480 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004631193580742227, + "loss": 3.0101, + "theoretical_loss": 4.191079686390996, + "tokens_seen": 274006016 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004631093279839519, + "loss": 3.117, + "theoretical_loss": 4.190960352646796, + "tokens_seen": 274071552 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046309929789368104, + "loss": 3.0705, + "theoretical_loss": 4.190841055421921, + "tokens_seen": 274137088 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004630892678034103, + "loss": 3.1212, + "theoretical_loss": 4.19072179469647, + "tokens_seen": 274202624 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004630792377131394, + "loss": 3.0454, + "theoretical_loss": 4.190602570450556, + "tokens_seen": 274268160 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046306920762286864, + "loss": 3.1203, + "theoretical_loss": 4.190483382664308, + "tokens_seen": 274333696 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004630591775325978, + "loss": 3.3216, + "theoretical_loss": 4.19036423131787, + "tokens_seen": 274399232 + }, + { + "epoch": 0.08, + "learning_rate": 0.000463049147442327, + "loss": 3.0616, + "theoretical_loss": 4.190245116391403, + "tokens_seen": 274464768 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004630391173520562, + "loss": 3.2246, + "theoretical_loss": 4.190126037865082, + "tokens_seen": 274530304 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046302908726178536, + "loss": 3.1843, + "theoretical_loss": 4.190006995719098, + "tokens_seen": 274595840 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046301905717151455, + "loss": 3.2058, + "theoretical_loss": 4.1898879899336565, + "tokens_seen": 274661376 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004630090270812438, + "loss": 3.1747, + "theoretical_loss": 4.189769020488981, + "tokens_seen": 274726912 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004629989969909729, + "loss": 3.0121, + "theoretical_loss": 4.189650087365309, + "tokens_seen": 274792448 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046298896690070214, + "loss": 3.2451, + "theoretical_loss": 4.189531190542893, + "tokens_seen": 274857984 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046297893681043127, + "loss": 2.9136, + "theoretical_loss": 4.189412330002001, + "tokens_seen": 274923520 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004629689067201605, + "loss": 3.0501, + "theoretical_loss": 4.189293505722918, + "tokens_seen": 274989056 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004629588766298897, + "loss": 3.3945, + "theoretical_loss": 4.189174717685942, + "tokens_seen": 275054592 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046294884653961887, + "loss": 3.1586, + "theoretical_loss": 4.189055965871389, + "tokens_seen": 275120128 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046293881644934805, + "loss": 3.069, + "theoretical_loss": 4.188937250259587, + "tokens_seen": 275185664 + }, + { + "epoch": 0.08, + "objective/train/docs_used": 167916, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.4955620765686035, + "objective/train/theoretical_loss": 4.188818570830883, + "objective/train/tokens_used": 295711200, + "theoretical_loss": 4.188818570830883, + "tokens_seen": 275251200 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004629287863590773, + "loss": 3.2733, + "theoretical_loss": 4.188818570830883, + "tokens_seen": 275251200 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004629187562688064, + "loss": 3.1768, + "theoretical_loss": 4.188699927565638, + "tokens_seen": 275316736 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046290872617853565, + "loss": 3.2311, + "theoretical_loss": 4.188581320444228, + "tokens_seen": 275382272 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046289869608826477, + "loss": 3.1863, + "theoretical_loss": 4.1884627494470426, + "tokens_seen": 275447808 + }, + { + "epoch": 0.08, + "learning_rate": 0.000462888665997994, + "loss": 3.0371, + "theoretical_loss": 4.1883442145544905, + "tokens_seen": 275513344 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004628786359077232, + "loss": 3.0025, + "theoretical_loss": 4.188225715746992, + "tokens_seen": 275578880 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046286860581745237, + "loss": 2.9066, + "theoretical_loss": 4.188107253004986, + "tokens_seen": 275644416 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046285857572718155, + "loss": 3.0672, + "theoretical_loss": 4.187988826308925, + "tokens_seen": 275709952 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046284854563691073, + "loss": 3.2419, + "theoretical_loss": 4.187870435639275, + "tokens_seen": 275775488 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004628385155466399, + "loss": 3.2879, + "theoretical_loss": 4.18775208097652, + "tokens_seen": 275841024 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046282848545636915, + "loss": 3.1249, + "theoretical_loss": 4.187633762301159, + "tokens_seen": 275906560 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004628184553660983, + "loss": 3.2244, + "theoretical_loss": 4.187515479593704, + "tokens_seen": 275972096 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004628084252758275, + "loss": 3.2093, + "theoretical_loss": 4.187397232834683, + "tokens_seen": 276037632 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046279839518555664, + "loss": 3.0838, + "theoretical_loss": 4.187279022004642, + "tokens_seen": 276103168 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004627883650952859, + "loss": 3.1865, + "theoretical_loss": 4.1871608470841375, + "tokens_seen": 276168704 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046277833500501505, + "loss": 3.1265, + "theoretical_loss": 4.1870427080537445, + "tokens_seen": 276234240 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046276830491474424, + "loss": 2.9422, + "theoretical_loss": 4.1869246048940525, + "tokens_seen": 276299776 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004627582748244734, + "loss": 3.1934, + "theoretical_loss": 4.186806537585666, + "tokens_seen": 276365312 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046274824473420265, + "loss": 3.0719, + "theoretical_loss": 4.186688506109202, + "tokens_seen": 276430848 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004627382146439318, + "loss": 3.0738, + "theoretical_loss": 4.186570510445296, + "tokens_seen": 276496384 + }, + { + "epoch": 0.08, + "learning_rate": 0.000462728184553661, + "loss": 2.9456, + "theoretical_loss": 4.186452550574599, + "tokens_seen": 276561920 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046271815446339014, + "loss": 3.1608, + "theoretical_loss": 4.186334626477774, + "tokens_seen": 276627456 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004627081243731194, + "loss": 3.2442, + "theoretical_loss": 4.186216738135501, + "tokens_seen": 276692992 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046269809428284856, + "loss": 3.2259, + "theoretical_loss": 4.186098885528473, + "tokens_seen": 276758528 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046268806419257774, + "loss": 3.1673, + "theoretical_loss": 4.185981068637401, + "tokens_seen": 276824064 + }, + { + "epoch": 0.08, + "objective/train/docs_used": 168545, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.0085372924804688, + "objective/train/theoretical_loss": 4.185863287443008, + "objective/train/tokens_used": 297349600, + "theoretical_loss": 4.185863287443008, + "tokens_seen": 276889600 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004626780341023069, + "loss": 3.2473, + "theoretical_loss": 4.185863287443008, + "tokens_seen": 276889600 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004626680040120361, + "loss": 3.0155, + "theoretical_loss": 4.185745541926035, + "tokens_seen": 276955136 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004626579739217653, + "loss": 3.2622, + "theoretical_loss": 4.185627832067237, + "tokens_seen": 277020672 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004626479438314945, + "loss": 3.1767, + "theoretical_loss": 4.1855101578473795, + "tokens_seen": 277086208 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046263791374122364, + "loss": 3.2087, + "theoretical_loss": 4.18539251924725, + "tokens_seen": 277151744 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004626278836509529, + "loss": 3.105, + "theoretical_loss": 4.185274916247646, + "tokens_seen": 277217280 + }, + { + "epoch": 0.08, + "learning_rate": 0.000462617853560682, + "loss": 3.1405, + "theoretical_loss": 4.185157348829383, + "tokens_seen": 277282816 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046260782347041124, + "loss": 3.0807, + "theoretical_loss": 4.185039816973289, + "tokens_seen": 277348352 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004625977933801404, + "loss": 3.2385, + "theoretical_loss": 4.184922320660207, + "tokens_seen": 277413888 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004625877632898696, + "loss": 3.2279, + "theoretical_loss": 4.184804859870997, + "tokens_seen": 277479424 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004625777331995988, + "loss": 3.3039, + "theoretical_loss": 4.184687434586531, + "tokens_seen": 277544960 + }, + { + "epoch": 0.08, + "learning_rate": 0.000462567703109328, + "loss": 3.2473, + "theoretical_loss": 4.184570044787698, + "tokens_seen": 277610496 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046255767301905715, + "loss": 2.8718, + "theoretical_loss": 4.1844526904554, + "tokens_seen": 277676032 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004625476429287864, + "loss": 3.1032, + "theoretical_loss": 4.184335371570556, + "tokens_seen": 277741568 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004625376128385155, + "loss": 2.8166, + "theoretical_loss": 4.184218088114097, + "tokens_seen": 277807104 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046252758274824475, + "loss": 2.9123, + "theoretical_loss": 4.1841008400669715, + "tokens_seen": 277872640 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004625175526579739, + "loss": 3.0332, + "theoretical_loss": 4.183983627410142, + "tokens_seen": 277938176 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004625075225677031, + "loss": 3.1231, + "theoretical_loss": 4.183866450124584, + "tokens_seen": 278003712 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004624974924774323, + "loss": 2.8858, + "theoretical_loss": 4.18374930819129, + "tokens_seen": 278069248 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046248746238716147, + "loss": 2.9387, + "theoretical_loss": 4.183632201591264, + "tokens_seen": 278134784 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046247743229689065, + "loss": 3.3533, + "theoretical_loss": 4.18351513030553, + "tokens_seen": 278200320 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004624674022066199, + "loss": 3.2685, + "theoretical_loss": 4.1833980943151206, + "tokens_seen": 278265856 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046245737211634907, + "loss": 3.1065, + "theoretical_loss": 4.183281093601087, + "tokens_seen": 278331392 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046244734202607825, + "loss": 2.9046, + "theoretical_loss": 4.183164128144495, + "tokens_seen": 278396928 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004624373119358075, + "loss": 3.0318, + "theoretical_loss": 4.183047197926422, + "tokens_seen": 278462464 + }, + { + "epoch": 0.08, + "objective/train/docs_used": 169743, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.9320547580718994, + "objective/train/theoretical_loss": 4.182930302927963, + "objective/train/tokens_used": 298988000, + "theoretical_loss": 4.182930302927963, + "tokens_seen": 278528000 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004624272818455366, + "loss": 3.0224, + "theoretical_loss": 4.182930302927963, + "tokens_seen": 278528000 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046241725175526585, + "loss": 2.9967, + "theoretical_loss": 4.182813443130227, + "tokens_seen": 278593536 + }, + { + "epoch": 0.08, + "learning_rate": 0.000462407221664995, + "loss": 3.1155, + "theoretical_loss": 4.182696618514337, + "tokens_seen": 278659072 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004623971915747242, + "loss": 3.0413, + "theoretical_loss": 4.18257982906143, + "tokens_seen": 278724608 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004623871614844534, + "loss": 3.0151, + "theoretical_loss": 4.1824630747526585, + "tokens_seen": 278790144 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046237713139418257, + "loss": 3.0908, + "theoretical_loss": 4.182346355569189, + "tokens_seen": 278855680 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046236710130391175, + "loss": 3.0478, + "theoretical_loss": 4.182229671492204, + "tokens_seen": 278921216 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046235707121364093, + "loss": 2.8684, + "theoretical_loss": 4.1821130225028975, + "tokens_seen": 278986752 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004623470411233701, + "loss": 2.9918, + "theoretical_loss": 4.1819964085824815, + "tokens_seen": 279052288 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046233701103309935, + "loss": 2.9214, + "theoretical_loss": 4.181879829712178, + "tokens_seen": 279117824 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004623269809428285, + "loss": 3.1783, + "theoretical_loss": 4.181763285873231, + "tokens_seen": 279183360 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004623169508525577, + "loss": 3.0012, + "theoretical_loss": 4.181646777046889, + "tokens_seen": 279248896 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046230692076228684, + "loss": 2.9717, + "theoretical_loss": 4.181530303214423, + "tokens_seen": 279314432 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004622968906720161, + "loss": 3.2372, + "theoretical_loss": 4.181413864357115, + "tokens_seen": 279379968 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046228686058174525, + "loss": 3.1612, + "theoretical_loss": 4.181297460456262, + "tokens_seen": 279445504 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046227683049147444, + "loss": 3.1548, + "theoretical_loss": 4.181181091493174, + "tokens_seen": 279511040 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004622668004012036, + "loss": 3.0632, + "theoretical_loss": 4.181064757449178, + "tokens_seen": 279576576 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046225677031093285, + "loss": 3.2773, + "theoretical_loss": 4.180948458305615, + "tokens_seen": 279642112 + }, + { + "epoch": 0.08, + "learning_rate": 0.000462246740220662, + "loss": 3.2008, + "theoretical_loss": 4.180832194043836, + "tokens_seen": 279707648 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004622367101303912, + "loss": 3.2201, + "theoretical_loss": 4.180715964645213, + "tokens_seen": 279773184 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046222668004012034, + "loss": 2.9943, + "theoretical_loss": 4.180599770091126, + "tokens_seen": 279838720 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004622166499498496, + "loss": 3.0171, + "theoretical_loss": 4.180483610362975, + "tokens_seen": 279904256 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046220661985957876, + "loss": 3.1411, + "theoretical_loss": 4.18036748544217, + "tokens_seen": 279969792 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046219658976930794, + "loss": 2.9301, + "theoretical_loss": 4.180251395310137, + "tokens_seen": 280035328 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004621865596790371, + "loss": 3.2077, + "theoretical_loss": 4.1801353399483165, + "tokens_seen": 280100864 + }, + { + "epoch": 0.08, + "objective/train/docs_used": 170343, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.683032274246216, + "objective/train/theoretical_loss": 4.180019319338163, + "objective/train/tokens_used": 300626400, + "theoretical_loss": 4.180019319338163, + "tokens_seen": 280166400 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004621765295887663, + "loss": 3.0922, + "theoretical_loss": 4.180019319338163, + "tokens_seen": 280166400 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004621664994984955, + "loss": 3.036, + "theoretical_loss": 4.179903333461144, + "tokens_seen": 280231936 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004621564694082247, + "loss": 3.2977, + "theoretical_loss": 4.179787382298744, + "tokens_seen": 280297472 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046214643931795384, + "loss": 3.0756, + "theoretical_loss": 4.179671465832458, + "tokens_seen": 280363008 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004621364092276831, + "loss": 3.2732, + "theoretical_loss": 4.179555584043799, + "tokens_seen": 280428544 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004621263791374122, + "loss": 3.1875, + "theoretical_loss": 4.17943973691429, + "tokens_seen": 280494080 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046211634904714144, + "loss": 2.9716, + "theoretical_loss": 4.179323924425472, + "tokens_seen": 280559616 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004621063189568706, + "loss": 2.944, + "theoretical_loss": 4.179208146558899, + "tokens_seen": 280625152 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004620962888665998, + "loss": 3.0199, + "theoretical_loss": 4.1790924032961385, + "tokens_seen": 280690688 + }, + { + "epoch": 0.09, + "learning_rate": 0.000462086258776329, + "loss": 3.0334, + "theoretical_loss": 4.178976694618772, + "tokens_seen": 280756224 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004620762286860582, + "loss": 3.1781, + "theoretical_loss": 4.178861020508395, + "tokens_seen": 280821760 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046206619859578735, + "loss": 2.7781, + "theoretical_loss": 4.178745380946619, + "tokens_seen": 280887296 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004620561685055166, + "loss": 2.8229, + "theoretical_loss": 4.178629775915066, + "tokens_seen": 280952832 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004620461384152457, + "loss": 3.3984, + "theoretical_loss": 4.178514205395376, + "tokens_seen": 281018368 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046203610832497495, + "loss": 3.2722, + "theoretical_loss": 4.178398669369201, + "tokens_seen": 281083904 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004620260782347041, + "loss": 3.287, + "theoretical_loss": 4.178283167818206, + "tokens_seen": 281149440 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004620160481444333, + "loss": 3.0895, + "theoretical_loss": 4.178167700724073, + "tokens_seen": 281214976 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004620060180541625, + "loss": 2.8691, + "theoretical_loss": 4.178052268068494, + "tokens_seen": 281280512 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046199598796389167, + "loss": 3.1857, + "theoretical_loss": 4.177936869833179, + "tokens_seen": 281346048 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046198595787362085, + "loss": 3.0906, + "theoretical_loss": 4.17782150599985, + "tokens_seen": 281411584 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004619759277833501, + "loss": 3.2616, + "theoretical_loss": 4.1777061765502435, + "tokens_seen": 281477120 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004619658976930792, + "loss": 3.12, + "theoretical_loss": 4.1775908814661085, + "tokens_seen": 281542656 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046195586760280845, + "loss": 2.9188, + "theoretical_loss": 4.17747562072921, + "tokens_seen": 281608192 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046194583751253763, + "loss": 3.178, + "theoretical_loss": 4.177360394321325, + "tokens_seen": 281673728 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004619358074222668, + "loss": 3.0027, + "theoretical_loss": 4.177245202224246, + "tokens_seen": 281739264 + }, + { + "epoch": 0.09, + "objective/train/docs_used": 171008, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.9614992141723633, + "objective/train/theoretical_loss": 4.17713004441978, + "objective/train/tokens_used": 302264800, + "theoretical_loss": 4.17713004441978, + "tokens_seen": 281804800 + }, + { + "epoch": 0.09, + "learning_rate": 0.000461925777331996, + "loss": 2.9775, + "theoretical_loss": 4.17713004441978, + "tokens_seen": 281804800 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004619157472417252, + "loss": 2.973, + "theoretical_loss": 4.177014920889745, + "tokens_seen": 281870336 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046190571715145435, + "loss": 3.0844, + "theoretical_loss": 4.176899831615974, + "tokens_seen": 281935872 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004618956870611836, + "loss": 3.0681, + "theoretical_loss": 4.176784776580316, + "tokens_seen": 282001408 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004618856569709127, + "loss": 3.2095, + "theoretical_loss": 4.176669755764632, + "tokens_seen": 282066944 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046187562688064195, + "loss": 3.12, + "theoretical_loss": 4.176554769150796, + "tokens_seen": 282132480 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004618655967903711, + "loss": 3.1457, + "theoretical_loss": 4.176439816720697, + "tokens_seen": 282198016 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004618555667001003, + "loss": 2.9781, + "theoretical_loss": 4.1763248984562376, + "tokens_seen": 282263552 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004618455366098295, + "loss": 3.1034, + "theoretical_loss": 4.176210014339335, + "tokens_seen": 282329088 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004618355065195587, + "loss": 3.1171, + "theoretical_loss": 4.17609516435192, + "tokens_seen": 282394624 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046182547642928786, + "loss": 3.2617, + "theoretical_loss": 4.1759803484759335, + "tokens_seen": 282460160 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046181544633901704, + "loss": 2.9321, + "theoretical_loss": 4.175865566693336, + "tokens_seen": 282525696 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004618054162487462, + "loss": 3.3135, + "theoretical_loss": 4.175750818986098, + "tokens_seen": 282591232 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046179538615847545, + "loss": 3.1274, + "theoretical_loss": 4.1756361053362046, + "tokens_seen": 282656768 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004617853560682046, + "loss": 3.0139, + "theoretical_loss": 4.1755214257256545, + "tokens_seen": 282722304 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004617753259779338, + "loss": 3.2311, + "theoretical_loss": 4.17540678013646, + "tokens_seen": 282787840 + }, + { + "epoch": 0.09, + "learning_rate": 0.000461765295887663, + "loss": 2.8269, + "theoretical_loss": 4.175292168550648, + "tokens_seen": 282853376 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004617552657973922, + "loss": 3.1649, + "theoretical_loss": 4.175177590950257, + "tokens_seen": 282918912 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046174523570712136, + "loss": 3.2356, + "theoretical_loss": 4.175063047317342, + "tokens_seen": 282984448 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046173520561685054, + "loss": 3.186, + "theoretical_loss": 4.174948537633968, + "tokens_seen": 283049984 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004617251755265797, + "loss": 2.9836, + "theoretical_loss": 4.174834061882218, + "tokens_seen": 283115520 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046171514543630896, + "loss": 3.2835, + "theoretical_loss": 4.1747196200441845, + "tokens_seen": 283181056 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046170511534603814, + "loss": 2.9382, + "theoretical_loss": 4.174605212101977, + "tokens_seen": 283246592 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004616950852557673, + "loss": 3.2108, + "theoretical_loss": 4.174490838037716, + "tokens_seen": 283312128 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004616850551654965, + "loss": 2.9375, + "theoretical_loss": 4.174376497833537, + "tokens_seen": 283377664 + }, + { + "epoch": 0.09, + "objective/train/docs_used": 172282, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.1862382888793945, + "objective/train/theoretical_loss": 4.174262191471587, + "objective/train/tokens_used": 303903200, + "theoretical_loss": 4.174262191471587, + "tokens_seen": 283443200 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004616750250752257, + "loss": 3.0131, + "theoretical_loss": 4.174262191471587, + "tokens_seen": 283443200 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004616649949849549, + "loss": 2.8988, + "theoretical_loss": 4.17414791893403, + "tokens_seen": 283508736 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046165496489468404, + "loss": 3.1517, + "theoretical_loss": 4.17403368020304, + "tokens_seen": 283574272 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004616449348044133, + "loss": 3.1693, + "theoretical_loss": 4.173919475260808, + "tokens_seen": 283639808 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004616349047141424, + "loss": 2.9719, + "theoretical_loss": 4.173805304089536, + "tokens_seen": 283705344 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046162487462387164, + "loss": 2.8531, + "theoretical_loss": 4.173691166671439, + "tokens_seen": 283770880 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004616148445336008, + "loss": 3.2185, + "theoretical_loss": 4.173577062988748, + "tokens_seen": 283836416 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046160481444333, + "loss": 2.8151, + "theoretical_loss": 4.173462993023706, + "tokens_seen": 283901952 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004615947843530592, + "loss": 3.07, + "theoretical_loss": 4.173348956758568, + "tokens_seen": 283967488 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004615847542627884, + "loss": 3.2179, + "theoretical_loss": 4.173234954175605, + "tokens_seen": 284033024 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046157472417251755, + "loss": 3.0998, + "theoretical_loss": 4.173120985257102, + "tokens_seen": 284098560 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004615646940822468, + "loss": 2.9929, + "theoretical_loss": 4.173007049985352, + "tokens_seen": 284164096 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004615546639919759, + "loss": 2.9733, + "theoretical_loss": 4.172893148342667, + "tokens_seen": 284229632 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046154463390170515, + "loss": 3.242, + "theoretical_loss": 4.172779280311372, + "tokens_seen": 284295168 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004615346038114343, + "loss": 3.1582, + "theoretical_loss": 4.172665445873801, + "tokens_seen": 284360704 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004615245737211635, + "loss": 2.8127, + "theoretical_loss": 4.172551645012307, + "tokens_seen": 284426240 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004615145436308927, + "loss": 3.2592, + "theoretical_loss": 4.1724378777092515, + "tokens_seen": 284491776 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046150451354062187, + "loss": 3.189, + "theoretical_loss": 4.172324143947012, + "tokens_seen": 284557312 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046149448345035105, + "loss": 3.3986, + "theoretical_loss": 4.172210443707979, + "tokens_seen": 284622848 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004614844533600803, + "loss": 3.0392, + "theoretical_loss": 4.1720967769745565, + "tokens_seen": 284688384 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004614744232698094, + "loss": 2.9162, + "theoretical_loss": 4.171983143729159, + "tokens_seen": 284753920 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046146439317953865, + "loss": 2.786, + "theoretical_loss": 4.1718695439542195, + "tokens_seen": 284819456 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046145436308926783, + "loss": 2.9771, + "theoretical_loss": 4.17175597763218, + "tokens_seen": 284884992 + }, + { + "epoch": 0.09, + "learning_rate": 0.000461444332998997, + "loss": 3.0589, + "theoretical_loss": 4.171642444745497, + "tokens_seen": 284950528 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004614343029087262, + "loss": 2.9479, + "theoretical_loss": 4.1715289452766395, + "tokens_seen": 285016064 + }, + { + "epoch": 0.09, + "objective/train/docs_used": 172985, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.2075891494750977, + "objective/train/theoretical_loss": 4.1714154792080915, + "objective/train/tokens_used": 305541600, + "theoretical_loss": 4.1714154792080915, + "tokens_seen": 285081600 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004614242728184554, + "loss": 3.0558, + "theoretical_loss": 4.1714154792080915, + "tokens_seen": 285081600 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046141424272818455, + "loss": 2.8283, + "theoretical_loss": 4.171302046522349, + "tokens_seen": 285147136 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004614042126379138, + "loss": 3.2928, + "theoretical_loss": 4.171188647201921, + "tokens_seen": 285212672 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004613941825476429, + "loss": 3.0267, + "theoretical_loss": 4.1710752812293315, + "tokens_seen": 285278208 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046138415245737215, + "loss": 3.3528, + "theoretical_loss": 4.170961948587115, + "tokens_seen": 285343744 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004613741223671013, + "loss": 3.1969, + "theoretical_loss": 4.17084864925782, + "tokens_seen": 285409280 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004613640922768305, + "loss": 3.0018, + "theoretical_loss": 4.1707353832240095, + "tokens_seen": 285474816 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004613540621865597, + "loss": 3.1038, + "theoretical_loss": 4.170622150468258, + "tokens_seen": 285540352 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004613440320962889, + "loss": 3.2723, + "theoretical_loss": 4.170508950973154, + "tokens_seen": 285605888 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046133400200601806, + "loss": 3.0653, + "theoretical_loss": 4.1703957847213, + "tokens_seen": 285671424 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046132397191574724, + "loss": 3.1866, + "theoretical_loss": 4.170282651695308, + "tokens_seen": 285736960 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004613139418254764, + "loss": 2.8471, + "theoretical_loss": 4.170169551877808, + "tokens_seen": 285802496 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046130391173520566, + "loss": 3.2179, + "theoretical_loss": 4.170056485251439, + "tokens_seen": 285868032 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004612938816449348, + "loss": 3.2434, + "theoretical_loss": 4.169943451798856, + "tokens_seen": 285933568 + }, + { + "epoch": 0.09, + "learning_rate": 0.000461283851554664, + "loss": 3.0074, + "theoretical_loss": 4.169830451502724, + "tokens_seen": 285999104 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004612738214643932, + "loss": 2.925, + "theoretical_loss": 4.169717484345725, + "tokens_seen": 286064640 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004612637913741224, + "loss": 3.2055, + "theoretical_loss": 4.1696045503105506, + "tokens_seen": 286130176 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046125376128385156, + "loss": 3.1824, + "theoretical_loss": 4.169491649379905, + "tokens_seen": 286195712 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046124373119358074, + "loss": 3.0065, + "theoretical_loss": 4.169378781536509, + "tokens_seen": 286261248 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004612337011033099, + "loss": 2.9547, + "theoretical_loss": 4.169265946763095, + "tokens_seen": 286326784 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046122367101303916, + "loss": 3.0051, + "theoretical_loss": 4.169153145042405, + "tokens_seen": 286392320 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004612136409227683, + "loss": 3.3204, + "theoretical_loss": 4.169040376357199, + "tokens_seen": 286457856 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004612036108324975, + "loss": 2.912, + "theoretical_loss": 4.168927640690246, + "tokens_seen": 286523392 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046119358074222665, + "loss": 3.1211, + "theoretical_loss": 4.16881493802433, + "tokens_seen": 286588928 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004611835506519559, + "loss": 3.0937, + "theoretical_loss": 4.168702268342248, + "tokens_seen": 286654464 + }, + { + "epoch": 0.09, + "objective/train/docs_used": 174327, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7830283641815186, + "objective/train/theoretical_loss": 4.168589631626808, + "objective/train/tokens_used": 307180000, + "theoretical_loss": 4.168589631626808, + "tokens_seen": 286720000 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046117352056168506, + "loss": 3.1613, + "theoretical_loss": 4.168589631626808, + "tokens_seen": 286720000 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046116349047141425, + "loss": 2.9701, + "theoretical_loss": 4.168477027860833, + "tokens_seen": 286785536 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004611534603811434, + "loss": 3.2039, + "theoretical_loss": 4.168364457027158, + "tokens_seen": 286851072 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004611434302908726, + "loss": 3.2299, + "theoretical_loss": 4.168251919108632, + "tokens_seen": 286916608 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004611334002006018, + "loss": 2.9995, + "theoretical_loss": 4.168139414088113, + "tokens_seen": 286982144 + }, + { + "epoch": 0.09, + "learning_rate": 0.000461123370110331, + "loss": 3.2146, + "theoretical_loss": 4.168026941948478, + "tokens_seen": 287047680 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046111334002006015, + "loss": 3.2024, + "theoretical_loss": 4.167914502672611, + "tokens_seen": 287113216 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004611033099297894, + "loss": 3.01, + "theoretical_loss": 4.1678020962434115, + "tokens_seen": 287178752 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046109327983951857, + "loss": 2.9861, + "theoretical_loss": 4.167689722643792, + "tokens_seen": 287244288 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046108324974924775, + "loss": 3.2729, + "theoretical_loss": 4.1675773818566775, + "tokens_seen": 287309824 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046107321965897693, + "loss": 2.978, + "theoretical_loss": 4.167465073865006, + "tokens_seen": 287375360 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004610631895687061, + "loss": 2.9117, + "theoretical_loss": 4.167352798651726, + "tokens_seen": 287440896 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004610531594784353, + "loss": 2.9516, + "theoretical_loss": 4.167240556199802, + "tokens_seen": 287506432 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004610431293881645, + "loss": 3.1987, + "theoretical_loss": 4.167128346492211, + "tokens_seen": 287571968 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046103309929789365, + "loss": 3.2585, + "theoretical_loss": 4.16701616951194, + "tokens_seen": 287637504 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004610230692076229, + "loss": 3.0668, + "theoretical_loss": 4.1669040252419896, + "tokens_seen": 287703040 + }, + { + "epoch": 0.09, + "learning_rate": 0.000461013039117352, + "loss": 3.0266, + "theoretical_loss": 4.166791913665375, + "tokens_seen": 287768576 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046100300902708125, + "loss": 2.9039, + "theoretical_loss": 4.166679834765123, + "tokens_seen": 287834112 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046099297893681043, + "loss": 3.1635, + "theoretical_loss": 4.166567788524272, + "tokens_seen": 287899648 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004609829488465396, + "loss": 3.3642, + "theoretical_loss": 4.166455774925875, + "tokens_seen": 287965184 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004609729187562688, + "loss": 3.0274, + "theoretical_loss": 4.166343793952995, + "tokens_seen": 288030720 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046096288866599803, + "loss": 2.8123, + "theoretical_loss": 4.166231845588712, + "tokens_seen": 288096256 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004609528585757272, + "loss": 3.1348, + "theoretical_loss": 4.166119929816113, + "tokens_seen": 288161792 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004609428284854564, + "loss": 3.3564, + "theoretical_loss": 4.166008046618303, + "tokens_seen": 288227328 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004609327983951856, + "loss": 2.9192, + "theoretical_loss": 4.1658961959783944, + "tokens_seen": 288292864 + }, + { + "epoch": 0.09, + "objective/train/docs_used": 174865, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.853527069091797, + "objective/train/theoretical_loss": 4.165784377879517, + "objective/train/tokens_used": 308818400, + "theoretical_loss": 4.165784377879517, + "tokens_seen": 288358400 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046092276830491475, + "loss": 2.9512, + "theoretical_loss": 4.165784377879517, + "tokens_seen": 288358400 + }, + { + "epoch": 0.09, + "learning_rate": 0.000460912738214644, + "loss": 2.9297, + "theoretical_loss": 4.165672592304811, + "tokens_seen": 288423936 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004609027081243731, + "loss": 3.3015, + "theoretical_loss": 4.165560839237429, + "tokens_seen": 288489472 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046089267803410235, + "loss": 2.881, + "theoretical_loss": 4.165449118660536, + "tokens_seen": 288555008 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004608826479438315, + "loss": 3.0803, + "theoretical_loss": 4.16533743055731, + "tokens_seen": 288620544 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004608726178535607, + "loss": 3.1305, + "theoretical_loss": 4.165225774910941, + "tokens_seen": 288686080 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004608625877632899, + "loss": 3.0305, + "theoretical_loss": 4.165114151704634, + "tokens_seen": 288751616 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004608525576730191, + "loss": 3.1447, + "theoretical_loss": 4.165002560921601, + "tokens_seen": 288817152 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046084252758274826, + "loss": 3.0493, + "theoretical_loss": 4.164891002545073, + "tokens_seen": 288882688 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046083249749247744, + "loss": 2.9415, + "theoretical_loss": 4.16477947655829, + "tokens_seen": 288948224 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004608224674022066, + "loss": 3.0108, + "theoretical_loss": 4.164667982944504, + "tokens_seen": 289013760 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046081243731193586, + "loss": 2.8006, + "theoretical_loss": 4.164556521686981, + "tokens_seen": 289079296 + }, + { + "epoch": 0.09, + "learning_rate": 0.000460802407221665, + "loss": 3.1075, + "theoretical_loss": 4.1644450927689975, + "tokens_seen": 289144832 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004607923771313942, + "loss": 2.8827, + "theoretical_loss": 4.164333696173846, + "tokens_seen": 289210368 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004607823470411234, + "loss": 3.1199, + "theoretical_loss": 4.164222331884827, + "tokens_seen": 289275904 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004607723169508526, + "loss": 2.983, + "theoretical_loss": 4.164110999885256, + "tokens_seen": 289341440 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046076228686058176, + "loss": 2.9234, + "theoretical_loss": 4.163999700158462, + "tokens_seen": 289406976 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046075225677031094, + "loss": 3.1027, + "theoretical_loss": 4.163888432687784, + "tokens_seen": 289472512 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004607422266800401, + "loss": 3.1702, + "theoretical_loss": 4.163777197456573, + "tokens_seen": 289538048 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046073219658976936, + "loss": 2.7635, + "theoretical_loss": 4.163665994448197, + "tokens_seen": 289603584 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004607221664994985, + "loss": 2.9246, + "theoretical_loss": 4.163554823646027, + "tokens_seen": 289669120 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004607121364092277, + "loss": 3.1985, + "theoretical_loss": 4.163443685033458, + "tokens_seen": 289734656 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046070210631895685, + "loss": 2.9788, + "theoretical_loss": 4.163332578593889, + "tokens_seen": 289800192 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004606920762286861, + "loss": 2.8548, + "theoretical_loss": 4.163221504310734, + "tokens_seen": 289865728 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046068204613841526, + "loss": 3.2104, + "theoretical_loss": 4.1631104621674195, + "tokens_seen": 289931264 + }, + { + "epoch": 0.09, + "objective/train/docs_used": 176137, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.9354758262634277, + "objective/train/theoretical_loss": 4.162999452147384, + "objective/train/tokens_used": 310456800, + "theoretical_loss": 4.162999452147384, + "tokens_seen": 289996800 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046067201604814445, + "loss": 3.0449, + "theoretical_loss": 4.162999452147384, + "tokens_seen": 289996800 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004606619859578736, + "loss": 3.2228, + "theoretical_loss": 4.1628884742340775, + "tokens_seen": 290062336 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004606519558676028, + "loss": 3.072, + "theoretical_loss": 4.162777528410963, + "tokens_seen": 290127872 + }, + { + "epoch": 0.09, + "learning_rate": 0.000460641925777332, + "loss": 3.2562, + "theoretical_loss": 4.162666614661518, + "tokens_seen": 290193408 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004606318956870612, + "loss": 3.1575, + "theoretical_loss": 4.162555732969227, + "tokens_seen": 290258944 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046062186559679035, + "loss": 3.1429, + "theoretical_loss": 4.162444883317591, + "tokens_seen": 290324480 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004606118355065196, + "loss": 3.0492, + "theoretical_loss": 4.162334065690123, + "tokens_seen": 290390016 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046060180541624877, + "loss": 2.7943, + "theoretical_loss": 4.162223280070345, + "tokens_seen": 290455552 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046059177532597795, + "loss": 3.0486, + "theoretical_loss": 4.1621125264417955, + "tokens_seen": 290521088 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046058174523570713, + "loss": 3.0196, + "theoretical_loss": 4.162001804788021, + "tokens_seen": 290586624 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004605717151454363, + "loss": 2.9805, + "theoretical_loss": 4.161891115092583, + "tokens_seen": 290652160 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004605616850551655, + "loss": 2.948, + "theoretical_loss": 4.161780457339055, + "tokens_seen": 290717696 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004605516549648947, + "loss": 3.0879, + "theoretical_loss": 4.161669831511022, + "tokens_seen": 290783232 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046054162487462385, + "loss": 2.9332, + "theoretical_loss": 4.16155923759208, + "tokens_seen": 290848768 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004605315947843531, + "loss": 3.0518, + "theoretical_loss": 4.161448675565838, + "tokens_seen": 290914304 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004605215646940822, + "loss": 2.7699, + "theoretical_loss": 4.161338145415918, + "tokens_seen": 290979840 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046051153460381145, + "loss": 2.8763, + "theoretical_loss": 4.161227647125955, + "tokens_seen": 291045376 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046050150451354063, + "loss": 2.8787, + "theoretical_loss": 4.161117180679591, + "tokens_seen": 291110912 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004604914744232698, + "loss": 3.0329, + "theoretical_loss": 4.161006746060488, + "tokens_seen": 291176448 + }, + { + "epoch": 0.09, + "learning_rate": 0.000460481444332999, + "loss": 3.0389, + "theoretical_loss": 4.160896343252311, + "tokens_seen": 291241984 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046047141424272823, + "loss": 3.0385, + "theoretical_loss": 4.160785972238745, + "tokens_seen": 291307520 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046046138415245736, + "loss": 3.2371, + "theoretical_loss": 4.160675633003484, + "tokens_seen": 291373056 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004604513540621866, + "loss": 3.1555, + "theoretical_loss": 4.16056532553023, + "tokens_seen": 291438592 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004604413239719157, + "loss": 3.1839, + "theoretical_loss": 4.160455049802706, + "tokens_seen": 291504128 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046043129388164495, + "loss": 3.2689, + "theoretical_loss": 4.1603448058046375, + "tokens_seen": 291569664 + }, + { + "epoch": 0.09, + "objective/train/docs_used": 176478, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.728947877883911, + "objective/train/theoretical_loss": 4.160234593519768, + "objective/train/tokens_used": 312095200, + "theoretical_loss": 4.160234593519768, + "tokens_seen": 291635200 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046042126379137414, + "loss": 2.9931, + "theoretical_loss": 4.160234593519768, + "tokens_seen": 291635200 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004604112337011033, + "loss": 3.1574, + "theoretical_loss": 4.160124412931852, + "tokens_seen": 291700736 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004604012036108325, + "loss": 3.243, + "theoretical_loss": 4.160014264024654, + "tokens_seen": 291766272 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004603911735205617, + "loss": 3.0241, + "theoretical_loss": 4.159904146781952, + "tokens_seen": 291831808 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046038114343029086, + "loss": 3.1666, + "theoretical_loss": 4.159794061187536, + "tokens_seen": 291897344 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004603711133400201, + "loss": 3.174, + "theoretical_loss": 4.1596840072252075, + "tokens_seen": 291962880 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004603610832497492, + "loss": 3.0524, + "theoretical_loss": 4.159573984878779, + "tokens_seen": 292028416 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046035105315947846, + "loss": 3.0647, + "theoretical_loss": 4.159463994132079, + "tokens_seen": 292093952 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004603410230692076, + "loss": 2.9996, + "theoretical_loss": 4.15935403496894, + "tokens_seen": 292159488 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004603309929789368, + "loss": 3.1838, + "theoretical_loss": 4.159244107373215, + "tokens_seen": 292225024 + }, + { + "epoch": 0.09, + "learning_rate": 0.000460320962888666, + "loss": 3.3803, + "theoretical_loss": 4.159134211328765, + "tokens_seen": 292290560 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004603109327983952, + "loss": 3.243, + "theoretical_loss": 4.159024346819461, + "tokens_seen": 292356096 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046030090270812436, + "loss": 3.0192, + "theoretical_loss": 4.158914513829189, + "tokens_seen": 292421632 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004602908726178536, + "loss": 3.0856, + "theoretical_loss": 4.158804712341845, + "tokens_seen": 292487168 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004602808425275827, + "loss": 3.0638, + "theoretical_loss": 4.158694942341338, + "tokens_seen": 292552704 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046027081243731196, + "loss": 3.1479, + "theoretical_loss": 4.1585852038115885, + "tokens_seen": 292618240 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004602607823470411, + "loss": 3.3467, + "theoretical_loss": 4.1584754967365285, + "tokens_seen": 292683776 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004602507522567703, + "loss": 2.9907, + "theoretical_loss": 4.1583658211001016, + "tokens_seen": 292749312 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004602407221664995, + "loss": 3.0226, + "theoretical_loss": 4.158256176886264, + "tokens_seen": 292814848 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004602306920762287, + "loss": 3.3639, + "theoretical_loss": 4.158146564078982, + "tokens_seen": 292880384 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046022066198595787, + "loss": 2.8426, + "theoretical_loss": 4.158036982662237, + "tokens_seen": 292945920 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046021063189568705, + "loss": 3.2016, + "theoretical_loss": 4.157927432620018, + "tokens_seen": 293011456 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004602006018054163, + "loss": 3.082, + "theoretical_loss": 4.157817913936329, + "tokens_seen": 293076992 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046019057171514546, + "loss": 3.0292, + "theoretical_loss": 4.157708426595184, + "tokens_seen": 293142528 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046018054162487465, + "loss": 3.0894, + "theoretical_loss": 4.157598970580608, + "tokens_seen": 293208064 + }, + { + "epoch": 0.09, + "objective/train/docs_used": 177825, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.9162099361419678, + "objective/train/theoretical_loss": 4.157489545876642, + "objective/train/tokens_used": 313733600, + "theoretical_loss": 4.157489545876642, + "tokens_seen": 293273600 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004601705115346038, + "loss": 2.8698, + "theoretical_loss": 4.157489545876642, + "tokens_seen": 293273600 + }, + { + "epoch": 0.09, + "learning_rate": 0.000460160481444333, + "loss": 3.0043, + "theoretical_loss": 4.157380152467333, + "tokens_seen": 293339136 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004601504513540622, + "loss": 3.1976, + "theoretical_loss": 4.157270790336742, + "tokens_seen": 293404672 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004601404212637914, + "loss": 3.1994, + "theoretical_loss": 4.157161459468944, + "tokens_seen": 293470208 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046013039117352055, + "loss": 2.9813, + "theoretical_loss": 4.157052159848023, + "tokens_seen": 293535744 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004601203610832498, + "loss": 3.2327, + "theoretical_loss": 4.156942891458074, + "tokens_seen": 293601280 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046011033099297897, + "loss": 3.2471, + "theoretical_loss": 4.156833654283207, + "tokens_seen": 293666816 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046010030090270815, + "loss": 2.8277, + "theoretical_loss": 4.15672444830754, + "tokens_seen": 293732352 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046009027081243733, + "loss": 3.1639, + "theoretical_loss": 4.156615273515205, + "tokens_seen": 293797888 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004600802407221665, + "loss": 2.9107, + "theoretical_loss": 4.156506129890344, + "tokens_seen": 293863424 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004600702106318957, + "loss": 3.0825, + "theoretical_loss": 4.156397017417111, + "tokens_seen": 293928960 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046006018054162493, + "loss": 2.8083, + "theoretical_loss": 4.156287936079675, + "tokens_seen": 293994496 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046005015045135405, + "loss": 2.7551, + "theoretical_loss": 4.156178885862209, + "tokens_seen": 294060032 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004600401203610833, + "loss": 3.2662, + "theoretical_loss": 4.156069866748906, + "tokens_seen": 294125568 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004600300902708124, + "loss": 3.1752, + "theoretical_loss": 4.155960878723965, + "tokens_seen": 294191104 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046002006018054165, + "loss": 3.3065, + "theoretical_loss": 4.155851921771598, + "tokens_seen": 294256640 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046001003009027083, + "loss": 3.0693, + "theoretical_loss": 4.155742995876029, + "tokens_seen": 294322176 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046, + "loss": 2.8647, + "theoretical_loss": 4.155634101021494, + "tokens_seen": 294387712 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004599899699097292, + "loss": 3.1685, + "theoretical_loss": 4.155525237192238, + "tokens_seen": 294453248 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045997993981945843, + "loss": 3.2802, + "theoretical_loss": 4.155416404372522, + "tokens_seen": 294518784 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045996990972918756, + "loss": 3.2273, + "theoretical_loss": 4.155307602546614, + "tokens_seen": 294584320 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004599598796389168, + "loss": 3.1666, + "theoretical_loss": 4.155198831698795, + "tokens_seen": 294649856 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004599498495486459, + "loss": 2.9214, + "theoretical_loss": 4.155090091813358, + "tokens_seen": 294715392 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045993981945837515, + "loss": 2.9529, + "theoretical_loss": 4.154981382874608, + "tokens_seen": 294780928 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045992978936810434, + "loss": 3.2296, + "theoretical_loss": 4.154872704866859, + "tokens_seen": 294846464 + }, + { + "epoch": 0.09, + "objective/train/docs_used": 179027, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.3487234115600586, + "objective/train/theoretical_loss": 4.15476405777444, + "objective/train/tokens_used": 315372000, + "theoretical_loss": 4.15476405777444, + "tokens_seen": 294912000 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004599197592778335, + "loss": 3.0837, + "theoretical_loss": 4.15476405777444, + "tokens_seen": 294912000 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004599097291875627, + "loss": 3.0564, + "theoretical_loss": 4.154655441581687, + "tokens_seen": 294977536 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004598996990972919, + "loss": 3.2977, + "theoretical_loss": 4.154546856272952, + "tokens_seen": 295043072 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045988966900702106, + "loss": 3.1702, + "theoretical_loss": 4.154438301832596, + "tokens_seen": 295108608 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004598796389167503, + "loss": 2.8826, + "theoretical_loss": 4.154329778244991, + "tokens_seen": 295174144 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004598696088264794, + "loss": 3.0928, + "theoretical_loss": 4.154221285494521, + "tokens_seen": 295239680 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045985957873620866, + "loss": 3.0669, + "theoretical_loss": 4.154112823565582, + "tokens_seen": 295305216 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004598495486459378, + "loss": 3.0981, + "theoretical_loss": 4.15400439244258, + "tokens_seen": 295370752 + }, + { + "epoch": 0.09, + "learning_rate": 0.000459839518555667, + "loss": 3.2081, + "theoretical_loss": 4.153895992109935, + "tokens_seen": 295436288 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004598294884653962, + "loss": 3.3336, + "theoretical_loss": 4.153787622552073, + "tokens_seen": 295501824 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004598194583751254, + "loss": 3.0078, + "theoretical_loss": 4.153679283753439, + "tokens_seen": 295567360 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045980942828485456, + "loss": 3.3019, + "theoretical_loss": 4.15357097569848, + "tokens_seen": 295632896 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004597993981945838, + "loss": 3.2259, + "theoretical_loss": 4.153462698371665, + "tokens_seen": 295698432 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004597893681043129, + "loss": 2.972, + "theoretical_loss": 4.1533544517574645, + "tokens_seen": 295763968 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045977933801404216, + "loss": 3.1422, + "theoretical_loss": 4.153246235840367, + "tokens_seen": 295829504 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004597693079237713, + "loss": 3.2295, + "theoretical_loss": 4.153138050604868, + "tokens_seen": 295895040 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004597592778335005, + "loss": 2.8786, + "theoretical_loss": 4.153029896035476, + "tokens_seen": 295960576 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004597492477432297, + "loss": 3.1914, + "theoretical_loss": 4.152921772116712, + "tokens_seen": 296026112 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004597392176529589, + "loss": 3.0829, + "theoretical_loss": 4.152813678833106, + "tokens_seen": 296091648 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045972918756268807, + "loss": 3.0331, + "theoretical_loss": 4.152705616169202, + "tokens_seen": 296157184 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045971915747241725, + "loss": 3.0882, + "theoretical_loss": 4.15259758410955, + "tokens_seen": 296222720 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045970912738214643, + "loss": 2.9775, + "theoretical_loss": 4.152489582638719, + "tokens_seen": 296288256 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045969909729187566, + "loss": 3.1396, + "theoretical_loss": 4.152381611741281, + "tokens_seen": 296353792 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004596890672016048, + "loss": 3.1199, + "theoretical_loss": 4.152273671401824, + "tokens_seen": 296419328 + }, + { + "epoch": 0.09, + "learning_rate": 0.000459679037111334, + "loss": 3.0917, + "theoretical_loss": 4.152165761604948, + "tokens_seen": 296484864 + }, + { + "epoch": 0.09, + "objective/train/docs_used": 179648, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.485643148422241, + "objective/train/theoretical_loss": 4.152057882335261, + "objective/train/tokens_used": 317010400, + "theoretical_loss": 4.152057882335261, + "tokens_seen": 296550400 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045966900702106315, + "loss": 3.1053, + "theoretical_loss": 4.152057882335261, + "tokens_seen": 296550400 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004596589769307924, + "loss": 3.1665, + "theoretical_loss": 4.151950033577383, + "tokens_seen": 296615936 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045964894684052157, + "loss": 2.8724, + "theoretical_loss": 4.151842215315947, + "tokens_seen": 296681472 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045963891675025075, + "loss": 2.9905, + "theoretical_loss": 4.151734427535594, + "tokens_seen": 296747008 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045962888665997993, + "loss": 3.1292, + "theoretical_loss": 4.151626670220979, + "tokens_seen": 296812544 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045961885656970917, + "loss": 3.0816, + "theoretical_loss": 4.151518943356768, + "tokens_seen": 296878080 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004596088264794383, + "loss": 3.1673, + "theoretical_loss": 4.151411246927636, + "tokens_seen": 296943616 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045959879638916753, + "loss": 3.1408, + "theoretical_loss": 4.15130358091827, + "tokens_seen": 297009152 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045958876629889666, + "loss": 3.1399, + "theoretical_loss": 4.151195945313369, + "tokens_seen": 297074688 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004595787362086259, + "loss": 3.2755, + "theoretical_loss": 4.151088340097642, + "tokens_seen": 297140224 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004595687061183551, + "loss": 3.0323, + "theoretical_loss": 4.15098076525581, + "tokens_seen": 297205760 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045955867602808425, + "loss": 3.1988, + "theoretical_loss": 4.150873220772604, + "tokens_seen": 297271296 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045954864593781344, + "loss": 3.0636, + "theoretical_loss": 4.150765706632766, + "tokens_seen": 297336832 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004595386158475426, + "loss": 3.3062, + "theoretical_loss": 4.1506582228210505, + "tokens_seen": 297402368 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004595285857572718, + "loss": 3.1741, + "theoretical_loss": 4.150550769322221, + "tokens_seen": 297467904 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045951855566700103, + "loss": 2.9926, + "theoretical_loss": 4.150443346121054, + "tokens_seen": 297533440 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045950852557673016, + "loss": 2.836, + "theoretical_loss": 4.150335953202336, + "tokens_seen": 297598976 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004594984954864594, + "loss": 2.992, + "theoretical_loss": 4.150228590550864, + "tokens_seen": 297664512 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004594884653961885, + "loss": 2.8378, + "theoretical_loss": 4.150121258151447, + "tokens_seen": 297730048 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045947843530591776, + "loss": 2.9939, + "theoretical_loss": 4.150013955988905, + "tokens_seen": 297795584 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045946840521564694, + "loss": 3.169, + "theoretical_loss": 4.149906684048068, + "tokens_seen": 297861120 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004594583751253761, + "loss": 3.0345, + "theoretical_loss": 4.1497994423137765, + "tokens_seen": 297926656 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045944834503510536, + "loss": 3.1162, + "theoretical_loss": 4.149692230770884, + "tokens_seen": 297992192 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045943831494483454, + "loss": 3.0695, + "theoretical_loss": 4.149585049404253, + "tokens_seen": 298057728 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004594282848545637, + "loss": 3.0501, + "theoretical_loss": 4.149477898198759, + "tokens_seen": 298123264 + }, + { + "epoch": 0.09, + "objective/train/docs_used": 180307, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.340719223022461, + "objective/train/theoretical_loss": 4.149370777139286, + "objective/train/tokens_used": 318648800, + "theoretical_loss": 4.149370777139286, + "tokens_seen": 298188800 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004594182547642929, + "loss": 2.9659, + "theoretical_loss": 4.149370777139286, + "tokens_seen": 298188800 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004594082246740221, + "loss": 3.0218, + "theoretical_loss": 4.14926368621073, + "tokens_seen": 298254336 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045939819458375126, + "loss": 3.2484, + "theoretical_loss": 4.149156625397998, + "tokens_seen": 298319872 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004593881644934805, + "loss": 2.8789, + "theoretical_loss": 4.149049594686008, + "tokens_seen": 298385408 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004593781344032096, + "loss": 2.7221, + "theoretical_loss": 4.1489425940596885, + "tokens_seen": 298450944 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045936810431293886, + "loss": 2.9175, + "theoretical_loss": 4.148835623503978, + "tokens_seen": 298516480 + }, + { + "epoch": 0.09, + "learning_rate": 0.000459358074222668, + "loss": 3.1918, + "theoretical_loss": 4.148728683003829, + "tokens_seen": 298582016 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004593480441323972, + "loss": 3.1617, + "theoretical_loss": 4.1486217725442005, + "tokens_seen": 298647552 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004593380140421264, + "loss": 3.0815, + "theoretical_loss": 4.148514892110065, + "tokens_seen": 298713088 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004593279839518556, + "loss": 2.9972, + "theoretical_loss": 4.148408041686406, + "tokens_seen": 298778624 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045931795386158476, + "loss": 3.0011, + "theoretical_loss": 4.148301221258217, + "tokens_seen": 298844160 + }, + { + "epoch": 0.09, + "learning_rate": 0.000459307923771314, + "loss": 3.2283, + "theoretical_loss": 4.148194430810502, + "tokens_seen": 298909696 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004592978936810431, + "loss": 3.034, + "theoretical_loss": 4.148087670328276, + "tokens_seen": 298975232 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045928786359077236, + "loss": 3.0931, + "theoretical_loss": 4.147980939796565, + "tokens_seen": 299040768 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004592778335005015, + "loss": 2.9644, + "theoretical_loss": 4.147874239200405, + "tokens_seen": 299106304 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004592678034102307, + "loss": 2.9799, + "theoretical_loss": 4.147767568524845, + "tokens_seen": 299171840 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004592577733199599, + "loss": 2.9674, + "theoretical_loss": 4.147660927754942, + "tokens_seen": 299237376 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004592477432296891, + "loss": 3.2242, + "theoretical_loss": 4.147554316875766, + "tokens_seen": 299302912 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045923771313941827, + "loss": 3.0345, + "theoretical_loss": 4.147447735872396, + "tokens_seen": 299368448 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045922768304914745, + "loss": 3.0913, + "theoretical_loss": 4.147341184729921, + "tokens_seen": 299433984 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045921765295887663, + "loss": 3.3066, + "theoretical_loss": 4.147234663433444, + "tokens_seen": 299499520 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045920762286860586, + "loss": 3.1276, + "theoretical_loss": 4.147128171968077, + "tokens_seen": 299565056 + }, + { + "epoch": 0.09, + "learning_rate": 0.000459197592778335, + "loss": 2.9992, + "theoretical_loss": 4.14702171031894, + "tokens_seen": 299630592 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004591875626880642, + "loss": 3.0702, + "theoretical_loss": 4.146915278471169, + "tokens_seen": 299696128 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045917753259779335, + "loss": 3.1761, + "theoretical_loss": 4.146808876409906, + "tokens_seen": 299761664 + }, + { + "epoch": 0.09, + "objective/train/docs_used": 181680, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.831494092941284, + "objective/train/theoretical_loss": 4.146702504120305, + "objective/train/tokens_used": 320287200, + "theoretical_loss": 4.146702504120305, + "tokens_seen": 299827200 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004591675025075226, + "loss": 3.0352, + "theoretical_loss": 4.146702504120305, + "tokens_seen": 299827200 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045915747241725177, + "loss": 2.9094, + "theoretical_loss": 4.146596161587532, + "tokens_seen": 299892736 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045914744232698095, + "loss": 2.7609, + "theoretical_loss": 4.146489848796763, + "tokens_seen": 299958272 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045913741223671013, + "loss": 2.9165, + "theoretical_loss": 4.146383565733184, + "tokens_seen": 300023808 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045912738214643937, + "loss": 3.1464, + "theoretical_loss": 4.146277312381991, + "tokens_seen": 300089344 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004591173520561685, + "loss": 3.0635, + "theoretical_loss": 4.1461710887283925, + "tokens_seen": 300154880 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045910732196589773, + "loss": 3.1717, + "theoretical_loss": 4.146064894757606, + "tokens_seen": 300220416 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045909729187562686, + "loss": 2.8361, + "theoretical_loss": 4.145958730454861, + "tokens_seen": 300285952 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004590872617853561, + "loss": 3.0251, + "theoretical_loss": 4.145852595805396, + "tokens_seen": 300351488 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004590772316950853, + "loss": 2.9362, + "theoretical_loss": 4.145746490794461, + "tokens_seen": 300417024 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045906720160481445, + "loss": 2.9819, + "theoretical_loss": 4.145640415407317, + "tokens_seen": 300482560 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045905717151454364, + "loss": 3.1455, + "theoretical_loss": 4.145534369629234, + "tokens_seen": 300548096 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004590471414242728, + "loss": 2.8992, + "theoretical_loss": 4.145428353445494, + "tokens_seen": 300613632 + }, + { + "epoch": 0.09, + "learning_rate": 0.000459037111334002, + "loss": 3.024, + "theoretical_loss": 4.145322366841389, + "tokens_seen": 300679168 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045902708124373123, + "loss": 2.8374, + "theoretical_loss": 4.145216409802221, + "tokens_seen": 300744704 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045901705115346036, + "loss": 2.9481, + "theoretical_loss": 4.145110482313304, + "tokens_seen": 300810240 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004590070210631896, + "loss": 2.9504, + "theoretical_loss": 4.1450045843599606, + "tokens_seen": 300875776 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004589969909729187, + "loss": 2.6389, + "theoretical_loss": 4.144898715927525, + "tokens_seen": 300941312 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045898696088264796, + "loss": 3.0004, + "theoretical_loss": 4.144792877001342, + "tokens_seen": 301006848 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045897693079237714, + "loss": 3.2357, + "theoretical_loss": 4.144687067566765, + "tokens_seen": 301072384 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004589669007021063, + "loss": 2.8041, + "theoretical_loss": 4.144581287609161, + "tokens_seen": 301137920 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004589568706118355, + "loss": 3.0882, + "theoretical_loss": 4.144475537113905, + "tokens_seen": 301203456 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045894684052156474, + "loss": 2.7924, + "theoretical_loss": 4.144369816066385, + "tokens_seen": 301268992 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045893681043129386, + "loss": 3.2243, + "theoretical_loss": 4.144264124451995, + "tokens_seen": 301334528 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004589267803410231, + "loss": 3.1834, + "theoretical_loss": 4.1441584622561445, + "tokens_seen": 301400064 + }, + { + "epoch": 0.09, + "objective/train/docs_used": 182385, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.5902211666107178, + "objective/train/theoretical_loss": 4.144052829464249, + "objective/train/tokens_used": 321925600, + "theoretical_loss": 4.144052829464249, + "tokens_seen": 301465600 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004589167502507522, + "loss": 3.2109, + "theoretical_loss": 4.144052829464249, + "tokens_seen": 301465600 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045890672016048146, + "loss": 2.8837, + "theoretical_loss": 4.143947226061737, + "tokens_seen": 301531136 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045889669007021064, + "loss": 2.9355, + "theoretical_loss": 4.143841652034048, + "tokens_seen": 301596672 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004588866599799398, + "loss": 3.1283, + "theoretical_loss": 4.143736107366629, + "tokens_seen": 301662208 + }, + { + "epoch": 0.09, + "learning_rate": 0.000458876629889669, + "loss": 3.1231, + "theoretical_loss": 4.14363059204494, + "tokens_seen": 301727744 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004588665997993982, + "loss": 2.8762, + "theoretical_loss": 4.14352510605445, + "tokens_seen": 301793280 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045885656970912737, + "loss": 3.0109, + "theoretical_loss": 4.143419649380639, + "tokens_seen": 301858816 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004588465396188566, + "loss": 3.0578, + "theoretical_loss": 4.143314222008997, + "tokens_seen": 301924352 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045883650952858573, + "loss": 3.0431, + "theoretical_loss": 4.143208823925024, + "tokens_seen": 301989888 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045882647943831496, + "loss": 2.9236, + "theoretical_loss": 4.143103455114231, + "tokens_seen": 302055424 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045881644934804415, + "loss": 3.1239, + "theoretical_loss": 4.142998115562139, + "tokens_seen": 302120960 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004588064192577733, + "loss": 3.0421, + "theoretical_loss": 4.14289280525428, + "tokens_seen": 302186496 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004587963891675025, + "loss": 3.3417, + "theoretical_loss": 4.142787524176194, + "tokens_seen": 302252032 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004587863590772317, + "loss": 3.1257, + "theoretical_loss": 4.142682272313435, + "tokens_seen": 302317568 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045877632898696087, + "loss": 3.215, + "theoretical_loss": 4.142577049651563, + "tokens_seen": 302383104 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004587662988966901, + "loss": 3.1011, + "theoretical_loss": 4.142471856176152, + "tokens_seen": 302448640 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045875626880641923, + "loss": 3.2312, + "theoretical_loss": 4.142366691872784, + "tokens_seen": 302514176 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045874623871614847, + "loss": 3.1098, + "theoretical_loss": 4.142261556727052, + "tokens_seen": 302579712 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004587362086258776, + "loss": 3.0405, + "theoretical_loss": 4.14215645072456, + "tokens_seen": 302645248 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045872617853560683, + "loss": 3.0807, + "theoretical_loss": 4.14205137385092, + "tokens_seen": 302710784 + }, + { + "epoch": 0.09, + "learning_rate": 0.000458716148445336, + "loss": 3.005, + "theoretical_loss": 4.141946326091756, + "tokens_seen": 302776320 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004587061183550652, + "loss": 3.0407, + "theoretical_loss": 4.141841307432703, + "tokens_seen": 302841856 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004586960882647944, + "loss": 3.0962, + "theoretical_loss": 4.1417363178594035, + "tokens_seen": 302907392 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045868605817452355, + "loss": 3.2176, + "theoretical_loss": 4.141631357357513, + "tokens_seen": 302972928 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004586760280842528, + "loss": 3.0142, + "theoretical_loss": 4.141526425912694, + "tokens_seen": 303038464 + }, + { + "epoch": 0.09, + "objective/train/docs_used": 183646, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.1002464294433594, + "objective/train/theoretical_loss": 4.141421523510623, + "objective/train/tokens_used": 323564000, + "theoretical_loss": 4.141421523510623, + "tokens_seen": 303104000 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045866599799398197, + "loss": 3.141, + "theoretical_loss": 4.141421523510623, + "tokens_seen": 303104000 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045865596790371115, + "loss": 3.1336, + "theoretical_loss": 4.141316650136983, + "tokens_seen": 303169536 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045864593781344033, + "loss": 3.0439, + "theoretical_loss": 4.14121180577747, + "tokens_seen": 303235072 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045863590772316957, + "loss": 3.0744, + "theoretical_loss": 4.141106990417789, + "tokens_seen": 303300608 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004586258776328987, + "loss": 3.2341, + "theoretical_loss": 4.141002204043654, + "tokens_seen": 303366144 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045861584754262793, + "loss": 2.8468, + "theoretical_loss": 4.140897446640793, + "tokens_seen": 303431680 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045860581745235706, + "loss": 3.2598, + "theoretical_loss": 4.1407927181949375, + "tokens_seen": 303497216 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004585957873620863, + "loss": 3.0394, + "theoretical_loss": 4.140688018691835, + "tokens_seen": 303562752 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004585857572718155, + "loss": 3.267, + "theoretical_loss": 4.140583348117241, + "tokens_seen": 303628288 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045857572718154465, + "loss": 3.037, + "theoretical_loss": 4.140478706456921, + "tokens_seen": 303693824 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045856569709127384, + "loss": 3.0885, + "theoretical_loss": 4.140374093696651, + "tokens_seen": 303759360 + }, + { + "epoch": 0.09, + "learning_rate": 0.000458555667001003, + "loss": 3.0503, + "theoretical_loss": 4.1402695098222155, + "tokens_seen": 303824896 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004585456369107322, + "loss": 3.2161, + "theoretical_loss": 4.140164954819412, + "tokens_seen": 303890432 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045853560682046143, + "loss": 3.098, + "theoretical_loss": 4.140060428674046, + "tokens_seen": 303955968 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045852557673019056, + "loss": 3.0109, + "theoretical_loss": 4.139955931371932, + "tokens_seen": 304021504 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004585155466399198, + "loss": 3.1894, + "theoretical_loss": 4.139851462898897, + "tokens_seen": 304087040 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004585055165496489, + "loss": 3.1719, + "theoretical_loss": 4.139747023240777, + "tokens_seen": 304152576 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045849548645937816, + "loss": 2.8868, + "theoretical_loss": 4.139642612383418, + "tokens_seen": 304218112 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045848545636910734, + "loss": 3.2134, + "theoretical_loss": 4.1395382303126755, + "tokens_seen": 304283648 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004584754262788365, + "loss": 3.0602, + "theoretical_loss": 4.139433877014415, + "tokens_seen": 304349184 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004584653961885657, + "loss": 2.9758, + "theoretical_loss": 4.139329552474514, + "tokens_seen": 304414720 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045845536609829494, + "loss": 3.1797, + "theoretical_loss": 4.139225256678857, + "tokens_seen": 304480256 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045844533600802406, + "loss": 3.0667, + "theoretical_loss": 4.139120989613341, + "tokens_seen": 304545792 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004584353059177533, + "loss": 2.8961, + "theoretical_loss": 4.1390167512638705, + "tokens_seen": 304611328 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004584252758274824, + "loss": 3.069, + "theoretical_loss": 4.138912541616363, + "tokens_seen": 304676864 + }, + { + "epoch": 0.09, + "objective/train/docs_used": 184075, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6668105125427246, + "objective/train/theoretical_loss": 4.138808360656742, + "objective/train/tokens_used": 325202400, + "theoretical_loss": 4.138808360656742, + "tokens_seen": 304742400 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045841524573721166, + "loss": 2.917, + "theoretical_loss": 4.138808360656742, + "tokens_seen": 304742400 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045840521564694084, + "loss": 3.0803, + "theoretical_loss": 4.138704208370944, + "tokens_seen": 304807936 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045839518555667, + "loss": 3.1018, + "theoretical_loss": 4.138600084744915, + "tokens_seen": 304873472 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004583851554663992, + "loss": 2.8826, + "theoretical_loss": 4.1384959897646105, + "tokens_seen": 304939008 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004583751253761284, + "loss": 3.032, + "theoretical_loss": 4.138391923415996, + "tokens_seen": 305004544 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045836509528585757, + "loss": 3.1799, + "theoretical_loss": 4.138287885685045, + "tokens_seen": 305070080 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004583550651955868, + "loss": 3.1447, + "theoretical_loss": 4.138183876557745, + "tokens_seen": 305135616 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045834503510531593, + "loss": 2.8712, + "theoretical_loss": 4.1380798960200895, + "tokens_seen": 305201152 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045833500501504516, + "loss": 3.1577, + "theoretical_loss": 4.137975944058083, + "tokens_seen": 305266688 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045832497492477435, + "loss": 2.8717, + "theoretical_loss": 4.137872020657742, + "tokens_seen": 305332224 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004583149448345035, + "loss": 3.2894, + "theoretical_loss": 4.1377681258050885, + "tokens_seen": 305397760 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004583049147442327, + "loss": 3.0479, + "theoretical_loss": 4.13766425948616, + "tokens_seen": 305463296 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004582948846539619, + "loss": 3.1308, + "theoretical_loss": 4.137560421686998, + "tokens_seen": 305528832 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045828485456369107, + "loss": 3.0632, + "theoretical_loss": 4.137456612393658, + "tokens_seen": 305594368 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004582748244734203, + "loss": 3.0022, + "theoretical_loss": 4.137352831592203, + "tokens_seen": 305659904 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045826479438314943, + "loss": 3.0474, + "theoretical_loss": 4.137249079268707, + "tokens_seen": 305725440 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045825476429287867, + "loss": 3.0966, + "theoretical_loss": 4.137145355409253, + "tokens_seen": 305790976 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004582447342026078, + "loss": 3.0445, + "theoretical_loss": 4.137041659999936, + "tokens_seen": 305856512 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045823470411233703, + "loss": 3.0454, + "theoretical_loss": 4.136937993026857, + "tokens_seen": 305922048 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004582246740220662, + "loss": 2.8907, + "theoretical_loss": 4.136834354476129, + "tokens_seen": 305987584 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004582146439317954, + "loss": 3.0305, + "theoretical_loss": 4.1367307443338746, + "tokens_seen": 306053120 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045820461384152457, + "loss": 3.0248, + "theoretical_loss": 4.136627162586226, + "tokens_seen": 306118656 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045819458375125375, + "loss": 3.2977, + "theoretical_loss": 4.136523609219327, + "tokens_seen": 306184192 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045818455366098294, + "loss": 2.917, + "theoretical_loss": 4.136420084219327, + "tokens_seen": 306249728 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045817452357071217, + "loss": 3.1805, + "theoretical_loss": 4.136316587572388, + "tokens_seen": 306315264 + }, + { + "epoch": 0.09, + "objective/train/docs_used": 185377, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.4130301475524902, + "objective/train/theoretical_loss": 4.136213119264681, + "objective/train/tokens_used": 326840800, + "theoretical_loss": 4.136213119264681, + "tokens_seen": 306380800 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004581644934804413, + "loss": 3.2706, + "theoretical_loss": 4.136213119264681, + "tokens_seen": 306380800 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045815446339017053, + "loss": 2.8039, + "theoretical_loss": 4.136109679282388, + "tokens_seen": 306446336 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004581444332998997, + "loss": 3.0117, + "theoretical_loss": 4.136006267611697, + "tokens_seen": 306511872 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004581344032096289, + "loss": 3.0833, + "theoretical_loss": 4.135902884238812, + "tokens_seen": 306577408 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004581243731193581, + "loss": 2.9987, + "theoretical_loss": 4.135799529149939, + "tokens_seen": 306642944 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045811434302908726, + "loss": 2.9467, + "theoretical_loss": 4.1356962023313, + "tokens_seen": 306708480 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045810431293881644, + "loss": 3.1567, + "theoretical_loss": 4.135592903769124, + "tokens_seen": 306774016 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004580942828485457, + "loss": 3.0652, + "theoretical_loss": 4.135489633449649, + "tokens_seen": 306839552 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004580842527582748, + "loss": 3.0627, + "theoretical_loss": 4.135386391359123, + "tokens_seen": 306905088 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045807422266800404, + "loss": 2.9053, + "theoretical_loss": 4.135283177483807, + "tokens_seen": 306970624 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045806419257773316, + "loss": 3.0322, + "theoretical_loss": 4.135179991809965, + "tokens_seen": 307036160 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004580541624874624, + "loss": 3.0044, + "theoretical_loss": 4.135076834323876, + "tokens_seen": 307101696 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004580441323971916, + "loss": 3.1982, + "theoretical_loss": 4.134973705011828, + "tokens_seen": 307167232 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045803410230692076, + "loss": 2.8279, + "theoretical_loss": 4.134870603860117, + "tokens_seen": 307232768 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045802407221664994, + "loss": 3.0219, + "theoretical_loss": 4.134767530855047, + "tokens_seen": 307298304 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004580140421263791, + "loss": 2.9982, + "theoretical_loss": 4.1346644859829365, + "tokens_seen": 307363840 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004580040120361083, + "loss": 3.1424, + "theoretical_loss": 4.1345614692301105, + "tokens_seen": 307429376 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045799398194583754, + "loss": 2.867, + "theoretical_loss": 4.134458480582902, + "tokens_seen": 307494912 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045798395185556667, + "loss": 3.1424, + "theoretical_loss": 4.134355520027657, + "tokens_seen": 307560448 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004579739217652959, + "loss": 3.0614, + "theoretical_loss": 4.134252587550728, + "tokens_seen": 307625984 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004579638916750251, + "loss": 3.178, + "theoretical_loss": 4.134149683138481, + "tokens_seen": 307691520 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045795386158475426, + "loss": 3.0396, + "theoretical_loss": 4.134046806777286, + "tokens_seen": 307757056 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004579438314944835, + "loss": 3.1608, + "theoretical_loss": 4.133943958453528, + "tokens_seen": 307822592 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004579338014042126, + "loss": 2.9014, + "theoretical_loss": 4.133841138153597, + "tokens_seen": 307888128 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045792377131394186, + "loss": 2.9882, + "theoretical_loss": 4.133738345863896, + "tokens_seen": 307953664 + }, + { + "epoch": 0.09, + "objective/train/docs_used": 186163, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.0626275539398193, + "objective/train/theoretical_loss": 4.133635581570836, + "objective/train/tokens_used": 328479200, + "theoretical_loss": 4.133635581570836, + "tokens_seen": 308019200 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045791374122367104, + "loss": 2.9341, + "theoretical_loss": 4.133635581570836, + "tokens_seen": 308019200 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004579037111334002, + "loss": 3.25, + "theoretical_loss": 4.133532845260836, + "tokens_seen": 308084736 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004578936810431294, + "loss": 3.28, + "theoretical_loss": 4.133430136920327, + "tokens_seen": 308150272 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004578836509528586, + "loss": 3.0361, + "theoretical_loss": 4.133327456535749, + "tokens_seen": 308215808 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045787362086258777, + "loss": 2.9337, + "theoretical_loss": 4.13322480409355, + "tokens_seen": 308281344 + }, + { + "epoch": 0.09, + "learning_rate": 0.000457863590772317, + "loss": 3.09, + "theoretical_loss": 4.133122179580189, + "tokens_seen": 308346880 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045785356068204613, + "loss": 3.3942, + "theoretical_loss": 4.133019582982134, + "tokens_seen": 308412416 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045784353059177536, + "loss": 3.116, + "theoretical_loss": 4.1329170142858604, + "tokens_seen": 308477952 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045783350050150455, + "loss": 2.8733, + "theoretical_loss": 4.132814473477857, + "tokens_seen": 308543488 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004578234704112337, + "loss": 3.119, + "theoretical_loss": 4.1327119605446185, + "tokens_seen": 308609024 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004578134403209629, + "loss": 2.8524, + "theoretical_loss": 4.132609475472651, + "tokens_seen": 308674560 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004578034102306921, + "loss": 3.1962, + "theoretical_loss": 4.132507018248469, + "tokens_seen": 308740096 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045779338014042127, + "loss": 2.9104, + "theoretical_loss": 4.132404588858597, + "tokens_seen": 308805632 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004577833500501505, + "loss": 2.8931, + "theoretical_loss": 4.132302187289568, + "tokens_seen": 308871168 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045777331995987963, + "loss": 3.0915, + "theoretical_loss": 4.132199813527926, + "tokens_seen": 308936704 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045776328986960887, + "loss": 2.8067, + "theoretical_loss": 4.132097467560223, + "tokens_seen": 309002240 + }, + { + "epoch": 0.09, + "learning_rate": 0.000457753259779338, + "loss": 3.0507, + "theoretical_loss": 4.1319951493730205, + "tokens_seen": 309067776 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045774322968906723, + "loss": 3.3508, + "theoretical_loss": 4.131892858952889, + "tokens_seen": 309133312 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004577331995987964, + "loss": 3.1086, + "theoretical_loss": 4.131790596286409, + "tokens_seen": 309198848 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004577231695085256, + "loss": 2.9348, + "theoretical_loss": 4.1316883613601725, + "tokens_seen": 309264384 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004577131394182548, + "loss": 2.9738, + "theoretical_loss": 4.131586154160775, + "tokens_seen": 309329920 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045770310932798395, + "loss": 3.277, + "theoretical_loss": 4.131483974674827, + "tokens_seen": 309395456 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045769307923771314, + "loss": 3.1956, + "theoretical_loss": 4.131381822888946, + "tokens_seen": 309460992 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045768304914744237, + "loss": 3.0944, + "theoretical_loss": 4.131279698789759, + "tokens_seen": 309526528 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004576730190571715, + "loss": 3.0388, + "theoretical_loss": 4.1311776023639, + "tokens_seen": 309592064 + }, + { + "epoch": 0.09, + "objective/train/docs_used": 187126, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.9592275619506836, + "objective/train/theoretical_loss": 4.131075533598018, + "objective/train/tokens_used": 330117600, + "theoretical_loss": 4.131075533598018, + "tokens_seen": 309657600 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045766298896690073, + "loss": 3.105, + "theoretical_loss": 4.131075533598018, + "tokens_seen": 309657600 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004576529588766299, + "loss": 3.1128, + "theoretical_loss": 4.130973492478766, + "tokens_seen": 309723136 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004576429287863591, + "loss": 3.2515, + "theoretical_loss": 4.130871478992807, + "tokens_seen": 309788672 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004576328986960883, + "loss": 2.9709, + "theoretical_loss": 4.130769493126817, + "tokens_seen": 309854208 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045762286860581746, + "loss": 3.087, + "theoretical_loss": 4.130667534867476, + "tokens_seen": 309919744 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045761283851554664, + "loss": 2.7197, + "theoretical_loss": 4.130565604201477, + "tokens_seen": 309985280 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004576028084252759, + "loss": 2.8904, + "theoretical_loss": 4.130463701115521, + "tokens_seen": 310050816 + }, + { + "epoch": 0.09, + "learning_rate": 0.000457592778335005, + "loss": 3.1747, + "theoretical_loss": 4.130361825596317, + "tokens_seen": 310116352 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045758274824473424, + "loss": 3.0938, + "theoretical_loss": 4.130259977630586, + "tokens_seen": 310181888 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045757271815446336, + "loss": 3.1166, + "theoretical_loss": 4.130158157205056, + "tokens_seen": 310247424 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004575626880641926, + "loss": 3.1422, + "theoretical_loss": 4.130056364306465, + "tokens_seen": 310312960 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004575526579739218, + "loss": 3.1079, + "theoretical_loss": 4.129954598921559, + "tokens_seen": 310378496 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045754262788365096, + "loss": 3.0701, + "theoretical_loss": 4.1298528610370955, + "tokens_seen": 310444032 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045753259779338014, + "loss": 3.0169, + "theoretical_loss": 4.12975115063984, + "tokens_seen": 310509568 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004575225677031093, + "loss": 2.9435, + "theoretical_loss": 4.129649467716565, + "tokens_seen": 310575104 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004575125376128385, + "loss": 3.1254, + "theoretical_loss": 4.1295478122540565, + "tokens_seen": 310640640 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045750250752256774, + "loss": 2.8771, + "theoretical_loss": 4.1294461842391055, + "tokens_seen": 310706176 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045749247743229687, + "loss": 3.0123, + "theoretical_loss": 4.129344583658516, + "tokens_seen": 310771712 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004574824473420261, + "loss": 2.8764, + "theoretical_loss": 4.1292430104990965, + "tokens_seen": 310837248 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004574724172517553, + "loss": 3.1251, + "theoretical_loss": 4.1291414647476685, + "tokens_seen": 310902784 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045746238716148446, + "loss": 3.1378, + "theoretical_loss": 4.129039946391062, + "tokens_seen": 310968320 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045745235707121364, + "loss": 3.0819, + "theoretical_loss": 4.128938455416115, + "tokens_seen": 311033856 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004574423269809428, + "loss": 3.3482, + "theoretical_loss": 4.128836991809674, + "tokens_seen": 311099392 + }, + { + "epoch": 0.09, + "learning_rate": 0.000457432296890672, + "loss": 3.1925, + "theoretical_loss": 4.128735555558597, + "tokens_seen": 311164928 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045742226680040124, + "loss": 3.1928, + "theoretical_loss": 4.128634146649748, + "tokens_seen": 311230464 + }, + { + "epoch": 0.09, + "objective/train/docs_used": 187875, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.9146547317504883, + "objective/train/theoretical_loss": 4.128532765070004, + "objective/train/tokens_used": 331756000, + "theoretical_loss": 4.128532765070004, + "tokens_seen": 311296000 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045741223671013037, + "loss": 2.9356, + "theoretical_loss": 4.128532765070004, + "tokens_seen": 311296000 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004574022066198596, + "loss": 2.9373, + "theoretical_loss": 4.128431410806247, + "tokens_seen": 311361536 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045739217652958873, + "loss": 3.181, + "theoretical_loss": 4.12833008384537, + "tokens_seen": 311427072 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045738214643931797, + "loss": 3.1765, + "theoretical_loss": 4.128228784174275, + "tokens_seen": 311492608 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045737211634904715, + "loss": 3.0631, + "theoretical_loss": 4.128127511779873, + "tokens_seen": 311558144 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045736208625877633, + "loss": 3.1609, + "theoretical_loss": 4.128026266649085, + "tokens_seen": 311623680 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004573520561685055, + "loss": 3.1894, + "theoretical_loss": 4.127925048768839, + "tokens_seen": 311689216 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045734202607823475, + "loss": 3.1843, + "theoretical_loss": 4.127823858126073, + "tokens_seen": 311754752 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045733199598796387, + "loss": 3.1684, + "theoretical_loss": 4.1277226947077335, + "tokens_seen": 311820288 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004573219658976931, + "loss": 3.0801, + "theoretical_loss": 4.127621558500778, + "tokens_seen": 311885824 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045731193580742223, + "loss": 3.2224, + "theoretical_loss": 4.12752044949217, + "tokens_seen": 311951360 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045730190571715147, + "loss": 3.3395, + "theoretical_loss": 4.127419367668884, + "tokens_seen": 312016896 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045729187562688065, + "loss": 3.0192, + "theoretical_loss": 4.127318313017904, + "tokens_seen": 312082432 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045728184553660983, + "loss": 3.0693, + "theoretical_loss": 4.12721728552622, + "tokens_seen": 312147968 + }, + { + "epoch": 0.09, + "learning_rate": 0.000457271815446339, + "loss": 2.7921, + "theoretical_loss": 4.1271162851808345, + "tokens_seen": 312213504 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004572617853560682, + "loss": 3.2569, + "theoretical_loss": 4.127015311968757, + "tokens_seen": 312279040 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004572517552657974, + "loss": 2.8277, + "theoretical_loss": 4.126914365877004, + "tokens_seen": 312344576 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004572417251755266, + "loss": 3.1511, + "theoretical_loss": 4.126813446892607, + "tokens_seen": 312410112 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045723169508525574, + "loss": 3.1329, + "theoretical_loss": 4.1267125550026, + "tokens_seen": 312475648 + }, + { + "epoch": 0.09, + "learning_rate": 0.000457221664994985, + "loss": 3.2022, + "theoretical_loss": 4.1266116901940295, + "tokens_seen": 312541184 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004572116349047141, + "loss": 2.8814, + "theoretical_loss": 4.126510852453949, + "tokens_seen": 312606720 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045720160481444334, + "loss": 3.2587, + "theoretical_loss": 4.126410041769423, + "tokens_seen": 312672256 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045719157472417257, + "loss": 2.9355, + "theoretical_loss": 4.126309258127524, + "tokens_seen": 312737792 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004571815446339017, + "loss": 3.0325, + "theoretical_loss": 4.126208501515331, + "tokens_seen": 312803328 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045717151454363093, + "loss": 3.2749, + "theoretical_loss": 4.126107771919935, + "tokens_seen": 312868864 + }, + { + "epoch": 0.09, + "objective/train/docs_used": 188927, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.28684401512146, + "objective/train/theoretical_loss": 4.126007069328436, + "objective/train/tokens_used": 333394400, + "theoretical_loss": 4.126007069328436, + "tokens_seen": 312934400 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004571614844533601, + "loss": 3.3178, + "theoretical_loss": 4.126007069328436, + "tokens_seen": 312934400 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004571514543630893, + "loss": 3.2018, + "theoretical_loss": 4.125906393727941, + "tokens_seen": 312999936 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004571414242728185, + "loss": 3.3575, + "theoretical_loss": 4.125805745105566, + "tokens_seen": 313065472 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045713139418254766, + "loss": 3.1319, + "theoretical_loss": 4.125705123448437, + "tokens_seen": 313131008 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045712136409227684, + "loss": 2.9203, + "theoretical_loss": 4.125604528743689, + "tokens_seen": 313196544 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004571113340020061, + "loss": 3.1521, + "theoretical_loss": 4.125503960978464, + "tokens_seen": 313262080 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004571013039117352, + "loss": 3.0665, + "theoretical_loss": 4.1254034201399135, + "tokens_seen": 313327616 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045709127382146444, + "loss": 3.2737, + "theoretical_loss": 4.125302906215199, + "tokens_seen": 313393152 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045708124373119356, + "loss": 3.2387, + "theoretical_loss": 4.12520241919149, + "tokens_seen": 313458688 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004570712136409228, + "loss": 3.1177, + "theoretical_loss": 4.125101959055965, + "tokens_seen": 313524224 + }, + { + "epoch": 0.1, + "learning_rate": 0.000457061183550652, + "loss": 2.888, + "theoretical_loss": 4.125001525795811, + "tokens_seen": 313589760 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045705115346038116, + "loss": 3.1711, + "theoretical_loss": 4.124901119398222, + "tokens_seen": 313655296 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045704112337011034, + "loss": 3.0909, + "theoretical_loss": 4.124800739850406, + "tokens_seen": 313720832 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004570310932798395, + "loss": 3.2564, + "theoretical_loss": 4.124700387139574, + "tokens_seen": 313786368 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004570210631895687, + "loss": 3.178, + "theoretical_loss": 4.12460006125295, + "tokens_seen": 313851904 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045701103309929794, + "loss": 2.9884, + "theoretical_loss": 4.124499762177764, + "tokens_seen": 313917440 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045700100300902707, + "loss": 3.4186, + "theoretical_loss": 4.124399489901254, + "tokens_seen": 313982976 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004569909729187563, + "loss": 3.1955, + "theoretical_loss": 4.124299244410672, + "tokens_seen": 314048512 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004569809428284855, + "loss": 3.1361, + "theoretical_loss": 4.124199025693272, + "tokens_seen": 314114048 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045697091273821466, + "loss": 3.2915, + "theoretical_loss": 4.124098833736321, + "tokens_seen": 314179584 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045696088264794384, + "loss": 3.2362, + "theoretical_loss": 4.123998668527094, + "tokens_seen": 314245120 + }, + { + "epoch": 0.1, + "learning_rate": 0.000456950852557673, + "loss": 3.2541, + "theoretical_loss": 4.123898530052874, + "tokens_seen": 314310656 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004569408224674022, + "loss": 3.1037, + "theoretical_loss": 4.123798418300953, + "tokens_seen": 314376192 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045693079237713144, + "loss": 2.967, + "theoretical_loss": 4.123698333258631, + "tokens_seen": 314441728 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045692076228686057, + "loss": 3.0382, + "theoretical_loss": 4.123598274913219, + "tokens_seen": 314507264 + }, + { + "epoch": 0.1, + "objective/train/docs_used": 189547, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.0788962841033936, + "objective/train/theoretical_loss": 4.123498243252032, + "objective/train/tokens_used": 335032800, + "theoretical_loss": 4.123498243252032, + "tokens_seen": 314572800 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004569107321965898, + "loss": 3.2229, + "theoretical_loss": 4.123498243252032, + "tokens_seen": 314572800 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045690070210631893, + "loss": 3.2057, + "theoretical_loss": 4.1233982382624, + "tokens_seen": 314638336 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045689067201604817, + "loss": 2.9642, + "theoretical_loss": 4.123298259931657, + "tokens_seen": 314703872 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045688064192577735, + "loss": 3.1263, + "theoretical_loss": 4.123198308247146, + "tokens_seen": 314769408 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045687061183550653, + "loss": 3.2155, + "theoretical_loss": 4.123098383196222, + "tokens_seen": 314834944 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004568605817452357, + "loss": 2.8863, + "theoretical_loss": 4.122998484766244, + "tokens_seen": 314900480 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045685055165496495, + "loss": 3.2068, + "theoretical_loss": 4.122898612944582, + "tokens_seen": 314966016 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045684052156469407, + "loss": 2.9666, + "theoretical_loss": 4.122798767718616, + "tokens_seen": 315031552 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004568304914744233, + "loss": 3.1796, + "theoretical_loss": 4.122698949075732, + "tokens_seen": 315097088 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045682046138415243, + "loss": 3.1554, + "theoretical_loss": 4.122599157003327, + "tokens_seen": 315162624 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045681043129388167, + "loss": 3.281, + "theoretical_loss": 4.1224993914888035, + "tokens_seen": 315228160 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045680040120361085, + "loss": 2.9982, + "theoretical_loss": 4.122399652519576, + "tokens_seen": 315293696 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045679037111334003, + "loss": 2.8952, + "theoretical_loss": 4.122299940083065, + "tokens_seen": 315359232 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004567803410230692, + "loss": 3.0766, + "theoretical_loss": 4.1222002541667, + "tokens_seen": 315424768 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004567703109327984, + "loss": 3.006, + "theoretical_loss": 4.122100594757921, + "tokens_seen": 315490304 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004567602808425276, + "loss": 3.1813, + "theoretical_loss": 4.122000961844175, + "tokens_seen": 315555840 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004567502507522568, + "loss": 3.1248, + "theoretical_loss": 4.121901355412917, + "tokens_seen": 315621376 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045674022066198594, + "loss": 3.301, + "theoretical_loss": 4.121801775451612, + "tokens_seen": 315686912 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004567301905717152, + "loss": 3.185, + "theoretical_loss": 4.121702221947732, + "tokens_seen": 315752448 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004567201604814443, + "loss": 3.1238, + "theoretical_loss": 4.121602694888759, + "tokens_seen": 315817984 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045671013039117354, + "loss": 3.4483, + "theoretical_loss": 4.121503194262183, + "tokens_seen": 315883520 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004567001003009027, + "loss": 2.9193, + "theoretical_loss": 4.121403720055502, + "tokens_seen": 315949056 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004566900702106319, + "loss": 3.1005, + "theoretical_loss": 4.121304272256222, + "tokens_seen": 316014592 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004566800401203611, + "loss": 3.1702, + "theoretical_loss": 4.121204850851861, + "tokens_seen": 316080128 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004566700100300903, + "loss": 3.0077, + "theoretical_loss": 4.121105455829939, + "tokens_seen": 316145664 + }, + { + "epoch": 0.1, + "objective/train/docs_used": 190733, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.068772554397583, + "objective/train/theoretical_loss": 4.121006087177992, + "objective/train/tokens_used": 336671200, + "theoretical_loss": 4.121006087177992, + "tokens_seen": 316211200 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045665997993981944, + "loss": 3.0643, + "theoretical_loss": 4.121006087177992, + "tokens_seen": 316211200 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004566499498495487, + "loss": 2.978, + "theoretical_loss": 4.120906744883559, + "tokens_seen": 316276736 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004566399197592778, + "loss": 3.1612, + "theoretical_loss": 4.120807428934189, + "tokens_seen": 316342272 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045662988966900704, + "loss": 3.1871, + "theoretical_loss": 4.120708139317441, + "tokens_seen": 316407808 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004566198595787362, + "loss": 3.1833, + "theoretical_loss": 4.12060887602088, + "tokens_seen": 316473344 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004566098294884654, + "loss": 3.1011, + "theoretical_loss": 4.120509639032081, + "tokens_seen": 316538880 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004565997993981946, + "loss": 3.1619, + "theoretical_loss": 4.120410428338628, + "tokens_seen": 316604416 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045658976930792376, + "loss": 3.2377, + "theoretical_loss": 4.120311243928111, + "tokens_seen": 316669952 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045657973921765294, + "loss": 3.1475, + "theoretical_loss": 4.120212085788131, + "tokens_seen": 316735488 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004565697091273822, + "loss": 2.9874, + "theoretical_loss": 4.120112953906296, + "tokens_seen": 316801024 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004565596790371113, + "loss": 3.0157, + "theoretical_loss": 4.120013848270222, + "tokens_seen": 316866560 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045654964894684054, + "loss": 3.1466, + "theoretical_loss": 4.119914768867536, + "tokens_seen": 316932096 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045653961885656967, + "loss": 3.1769, + "theoretical_loss": 4.11981571568587, + "tokens_seen": 316997632 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004565295887662989, + "loss": 3.0504, + "theoretical_loss": 4.119716688712866, + "tokens_seen": 317063168 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004565195586760281, + "loss": 3.0305, + "theoretical_loss": 4.119617687936175, + "tokens_seen": 317128704 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045650952858575727, + "loss": 3.1897, + "theoretical_loss": 4.119518713343455, + "tokens_seen": 317194240 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045649949849548645, + "loss": 2.8688, + "theoretical_loss": 4.119419764922374, + "tokens_seen": 317259776 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004564894684052157, + "loss": 3.2496, + "theoretical_loss": 4.119320842660606, + "tokens_seen": 317325312 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004564794383149448, + "loss": 2.9978, + "theoretical_loss": 4.119221946545836, + "tokens_seen": 317390848 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045646940822467405, + "loss": 3.1207, + "theoretical_loss": 4.119123076565755, + "tokens_seen": 317456384 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045645937813440317, + "loss": 2.931, + "theoretical_loss": 4.119024232708064, + "tokens_seen": 317521920 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004564493480441324, + "loss": 3.1931, + "theoretical_loss": 4.118925414960472, + "tokens_seen": 317587456 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045643931795386164, + "loss": 3.3486, + "theoretical_loss": 4.118826623310696, + "tokens_seen": 317652992 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045642928786359077, + "loss": 3.1451, + "theoretical_loss": 4.11872785774646, + "tokens_seen": 317718528 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045641925777332, + "loss": 3.0218, + "theoretical_loss": 4.1186291182555, + "tokens_seen": 317784064 + }, + { + "epoch": 0.1, + "objective/train/docs_used": 191330, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.22560453414917, + "objective/train/theoretical_loss": 4.118530404825556, + "objective/train/tokens_used": 338309600, + "theoretical_loss": 4.118530404825556, + "tokens_seen": 317849600 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045640922768304913, + "loss": 3.0718, + "theoretical_loss": 4.118530404825556, + "tokens_seen": 317849600 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045639919759277837, + "loss": 2.9003, + "theoretical_loss": 4.11843171744438, + "tokens_seen": 317915136 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045638916750250755, + "loss": 3.1771, + "theoretical_loss": 4.118333056099728, + "tokens_seen": 317980672 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045637913741223673, + "loss": 2.9972, + "theoretical_loss": 4.11823442077937, + "tokens_seen": 318046208 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004563691073219659, + "loss": 3.1471, + "theoretical_loss": 4.1181358114710775, + "tokens_seen": 318111744 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045635907723169515, + "loss": 3.0852, + "theoretical_loss": 4.1180372281626365, + "tokens_seen": 318177280 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045634904714142427, + "loss": 3.123, + "theoretical_loss": 4.117938670841838, + "tokens_seen": 318242816 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004563390170511535, + "loss": 3.2831, + "theoretical_loss": 4.117840139496482, + "tokens_seen": 318308352 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045632898696088263, + "loss": 3.1855, + "theoretical_loss": 4.117741634114376, + "tokens_seen": 318373888 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045631895687061187, + "loss": 3.2221, + "theoretical_loss": 4.1176431546833365, + "tokens_seen": 318439424 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045630892678034105, + "loss": 2.9736, + "theoretical_loss": 4.117544701191187, + "tokens_seen": 318504960 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045629889669007023, + "loss": 3.2034, + "theoretical_loss": 4.117446273625763, + "tokens_seen": 318570496 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004562888665997994, + "loss": 3.1808, + "theoretical_loss": 4.117347871974903, + "tokens_seen": 318636032 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004562788365095286, + "loss": 2.9921, + "theoretical_loss": 4.1172494962264565, + "tokens_seen": 318701568 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004562688064192578, + "loss": 3.045, + "theoretical_loss": 4.117151146368282, + "tokens_seen": 318767104 + }, + { + "epoch": 0.1, + "learning_rate": 0.000456258776328987, + "loss": 3.1971, + "theoretical_loss": 4.117052822388243, + "tokens_seen": 318832640 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045624874623871614, + "loss": 3.4411, + "theoretical_loss": 4.116954524274216, + "tokens_seen": 318898176 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004562387161484454, + "loss": 2.8718, + "theoretical_loss": 4.11685625201408, + "tokens_seen": 318963712 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004562286860581745, + "loss": 3.1621, + "theoretical_loss": 4.116758005595727, + "tokens_seen": 319029248 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045621865596790374, + "loss": 3.1288, + "theoretical_loss": 4.116659785007055, + "tokens_seen": 319094784 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004562086258776329, + "loss": 3.2477, + "theoretical_loss": 4.116561590235969, + "tokens_seen": 319160320 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004561985957873621, + "loss": 3.1955, + "theoretical_loss": 4.116463421270385, + "tokens_seen": 319225856 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004561885656970913, + "loss": 3.0558, + "theoretical_loss": 4.116365278098225, + "tokens_seen": 319291392 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004561785356068205, + "loss": 3.2387, + "theoretical_loss": 4.116267160707421, + "tokens_seen": 319356928 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045616850551654964, + "loss": 3.1042, + "theoretical_loss": 4.11616906908591, + "tokens_seen": 319422464 + }, + { + "epoch": 0.1, + "objective/train/docs_used": 192764, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.9873268604278564, + "objective/train/theoretical_loss": 4.11607100322164, + "objective/train/tokens_used": 339948000, + "theoretical_loss": 4.11607100322164, + "tokens_seen": 319488000 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004561584754262789, + "loss": 3.0405, + "theoretical_loss": 4.11607100322164, + "tokens_seen": 319488000 + }, + { + "epoch": 0.1, + "learning_rate": 0.000456148445336008, + "loss": 3.2693, + "theoretical_loss": 4.115972963102565, + "tokens_seen": 319553536 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045613841524573724, + "loss": 3.3693, + "theoretical_loss": 4.11587494871665, + "tokens_seen": 319619072 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004561283851554664, + "loss": 3.0797, + "theoretical_loss": 4.115776960051864, + "tokens_seen": 319684608 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004561183550651956, + "loss": 3.2081, + "theoretical_loss": 4.11567899709619, + "tokens_seen": 319750144 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004561083249749248, + "loss": 2.9423, + "theoretical_loss": 4.115581059837612, + "tokens_seen": 319815680 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045609829488465396, + "loss": 3.1653, + "theoretical_loss": 4.115483148264127, + "tokens_seen": 319881216 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045608826479438314, + "loss": 3.2942, + "theoretical_loss": 4.115385262363739, + "tokens_seen": 319946752 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004560782347041124, + "loss": 2.9012, + "theoretical_loss": 4.1152874021244585, + "tokens_seen": 320012288 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004560682046138415, + "loss": 3.203, + "theoretical_loss": 4.115189567534307, + "tokens_seen": 320077824 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045605817452357074, + "loss": 3.3119, + "theoretical_loss": 4.115091758581309, + "tokens_seen": 320143360 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045604814443329987, + "loss": 2.9729, + "theoretical_loss": 4.114993975253505, + "tokens_seen": 320208896 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004560381143430291, + "loss": 3.0069, + "theoretical_loss": 4.114896217538935, + "tokens_seen": 320274432 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004560280842527583, + "loss": 3.3147, + "theoretical_loss": 4.114798485425652, + "tokens_seen": 320339968 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045601805416248747, + "loss": 3.0634, + "theoretical_loss": 4.114700778901717, + "tokens_seen": 320405504 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045600802407221665, + "loss": 3.2693, + "theoretical_loss": 4.114603097955197, + "tokens_seen": 320471040 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004559979939819459, + "loss": 3.1818, + "theoretical_loss": 4.114505442574167, + "tokens_seen": 320536576 + }, + { + "epoch": 0.1, + "learning_rate": 0.000455987963891675, + "loss": 2.9524, + "theoretical_loss": 4.1144078127467125, + "tokens_seen": 320602112 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045597793380140425, + "loss": 3.1601, + "theoretical_loss": 4.114310208460924, + "tokens_seen": 320667648 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045596790371113337, + "loss": 3.1247, + "theoretical_loss": 4.114212629704902, + "tokens_seen": 320733184 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004559578736208626, + "loss": 3.2691, + "theoretical_loss": 4.114115076466755, + "tokens_seen": 320798720 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004559478435305918, + "loss": 2.8277, + "theoretical_loss": 4.114017548734598, + "tokens_seen": 320864256 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045593781344032097, + "loss": 2.9319, + "theoretical_loss": 4.113920046496554, + "tokens_seen": 320929792 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045592778335005015, + "loss": 3.2766, + "theoretical_loss": 4.113822569740757, + "tokens_seen": 320995328 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045591775325977933, + "loss": 3.0531, + "theoretical_loss": 4.113725118455344, + "tokens_seen": 321060864 + }, + { + "epoch": 0.1, + "objective/train/docs_used": 193449, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.546736001968384, + "objective/train/theoretical_loss": 4.113627692628464, + "objective/train/tokens_used": 341586400, + "theoretical_loss": 4.113627692628464, + "tokens_seen": 321126400 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004559077231695085, + "loss": 3.0229, + "theoretical_loss": 4.113627692628464, + "tokens_seen": 321126400 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045589769307923775, + "loss": 2.9446, + "theoretical_loss": 4.113530292248273, + "tokens_seen": 321191936 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004558876629889669, + "loss": 3.0691, + "theoretical_loss": 4.113432917302934, + "tokens_seen": 321257472 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004558776328986961, + "loss": 3.1146, + "theoretical_loss": 4.113335567780618, + "tokens_seen": 321323008 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045586760280842524, + "loss": 3.1023, + "theoretical_loss": 4.113238243669504, + "tokens_seen": 321388544 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004558575727181545, + "loss": 3.0519, + "theoretical_loss": 4.113140944957781, + "tokens_seen": 321454080 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045584754262788365, + "loss": 3.2203, + "theoretical_loss": 4.113043671633641, + "tokens_seen": 321519616 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045583751253761284, + "loss": 2.9379, + "theoretical_loss": 4.11294642368529, + "tokens_seen": 321585152 + }, + { + "epoch": 0.1, + "learning_rate": 0.000455827482447342, + "loss": 2.9221, + "theoretical_loss": 4.112849201100938, + "tokens_seen": 321650688 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045581745235707125, + "loss": 3.3093, + "theoretical_loss": 4.1127520038688035, + "tokens_seen": 321716224 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004558074222668004, + "loss": 3.0509, + "theoretical_loss": 4.112654831977112, + "tokens_seen": 321781760 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004557973921765296, + "loss": 3.0694, + "theoretical_loss": 4.1125576854141, + "tokens_seen": 321847296 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045578736208625874, + "loss": 2.9662, + "theoretical_loss": 4.112460564168009, + "tokens_seen": 321912832 + }, + { + "epoch": 0.1, + "learning_rate": 0.000455777331995988, + "loss": 3.1083, + "theoretical_loss": 4.112363468227088, + "tokens_seen": 321978368 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045576730190571716, + "loss": 2.994, + "theoretical_loss": 4.112266397579598, + "tokens_seen": 322043904 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045575727181544634, + "loss": 3.0595, + "theoretical_loss": 4.112169352213801, + "tokens_seen": 322109440 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004557472417251755, + "loss": 3.069, + "theoretical_loss": 4.1120723321179735, + "tokens_seen": 322174976 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004557372116349047, + "loss": 2.8909, + "theoretical_loss": 4.111975337280397, + "tokens_seen": 322240512 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004557271815446339, + "loss": 3.1728, + "theoretical_loss": 4.111878367689359, + "tokens_seen": 322306048 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004557171514543631, + "loss": 3.1802, + "theoretical_loss": 4.1117814233331575, + "tokens_seen": 322371584 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045570712136409224, + "loss": 3.341, + "theoretical_loss": 4.111684504200099, + "tokens_seen": 322437120 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004556970912738215, + "loss": 3.0028, + "theoretical_loss": 4.111587610278494, + "tokens_seen": 322502656 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004556870611835507, + "loss": 3.0722, + "theoretical_loss": 4.111490741556663, + "tokens_seen": 322568192 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045567703109327984, + "loss": 3.1757, + "theoretical_loss": 4.1113938980229365, + "tokens_seen": 322633728 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004556670010030091, + "loss": 3.148, + "theoretical_loss": 4.11129707966565, + "tokens_seen": 322699264 + }, + { + "epoch": 0.1, + "objective/train/docs_used": 194727, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.387052297592163, + "objective/train/theoretical_loss": 4.111200286473145, + "objective/train/tokens_used": 343224800, + "theoretical_loss": 4.111200286473145, + "tokens_seen": 322764800 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004556569709127382, + "loss": 2.9313, + "theoretical_loss": 4.111200286473145, + "tokens_seen": 322764800 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045564694082246744, + "loss": 3.0901, + "theoretical_loss": 4.111103518433776, + "tokens_seen": 322830336 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004556369107321966, + "loss": 2.9013, + "theoretical_loss": 4.111006775535901, + "tokens_seen": 322895872 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004556268806419258, + "loss": 3.0106, + "theoretical_loss": 4.110910057767887, + "tokens_seen": 322961408 + }, + { + "epoch": 0.1, + "learning_rate": 0.000455616850551655, + "loss": 2.8652, + "theoretical_loss": 4.110813365118109, + "tokens_seen": 323026944 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045560682046138416, + "loss": 3.0251, + "theoretical_loss": 4.110716697574951, + "tokens_seen": 323092480 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045559679037111334, + "loss": 3.1885, + "theoretical_loss": 4.110620055126802, + "tokens_seen": 323158016 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004555867602808426, + "loss": 2.9144, + "theoretical_loss": 4.110523437762059, + "tokens_seen": 323223552 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004555767301905717, + "loss": 2.989, + "theoretical_loss": 4.11042684546913, + "tokens_seen": 323289088 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045556670010030094, + "loss": 3.1318, + "theoretical_loss": 4.110330278236427, + "tokens_seen": 323354624 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045555667001003007, + "loss": 2.9242, + "theoretical_loss": 4.110233736052372, + "tokens_seen": 323420160 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004555466399197593, + "loss": 3.0721, + "theoretical_loss": 4.110137218905393, + "tokens_seen": 323485696 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004555366098294885, + "loss": 3.2357, + "theoretical_loss": 4.110040726783927, + "tokens_seen": 323551232 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045552657973921767, + "loss": 3.27, + "theoretical_loss": 4.109944259676419, + "tokens_seen": 323616768 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045551654964894685, + "loss": 3.2144, + "theoretical_loss": 4.109847817571319, + "tokens_seen": 323682304 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004555065195586761, + "loss": 3.1028, + "theoretical_loss": 4.109751400457089, + "tokens_seen": 323747840 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004554964894684052, + "loss": 3.1743, + "theoretical_loss": 4.109655008322195, + "tokens_seen": 323813376 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045548645937813445, + "loss": 3.3165, + "theoretical_loss": 4.109558641155112, + "tokens_seen": 323878912 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045547642928786357, + "loss": 2.7415, + "theoretical_loss": 4.109462298944322, + "tokens_seen": 323944448 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004554663991975928, + "loss": 3.1486, + "theoretical_loss": 4.109365981678316, + "tokens_seen": 324009984 + }, + { + "epoch": 0.1, + "learning_rate": 0.000455456369107322, + "loss": 3.0672, + "theoretical_loss": 4.109269689345592, + "tokens_seen": 324075520 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045544633901705117, + "loss": 2.6652, + "theoretical_loss": 4.109173421934654, + "tokens_seen": 324141056 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045543630892678035, + "loss": 2.986, + "theoretical_loss": 4.109077179434016, + "tokens_seen": 324206592 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045542627883650953, + "loss": 3.0899, + "theoretical_loss": 4.1089809618321995, + "tokens_seen": 324272128 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004554162487462387, + "loss": 3.0958, + "theoretical_loss": 4.108884769117731, + "tokens_seen": 324337664 + }, + { + "epoch": 0.1, + "objective/train/docs_used": 195463, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.303276300430298, + "objective/train/theoretical_loss": 4.108788601279149, + "objective/train/tokens_used": 344863200, + "theoretical_loss": 4.108788601279149, + "tokens_seen": 324403200 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045540621865596795, + "loss": 3.0587, + "theoretical_loss": 4.108788601279149, + "tokens_seen": 324403200 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004553961885656971, + "loss": 3.0702, + "theoretical_loss": 4.108692458304994, + "tokens_seen": 324468736 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004553861584754263, + "loss": 2.9585, + "theoretical_loss": 4.108596340183819, + "tokens_seen": 324534272 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045537612838515544, + "loss": 3.0077, + "theoretical_loss": 4.108500246904184, + "tokens_seen": 324599808 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004553660982948847, + "loss": 2.9844, + "theoretical_loss": 4.108404178454651, + "tokens_seen": 324665344 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045535606820461385, + "loss": 3.0598, + "theoretical_loss": 4.1083081348237975, + "tokens_seen": 324730880 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045534603811434304, + "loss": 2.829, + "theoretical_loss": 4.108212116000203, + "tokens_seen": 324796416 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004553360080240722, + "loss": 3.0486, + "theoretical_loss": 4.108116121972457, + "tokens_seen": 324861952 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045532597793380145, + "loss": 3.0607, + "theoretical_loss": 4.108020152729157, + "tokens_seen": 324927488 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004553159478435306, + "loss": 3.0999, + "theoretical_loss": 4.107924208258905, + "tokens_seen": 324993024 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004553059177532598, + "loss": 2.9379, + "theoretical_loss": 4.107828288550314, + "tokens_seen": 325058560 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045529588766298894, + "loss": 3.0583, + "theoretical_loss": 4.107732393592003, + "tokens_seen": 325124096 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004552858575727182, + "loss": 3.1322, + "theoretical_loss": 4.107636523372598, + "tokens_seen": 325189632 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045527582748244736, + "loss": 2.8593, + "theoretical_loss": 4.107540677880734, + "tokens_seen": 325255168 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045526579739217654, + "loss": 3.0679, + "theoretical_loss": 4.107444857105052, + "tokens_seen": 325320704 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004552557673019057, + "loss": 2.9312, + "theoretical_loss": 4.107349061034201, + "tokens_seen": 325386240 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004552457372116349, + "loss": 2.8763, + "theoretical_loss": 4.107253289656838, + "tokens_seen": 325451776 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004552357071213641, + "loss": 3.0886, + "theoretical_loss": 4.107157542961628, + "tokens_seen": 325517312 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004552256770310933, + "loss": 2.9611, + "theoretical_loss": 4.10706182093724, + "tokens_seen": 325582848 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045521564694082244, + "loss": 3.0349, + "theoretical_loss": 4.106966123572356, + "tokens_seen": 325648384 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004552056168505517, + "loss": 2.9213, + "theoretical_loss": 4.106870450855661, + "tokens_seen": 325713920 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004551955867602808, + "loss": 3.0238, + "theoretical_loss": 4.106774802775849, + "tokens_seen": 325779456 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045518555667001004, + "loss": 2.7434, + "theoretical_loss": 4.106679179321622, + "tokens_seen": 325844992 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004551755265797392, + "loss": 2.8713, + "theoretical_loss": 4.106583580481689, + "tokens_seen": 325910528 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004551654964894684, + "loss": 3.0953, + "theoretical_loss": 4.106488006244767, + "tokens_seen": 325976064 + }, + { + "epoch": 0.1, + "objective/train/docs_used": 196731, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.2853167057037354, + "objective/train/theoretical_loss": 4.106392456599577, + "objective/train/tokens_used": 346501600, + "theoretical_loss": 4.106392456599577, + "tokens_seen": 326041600 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004551554663991976, + "loss": 3.0812, + "theoretical_loss": 4.106392456599577, + "tokens_seen": 326041600 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004551454363089268, + "loss": 3.1486, + "theoretical_loss": 4.106296931534854, + "tokens_seen": 326107136 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045513540621865595, + "loss": 2.9492, + "theoretical_loss": 4.106201431039335, + "tokens_seen": 326172672 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004551253761283852, + "loss": 3.0063, + "theoretical_loss": 4.106105955101766, + "tokens_seen": 326238208 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004551153460381143, + "loss": 3.1113, + "theoretical_loss": 4.1060105037109, + "tokens_seen": 326303744 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045510531594784354, + "loss": 3.1401, + "theoretical_loss": 4.105915076855499, + "tokens_seen": 326369280 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004550952858575727, + "loss": 3.1324, + "theoretical_loss": 4.105819674524332, + "tokens_seen": 326434816 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004550852557673019, + "loss": 3.1616, + "theoretical_loss": 4.105724296706172, + "tokens_seen": 326500352 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004550752256770311, + "loss": 3.1111, + "theoretical_loss": 4.105628943389805, + "tokens_seen": 326565888 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045506519558676027, + "loss": 3.0011, + "theoretical_loss": 4.1055336145640196, + "tokens_seen": 326631424 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045505516549648945, + "loss": 3.0663, + "theoretical_loss": 4.105438310217615, + "tokens_seen": 326696960 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004550451354062187, + "loss": 3.0075, + "theoretical_loss": 4.105343030339395, + "tokens_seen": 326762496 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004550351053159478, + "loss": 3.0593, + "theoretical_loss": 4.1052477749181735, + "tokens_seen": 326828032 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045502507522567705, + "loss": 2.9374, + "theoretical_loss": 4.10515254394277, + "tokens_seen": 326893568 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045501504513540623, + "loss": 2.846, + "theoretical_loss": 4.1050573374020125, + "tokens_seen": 326959104 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004550050150451354, + "loss": 3.0197, + "theoretical_loss": 4.104962155284734, + "tokens_seen": 327024640 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004549949849548646, + "loss": 2.945, + "theoretical_loss": 4.104866997579778, + "tokens_seen": 327090176 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045498495486459377, + "loss": 3.1306, + "theoretical_loss": 4.104771864275993, + "tokens_seen": 327155712 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045497492477432295, + "loss": 2.9276, + "theoretical_loss": 4.104676755362237, + "tokens_seen": 327221248 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004549648946840522, + "loss": 3.023, + "theoretical_loss": 4.104581670827372, + "tokens_seen": 327286784 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045495486459378137, + "loss": 3.0762, + "theoretical_loss": 4.10448661066027, + "tokens_seen": 327352320 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045494483450351055, + "loss": 3.1613, + "theoretical_loss": 4.104391574849812, + "tokens_seen": 327417856 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045493480441323973, + "loss": 3.0041, + "theoretical_loss": 4.10429656338488, + "tokens_seen": 327483392 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004549247743229689, + "loss": 2.8469, + "theoretical_loss": 4.104201576254369, + "tokens_seen": 327548928 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045491474423269815, + "loss": 3.1408, + "theoretical_loss": 4.10410661344718, + "tokens_seen": 327614464 + }, + { + "debugging/Self-BLEU-5": 0.48445945981218463, + "debugging/distinct-1-grams": 0.7948268571619983, + "debugging/distinct-2-grams": 0.9739697199580383, + "debugging/entropy-1-grams": 6.01135557249139, + "debugging/entropy-2-grams": 6.827639403579008, + "debugging/length": 512.9230769230769, + "debugging/num_segments": 13, + "debugging/score": 0.008299263930001547, + "debugging/score_std": 0.00590965264802656, + "epoch": 0.1, + "objective/train/docs_used": 197513, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.126281499862671, + "objective/train/theoretical_loss": 4.10401167495222, + "objective/train/tokens_used": 348140000, + "theoretical_loss": 4.10401167495222, + "tokens_seen": 327680000 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004549047141424273, + "loss": 3.1336, + "theoretical_loss": 4.10401167495222, + "tokens_seen": 327680000 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004548946840521565, + "loss": 2.9133, + "theoretical_loss": 4.103916760758405, + "tokens_seen": 327745536 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045488465396188564, + "loss": 2.9391, + "theoretical_loss": 4.103821870854656, + "tokens_seen": 327811072 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004548746238716149, + "loss": 2.6436, + "theoretical_loss": 4.103727005229903, + "tokens_seen": 327876608 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045486459378134405, + "loss": 2.9384, + "theoretical_loss": 4.103632163873083, + "tokens_seen": 327942144 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045485456369107324, + "loss": 3.2155, + "theoretical_loss": 4.10353734677314, + "tokens_seen": 328007680 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004548445336008024, + "loss": 2.9405, + "theoretical_loss": 4.103442553919026, + "tokens_seen": 328073216 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045483450351053165, + "loss": 3.0441, + "theoretical_loss": 4.1033477852996985, + "tokens_seen": 328138752 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004548244734202608, + "loss": 3.2765, + "theoretical_loss": 4.103253040904124, + "tokens_seen": 328204288 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045481444332999, + "loss": 3.143, + "theoretical_loss": 4.103158320721276, + "tokens_seen": 328269824 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045480441323971914, + "loss": 3.1284, + "theoretical_loss": 4.103063624740133, + "tokens_seen": 328335360 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004547943831494484, + "loss": 3.1121, + "theoretical_loss": 4.102968952949684, + "tokens_seen": 328400896 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045478435305917756, + "loss": 3.0309, + "theoretical_loss": 4.102874305338923, + "tokens_seen": 328466432 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045477432296890674, + "loss": 3.1, + "theoretical_loss": 4.102779681896852, + "tokens_seen": 328531968 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004547642928786359, + "loss": 3.1401, + "theoretical_loss": 4.10268508261248, + "tokens_seen": 328597504 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004547542627883651, + "loss": 3.0456, + "theoretical_loss": 4.102590507474824, + "tokens_seen": 328663040 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004547442326980943, + "loss": 3.1836, + "theoretical_loss": 4.1024959564729055, + "tokens_seen": 328728576 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004547342026078235, + "loss": 3.0008, + "theoretical_loss": 4.102401429595758, + "tokens_seen": 328794112 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045472417251755264, + "loss": 3.1338, + "theoretical_loss": 4.102306926832417, + "tokens_seen": 328859648 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004547141424272819, + "loss": 2.7724, + "theoretical_loss": 4.102212448171928, + "tokens_seen": 328925184 + }, + { + "epoch": 0.1, + "learning_rate": 0.000454704112337011, + "loss": 3.0015, + "theoretical_loss": 4.1021179936033425, + "tokens_seen": 328990720 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045469408224674024, + "loss": 3.0279, + "theoretical_loss": 4.102023563115721, + "tokens_seen": 329056256 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004546840521564694, + "loss": 3.0525, + "theoretical_loss": 4.10192915669813, + "tokens_seen": 329121792 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004546740220661986, + "loss": 3.1069, + "theoretical_loss": 4.1018347743396415, + "tokens_seen": 329187328 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004546639919759278, + "loss": 3.1062, + "theoretical_loss": 4.101740416029338, + "tokens_seen": 329252864 + }, + { + "epoch": 0.1, + "objective/train/docs_used": 198138, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.3793585300445557, + "objective/train/theoretical_loss": 4.101646081756305, + "objective/train/tokens_used": 349778400, + "theoretical_loss": 4.101646081756305, + "tokens_seen": 329318400 + }, + { + "epoch": 0.1, + "learning_rate": 0.000454653961885657, + "loss": 3.1387, + "theoretical_loss": 4.101646081756305, + "tokens_seen": 329318400 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045464393179538615, + "loss": 3.2323, + "theoretical_loss": 4.101551771509641, + "tokens_seen": 329383936 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004546339017051154, + "loss": 3.1388, + "theoretical_loss": 4.101457485278444, + "tokens_seen": 329449472 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004546238716148445, + "loss": 3.0909, + "theoretical_loss": 4.101363223051826, + "tokens_seen": 329515008 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045461384152457374, + "loss": 3.0194, + "theoretical_loss": 4.101268984818901, + "tokens_seen": 329580544 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004546038114343029, + "loss": 3.0234, + "theoretical_loss": 4.101174770568795, + "tokens_seen": 329646080 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004545937813440321, + "loss": 3.0885, + "theoretical_loss": 4.1010805802906365, + "tokens_seen": 329711616 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004545837512537613, + "loss": 3.1483, + "theoretical_loss": 4.100986413973564, + "tokens_seen": 329777152 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045457372116349047, + "loss": 3.0767, + "theoretical_loss": 4.100892271606721, + "tokens_seen": 329842688 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045456369107321965, + "loss": 2.9534, + "theoretical_loss": 4.1007981531792606, + "tokens_seen": 329908224 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004545536609829489, + "loss": 3.075, + "theoretical_loss": 4.100704058680341, + "tokens_seen": 329973760 + }, + { + "epoch": 0.1, + "learning_rate": 0.000454543630892678, + "loss": 3.0404, + "theoretical_loss": 4.1006099880991265, + "tokens_seen": 330039296 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045453360080240725, + "loss": 2.7094, + "theoretical_loss": 4.100515941424792, + "tokens_seen": 330104832 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045452357071213643, + "loss": 2.6964, + "theoretical_loss": 4.100421918646517, + "tokens_seen": 330170368 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004545135406218656, + "loss": 2.9502, + "theoretical_loss": 4.1003279197534885, + "tokens_seen": 330235904 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004545035105315948, + "loss": 2.9108, + "theoretical_loss": 4.100233944734899, + "tokens_seen": 330301440 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045449348044132397, + "loss": 3.0213, + "theoretical_loss": 4.100139993579952, + "tokens_seen": 330366976 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045448345035105315, + "loss": 3.0134, + "theoretical_loss": 4.100046066277853, + "tokens_seen": 330432512 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004544734202607824, + "loss": 3.0427, + "theoretical_loss": 4.09995216281782, + "tokens_seen": 330498048 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004544633901705115, + "loss": 3.0521, + "theoretical_loss": 4.0998582831890715, + "tokens_seen": 330563584 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045445336008024075, + "loss": 2.8685, + "theoretical_loss": 4.0997644273808405, + "tokens_seen": 330629120 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004544433299899699, + "loss": 2.8279, + "theoretical_loss": 4.09967059538236, + "tokens_seen": 330694656 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004544332998996991, + "loss": 3.0582, + "theoretical_loss": 4.099576787182874, + "tokens_seen": 330760192 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004544232698094283, + "loss": 3.0239, + "theoretical_loss": 4.099483002771633, + "tokens_seen": 330825728 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004544132397191575, + "loss": 3.3214, + "theoretical_loss": 4.099389242137894, + "tokens_seen": 330891264 + }, + { + "epoch": 0.1, + "objective/train/docs_used": 199416, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.3213510513305664, + "objective/train/theoretical_loss": 4.099295505270921, + "objective/train/tokens_used": 351416800, + "theoretical_loss": 4.099295505270921, + "tokens_seen": 330956800 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045440320962888666, + "loss": 3.0776, + "theoretical_loss": 4.099295505270921, + "tokens_seen": 330956800 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045439317953861584, + "loss": 3.0486, + "theoretical_loss": 4.099201792159985, + "tokens_seen": 331022336 + }, + { + "epoch": 0.1, + "learning_rate": 0.000454383149448345, + "loss": 3.0615, + "theoretical_loss": 4.099108102794363, + "tokens_seen": 331087872 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045437311935807425, + "loss": 3.0662, + "theoretical_loss": 4.099014437163342, + "tokens_seen": 331153408 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004543630892678034, + "loss": 3.2298, + "theoretical_loss": 4.098920795256213, + "tokens_seen": 331218944 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004543530591775326, + "loss": 3.1568, + "theoretical_loss": 4.098827177062273, + "tokens_seen": 331284480 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004543430290872618, + "loss": 3.0858, + "theoretical_loss": 4.098733582570831, + "tokens_seen": 331350016 + }, + { + "epoch": 0.1, + "learning_rate": 0.000454332998996991, + "loss": 2.9941, + "theoretical_loss": 4.098640011771198, + "tokens_seen": 331415552 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045432296890672016, + "loss": 3.0898, + "theoretical_loss": 4.098546464652693, + "tokens_seen": 331481088 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045431293881644934, + "loss": 3.1252, + "theoretical_loss": 4.098452941204643, + "tokens_seen": 331546624 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004543029087261785, + "loss": 2.9155, + "theoretical_loss": 4.098359441416383, + "tokens_seen": 331612160 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045429287863590776, + "loss": 3.0528, + "theoretical_loss": 4.0982659652772515, + "tokens_seen": 331677696 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004542828485456369, + "loss": 3.0785, + "theoretical_loss": 4.098172512776597, + "tokens_seen": 331743232 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004542728184553661, + "loss": 2.7989, + "theoretical_loss": 4.098079083903773, + "tokens_seen": 331808768 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045426278836509525, + "loss": 3.0039, + "theoretical_loss": 4.097985678648142, + "tokens_seen": 331874304 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004542527582748245, + "loss": 3.0068, + "theoretical_loss": 4.09789229699907, + "tokens_seen": 331939840 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045424272818455366, + "loss": 2.8917, + "theoretical_loss": 4.097798938945933, + "tokens_seen": 332005376 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045423269809428284, + "loss": 2.9179, + "theoretical_loss": 4.097705604478112, + "tokens_seen": 332070912 + }, + { + "epoch": 0.1, + "learning_rate": 0.000454222668004012, + "loss": 2.9804, + "theoretical_loss": 4.097612293584998, + "tokens_seen": 332136448 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004542126379137412, + "loss": 3.2332, + "theoretical_loss": 4.0975190062559825, + "tokens_seen": 332201984 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045420260782347044, + "loss": 2.897, + "theoretical_loss": 4.097425742480472, + "tokens_seen": 332267520 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004541925777331996, + "loss": 2.9868, + "theoretical_loss": 4.097332502247873, + "tokens_seen": 332333056 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004541825476429288, + "loss": 2.9921, + "theoretical_loss": 4.0972392855476025, + "tokens_seen": 332398592 + }, + { + "epoch": 0.1, + "learning_rate": 0.000454172517552658, + "loss": 3.0413, + "theoretical_loss": 4.097146092369084, + "tokens_seen": 332464128 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004541624874623872, + "loss": 2.9296, + "theoretical_loss": 4.097052922701746, + "tokens_seen": 332529664 + }, + { + "epoch": 0.1, + "objective/train/docs_used": 200524, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.3535284996032715, + "objective/train/theoretical_loss": 4.0969597765350265, + "objective/train/tokens_used": 353055200, + "theoretical_loss": 4.0969597765350265, + "tokens_seen": 332595200 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045415245737211635, + "loss": 3.0659, + "theoretical_loss": 4.0969597765350265, + "tokens_seen": 332595200 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004541424272818456, + "loss": 3.0972, + "theoretical_loss": 4.096866653858368, + "tokens_seen": 332660736 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004541323971915747, + "loss": 3.0509, + "theoretical_loss": 4.09677355466122, + "tokens_seen": 332726272 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045412236710130395, + "loss": 2.9711, + "theoretical_loss": 4.096680478933042, + "tokens_seen": 332791808 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004541123370110331, + "loss": 2.9232, + "theoretical_loss": 4.0965874266632945, + "tokens_seen": 332857344 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004541023069207623, + "loss": 2.878, + "theoretical_loss": 4.09649439784145, + "tokens_seen": 332922880 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004540922768304915, + "loss": 2.9766, + "theoretical_loss": 4.096401392456988, + "tokens_seen": 332988416 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045408224674022067, + "loss": 3.126, + "theoretical_loss": 4.09630841049939, + "tokens_seen": 333053952 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045407221664994985, + "loss": 2.996, + "theoretical_loss": 4.096215451958146, + "tokens_seen": 333119488 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004540621865596791, + "loss": 3.0112, + "theoretical_loss": 4.096122516822757, + "tokens_seen": 333185024 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004540521564694082, + "loss": 3.0493, + "theoretical_loss": 4.096029605082726, + "tokens_seen": 333250560 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045404212637913745, + "loss": 3.0256, + "theoretical_loss": 4.095936716727564, + "tokens_seen": 333316096 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045403209628886663, + "loss": 3.0633, + "theoretical_loss": 4.095843851746791, + "tokens_seen": 333381632 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004540220661985958, + "loss": 2.9775, + "theoretical_loss": 4.095751010129929, + "tokens_seen": 333447168 + }, + { + "epoch": 0.1, + "learning_rate": 0.000454012036108325, + "loss": 2.9432, + "theoretical_loss": 4.095658191866512, + "tokens_seen": 333512704 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045400200601805417, + "loss": 3.3982, + "theoretical_loss": 4.0955653969460775, + "tokens_seen": 333578240 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045399197592778335, + "loss": 2.9339, + "theoretical_loss": 4.095472625358171, + "tokens_seen": 333643776 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004539819458375126, + "loss": 3.1546, + "theoretical_loss": 4.095379877092343, + "tokens_seen": 333709312 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004539719157472417, + "loss": 2.8311, + "theoretical_loss": 4.095287152138154, + "tokens_seen": 333774848 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045396188565697095, + "loss": 3.1384, + "theoretical_loss": 4.0951944504851685, + "tokens_seen": 333840384 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004539518555667001, + "loss": 3.1351, + "theoretical_loss": 4.095101772122959, + "tokens_seen": 333905920 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004539418254764293, + "loss": 2.9479, + "theoretical_loss": 4.095009117041102, + "tokens_seen": 333971456 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004539317953861585, + "loss": 2.8694, + "theoretical_loss": 4.094916485229186, + "tokens_seen": 334036992 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004539217652958877, + "loss": 2.965, + "theoretical_loss": 4.094823876676802, + "tokens_seen": 334102528 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045391173520561686, + "loss": 3.126, + "theoretical_loss": 4.094731291373548, + "tokens_seen": 334168064 + }, + { + "epoch": 0.1, + "objective/train/docs_used": 201196, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.88828706741333, + "objective/train/theoretical_loss": 4.094638729309031, + "objective/train/tokens_used": 354693600, + "theoretical_loss": 4.094638729309031, + "tokens_seen": 334233600 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045390170511534604, + "loss": 3.0232, + "theoretical_loss": 4.094638729309031, + "tokens_seen": 334233600 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004538916750250752, + "loss": 3.1237, + "theoretical_loss": 4.094546190472862, + "tokens_seen": 334299136 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045388164493480445, + "loss": 3.3401, + "theoretical_loss": 4.09445367485466, + "tokens_seen": 334364672 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004538716148445336, + "loss": 2.9893, + "theoretical_loss": 4.09436118244405, + "tokens_seen": 334430208 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004538615847542628, + "loss": 3.0786, + "theoretical_loss": 4.094268713230667, + "tokens_seen": 334495744 + }, + { + "epoch": 0.1, + "learning_rate": 0.000453851554663992, + "loss": 2.9697, + "theoretical_loss": 4.094176267204148, + "tokens_seen": 334561280 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004538415245737212, + "loss": 3.1148, + "theoretical_loss": 4.094083844354137, + "tokens_seen": 334626816 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045383149448345036, + "loss": 2.9482, + "theoretical_loss": 4.093991444670289, + "tokens_seen": 334692352 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045382146439317954, + "loss": 3.0128, + "theoretical_loss": 4.093899068142262, + "tokens_seen": 334757888 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004538114343029087, + "loss": 3.0838, + "theoretical_loss": 4.093806714759721, + "tokens_seen": 334823424 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045380140421263796, + "loss": 2.8237, + "theoretical_loss": 4.093714384512337, + "tokens_seen": 334888960 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004537913741223671, + "loss": 3.0851, + "theoretical_loss": 4.093622077389791, + "tokens_seen": 334954496 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004537813440320963, + "loss": 3.2397, + "theoretical_loss": 4.093529793381768, + "tokens_seen": 335020032 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045377131394182545, + "loss": 3.0707, + "theoretical_loss": 4.093437532477958, + "tokens_seen": 335085568 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004537612838515547, + "loss": 3.045, + "theoretical_loss": 4.093345294668063, + "tokens_seen": 335151104 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045375125376128386, + "loss": 3.3077, + "theoretical_loss": 4.0932530799417846, + "tokens_seen": 335216640 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045374122367101304, + "loss": 3.0837, + "theoretical_loss": 4.093160888288837, + "tokens_seen": 335282176 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004537311935807422, + "loss": 3.2719, + "theoretical_loss": 4.0930687196989375, + "tokens_seen": 335347712 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004537211634904714, + "loss": 3.0235, + "theoretical_loss": 4.0929765741618125, + "tokens_seen": 335413248 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004537111334002006, + "loss": 3.0189, + "theoretical_loss": 4.092884451667191, + "tokens_seen": 335478784 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004537011033099298, + "loss": 3.0861, + "theoretical_loss": 4.092792352204814, + "tokens_seen": 335544320 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045369107321965895, + "loss": 3.0651, + "theoretical_loss": 4.092700275764424, + "tokens_seen": 335609856 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004536810431293882, + "loss": 2.9012, + "theoretical_loss": 4.092608222335774, + "tokens_seen": 335675392 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045367101303911737, + "loss": 3.0509, + "theoretical_loss": 4.092516191908621, + "tokens_seen": 335740928 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045366098294884655, + "loss": 3.3312, + "theoretical_loss": 4.09242418447273, + "tokens_seen": 335806464 + }, + { + "epoch": 0.1, + "objective/train/docs_used": 202460, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.067107677459717, + "objective/train/theoretical_loss": 4.092332200017871, + "objective/train/tokens_used": 356332000, + "theoretical_loss": 4.092332200017871, + "tokens_seen": 335872000 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045365095285857573, + "loss": 2.9362, + "theoretical_loss": 4.092332200017871, + "tokens_seen": 335872000 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004536409227683049, + "loss": 3.3125, + "theoretical_loss": 4.092240238533822, + "tokens_seen": 335937536 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004536308926780341, + "loss": 3.1253, + "theoretical_loss": 4.092148300010367, + "tokens_seen": 336003072 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004536208625877633, + "loss": 3.0043, + "theoretical_loss": 4.092056384437297, + "tokens_seen": 336068608 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045361083249749245, + "loss": 2.907, + "theoretical_loss": 4.091964491804409, + "tokens_seen": 336134144 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004536008024072217, + "loss": 3.0272, + "theoretical_loss": 4.091872622101506, + "tokens_seen": 336199680 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004535907723169508, + "loss": 3.1327, + "theoretical_loss": 4.091780775318399, + "tokens_seen": 336265216 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045358074222668005, + "loss": 3.0913, + "theoretical_loss": 4.091688951444904, + "tokens_seen": 336330752 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045357071213640923, + "loss": 3.0405, + "theoretical_loss": 4.091597150470845, + "tokens_seen": 336396288 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004535606820461384, + "loss": 3.3047, + "theoretical_loss": 4.091505372386051, + "tokens_seen": 336461824 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004535506519558676, + "loss": 3.1592, + "theoretical_loss": 4.091413617180358, + "tokens_seen": 336527360 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045354062186559683, + "loss": 2.9044, + "theoretical_loss": 4.091321884843609, + "tokens_seen": 336592896 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045353059177532596, + "loss": 3.268, + "theoretical_loss": 4.091230175365653, + "tokens_seen": 336658432 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004535205616850552, + "loss": 3.1605, + "theoretical_loss": 4.0911384887363464, + "tokens_seen": 336723968 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004535105315947843, + "loss": 2.8838, + "theoretical_loss": 4.091046824945551, + "tokens_seen": 336789504 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045350050150451355, + "loss": 3.2622, + "theoretical_loss": 4.0909551839831355, + "tokens_seen": 336855040 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045349047141424274, + "loss": 3.1742, + "theoretical_loss": 4.090863565838974, + "tokens_seen": 336920576 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004534804413239719, + "loss": 2.833, + "theoretical_loss": 4.090771970502948, + "tokens_seen": 336986112 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004534704112337011, + "loss": 3.2279, + "theoretical_loss": 4.090680397964947, + "tokens_seen": 337051648 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004534603811434303, + "loss": 2.9885, + "theoretical_loss": 4.090588848214865, + "tokens_seen": 337117184 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004534503510531595, + "loss": 3.0721, + "theoretical_loss": 4.0904973212426015, + "tokens_seen": 337182720 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004534403209628887, + "loss": 2.9347, + "theoretical_loss": 4.090405817038065, + "tokens_seen": 337248256 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004534302908726179, + "loss": 3.059, + "theoretical_loss": 4.090314335591169, + "tokens_seen": 337313792 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045342026078234706, + "loss": 2.9167, + "theoretical_loss": 4.090222876891834, + "tokens_seen": 337379328 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045341023069207624, + "loss": 3.2547, + "theoretical_loss": 4.090131440929985, + "tokens_seen": 337444864 + }, + { + "epoch": 0.1, + "objective/train/docs_used": 203033, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.1972439289093018, + "objective/train/theoretical_loss": 4.090040027695556, + "objective/train/tokens_used": 357970400, + "theoretical_loss": 4.090040027695556, + "tokens_seen": 337510400 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004534002006018054, + "loss": 3.2806, + "theoretical_loss": 4.090040027695556, + "tokens_seen": 337510400 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045339017051153465, + "loss": 2.833, + "theoretical_loss": 4.0899486371784874, + "tokens_seen": 337575936 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004533801404212638, + "loss": 3.014, + "theoretical_loss": 4.089857269368725, + "tokens_seen": 337641472 + }, + { + "epoch": 0.1, + "learning_rate": 0.000453370110330993, + "loss": 2.9357, + "theoretical_loss": 4.089765924256218, + "tokens_seen": 337707008 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004533600802407222, + "loss": 3.1723, + "theoretical_loss": 4.089674601830929, + "tokens_seen": 337772544 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004533500501504514, + "loss": 2.8899, + "theoretical_loss": 4.08958330208282, + "tokens_seen": 337838080 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045334002006018056, + "loss": 3.1651, + "theoretical_loss": 4.089492025001864, + "tokens_seen": 337903616 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045332998996990974, + "loss": 3.095, + "theoretical_loss": 4.089400770578038, + "tokens_seen": 337969152 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004533199598796389, + "loss": 3.2317, + "theoretical_loss": 4.089309538801327, + "tokens_seen": 338034688 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045330992978936816, + "loss": 2.8262, + "theoretical_loss": 4.08921832966172, + "tokens_seen": 338100224 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004532998996990973, + "loss": 2.9007, + "theoretical_loss": 4.0891271431492155, + "tokens_seen": 338165760 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004532898696088265, + "loss": 3.2057, + "theoretical_loss": 4.089035979253816, + "tokens_seen": 338231296 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045327983951855565, + "loss": 2.9183, + "theoretical_loss": 4.08894483796553, + "tokens_seen": 338296832 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004532698094282849, + "loss": 3.1165, + "theoretical_loss": 4.0888537192743755, + "tokens_seen": 338362368 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045325977933801406, + "loss": 3.095, + "theoretical_loss": 4.088762623170373, + "tokens_seen": 338427904 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045324974924774324, + "loss": 2.7454, + "theoretical_loss": 4.088671549643553, + "tokens_seen": 338493440 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004532397191574724, + "loss": 3.0204, + "theoretical_loss": 4.088580498683948, + "tokens_seen": 338558976 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004532296890672016, + "loss": 3.3193, + "theoretical_loss": 4.088489470281601, + "tokens_seen": 338624512 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004532196589769308, + "loss": 2.7623, + "theoretical_loss": 4.088398464426559, + "tokens_seen": 338690048 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045320962888666, + "loss": 3.1227, + "theoretical_loss": 4.088307481108876, + "tokens_seen": 338755584 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045319959879638915, + "loss": 3.0064, + "theoretical_loss": 4.088216520318612, + "tokens_seen": 338821120 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004531895687061184, + "loss": 3.0352, + "theoretical_loss": 4.0881255820458335, + "tokens_seen": 338886656 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045317953861584757, + "loss": 3.0182, + "theoretical_loss": 4.088034666280614, + "tokens_seen": 338952192 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045316950852557675, + "loss": 3.1979, + "theoretical_loss": 4.087943773013032, + "tokens_seen": 339017728 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045315947843530593, + "loss": 3.0418, + "theoretical_loss": 4.087852902233173, + "tokens_seen": 339083264 + }, + { + "epoch": 0.1, + "objective/train/docs_used": 204232, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7819879055023193, + "objective/train/theoretical_loss": 4.087762053931129, + "objective/train/tokens_used": 359608800, + "theoretical_loss": 4.087762053931129, + "tokens_seen": 339148800 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004531494483450351, + "loss": 3.0312, + "theoretical_loss": 4.087762053931129, + "tokens_seen": 339148800 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004531394182547643, + "loss": 3.0391, + "theoretical_loss": 4.087671228096997, + "tokens_seen": 339214336 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004531293881644935, + "loss": 2.9776, + "theoretical_loss": 4.087580424720882, + "tokens_seen": 339279872 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045311935807422265, + "loss": 3.0434, + "theoretical_loss": 4.087489643792894, + "tokens_seen": 339345408 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004531093279839519, + "loss": 2.947, + "theoretical_loss": 4.08739888530315, + "tokens_seen": 339410944 + }, + { + "epoch": 0.1, + "learning_rate": 0.000453099297893681, + "loss": 2.985, + "theoretical_loss": 4.087308149241774, + "tokens_seen": 339476480 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045308926780341025, + "loss": 2.8241, + "theoretical_loss": 4.087217435598894, + "tokens_seen": 339542016 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045307923771313943, + "loss": 3.0844, + "theoretical_loss": 4.087126744364646, + "tokens_seen": 339607552 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004530692076228686, + "loss": 2.952, + "theoretical_loss": 4.087036075529172, + "tokens_seen": 339673088 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004530591775325978, + "loss": 3.0252, + "theoretical_loss": 4.086945429082618, + "tokens_seen": 339738624 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045304914744232703, + "loss": 3.1182, + "theoretical_loss": 4.086854805015141, + "tokens_seen": 339804160 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045303911735205616, + "loss": 3.1047, + "theoretical_loss": 4.086764203316902, + "tokens_seen": 339869696 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004530290872617854, + "loss": 2.9837, + "theoretical_loss": 4.086673623978064, + "tokens_seen": 339935232 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004530190571715145, + "loss": 3.0321, + "theoretical_loss": 4.086583066988802, + "tokens_seen": 340000768 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045300902708124375, + "loss": 3.1585, + "theoretical_loss": 4.086492532339296, + "tokens_seen": 340066304 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045299899699097294, + "loss": 3.2067, + "theoretical_loss": 4.0864020200197295, + "tokens_seen": 340131840 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004529889669007021, + "loss": 3.0021, + "theoretical_loss": 4.086311530020296, + "tokens_seen": 340197376 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004529789368104313, + "loss": 3.1524, + "theoretical_loss": 4.086221062331192, + "tokens_seen": 340262912 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004529689067201605, + "loss": 2.9693, + "theoretical_loss": 4.086130616942621, + "tokens_seen": 340328448 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045295887662988966, + "loss": 2.9457, + "theoretical_loss": 4.086040193844794, + "tokens_seen": 340393984 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004529488465396189, + "loss": 2.8561, + "theoretical_loss": 4.085949793027927, + "tokens_seen": 340459520 + }, + { + "epoch": 0.1, + "learning_rate": 0.000452938816449348, + "loss": 3.2566, + "theoretical_loss": 4.0858594144822415, + "tokens_seen": 340525056 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045292878635907726, + "loss": 2.9455, + "theoretical_loss": 4.085769058197968, + "tokens_seen": 340590592 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004529187562688064, + "loss": 2.8537, + "theoretical_loss": 4.085678724165341, + "tokens_seen": 340656128 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004529087261785356, + "loss": 3.2748, + "theoretical_loss": 4.0855884123746, + "tokens_seen": 340721664 + }, + { + "epoch": 0.1, + "objective/train/docs_used": 205032, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.317850112915039, + "objective/train/theoretical_loss": 4.085498122815992, + "objective/train/tokens_used": 361247200, + "theoretical_loss": 4.085498122815992, + "tokens_seen": 340787200 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004528986960882648, + "loss": 3.1587, + "theoretical_loss": 4.085498122815992, + "tokens_seen": 340787200 + }, + { + "epoch": 0.1, + "learning_rate": 0.000452888665997994, + "loss": 2.7753, + "theoretical_loss": 4.085407855479772, + "tokens_seen": 340852736 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045287863590772316, + "loss": 3.1143, + "theoretical_loss": 4.085317610356199, + "tokens_seen": 340918272 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004528686058174524, + "loss": 3.1908, + "theoretical_loss": 4.085227387435538, + "tokens_seen": 340983808 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004528585757271815, + "loss": 2.8026, + "theoretical_loss": 4.08513718670806, + "tokens_seen": 341049344 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045284854563691076, + "loss": 3.0414, + "theoretical_loss": 4.085047008164044, + "tokens_seen": 341114880 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004528385155466399, + "loss": 2.9043, + "theoretical_loss": 4.084956851793773, + "tokens_seen": 341180416 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004528284854563691, + "loss": 3.2055, + "theoretical_loss": 4.0848667175875395, + "tokens_seen": 341245952 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004528184553660983, + "loss": 2.8475, + "theoretical_loss": 4.0847766055356365, + "tokens_seen": 341311488 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004528084252758275, + "loss": 3.0822, + "theoretical_loss": 4.084686515628368, + "tokens_seen": 341377024 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045279839518555667, + "loss": 2.5604, + "theoretical_loss": 4.0845964478560415, + "tokens_seen": 341442560 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045278836509528585, + "loss": 3.2325, + "theoretical_loss": 4.084506402208972, + "tokens_seen": 341508096 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045277833500501503, + "loss": 3.2722, + "theoretical_loss": 4.0844163786774805, + "tokens_seen": 341573632 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045276830491474426, + "loss": 3.2708, + "theoretical_loss": 4.084326377251894, + "tokens_seen": 341639168 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004527582748244734, + "loss": 2.9948, + "theoretical_loss": 4.084236397922544, + "tokens_seen": 341704704 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004527482447342026, + "loss": 3.1429, + "theoretical_loss": 4.084146440679769, + "tokens_seen": 341770240 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045273821464393175, + "loss": 2.9452, + "theoretical_loss": 4.0840565055139155, + "tokens_seen": 341835776 + }, + { + "epoch": 0.1, + "learning_rate": 0.000452728184553661, + "loss": 3.001, + "theoretical_loss": 4.083966592415333, + "tokens_seen": 341901312 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045271815446339017, + "loss": 2.9977, + "theoretical_loss": 4.083876701374381, + "tokens_seen": 341966848 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045270812437311935, + "loss": 3.1103, + "theoretical_loss": 4.08378683238142, + "tokens_seen": 342032384 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004526980942828486, + "loss": 2.926, + "theoretical_loss": 4.08369698542682, + "tokens_seen": 342097920 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045268806419257777, + "loss": 2.8634, + "theoretical_loss": 4.083607160500957, + "tokens_seen": 342163456 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045267803410230695, + "loss": 3.2284, + "theoretical_loss": 4.083517357594212, + "tokens_seen": 342228992 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045266800401203613, + "loss": 3.2356, + "theoretical_loss": 4.0834275766969705, + "tokens_seen": 342294528 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004526579739217653, + "loss": 3.1546, + "theoretical_loss": 4.083337817799629, + "tokens_seen": 342360064 + }, + { + "epoch": 0.1, + "objective/train/docs_used": 206109, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.1210689544677734, + "objective/train/theoretical_loss": 4.083248080892584, + "objective/train/tokens_used": 362885600, + "theoretical_loss": 4.083248080892584, + "tokens_seen": 342425600 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004526479438314945, + "loss": 3.2345, + "theoretical_loss": 4.083248080892584, + "tokens_seen": 342425600 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004526379137412237, + "loss": 3.1819, + "theoretical_loss": 4.083158365966243, + "tokens_seen": 342491136 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045262788365095285, + "loss": 2.7943, + "theoretical_loss": 4.083068673011017, + "tokens_seen": 342556672 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004526178535606821, + "loss": 2.9256, + "theoretical_loss": 4.082979002017321, + "tokens_seen": 342622208 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004526078234704112, + "loss": 2.8331, + "theoretical_loss": 4.082889352975583, + "tokens_seen": 342687744 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045259779338014045, + "loss": 3.0588, + "theoretical_loss": 4.082799725876228, + "tokens_seen": 342753280 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045258776328986963, + "loss": 3.0001, + "theoretical_loss": 4.082710120709694, + "tokens_seen": 342818816 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004525777331995988, + "loss": 2.7877, + "theoretical_loss": 4.082620537466422, + "tokens_seen": 342884352 + }, + { + "epoch": 0.1, + "learning_rate": 0.000452567703109328, + "loss": 3.0808, + "theoretical_loss": 4.08253097613686, + "tokens_seen": 342949888 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045255767301905723, + "loss": 3.1626, + "theoretical_loss": 4.082441436711459, + "tokens_seen": 343015424 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045254764292878636, + "loss": 3.0913, + "theoretical_loss": 4.082351919180682, + "tokens_seen": 343080960 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004525376128385156, + "loss": 2.9768, + "theoretical_loss": 4.082262423534992, + "tokens_seen": 343146496 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004525275827482447, + "loss": 3.1877, + "theoretical_loss": 4.0821729497648604, + "tokens_seen": 343212032 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045251755265797395, + "loss": 3.0024, + "theoretical_loss": 4.082083497860766, + "tokens_seen": 343277568 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045250752256770314, + "loss": 2.7657, + "theoretical_loss": 4.0819940678131905, + "tokens_seen": 343343104 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004524974924774323, + "loss": 2.9385, + "theoretical_loss": 4.081904659612624, + "tokens_seen": 343408640 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004524874623871615, + "loss": 2.8899, + "theoretical_loss": 4.081815273249561, + "tokens_seen": 343474176 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004524774322968907, + "loss": 3.1788, + "theoretical_loss": 4.081725908714505, + "tokens_seen": 343539712 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045246740220661986, + "loss": 3.1825, + "theoretical_loss": 4.081636565997959, + "tokens_seen": 343605248 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004524573721163491, + "loss": 2.9514, + "theoretical_loss": 4.08154724509044, + "tokens_seen": 343670784 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004524473420260782, + "loss": 2.9563, + "theoretical_loss": 4.081457945982465, + "tokens_seen": 343736320 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045243731193580746, + "loss": 3.3829, + "theoretical_loss": 4.08136866866456, + "tokens_seen": 343801856 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004524272818455366, + "loss": 3.008, + "theoretical_loss": 4.081279413127254, + "tokens_seen": 343867392 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004524172517552658, + "loss": 3.2083, + "theoretical_loss": 4.0811901793610845, + "tokens_seen": 343932928 + }, + { + "epoch": 0.1, + "learning_rate": 0.000452407221664995, + "loss": 3.0354, + "theoretical_loss": 4.081100967356594, + "tokens_seen": 343998464 + }, + { + "epoch": 0.1, + "objective/train/docs_used": 206760, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.942789316177368, + "objective/train/theoretical_loss": 4.081011777104333, + "objective/train/tokens_used": 364524000, + "theoretical_loss": 4.081011777104333, + "tokens_seen": 344064000 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004523971915747242, + "loss": 2.8829, + "theoretical_loss": 4.081011777104333, + "tokens_seen": 344064000 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045238716148445336, + "loss": 3.1074, + "theoretical_loss": 4.080922608594854, + "tokens_seen": 344129536 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004523771313941826, + "loss": 3.2423, + "theoretical_loss": 4.080833461818718, + "tokens_seen": 344195072 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004523671013039117, + "loss": 2.9642, + "theoretical_loss": 4.08074433676649, + "tokens_seen": 344260608 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045235707121364096, + "loss": 3.012, + "theoretical_loss": 4.080655233428745, + "tokens_seen": 344326144 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004523470411233701, + "loss": 3.0892, + "theoretical_loss": 4.0805661517960585, + "tokens_seen": 344391680 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004523370110330993, + "loss": 2.9741, + "theoretical_loss": 4.080477091859015, + "tokens_seen": 344457216 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004523269809428285, + "loss": 3.0063, + "theoretical_loss": 4.080388053608205, + "tokens_seen": 344522752 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004523169508525577, + "loss": 2.877, + "theoretical_loss": 4.080299037034224, + "tokens_seen": 344588288 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045230692076228687, + "loss": 2.8735, + "theoretical_loss": 4.080210042127672, + "tokens_seen": 344653824 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045229689067201605, + "loss": 3.1439, + "theoretical_loss": 4.0801210688791585, + "tokens_seen": 344719360 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045228686058174523, + "loss": 3.1145, + "theoretical_loss": 4.080032117279296, + "tokens_seen": 344784896 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045227683049147446, + "loss": 2.9046, + "theoretical_loss": 4.079943187318703, + "tokens_seen": 344850432 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004522668004012036, + "loss": 2.8064, + "theoretical_loss": 4.079854278988005, + "tokens_seen": 344915968 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004522567703109328, + "loss": 3.1509, + "theoretical_loss": 4.0797653922778325, + "tokens_seen": 344981504 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045224674022066195, + "loss": 3.0352, + "theoretical_loss": 4.0796765271788225, + "tokens_seen": 345047040 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004522367101303912, + "loss": 2.8234, + "theoretical_loss": 4.079587683681617, + "tokens_seen": 345112576 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045222668004012037, + "loss": 2.8664, + "theoretical_loss": 4.079498861776865, + "tokens_seen": 345178112 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045221664994984955, + "loss": 3.0486, + "theoretical_loss": 4.07941006145522, + "tokens_seen": 345243648 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045220661985957873, + "loss": 3.0185, + "theoretical_loss": 4.079321282707342, + "tokens_seen": 345309184 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045219658976930797, + "loss": 2.9107, + "theoretical_loss": 4.0792325255238975, + "tokens_seen": 345374720 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004521865596790371, + "loss": 2.9138, + "theoretical_loss": 4.079143789895556, + "tokens_seen": 345440256 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045217652958876633, + "loss": 2.8814, + "theoretical_loss": 4.079055075812997, + "tokens_seen": 345505792 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045216649949849546, + "loss": 2.9382, + "theoretical_loss": 4.078966383266904, + "tokens_seen": 345571328 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004521564694082247, + "loss": 2.6713, + "theoretical_loss": 4.078877712247963, + "tokens_seen": 345636864 + }, + { + "epoch": 0.1, + "objective/train/docs_used": 207336, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.917886972427368, + "objective/train/theoretical_loss": 4.078789062746872, + "objective/train/tokens_used": 366162400, + "theoretical_loss": 4.078789062746872, + "tokens_seen": 345702400 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045214643931795387, + "loss": 3.0, + "theoretical_loss": 4.078789062746872, + "tokens_seen": 345702400 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045213640922768305, + "loss": 3.3253, + "theoretical_loss": 4.07870043475433, + "tokens_seen": 345767936 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045212637913741223, + "loss": 3.0843, + "theoretical_loss": 4.078611828261042, + "tokens_seen": 345833472 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004521163490471414, + "loss": 2.8403, + "theoretical_loss": 4.078523243257723, + "tokens_seen": 345899008 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004521063189568706, + "loss": 3.1081, + "theoretical_loss": 4.078434679735088, + "tokens_seen": 345964544 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045209628886659983, + "loss": 2.8655, + "theoretical_loss": 4.078346137683862, + "tokens_seen": 346030080 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045208625877632896, + "loss": 2.9257, + "theoretical_loss": 4.078257617094776, + "tokens_seen": 346095616 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004520762286860582, + "loss": 2.6188, + "theoretical_loss": 4.078169117958562, + "tokens_seen": 346161152 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004520661985957873, + "loss": 2.6858, + "theoretical_loss": 4.078080640265963, + "tokens_seen": 346226688 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045205616850551656, + "loss": 2.989, + "theoretical_loss": 4.077992184007725, + "tokens_seen": 346292224 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045204613841524574, + "loss": 2.8928, + "theoretical_loss": 4.0779037491746, + "tokens_seen": 346357760 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004520361083249749, + "loss": 3.1371, + "theoretical_loss": 4.077815335757347, + "tokens_seen": 346423296 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004520260782347041, + "loss": 3.3293, + "theoretical_loss": 4.077726943746729, + "tokens_seen": 346488832 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045201604814443334, + "loss": 2.6778, + "theoretical_loss": 4.077638573133515, + "tokens_seen": 346554368 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045200601805416246, + "loss": 2.8839, + "theoretical_loss": 4.077550223908482, + "tokens_seen": 346619904 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004519959879638917, + "loss": 3.0546, + "theoretical_loss": 4.0774618960624105, + "tokens_seen": 346685440 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004519859578736208, + "loss": 2.9795, + "theoretical_loss": 4.077373589586087, + "tokens_seen": 346750976 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045197592778335006, + "loss": 2.9755, + "theoretical_loss": 4.077285304470304, + "tokens_seen": 346816512 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045196589769307924, + "loss": 3.0244, + "theoretical_loss": 4.077197040705859, + "tokens_seen": 346882048 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004519558676028084, + "loss": 3.0215, + "theoretical_loss": 4.077108798283557, + "tokens_seen": 346947584 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045194583751253766, + "loss": 2.9532, + "theoretical_loss": 4.077020577194207, + "tokens_seen": 347013120 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004519358074222668, + "loss": 3.255, + "theoretical_loss": 4.076932377428625, + "tokens_seen": 347078656 + }, + { + "epoch": 0.11, + "learning_rate": 0.000451925777331996, + "loss": 3.0683, + "theoretical_loss": 4.076844198977629, + "tokens_seen": 347144192 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004519157472417252, + "loss": 2.9093, + "theoretical_loss": 4.076756041832049, + "tokens_seen": 347209728 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004519057171514544, + "loss": 2.9952, + "theoretical_loss": 4.076667905982717, + "tokens_seen": 347275264 + }, + { + "epoch": 0.11, + "objective/train/docs_used": 208796, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.4525599479675293, + "objective/train/theoretical_loss": 4.076579791420469, + "objective/train/tokens_used": 367800800, + "theoretical_loss": 4.076579791420469, + "tokens_seen": 347340800 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045189568706118356, + "loss": 3.0439, + "theoretical_loss": 4.076579791420469, + "tokens_seen": 347340800 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004518856569709128, + "loss": 3.053, + "theoretical_loss": 4.0764916981361505, + "tokens_seen": 347406336 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004518756268806419, + "loss": 2.7657, + "theoretical_loss": 4.07640362612061, + "tokens_seen": 347471872 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045186559679037116, + "loss": 2.9555, + "theoretical_loss": 4.076315575364702, + "tokens_seen": 347537408 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004518555667001003, + "loss": 2.7876, + "theoretical_loss": 4.076227545859288, + "tokens_seen": 347602944 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004518455366098295, + "loss": 3.0534, + "theoretical_loss": 4.076139537595234, + "tokens_seen": 347668480 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004518355065195587, + "loss": 3.1265, + "theoretical_loss": 4.076051550563411, + "tokens_seen": 347734016 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004518254764292879, + "loss": 2.9558, + "theoretical_loss": 4.075963584754698, + "tokens_seen": 347799552 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045181544633901707, + "loss": 2.9576, + "theoretical_loss": 4.075875640159978, + "tokens_seen": 347865088 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045180541624874625, + "loss": 2.8257, + "theoretical_loss": 4.075787716770139, + "tokens_seen": 347930624 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045179538615847543, + "loss": 2.5727, + "theoretical_loss": 4.075699814576076, + "tokens_seen": 347996160 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045178535606820466, + "loss": 2.993, + "theoretical_loss": 4.075611933568689, + "tokens_seen": 348061696 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004517753259779338, + "loss": 2.8647, + "theoretical_loss": 4.075524073738883, + "tokens_seen": 348127232 + }, + { + "epoch": 0.11, + "learning_rate": 0.000451765295887663, + "loss": 2.9692, + "theoretical_loss": 4.0754362350775715, + "tokens_seen": 348192768 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045175526579739215, + "loss": 2.9487, + "theoretical_loss": 4.075348417575669, + "tokens_seen": 348258304 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004517452357071214, + "loss": 3.1893, + "theoretical_loss": 4.075260621224099, + "tokens_seen": 348323840 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045173520561685057, + "loss": 2.8631, + "theoretical_loss": 4.075172846013789, + "tokens_seen": 348389376 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045172517552657975, + "loss": 2.7835, + "theoretical_loss": 4.075085091935675, + "tokens_seen": 348454912 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045171514543630893, + "loss": 3.0199, + "theoretical_loss": 4.074997358980694, + "tokens_seen": 348520448 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045170511534603817, + "loss": 2.9201, + "theoretical_loss": 4.0749096471397905, + "tokens_seen": 348585984 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004516950852557673, + "loss": 2.5913, + "theoretical_loss": 4.074821956403917, + "tokens_seen": 348651520 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045168505516549653, + "loss": 3.0839, + "theoretical_loss": 4.074734286764029, + "tokens_seen": 348717056 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045167502507522566, + "loss": 2.8769, + "theoretical_loss": 4.074646638211087, + "tokens_seen": 348782592 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004516649949849549, + "loss": 2.9982, + "theoretical_loss": 4.074559010736059, + "tokens_seen": 348848128 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045165496489468407, + "loss": 3.1578, + "theoretical_loss": 4.074471404329918, + "tokens_seen": 348913664 + }, + { + "epoch": 0.11, + "objective/train/docs_used": 209597, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.9342572689056396, + "objective/train/theoretical_loss": 4.074383818983643, + "objective/train/tokens_used": 369439200, + "theoretical_loss": 4.074383818983643, + "tokens_seen": 348979200 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045164493480441325, + "loss": 2.8375, + "theoretical_loss": 4.074383818983643, + "tokens_seen": 348979200 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045163490471414244, + "loss": 3.1525, + "theoretical_loss": 4.074296254688216, + "tokens_seen": 349044736 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004516248746238716, + "loss": 2.993, + "theoretical_loss": 4.074208711434628, + "tokens_seen": 349110272 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004516148445336008, + "loss": 3.0674, + "theoretical_loss": 4.074121189213873, + "tokens_seen": 349175808 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045160481444333003, + "loss": 2.9211, + "theoretical_loss": 4.074033688016953, + "tokens_seen": 349241344 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045159478435305916, + "loss": 2.5875, + "theoretical_loss": 4.073946207834872, + "tokens_seen": 349306880 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004515847542627884, + "loss": 2.938, + "theoretical_loss": 4.073858748658644, + "tokens_seen": 349372416 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004515747241725175, + "loss": 2.9499, + "theoretical_loss": 4.073771310479284, + "tokens_seen": 349437952 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045156469408224676, + "loss": 2.7625, + "theoretical_loss": 4.073683893287816, + "tokens_seen": 349503488 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045155466399197594, + "loss": 2.8778, + "theoretical_loss": 4.073596497075267, + "tokens_seen": 349569024 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004515446339017051, + "loss": 2.9709, + "theoretical_loss": 4.073509121832672, + "tokens_seen": 349634560 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004515346038114343, + "loss": 2.8745, + "theoretical_loss": 4.07342176755107, + "tokens_seen": 349700096 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045152457372116354, + "loss": 2.9335, + "theoretical_loss": 4.073334434221506, + "tokens_seen": 349765632 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045151454363089266, + "loss": 3.1618, + "theoretical_loss": 4.073247121835028, + "tokens_seen": 349831168 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004515045135406219, + "loss": 3.157, + "theoretical_loss": 4.073159830382695, + "tokens_seen": 349896704 + }, + { + "epoch": 0.11, + "learning_rate": 0.000451494483450351, + "loss": 3.0091, + "theoretical_loss": 4.073072559855565, + "tokens_seen": 349962240 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045148445336008026, + "loss": 2.7699, + "theoretical_loss": 4.072985310244707, + "tokens_seen": 350027776 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045147442326980944, + "loss": 2.952, + "theoretical_loss": 4.072898081541193, + "tokens_seen": 350093312 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004514643931795386, + "loss": 2.8273, + "theoretical_loss": 4.0728108737361, + "tokens_seen": 350158848 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004514543630892678, + "loss": 2.8661, + "theoretical_loss": 4.0727236868205114, + "tokens_seen": 350224384 + }, + { + "epoch": 0.11, + "learning_rate": 0.000451444332998997, + "loss": 2.9318, + "theoretical_loss": 4.072636520785515, + "tokens_seen": 350289920 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045143430290872617, + "loss": 2.9923, + "theoretical_loss": 4.072549375622207, + "tokens_seen": 350355456 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004514242728184554, + "loss": 3.0223, + "theoretical_loss": 4.072462251321684, + "tokens_seen": 350420992 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045141424272818453, + "loss": 2.9196, + "theoretical_loss": 4.072375147875054, + "tokens_seen": 350486528 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045140421263791376, + "loss": 2.7976, + "theoretical_loss": 4.072288065273426, + "tokens_seen": 350552064 + }, + { + "epoch": 0.11, + "objective/train/docs_used": 210243, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.0284438133239746, + "objective/train/theoretical_loss": 4.0722010035079155, + "objective/train/tokens_used": 371077600, + "theoretical_loss": 4.0722010035079155, + "tokens_seen": 350617600 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045139418254764294, + "loss": 3.1769, + "theoretical_loss": 4.0722010035079155, + "tokens_seen": 350617600 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004513841524573721, + "loss": 2.8403, + "theoretical_loss": 4.072113962569645, + "tokens_seen": 350683136 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004513741223671013, + "loss": 3.0464, + "theoretical_loss": 4.07202694244974, + "tokens_seen": 350748672 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004513640922768305, + "loss": 2.6278, + "theoretical_loss": 4.071939943139334, + "tokens_seen": 350814208 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045135406218655967, + "loss": 2.9517, + "theoretical_loss": 4.0718529646295645, + "tokens_seen": 350879744 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004513440320962889, + "loss": 2.923, + "theoretical_loss": 4.071766006911574, + "tokens_seen": 350945280 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045133400200601803, + "loss": 3.0396, + "theoretical_loss": 4.071679069976511, + "tokens_seen": 351010816 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045132397191574727, + "loss": 2.9749, + "theoretical_loss": 4.07159215381553, + "tokens_seen": 351076352 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004513139418254764, + "loss": 2.9877, + "theoretical_loss": 4.07150525841979, + "tokens_seen": 351141888 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045130391173520563, + "loss": 2.9862, + "theoretical_loss": 4.071418383780456, + "tokens_seen": 351207424 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004512938816449348, + "loss": 2.9886, + "theoretical_loss": 4.071331529888698, + "tokens_seen": 351272960 + }, + { + "epoch": 0.11, + "learning_rate": 0.000451283851554664, + "loss": 2.7883, + "theoretical_loss": 4.071244696735692, + "tokens_seen": 351338496 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045127382146439317, + "loss": 2.8326, + "theoretical_loss": 4.071157884312619, + "tokens_seen": 351404032 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045126379137412235, + "loss": 2.8969, + "theoretical_loss": 4.071071092610664, + "tokens_seen": 351469568 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045125376128385153, + "loss": 2.9976, + "theoretical_loss": 4.070984321621021, + "tokens_seen": 351535104 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045124373119358077, + "loss": 2.9044, + "theoretical_loss": 4.070897571334886, + "tokens_seen": 351600640 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004512337011033099, + "loss": 3.0543, + "theoretical_loss": 4.0708108417434605, + "tokens_seen": 351666176 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045122367101303913, + "loss": 2.9974, + "theoretical_loss": 4.070724132837954, + "tokens_seen": 351731712 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004512136409227683, + "loss": 2.7005, + "theoretical_loss": 4.07063744460958, + "tokens_seen": 351797248 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004512036108324975, + "loss": 3.0464, + "theoretical_loss": 4.070550777049555, + "tokens_seen": 351862784 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045119358074222673, + "loss": 2.7318, + "theoretical_loss": 4.070464130149106, + "tokens_seen": 351928320 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045118355065195586, + "loss": 2.7703, + "theoretical_loss": 4.07037750389946, + "tokens_seen": 351993856 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004511735205616851, + "loss": 2.9584, + "theoretical_loss": 4.070290898291852, + "tokens_seen": 352059392 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004511634904714143, + "loss": 2.9322, + "theoretical_loss": 4.070204313317523, + "tokens_seen": 352124928 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045115346038114345, + "loss": 2.9116, + "theoretical_loss": 4.07011774896772, + "tokens_seen": 352190464 + }, + { + "epoch": 0.11, + "objective/train/docs_used": 211560, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.8025479316711426, + "objective/train/theoretical_loss": 4.07003120523369, + "objective/train/tokens_used": 372716000, + "theoretical_loss": 4.07003120523369, + "tokens_seen": 352256000 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045114343029087264, + "loss": 2.7873, + "theoretical_loss": 4.07003120523369, + "tokens_seen": 352256000 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004511334002006018, + "loss": 2.8168, + "theoretical_loss": 4.069944682106691, + "tokens_seen": 352321536 + }, + { + "epoch": 0.11, + "learning_rate": 0.000451123370110331, + "loss": 3.1558, + "theoretical_loss": 4.0698581795779845, + "tokens_seen": 352387072 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045111334002006023, + "loss": 3.003, + "theoretical_loss": 4.069771697638838, + "tokens_seen": 352452608 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045110330992978936, + "loss": 3.0864, + "theoretical_loss": 4.069685236280522, + "tokens_seen": 352518144 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004510932798395186, + "loss": 2.8406, + "theoretical_loss": 4.069598795494315, + "tokens_seen": 352583680 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004510832497492477, + "loss": 2.908, + "theoretical_loss": 4.0695123752715, + "tokens_seen": 352649216 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045107321965897696, + "loss": 3.1474, + "theoretical_loss": 4.069425975603364, + "tokens_seen": 352714752 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045106318956870614, + "loss": 2.9216, + "theoretical_loss": 4.069339596481201, + "tokens_seen": 352780288 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004510531594784353, + "loss": 2.8176, + "theoretical_loss": 4.0692532378963095, + "tokens_seen": 352845824 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004510431293881645, + "loss": 3.0656, + "theoretical_loss": 4.069166899839994, + "tokens_seen": 352911360 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045103309929789374, + "loss": 3.0997, + "theoretical_loss": 4.069080582303563, + "tokens_seen": 352976896 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045102306920762286, + "loss": 3.0375, + "theoretical_loss": 4.068994285278332, + "tokens_seen": 353042432 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004510130391173521, + "loss": 2.919, + "theoretical_loss": 4.068908008755619, + "tokens_seen": 353107968 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004510030090270812, + "loss": 3.0014, + "theoretical_loss": 4.068821752726751, + "tokens_seen": 353173504 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045099297893681046, + "loss": 2.8295, + "theoretical_loss": 4.0687355171830575, + "tokens_seen": 353239040 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045098294884653964, + "loss": 2.8025, + "theoretical_loss": 4.068649302115875, + "tokens_seen": 353304576 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004509729187562688, + "loss": 2.7734, + "theoretical_loss": 4.068563107516544, + "tokens_seen": 353370112 + }, + { + "epoch": 0.11, + "learning_rate": 0.000450962888665998, + "loss": 2.8549, + "theoretical_loss": 4.068476933376411, + "tokens_seen": 353435648 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004509528585757272, + "loss": 2.7952, + "theoretical_loss": 4.068390779686826, + "tokens_seen": 353501184 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045094282848545637, + "loss": 3.0685, + "theoretical_loss": 4.068304646439148, + "tokens_seen": 353566720 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004509327983951856, + "loss": 2.982, + "theoretical_loss": 4.068218533624737, + "tokens_seen": 353632256 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045092276830491473, + "loss": 3.0206, + "theoretical_loss": 4.068132441234962, + "tokens_seen": 353697792 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045091273821464396, + "loss": 2.8828, + "theoretical_loss": 4.068046369261194, + "tokens_seen": 353763328 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045090270812437314, + "loss": 2.9229, + "theoretical_loss": 4.067960317694812, + "tokens_seen": 353828864 + }, + { + "epoch": 0.11, + "objective/train/docs_used": 212189, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.790320634841919, + "objective/train/theoretical_loss": 4.067874286527197, + "objective/train/tokens_used": 374354400, + "theoretical_loss": 4.067874286527197, + "tokens_seen": 353894400 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004508926780341023, + "loss": 2.9652, + "theoretical_loss": 4.067874286527197, + "tokens_seen": 353894400 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004508826479438315, + "loss": 2.7248, + "theoretical_loss": 4.06778827574974, + "tokens_seen": 353959936 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004508726178535607, + "loss": 2.7593, + "theoretical_loss": 4.067702285353833, + "tokens_seen": 354025472 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045086258776328987, + "loss": 2.9225, + "theoretical_loss": 4.0676163153308735, + "tokens_seen": 354091008 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004508525576730191, + "loss": 2.8178, + "theoretical_loss": 4.067530365672267, + "tokens_seen": 354156544 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045084252758274823, + "loss": 2.9279, + "theoretical_loss": 4.067444436369422, + "tokens_seen": 354222080 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045083249749247747, + "loss": 2.9322, + "theoretical_loss": 4.067358527413753, + "tokens_seen": 354287616 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004508224674022066, + "loss": 2.8184, + "theoretical_loss": 4.06727263879668, + "tokens_seen": 354353152 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045081243731193583, + "loss": 2.8114, + "theoretical_loss": 4.067186770509627, + "tokens_seen": 354418688 + }, + { + "epoch": 0.11, + "learning_rate": 0.000450802407221665, + "loss": 3.1263, + "theoretical_loss": 4.067100922544023, + "tokens_seen": 354484224 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004507923771313942, + "loss": 2.7697, + "theoretical_loss": 4.067015094891306, + "tokens_seen": 354549760 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045078234704112337, + "loss": 2.6973, + "theoretical_loss": 4.066929287542914, + "tokens_seen": 354615296 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045077231695085255, + "loss": 3.1394, + "theoretical_loss": 4.066843500490292, + "tokens_seen": 354680832 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045076228686058173, + "loss": 3.2258, + "theoretical_loss": 4.066757733724893, + "tokens_seen": 354746368 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045075225677031097, + "loss": 3.1463, + "theoretical_loss": 4.066671987238171, + "tokens_seen": 354811904 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004507422266800401, + "loss": 2.7487, + "theoretical_loss": 4.0665862610215875, + "tokens_seen": 354877440 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045073219658976933, + "loss": 2.6251, + "theoretical_loss": 4.066500555066609, + "tokens_seen": 354942976 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004507221664994985, + "loss": 2.9068, + "theoretical_loss": 4.066414869364707, + "tokens_seen": 355008512 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004507121364092277, + "loss": 3.0232, + "theoretical_loss": 4.066329203907358, + "tokens_seen": 355074048 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004507021063189569, + "loss": 3.1146, + "theoretical_loss": 4.066243558686043, + "tokens_seen": 355139584 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045069207622868606, + "loss": 2.8921, + "theoretical_loss": 4.066157933692249, + "tokens_seen": 355205120 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045068204613841524, + "loss": 2.8724, + "theoretical_loss": 4.066072328917469, + "tokens_seen": 355270656 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004506720160481445, + "loss": 2.9334, + "theoretical_loss": 4.0659867443532, + "tokens_seen": 355336192 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004506619859578736, + "loss": 2.7602, + "theoretical_loss": 4.065901179990943, + "tokens_seen": 355401728 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045065195586760284, + "loss": 2.9236, + "theoretical_loss": 4.065815635822206, + "tokens_seen": 355467264 + }, + { + "epoch": 0.11, + "objective/train/docs_used": 213521, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.03275203704834, + "objective/train/theoretical_loss": 4.065730111838502, + "objective/train/tokens_used": 375992800, + "theoretical_loss": 4.065730111838502, + "tokens_seen": 355532800 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045064192577733196, + "loss": 2.9477, + "theoretical_loss": 4.065730111838502, + "tokens_seen": 355532800 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004506318956870612, + "loss": 3.1333, + "theoretical_loss": 4.0656446080313495, + "tokens_seen": 355598336 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004506218655967904, + "loss": 2.9186, + "theoretical_loss": 4.065559124392269, + "tokens_seen": 355663872 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045061183550651956, + "loss": 2.8825, + "theoretical_loss": 4.065473660912791, + "tokens_seen": 355729408 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045060180541624874, + "loss": 2.9473, + "theoretical_loss": 4.065388217584447, + "tokens_seen": 355794944 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004505917753259779, + "loss": 2.4189, + "theoretical_loss": 4.065302794398775, + "tokens_seen": 355860480 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004505817452357071, + "loss": 3.0602, + "theoretical_loss": 4.0652173913473195, + "tokens_seen": 355926016 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045057171514543634, + "loss": 2.7762, + "theoretical_loss": 4.065132008421628, + "tokens_seen": 355991552 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045056168505516547, + "loss": 2.7655, + "theoretical_loss": 4.065046645613254, + "tokens_seen": 356057088 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004505516549648947, + "loss": 2.8888, + "theoretical_loss": 4.0649613029137575, + "tokens_seen": 356122624 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004505416248746239, + "loss": 3.0059, + "theoretical_loss": 4.0648759803147, + "tokens_seen": 356188160 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045053159478435306, + "loss": 2.9961, + "theoretical_loss": 4.064790677807652, + "tokens_seen": 356253696 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045052156469408224, + "loss": 2.9662, + "theoretical_loss": 4.064705395384187, + "tokens_seen": 356319232 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004505115346038114, + "loss": 2.9933, + "theoretical_loss": 4.064620133035883, + "tokens_seen": 356384768 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004505015045135406, + "loss": 3.1455, + "theoretical_loss": 4.0645348907543255, + "tokens_seen": 356450304 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045049147442326984, + "loss": 2.8921, + "theoretical_loss": 4.064449668531103, + "tokens_seen": 356515840 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045048144433299897, + "loss": 2.47, + "theoretical_loss": 4.064364466357809, + "tokens_seen": 356581376 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004504714142427282, + "loss": 2.7911, + "theoretical_loss": 4.064279284226044, + "tokens_seen": 356646912 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045046138415245733, + "loss": 2.9327, + "theoretical_loss": 4.064194122127411, + "tokens_seen": 356712448 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045045135406218657, + "loss": 2.9272, + "theoretical_loss": 4.06410898005352, + "tokens_seen": 356777984 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004504413239719158, + "loss": 2.6876, + "theoretical_loss": 4.064023857995986, + "tokens_seen": 356843520 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045043129388164493, + "loss": 2.8632, + "theoretical_loss": 4.0639387559464275, + "tokens_seen": 356909056 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045042126379137416, + "loss": 2.9527, + "theoretical_loss": 4.063853673896469, + "tokens_seen": 356974592 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045041123370110334, + "loss": 2.5967, + "theoretical_loss": 4.063768611837742, + "tokens_seen": 357040128 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004504012036108325, + "loss": 2.9134, + "theoretical_loss": 4.063683569761879, + "tokens_seen": 357105664 + }, + { + "epoch": 0.11, + "objective/train/docs_used": 214188, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.2600982189178467, + "objective/train/theoretical_loss": 4.063598547660519, + "objective/train/tokens_used": 377631200, + "theoretical_loss": 4.063598547660519, + "tokens_seen": 357171200 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004503911735205617, + "loss": 2.807, + "theoretical_loss": 4.063598547660519, + "tokens_seen": 357171200 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004503811434302909, + "loss": 2.7391, + "theoretical_loss": 4.06351354552531, + "tokens_seen": 357236736 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045037111334002007, + "loss": 3.0373, + "theoretical_loss": 4.063428563347898, + "tokens_seen": 357302272 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004503610832497493, + "loss": 2.9072, + "theoretical_loss": 4.063343601119941, + "tokens_seen": 357367808 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045035105315947843, + "loss": 3.1686, + "theoretical_loss": 4.063258658833096, + "tokens_seen": 357433344 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045034102306920767, + "loss": 2.9887, + "theoretical_loss": 4.06317373647903, + "tokens_seen": 357498880 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004503309929789368, + "loss": 2.9996, + "theoretical_loss": 4.0630888340494105, + "tokens_seen": 357564416 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045032096288866603, + "loss": 2.821, + "theoretical_loss": 4.063003951535913, + "tokens_seen": 357629952 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004503109327983952, + "loss": 2.699, + "theoretical_loss": 4.06291908893022, + "tokens_seen": 357695488 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004503009027081244, + "loss": 2.8687, + "theoretical_loss": 4.062834246224012, + "tokens_seen": 357761024 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045029087261785357, + "loss": 3.2984, + "theoretical_loss": 4.062749423408981, + "tokens_seen": 357826560 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045028084252758275, + "loss": 3.1831, + "theoretical_loss": 4.062664620476822, + "tokens_seen": 357892096 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045027081243731193, + "loss": 2.9771, + "theoretical_loss": 4.062579837419235, + "tokens_seen": 357957632 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045026078234704117, + "loss": 3.0185, + "theoretical_loss": 4.062495074227923, + "tokens_seen": 358023168 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004502507522567703, + "loss": 2.9462, + "theoretical_loss": 4.062410330894597, + "tokens_seen": 358088704 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045024072216649953, + "loss": 3.2081, + "theoretical_loss": 4.062325607410972, + "tokens_seen": 358154240 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004502306920762287, + "loss": 2.8936, + "theoretical_loss": 4.062240903768767, + "tokens_seen": 358219776 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004502206619859579, + "loss": 2.9138, + "theoretical_loss": 4.062156219959705, + "tokens_seen": 358285312 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004502106318956871, + "loss": 3.1519, + "theoretical_loss": 4.062071555975519, + "tokens_seen": 358350848 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045020060180541626, + "loss": 3.0064, + "theoretical_loss": 4.061986911807942, + "tokens_seen": 358416384 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045019057171514544, + "loss": 2.9092, + "theoretical_loss": 4.061902287448714, + "tokens_seen": 358481920 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004501805416248747, + "loss": 2.8254, + "theoretical_loss": 4.061817682889577, + "tokens_seen": 358547456 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004501705115346038, + "loss": 3.0591, + "theoretical_loss": 4.061733098122284, + "tokens_seen": 358612992 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045016048144433304, + "loss": 2.9911, + "theoretical_loss": 4.061648533138587, + "tokens_seen": 358678528 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045015045135406216, + "loss": 2.8058, + "theoretical_loss": 4.0615639879302465, + "tokens_seen": 358744064 + }, + { + "epoch": 0.11, + "objective/train/docs_used": 215284, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.2827391624450684, + "objective/train/theoretical_loss": 4.061479462489026, + "objective/train/tokens_used": 379269600, + "theoretical_loss": 4.061479462489026, + "tokens_seen": 358809600 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004501404212637914, + "loss": 3.1118, + "theoretical_loss": 4.061479462489026, + "tokens_seen": 358809600 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004501303911735206, + "loss": 3.0285, + "theoretical_loss": 4.061394956806694, + "tokens_seen": 358875136 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045012036108324976, + "loss": 2.908, + "theoretical_loss": 4.061310470875027, + "tokens_seen": 358940672 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045011033099297894, + "loss": 2.7175, + "theoretical_loss": 4.0612260046858015, + "tokens_seen": 359006208 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004501003009027081, + "loss": 2.8343, + "theoretical_loss": 4.061141558230803, + "tokens_seen": 359071744 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004500902708124373, + "loss": 3.1498, + "theoretical_loss": 4.06105713150182, + "tokens_seen": 359137280 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045008024072216654, + "loss": 3.0164, + "theoretical_loss": 4.060972724490646, + "tokens_seen": 359202816 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045007021063189567, + "loss": 3.1507, + "theoretical_loss": 4.0608883371890805, + "tokens_seen": 359268352 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004500601805416249, + "loss": 3.0005, + "theoretical_loss": 4.060803969588925, + "tokens_seen": 359333888 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004500501504513541, + "loss": 2.9544, + "theoretical_loss": 4.06071962168199, + "tokens_seen": 359399424 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045004012036108326, + "loss": 2.9804, + "theoretical_loss": 4.060635293460089, + "tokens_seen": 359464960 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045003009027081244, + "loss": 2.7423, + "theoretical_loss": 4.060550984915039, + "tokens_seen": 359530496 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004500200601805416, + "loss": 2.9822, + "theoretical_loss": 4.060466696038664, + "tokens_seen": 359596032 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004500100300902708, + "loss": 3.087, + "theoretical_loss": 4.060382426822792, + "tokens_seen": 359661568 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045000000000000004, + "loss": 2.9881, + "theoretical_loss": 4.060298177259256, + "tokens_seen": 359727104 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044998996990972917, + "loss": 3.087, + "theoretical_loss": 4.060213947339894, + "tokens_seen": 359792640 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004499799398194584, + "loss": 2.8326, + "theoretical_loss": 4.060129737056547, + "tokens_seen": 359858176 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044996990972918753, + "loss": 3.111, + "theoretical_loss": 4.060045546401065, + "tokens_seen": 359923712 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044995987963891677, + "loss": 2.9913, + "theoretical_loss": 4.0599613753653, + "tokens_seen": 359989248 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044994984954864595, + "loss": 2.8806, + "theoretical_loss": 4.0598772239411085, + "tokens_seen": 360054784 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044993981945837513, + "loss": 3.0183, + "theoretical_loss": 4.059793092120353, + "tokens_seen": 360120320 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004499297893681043, + "loss": 2.9745, + "theoretical_loss": 4.0597089798949, + "tokens_seen": 360185856 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044991975927783355, + "loss": 2.7412, + "theoretical_loss": 4.059624887256622, + "tokens_seen": 360251392 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044990972918756267, + "loss": 2.9278, + "theoretical_loss": 4.059540814197396, + "tokens_seen": 360316928 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004498996990972919, + "loss": 3.0531, + "theoretical_loss": 4.059456760709104, + "tokens_seen": 360382464 + }, + { + "epoch": 0.11, + "objective/train/docs_used": 215704, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.128261089324951, + "objective/train/theoretical_loss": 4.05937272678363, + "objective/train/tokens_used": 380908000, + "theoretical_loss": 4.05937272678363, + "tokens_seen": 360448000 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044988966900702103, + "loss": 2.8879, + "theoretical_loss": 4.05937272678363, + "tokens_seen": 360448000 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044987963891675027, + "loss": 2.966, + "theoretical_loss": 4.059288712412868, + "tokens_seen": 360513536 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044986960882647945, + "loss": 2.8885, + "theoretical_loss": 4.059204717588712, + "tokens_seen": 360579072 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044985957873620863, + "loss": 3.1517, + "theoretical_loss": 4.059120742303065, + "tokens_seen": 360644608 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004498495486459378, + "loss": 2.8481, + "theoretical_loss": 4.059036786547831, + "tokens_seen": 360710144 + }, + { + "epoch": 0.11, + "learning_rate": 0.000449839518555667, + "loss": 2.8905, + "theoretical_loss": 4.058952850314922, + "tokens_seen": 360775680 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004498294884653962, + "loss": 3.1202, + "theoretical_loss": 4.058868933596252, + "tokens_seen": 360841216 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004498194583751254, + "loss": 2.9578, + "theoretical_loss": 4.0587850363837426, + "tokens_seen": 360906752 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044980942828485454, + "loss": 2.9792, + "theoretical_loss": 4.058701158669318, + "tokens_seen": 360972288 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044979939819458377, + "loss": 3.1683, + "theoretical_loss": 4.058617300444908, + "tokens_seen": 361037824 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004497893681043129, + "loss": 2.7097, + "theoretical_loss": 4.0585334617024476, + "tokens_seen": 361103360 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044977933801404213, + "loss": 2.9485, + "theoretical_loss": 4.058449642433875, + "tokens_seen": 361168896 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004497693079237713, + "loss": 2.8668, + "theoretical_loss": 4.058365842631137, + "tokens_seen": 361234432 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004497592778335005, + "loss": 2.9666, + "theoretical_loss": 4.05828206228618, + "tokens_seen": 361299968 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004497492477432297, + "loss": 3.0179, + "theoretical_loss": 4.058198301390959, + "tokens_seen": 361365504 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004497392176529589, + "loss": 2.8444, + "theoretical_loss": 4.058114559937433, + "tokens_seen": 361431040 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044972918756268804, + "loss": 2.5265, + "theoretical_loss": 4.058030837917565, + "tokens_seen": 361496576 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004497191574724173, + "loss": 2.786, + "theoretical_loss": 4.057947135323323, + "tokens_seen": 361562112 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004497091273821464, + "loss": 2.9223, + "theoretical_loss": 4.057863452146679, + "tokens_seen": 361627648 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044969909729187564, + "loss": 2.8798, + "theoretical_loss": 4.057779788379613, + "tokens_seen": 361693184 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004496890672016049, + "loss": 3.0724, + "theoretical_loss": 4.057696144014106, + "tokens_seen": 361758720 + }, + { + "epoch": 0.11, + "learning_rate": 0.000449679037111334, + "loss": 3.0366, + "theoretical_loss": 4.057612519042145, + "tokens_seen": 361824256 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044966900702106324, + "loss": 3.082, + "theoretical_loss": 4.057528913455723, + "tokens_seen": 361889792 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044965897693079236, + "loss": 2.8251, + "theoretical_loss": 4.057445327246836, + "tokens_seen": 361955328 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004496489468405216, + "loss": 2.9818, + "theoretical_loss": 4.0573617604074865, + "tokens_seen": 362020864 + }, + { + "epoch": 0.11, + "objective/train/docs_used": 216313, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.9674389362335205, + "objective/train/theoretical_loss": 4.0572782129296785, + "objective/train/tokens_used": 382546400, + "theoretical_loss": 4.0572782129296785, + "tokens_seen": 362086400 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004496389167502508, + "loss": 3.1386, + "theoretical_loss": 4.0572782129296785, + "tokens_seen": 362086400 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044962888665997996, + "loss": 2.9707, + "theoretical_loss": 4.057194684805426, + "tokens_seen": 362151936 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044961885656970914, + "loss": 3.2148, + "theoretical_loss": 4.057111176026742, + "tokens_seen": 362217472 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004496088264794383, + "loss": 2.8784, + "theoretical_loss": 4.0570276865856485, + "tokens_seen": 362283008 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004495987963891675, + "loss": 2.8436, + "theoretical_loss": 4.056944216474171, + "tokens_seen": 362348544 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044958876629889674, + "loss": 2.9186, + "theoretical_loss": 4.056860765684338, + "tokens_seen": 362414080 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044957873620862587, + "loss": 3.0897, + "theoretical_loss": 4.056777334208186, + "tokens_seen": 362479616 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004495687061183551, + "loss": 3.0256, + "theoretical_loss": 4.056693922037753, + "tokens_seen": 362545152 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004495586760280843, + "loss": 2.9281, + "theoretical_loss": 4.056610529165083, + "tokens_seen": 362610688 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044954864593781346, + "loss": 2.8178, + "theoretical_loss": 4.056527155582226, + "tokens_seen": 362676224 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044953861584754264, + "loss": 3.097, + "theoretical_loss": 4.056443801281235, + "tokens_seen": 362741760 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004495285857572718, + "loss": 3.0271, + "theoretical_loss": 4.056360466254168, + "tokens_seen": 362807296 + }, + { + "epoch": 0.11, + "learning_rate": 0.000449518555667001, + "loss": 3.0994, + "theoretical_loss": 4.056277150493088, + "tokens_seen": 362872832 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044950852557673024, + "loss": 3.075, + "theoretical_loss": 4.056193853990063, + "tokens_seen": 362938368 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044949849548645937, + "loss": 2.8864, + "theoretical_loss": 4.056110576737166, + "tokens_seen": 363003904 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004494884653961886, + "loss": 2.9704, + "theoretical_loss": 4.056027318726472, + "tokens_seen": 363069440 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044947843530591773, + "loss": 3.164, + "theoretical_loss": 4.055944079950065, + "tokens_seen": 363134976 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044946840521564697, + "loss": 2.7436, + "theoretical_loss": 4.0558608604000295, + "tokens_seen": 363200512 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044945837512537615, + "loss": 3.0883, + "theoretical_loss": 4.055777660068458, + "tokens_seen": 363266048 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044944834503510533, + "loss": 3.1314, + "theoretical_loss": 4.055694478947445, + "tokens_seen": 363331584 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004494383149448345, + "loss": 2.8359, + "theoretical_loss": 4.055611317029093, + "tokens_seen": 363397120 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044942828485456375, + "loss": 2.9758, + "theoretical_loss": 4.055528174305505, + "tokens_seen": 363462656 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044941825476429287, + "loss": 3.1232, + "theoretical_loss": 4.055445050768792, + "tokens_seen": 363528192 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004494082246740221, + "loss": 3.1773, + "theoretical_loss": 4.055361946411068, + "tokens_seen": 363593728 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044939819458375123, + "loss": 3.1087, + "theoretical_loss": 4.055278861224452, + "tokens_seen": 363659264 + }, + { + "epoch": 0.11, + "objective/train/docs_used": 217479, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.9815425872802734, + "objective/train/theoretical_loss": 4.055195795201069, + "objective/train/tokens_used": 384184800, + "theoretical_loss": 4.055195795201069, + "tokens_seen": 363724800 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044938816449348047, + "loss": 3.1028, + "theoretical_loss": 4.055195795201069, + "tokens_seen": 363724800 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044937813440320965, + "loss": 3.0281, + "theoretical_loss": 4.055112748333046, + "tokens_seen": 363790336 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044936810431293883, + "loss": 2.8046, + "theoretical_loss": 4.0550297206125165, + "tokens_seen": 363855872 + }, + { + "epoch": 0.11, + "learning_rate": 0.000449358074222668, + "loss": 3.2075, + "theoretical_loss": 4.054946712031619, + "tokens_seen": 363921408 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004493480441323972, + "loss": 2.7952, + "theoretical_loss": 4.054863722582494, + "tokens_seen": 363986944 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004493380140421264, + "loss": 2.9203, + "theoretical_loss": 4.054780752257291, + "tokens_seen": 364052480 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004493279839518556, + "loss": 3.0515, + "theoretical_loss": 4.05469780104816, + "tokens_seen": 364118016 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044931795386158474, + "loss": 3.0824, + "theoretical_loss": 4.054614868947258, + "tokens_seen": 364183552 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044930792377131397, + "loss": 2.9547, + "theoretical_loss": 4.054531955946745, + "tokens_seen": 364249088 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004492978936810431, + "loss": 3.0873, + "theoretical_loss": 4.054449062038787, + "tokens_seen": 364314624 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044928786359077234, + "loss": 2.9719, + "theoretical_loss": 4.054366187215556, + "tokens_seen": 364380160 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004492778335005015, + "loss": 3.0504, + "theoretical_loss": 4.0542833314692235, + "tokens_seen": 364445696 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004492678034102307, + "loss": 3.0754, + "theoretical_loss": 4.054200494791972, + "tokens_seen": 364511232 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004492577733199599, + "loss": 3.0383, + "theoretical_loss": 4.054117677175983, + "tokens_seen": 364576768 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004492477432296891, + "loss": 2.9937, + "theoretical_loss": 4.054034878613447, + "tokens_seen": 364642304 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044923771313941824, + "loss": 3.1713, + "theoretical_loss": 4.053952099096556, + "tokens_seen": 364707840 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004492276830491475, + "loss": 2.9236, + "theoretical_loss": 4.053869338617508, + "tokens_seen": 364773376 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004492176529588766, + "loss": 2.9382, + "theoretical_loss": 4.053786597168506, + "tokens_seen": 364838912 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044920762286860584, + "loss": 3.0184, + "theoretical_loss": 4.053703874741757, + "tokens_seen": 364904448 + }, + { + "epoch": 0.11, + "learning_rate": 0.000449197592778335, + "loss": 2.9074, + "theoretical_loss": 4.053621171329472, + "tokens_seen": 364969984 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004491875626880642, + "loss": 2.848, + "theoretical_loss": 4.053538486923868, + "tokens_seen": 365035520 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004491775325977934, + "loss": 2.8162, + "theoretical_loss": 4.053455821517165, + "tokens_seen": 365101056 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044916750250752256, + "loss": 2.7435, + "theoretical_loss": 4.053373175101589, + "tokens_seen": 365166592 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044915747241725174, + "loss": 3.0683, + "theoretical_loss": 4.053290547669369, + "tokens_seen": 365232128 + }, + { + "epoch": 0.11, + "learning_rate": 0.000449147442326981, + "loss": 3.0496, + "theoretical_loss": 4.05320793921274, + "tokens_seen": 365297664 + }, + { + "epoch": 0.11, + "objective/train/docs_used": 218231, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.985569715499878, + "objective/train/theoretical_loss": 4.053125349723942, + "objective/train/tokens_used": 385823200, + "theoretical_loss": 4.053125349723942, + "tokens_seen": 365363200 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004491374122367101, + "loss": 3.1315, + "theoretical_loss": 4.053125349723942, + "tokens_seen": 365363200 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044912738214643934, + "loss": 2.9301, + "theoretical_loss": 4.053042779195217, + "tokens_seen": 365428736 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044911735205616847, + "loss": 3.1535, + "theoretical_loss": 4.052960227618815, + "tokens_seen": 365494272 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004491073219658977, + "loss": 3.1944, + "theoretical_loss": 4.052877694986987, + "tokens_seen": 365559808 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004490972918756269, + "loss": 2.9058, + "theoretical_loss": 4.052795181291992, + "tokens_seen": 365625344 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044908726178535607, + "loss": 2.9373, + "theoretical_loss": 4.05271268652609, + "tokens_seen": 365690880 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044907723169508525, + "loss": 2.9022, + "theoretical_loss": 4.052630210681549, + "tokens_seen": 365756416 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004490672016048145, + "loss": 2.8118, + "theoretical_loss": 4.052547753750639, + "tokens_seen": 365821952 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004490571715145436, + "loss": 2.7279, + "theoretical_loss": 4.0524653157256365, + "tokens_seen": 365887488 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044904714142427284, + "loss": 3.2046, + "theoretical_loss": 4.05238289659882, + "tokens_seen": 365953024 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044903711133400197, + "loss": 2.7934, + "theoretical_loss": 4.052300496362475, + "tokens_seen": 366018560 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004490270812437312, + "loss": 2.9941, + "theoretical_loss": 4.052218115008892, + "tokens_seen": 366084096 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004490170511534604, + "loss": 3.0412, + "theoretical_loss": 4.052135752530362, + "tokens_seen": 366149632 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044900702106318957, + "loss": 3.1504, + "theoretical_loss": 4.0520534089191855, + "tokens_seen": 366215168 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044899699097291875, + "loss": 2.7261, + "theoretical_loss": 4.051971084167663, + "tokens_seen": 366280704 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044898696088264793, + "loss": 3.1316, + "theoretical_loss": 4.051888778268103, + "tokens_seen": 366346240 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004489769307923771, + "loss": 2.6576, + "theoretical_loss": 4.051806491212817, + "tokens_seen": 366411776 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044896690070210635, + "loss": 2.9336, + "theoretical_loss": 4.051724222994122, + "tokens_seen": 366477312 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004489568706118355, + "loss": 2.7274, + "theoretical_loss": 4.051641973604337, + "tokens_seen": 366542848 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004489468405215647, + "loss": 3.1474, + "theoretical_loss": 4.051559743035789, + "tokens_seen": 366608384 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044893681043129395, + "loss": 2.841, + "theoretical_loss": 4.051477531280806, + "tokens_seen": 366673920 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044892678034102307, + "loss": 3.0319, + "theoretical_loss": 4.051395338331723, + "tokens_seen": 366739456 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004489167502507523, + "loss": 2.9761, + "theoretical_loss": 4.05131316418088, + "tokens_seen": 366804992 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044890672016048143, + "loss": 3.1121, + "theoretical_loss": 4.051231008820618, + "tokens_seen": 366870528 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044889669007021067, + "loss": 3.01, + "theoretical_loss": 4.051148872243286, + "tokens_seen": 366936064 + }, + { + "epoch": 0.11, + "objective/train/docs_used": 219467, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.9876043796539307, + "objective/train/theoretical_loss": 4.051066754441235, + "objective/train/tokens_used": 387461600, + "theoretical_loss": 4.051066754441235, + "tokens_seen": 367001600 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044888665997993985, + "loss": 3.0609, + "theoretical_loss": 4.051066754441235, + "tokens_seen": 367001600 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044887662988966903, + "loss": 2.9901, + "theoretical_loss": 4.050984655406824, + "tokens_seen": 367067136 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004488665997993982, + "loss": 2.9271, + "theoretical_loss": 4.050902575132412, + "tokens_seen": 367132672 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004488565697091274, + "loss": 3.0142, + "theoretical_loss": 4.050820513610366, + "tokens_seen": 367198208 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004488465396188566, + "loss": 3.2082, + "theoretical_loss": 4.050738470833054, + "tokens_seen": 367263744 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004488365095285858, + "loss": 3.1262, + "theoretical_loss": 4.0506564467928525, + "tokens_seen": 367329280 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044882647943831494, + "loss": 3.0844, + "theoretical_loss": 4.05057444148214, + "tokens_seen": 367394816 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004488164493480442, + "loss": 2.9878, + "theoretical_loss": 4.0504924548933, + "tokens_seen": 367460352 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004488064192577733, + "loss": 3.0918, + "theoretical_loss": 4.05041048701872, + "tokens_seen": 367525888 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044879638916750254, + "loss": 3.0793, + "theoretical_loss": 4.050328537850792, + "tokens_seen": 367591424 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004487863590772317, + "loss": 2.9909, + "theoretical_loss": 4.050246607381913, + "tokens_seen": 367656960 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004487763289869609, + "loss": 2.8988, + "theoretical_loss": 4.0501646956044866, + "tokens_seen": 367722496 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004487662988966901, + "loss": 3.0084, + "theoretical_loss": 4.050082802510915, + "tokens_seen": 367788032 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004487562688064193, + "loss": 3.0207, + "theoretical_loss": 4.050000928093611, + "tokens_seen": 367853568 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044874623871614844, + "loss": 3.0967, + "theoretical_loss": 4.049919072344987, + "tokens_seen": 367919104 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004487362086258777, + "loss": 2.9055, + "theoretical_loss": 4.049837235257463, + "tokens_seen": 367984640 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004487261785356068, + "loss": 3.1636, + "theoretical_loss": 4.049755416823463, + "tokens_seen": 368050176 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044871614844533604, + "loss": 2.9096, + "theoretical_loss": 4.049673617035414, + "tokens_seen": 368115712 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004487061183550652, + "loss": 2.9354, + "theoretical_loss": 4.0495918358857494, + "tokens_seen": 368181248 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004486960882647944, + "loss": 3.0377, + "theoretical_loss": 4.049510073366903, + "tokens_seen": 368246784 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004486860581745236, + "loss": 3.1498, + "theoretical_loss": 4.04942832947132, + "tokens_seen": 368312320 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044867602808425276, + "loss": 2.7382, + "theoretical_loss": 4.049346604191443, + "tokens_seen": 368377856 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044866599799398194, + "loss": 2.9834, + "theoretical_loss": 4.049264897519723, + "tokens_seen": 368443392 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004486559679037112, + "loss": 2.9794, + "theoretical_loss": 4.049183209448615, + "tokens_seen": 368508928 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004486459378134403, + "loss": 2.8414, + "theoretical_loss": 4.049101539970576, + "tokens_seen": 368574464 + }, + { + "epoch": 0.11, + "objective/train/docs_used": 220098, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6181623935699463, + "objective/train/theoretical_loss": 4.04901988907807, + "objective/train/tokens_used": 389100000, + "theoretical_loss": 4.04901988907807, + "tokens_seen": 368640000 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044863590772316954, + "loss": 2.9643, + "theoretical_loss": 4.04901988907807, + "tokens_seen": 368640000 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044862587763289867, + "loss": 2.9164, + "theoretical_loss": 4.048938256763565, + "tokens_seen": 368705536 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004486158475426279, + "loss": 2.8644, + "theoretical_loss": 4.048856643019532, + "tokens_seen": 368771072 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004486058174523571, + "loss": 2.893, + "theoretical_loss": 4.048775047838447, + "tokens_seen": 368836608 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044859578736208627, + "loss": 2.8661, + "theoretical_loss": 4.048693471212792, + "tokens_seen": 368902144 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044858575727181545, + "loss": 3.1433, + "theoretical_loss": 4.048611913135052, + "tokens_seen": 368967680 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004485757271815447, + "loss": 2.8444, + "theoretical_loss": 4.048530373597716, + "tokens_seen": 369033216 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004485656970912738, + "loss": 3.0212, + "theoretical_loss": 4.048448852593277, + "tokens_seen": 369098752 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044855566700100304, + "loss": 2.931, + "theoretical_loss": 4.048367350114235, + "tokens_seen": 369164288 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044854563691073217, + "loss": 2.7455, + "theoretical_loss": 4.0482858661530905, + "tokens_seen": 369229824 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004485356068204614, + "loss": 3.1433, + "theoretical_loss": 4.048204400702351, + "tokens_seen": 369295360 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004485255767301906, + "loss": 2.7918, + "theoretical_loss": 4.04812295375453, + "tokens_seen": 369360896 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044851554663991977, + "loss": 2.8951, + "theoretical_loss": 4.04804152530214, + "tokens_seen": 369426432 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044850551654964895, + "loss": 3.2253, + "theoretical_loss": 4.047960115337702, + "tokens_seen": 369491968 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044849548645937813, + "loss": 2.9449, + "theoretical_loss": 4.047878723853741, + "tokens_seen": 369557504 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004484854563691073, + "loss": 2.825, + "theoretical_loss": 4.047797350842786, + "tokens_seen": 369623040 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044847542627883655, + "loss": 2.8463, + "theoretical_loss": 4.047715996297369, + "tokens_seen": 369688576 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004484653961885657, + "loss": 3.0055, + "theoretical_loss": 4.0476346602100275, + "tokens_seen": 369754112 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004484553660982949, + "loss": 2.9229, + "theoretical_loss": 4.047553342573304, + "tokens_seen": 369819648 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044844533600802404, + "loss": 2.9315, + "theoretical_loss": 4.047472043379743, + "tokens_seen": 369885184 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044843530591775327, + "loss": 3.1268, + "theoretical_loss": 4.047390762621897, + "tokens_seen": 369950720 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044842527582748245, + "loss": 3.0089, + "theoretical_loss": 4.04730950029232, + "tokens_seen": 370016256 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044841524573721163, + "loss": 2.8166, + "theoretical_loss": 4.047228256383569, + "tokens_seen": 370081792 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004484052156469408, + "loss": 3.1335, + "theoretical_loss": 4.04714703088821, + "tokens_seen": 370147328 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044839518555667005, + "loss": 2.7896, + "theoretical_loss": 4.0470658237988095, + "tokens_seen": 370212864 + }, + { + "epoch": 0.11, + "objective/train/docs_used": 221122, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.769576072692871, + "objective/train/theoretical_loss": 4.04698463510794, + "objective/train/tokens_used": 390738400, + "theoretical_loss": 4.04698463510794, + "tokens_seen": 370278400 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004483851554663992, + "loss": 3.0609, + "theoretical_loss": 4.04698463510794, + "tokens_seen": 370278400 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004483751253761284, + "loss": 2.8521, + "theoretical_loss": 4.046903464808177, + "tokens_seen": 370343936 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044836509528585754, + "loss": 2.8761, + "theoretical_loss": 4.046822312892102, + "tokens_seen": 370409472 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004483550651955868, + "loss": 3.1948, + "theoretical_loss": 4.046741179352299, + "tokens_seen": 370475008 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044834503510531596, + "loss": 3.2596, + "theoretical_loss": 4.046660064181359, + "tokens_seen": 370540544 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044833500501504514, + "loss": 3.0566, + "theoretical_loss": 4.0465789673718735, + "tokens_seen": 370606080 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004483249749247743, + "loss": 3.0197, + "theoretical_loss": 4.0464978889164405, + "tokens_seen": 370671616 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004483149448345035, + "loss": 3.0212, + "theoretical_loss": 4.046416828807663, + "tokens_seen": 370737152 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004483049147442327, + "loss": 2.9466, + "theoretical_loss": 4.046335787038148, + "tokens_seen": 370802688 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004482948846539619, + "loss": 3.0579, + "theoretical_loss": 4.046254763600503, + "tokens_seen": 370868224 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044828485456369104, + "loss": 3.0724, + "theoretical_loss": 4.046173758487347, + "tokens_seen": 370933760 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004482748244734203, + "loss": 3.0452, + "theoretical_loss": 4.046092771691297, + "tokens_seen": 370999296 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004482647943831494, + "loss": 3.1088, + "theoretical_loss": 4.046011803204976, + "tokens_seen": 371064832 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044825476429287864, + "loss": 2.7885, + "theoretical_loss": 4.045930853021014, + "tokens_seen": 371130368 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004482447342026078, + "loss": 3.0081, + "theoretical_loss": 4.045849921132042, + "tokens_seen": 371195904 + }, + { + "epoch": 0.11, + "learning_rate": 0.000448234704112337, + "loss": 3.1395, + "theoretical_loss": 4.045769007530695, + "tokens_seen": 371261440 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004482246740220662, + "loss": 3.1214, + "theoretical_loss": 4.045688112209615, + "tokens_seen": 371326976 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004482146439317954, + "loss": 3.1716, + "theoretical_loss": 4.045607235161447, + "tokens_seen": 371392512 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044820461384152455, + "loss": 3.0318, + "theoretical_loss": 4.04552637637884, + "tokens_seen": 371458048 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004481945837512538, + "loss": 2.9928, + "theoretical_loss": 4.045445535854446, + "tokens_seen": 371523584 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044818455366098296, + "loss": 3.0741, + "theoretical_loss": 4.045364713580923, + "tokens_seen": 371589120 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044817452357071214, + "loss": 2.9073, + "theoretical_loss": 4.045283909550935, + "tokens_seen": 371654656 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004481644934804414, + "loss": 3.2099, + "theoretical_loss": 4.045203123757146, + "tokens_seen": 371720192 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004481544633901705, + "loss": 3.1644, + "theoretical_loss": 4.045122356192227, + "tokens_seen": 371785728 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044814443329989974, + "loss": 2.901, + "theoretical_loss": 4.0450416068488515, + "tokens_seen": 371851264 + }, + { + "epoch": 0.11, + "objective/train/docs_used": 221664, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.9370317459106445, + "objective/train/theoretical_loss": 4.044960875719701, + "objective/train/tokens_used": 392376800, + "theoretical_loss": 4.044960875719701, + "tokens_seen": 371916800 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044813440320962887, + "loss": 3.047, + "theoretical_loss": 4.044960875719701, + "tokens_seen": 371916800 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004481243731193581, + "loss": 2.9598, + "theoretical_loss": 4.0448801627974555, + "tokens_seen": 371982336 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004481143430290873, + "loss": 3.0133, + "theoretical_loss": 4.044799468074804, + "tokens_seen": 372047872 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044810431293881647, + "loss": 2.9479, + "theoretical_loss": 4.044718791544438, + "tokens_seen": 372113408 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044809428284854565, + "loss": 3.0323, + "theoretical_loss": 4.044638133199052, + "tokens_seen": 372178944 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004480842527582749, + "loss": 3.0212, + "theoretical_loss": 4.044557493031347, + "tokens_seen": 372244480 + }, + { + "epoch": 0.11, + "learning_rate": 0.000448074222668004, + "loss": 3.1551, + "theoretical_loss": 4.044476871034027, + "tokens_seen": 372310016 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044806419257773324, + "loss": 3.0697, + "theoretical_loss": 4.0443962671998, + "tokens_seen": 372375552 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044805416248746237, + "loss": 2.8708, + "theoretical_loss": 4.044315681521379, + "tokens_seen": 372441088 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004480441323971916, + "loss": 3.1652, + "theoretical_loss": 4.04423511399148, + "tokens_seen": 372506624 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004480341023069208, + "loss": 2.9517, + "theoretical_loss": 4.044154564602826, + "tokens_seen": 372572160 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044802407221664997, + "loss": 2.8858, + "theoretical_loss": 4.044074033348139, + "tokens_seen": 372637696 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044801404212637915, + "loss": 3.1258, + "theoretical_loss": 4.043993520220151, + "tokens_seen": 372703232 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044800401203610833, + "loss": 3.1399, + "theoretical_loss": 4.0439130252115945, + "tokens_seen": 372768768 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004479939819458375, + "loss": 2.8798, + "theoretical_loss": 4.043832548315207, + "tokens_seen": 372834304 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044798395185556675, + "loss": 3.1893, + "theoretical_loss": 4.043752089523732, + "tokens_seen": 372899840 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004479739217652959, + "loss": 2.8171, + "theoretical_loss": 4.043671648829913, + "tokens_seen": 372965376 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004479638916750251, + "loss": 2.9128, + "theoretical_loss": 4.043591226226503, + "tokens_seen": 373030912 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044795386158475424, + "loss": 2.9054, + "theoretical_loss": 4.0435108217062545, + "tokens_seen": 373096448 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044794383149448347, + "loss": 3.1926, + "theoretical_loss": 4.043430435261927, + "tokens_seen": 373161984 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044793380140421265, + "loss": 3.2438, + "theoretical_loss": 4.043350066886283, + "tokens_seen": 373227520 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044792377131394183, + "loss": 3.2269, + "theoretical_loss": 4.043269716572091, + "tokens_seen": 373293056 + }, + { + "epoch": 0.11, + "learning_rate": 0.000447913741223671, + "loss": 2.9647, + "theoretical_loss": 4.04318938431212, + "tokens_seen": 373358592 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044790371113340025, + "loss": 3.0139, + "theoretical_loss": 4.043109070099146, + "tokens_seen": 373424128 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004478936810431294, + "loss": 2.8018, + "theoretical_loss": 4.043028773925949, + "tokens_seen": 373489664 + }, + { + "epoch": 0.11, + "objective/train/docs_used": 222906, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7618439197540283, + "objective/train/theoretical_loss": 4.042948495785312, + "objective/train/tokens_used": 394015200, + "theoretical_loss": 4.042948495785312, + "tokens_seen": 373555200 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004478836509528586, + "loss": 2.948, + "theoretical_loss": 4.042948495785312, + "tokens_seen": 373555200 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044787362086258774, + "loss": 2.9264, + "theoretical_loss": 4.042868235670023, + "tokens_seen": 373620736 + }, + { + "epoch": 0.11, + "learning_rate": 0.000447863590772317, + "loss": 3.0927, + "theoretical_loss": 4.042787993572875, + "tokens_seen": 373686272 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044785356068204616, + "loss": 2.9434, + "theoretical_loss": 4.042707769486663, + "tokens_seen": 373751808 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044784353059177534, + "loss": 3.185, + "theoretical_loss": 4.042627563404187, + "tokens_seen": 373817344 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004478335005015045, + "loss": 2.9869, + "theoretical_loss": 4.042547375318251, + "tokens_seen": 373882880 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004478234704112337, + "loss": 2.7904, + "theoretical_loss": 4.042467205221665, + "tokens_seen": 373948416 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004478134403209629, + "loss": 3.1134, + "theoretical_loss": 4.042387053107241, + "tokens_seen": 374013952 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004478034102306921, + "loss": 2.7583, + "theoretical_loss": 4.042306918967795, + "tokens_seen": 374079488 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044779338014042124, + "loss": 3.1288, + "theoretical_loss": 4.042226802796149, + "tokens_seen": 374145024 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004477833500501505, + "loss": 2.8818, + "theoretical_loss": 4.042146704585127, + "tokens_seen": 374210560 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004477733199598796, + "loss": 2.9968, + "theoretical_loss": 4.042066624327559, + "tokens_seen": 374276096 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044776328986960884, + "loss": 2.747, + "theoretical_loss": 4.041986562016278, + "tokens_seen": 374341632 + }, + { + "epoch": 0.11, + "learning_rate": 0.000447753259779338, + "loss": 3.0316, + "theoretical_loss": 4.041906517644121, + "tokens_seen": 374407168 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004477432296890672, + "loss": 3.0119, + "theoretical_loss": 4.0418264912039295, + "tokens_seen": 374472704 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004477331995987964, + "loss": 3.0869, + "theoretical_loss": 4.04174648268855, + "tokens_seen": 374538240 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004477231695085256, + "loss": 2.9858, + "theoretical_loss": 4.041666492090831, + "tokens_seen": 374603776 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044771313941825475, + "loss": 3.1753, + "theoretical_loss": 4.041586519403627, + "tokens_seen": 374669312 + }, + { + "epoch": 0.11, + "learning_rate": 0.000447703109327984, + "loss": 3.0959, + "theoretical_loss": 4.041506564619796, + "tokens_seen": 374734848 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004476930792377131, + "loss": 3.1061, + "theoretical_loss": 4.0414266277322, + "tokens_seen": 374800384 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044768304914744234, + "loss": 3.177, + "theoretical_loss": 4.041346708733704, + "tokens_seen": 374865920 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004476730190571715, + "loss": 3.0699, + "theoretical_loss": 4.041266807617179, + "tokens_seen": 374931456 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004476629889669007, + "loss": 3.0341, + "theoretical_loss": 4.041186924375499, + "tokens_seen": 374996992 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004476529588766299, + "loss": 3.051, + "theoretical_loss": 4.041107059001544, + "tokens_seen": 375062528 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044764292878635907, + "loss": 2.8616, + "theoretical_loss": 4.041027211488194, + "tokens_seen": 375128064 + }, + { + "epoch": 0.11, + "objective/train/docs_used": 223164, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.8120296001434326, + "objective/train/theoretical_loss": 4.040947381828337, + "objective/train/tokens_used": 395653600, + "theoretical_loss": 4.040947381828337, + "tokens_seen": 375193600 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044763289869608825, + "loss": 2.9899, + "theoretical_loss": 4.040947381828337, + "tokens_seen": 375193600 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004476228686058175, + "loss": 2.9751, + "theoretical_loss": 4.040867570014862, + "tokens_seen": 375259136 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004476128385155466, + "loss": 2.9461, + "theoretical_loss": 4.040787776040666, + "tokens_seen": 375324672 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044760280842527585, + "loss": 2.9823, + "theoretical_loss": 4.040707999898647, + "tokens_seen": 375390208 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044759277833500503, + "loss": 2.8336, + "theoretical_loss": 4.040628241581706, + "tokens_seen": 375455744 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004475827482447342, + "loss": 3.0432, + "theoretical_loss": 4.0405485010827515, + "tokens_seen": 375521280 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004475727181544634, + "loss": 3.0058, + "theoretical_loss": 4.040468778394694, + "tokens_seen": 375586816 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044756268806419257, + "loss": 2.9112, + "theoretical_loss": 4.040389073510449, + "tokens_seen": 375652352 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044755265797392175, + "loss": 3.0086, + "theoretical_loss": 4.040309386422934, + "tokens_seen": 375717888 + }, + { + "epoch": 0.11, + "learning_rate": 0.000447542627883651, + "loss": 3.1287, + "theoretical_loss": 4.040229717125074, + "tokens_seen": 375783424 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004475325977933801, + "loss": 3.2131, + "theoretical_loss": 4.0401500656097955, + "tokens_seen": 375848960 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044752256770310935, + "loss": 2.7229, + "theoretical_loss": 4.040070431870029, + "tokens_seen": 375914496 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004475125376128385, + "loss": 2.9755, + "theoretical_loss": 4.03999081589871, + "tokens_seen": 375980032 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004475025075225677, + "loss": 2.8314, + "theoretical_loss": 4.039911217688777, + "tokens_seen": 376045568 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004474924774322969, + "loss": 3.4235, + "theoretical_loss": 4.039831637233176, + "tokens_seen": 376111104 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004474824473420261, + "loss": 2.7758, + "theoretical_loss": 4.039752074524851, + "tokens_seen": 376176640 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044747241725175526, + "loss": 2.9362, + "theoretical_loss": 4.039672529556755, + "tokens_seen": 376242176 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044746238716148444, + "loss": 2.7325, + "theoretical_loss": 4.039593002321844, + "tokens_seen": 376307712 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004474523570712136, + "loss": 3.1786, + "theoretical_loss": 4.039513492813076, + "tokens_seen": 376373248 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044744232698094285, + "loss": 3.0039, + "theoretical_loss": 4.039434001023415, + "tokens_seen": 376438784 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044743229689067204, + "loss": 3.2286, + "theoretical_loss": 4.039354526945829, + "tokens_seen": 376504320 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004474222668004012, + "loss": 3.1581, + "theoretical_loss": 4.039275070573288, + "tokens_seen": 376569856 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044741223671013045, + "loss": 3.1757, + "theoretical_loss": 4.039195631898768, + "tokens_seen": 376635392 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004474022066198596, + "loss": 2.8833, + "theoretical_loss": 4.03911621091525, + "tokens_seen": 376700928 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004473921765295888, + "loss": 2.9509, + "theoretical_loss": 4.039036807615716, + "tokens_seen": 376766464 + }, + { + "epoch": 0.11, + "objective/train/docs_used": 224842, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.646427869796753, + "objective/train/theoretical_loss": 4.038957421993153, + "objective/train/tokens_used": 397292000, + "theoretical_loss": 4.038957421993153, + "tokens_seen": 376832000 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044738214643931794, + "loss": 2.9259, + "theoretical_loss": 4.038957421993153, + "tokens_seen": 376832000 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004473721163490472, + "loss": 3.0552, + "theoretical_loss": 4.038878054040554, + "tokens_seen": 376897536 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044736208625877636, + "loss": 2.8755, + "theoretical_loss": 4.038798703750913, + "tokens_seen": 376963072 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044735205616850554, + "loss": 2.7265, + "theoretical_loss": 4.038719371117231, + "tokens_seen": 377028608 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004473420260782347, + "loss": 2.9808, + "theoretical_loss": 4.038640056132509, + "tokens_seen": 377094144 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004473319959879639, + "loss": 2.9284, + "theoretical_loss": 4.038560758789758, + "tokens_seen": 377159680 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004473219658976931, + "loss": 3.0299, + "theoretical_loss": 4.0384814790819865, + "tokens_seen": 377225216 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004473119358074223, + "loss": 3.0097, + "theoretical_loss": 4.038402217002211, + "tokens_seen": 377290752 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044730190571715144, + "loss": 2.9468, + "theoretical_loss": 4.038322972543451, + "tokens_seen": 377356288 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004472918756268807, + "loss": 2.969, + "theoretical_loss": 4.03824374569873, + "tokens_seen": 377421824 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004472818455366098, + "loss": 3.2655, + "theoretical_loss": 4.038164536461075, + "tokens_seen": 377487360 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044727181544633904, + "loss": 3.058, + "theoretical_loss": 4.038085344823516, + "tokens_seen": 377552896 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004472617853560682, + "loss": 3.2178, + "theoretical_loss": 4.038006170779091, + "tokens_seen": 377618432 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004472517552657974, + "loss": 3.0786, + "theoretical_loss": 4.037927014320838, + "tokens_seen": 377683968 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004472417251755266, + "loss": 3.0093, + "theoretical_loss": 4.037847875441798, + "tokens_seen": 377749504 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004472316950852558, + "loss": 2.8994, + "theoretical_loss": 4.037768754135023, + "tokens_seen": 377815040 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044722166499498495, + "loss": 3.121, + "theoretical_loss": 4.037689650393559, + "tokens_seen": 377880576 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004472116349047142, + "loss": 3.0612, + "theoretical_loss": 4.037610564210464, + "tokens_seen": 377946112 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004472016048144433, + "loss": 2.9868, + "theoretical_loss": 4.037531495578797, + "tokens_seen": 378011648 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044719157472417254, + "loss": 3.0085, + "theoretical_loss": 4.03745244449162, + "tokens_seen": 378077184 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004471815446339017, + "loss": 2.8586, + "theoretical_loss": 4.037373410942001, + "tokens_seen": 378142720 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004471715145436309, + "loss": 2.9787, + "theoretical_loss": 4.037294394923009, + "tokens_seen": 378208256 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004471614844533601, + "loss": 3.0486, + "theoretical_loss": 4.037215396427721, + "tokens_seen": 378273792 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044715145436308927, + "loss": 2.9491, + "theoretical_loss": 4.037136415449214, + "tokens_seen": 378339328 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044714142427281845, + "loss": 2.8674, + "theoretical_loss": 4.037057451980571, + "tokens_seen": 378404864 + }, + { + "epoch": 0.11, + "objective/train/docs_used": 225570, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.1136412620544434, + "objective/train/theoretical_loss": 4.036978506014879, + "objective/train/tokens_used": 398930400, + "theoretical_loss": 4.036978506014879, + "tokens_seen": 378470400 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004471313941825477, + "loss": 3.0621, + "theoretical_loss": 4.036978506014879, + "tokens_seen": 378470400 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004471213640922768, + "loss": 3.225, + "theoretical_loss": 4.036899577545229, + "tokens_seen": 378535936 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044711133400200605, + "loss": 3.1221, + "theoretical_loss": 4.036820666564714, + "tokens_seen": 378601472 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044710130391173523, + "loss": 3.051, + "theoretical_loss": 4.036741773066434, + "tokens_seen": 378667008 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004470912738214644, + "loss": 3.1714, + "theoretical_loss": 4.03666289704349, + "tokens_seen": 378732544 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004470812437311936, + "loss": 2.8334, + "theoretical_loss": 4.036584038488988, + "tokens_seen": 378798080 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044707121364092277, + "loss": 3.0901, + "theoretical_loss": 4.03650519739604, + "tokens_seen": 378863616 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044706118355065195, + "loss": 2.9578, + "theoretical_loss": 4.036426373757759, + "tokens_seen": 378929152 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004470511534603812, + "loss": 2.8524, + "theoretical_loss": 4.036347567567262, + "tokens_seen": 378994688 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004470411233701103, + "loss": 3.1457, + "theoretical_loss": 4.0362687788176705, + "tokens_seen": 379060224 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044703109327983955, + "loss": 2.9946, + "theoretical_loss": 4.036190007502112, + "tokens_seen": 379125760 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004470210631895687, + "loss": 2.8028, + "theoretical_loss": 4.036111253613715, + "tokens_seen": 379191296 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004470110330992979, + "loss": 3.1217, + "theoretical_loss": 4.036032517145614, + "tokens_seen": 379256832 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004470010030090271, + "loss": 3.0045, + "theoretical_loss": 4.0359537980909455, + "tokens_seen": 379322368 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004469909729187563, + "loss": 3.0265, + "theoretical_loss": 4.03587509644285, + "tokens_seen": 379387904 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044698094282848546, + "loss": 3.0686, + "theoretical_loss": 4.0357964121944745, + "tokens_seen": 379453440 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044697091273821464, + "loss": 2.9145, + "theoretical_loss": 4.035717745338967, + "tokens_seen": 379518976 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004469608826479438, + "loss": 2.8989, + "theoretical_loss": 4.035639095869481, + "tokens_seen": 379584512 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044695085255767305, + "loss": 2.8647, + "theoretical_loss": 4.035560463779173, + "tokens_seen": 379650048 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004469408224674022, + "loss": 3.0145, + "theoretical_loss": 4.0354818490612026, + "tokens_seen": 379715584 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004469307923771314, + "loss": 3.1178, + "theoretical_loss": 4.035403251708736, + "tokens_seen": 379781120 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004469207622868606, + "loss": 2.9392, + "theoretical_loss": 4.035324671714942, + "tokens_seen": 379846656 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004469107321965898, + "loss": 3.0735, + "theoretical_loss": 4.0352461090729905, + "tokens_seen": 379912192 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044690070210631896, + "loss": 2.8095, + "theoretical_loss": 4.03516756377606, + "tokens_seen": 379977728 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044689067201604814, + "loss": 3.1436, + "theoretical_loss": 4.035089035817329, + "tokens_seen": 380043264 + }, + { + "epoch": 0.12, + "objective/train/docs_used": 226798, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.004202365875244, + "objective/train/theoretical_loss": 4.035010525189982, + "objective/train/tokens_used": 400568800, + "theoretical_loss": 4.035010525189982, + "tokens_seen": 380108800 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004468806419257773, + "loss": 2.8807, + "theoretical_loss": 4.035010525189982, + "tokens_seen": 380108800 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044687061183550656, + "loss": 2.8744, + "theoretical_loss": 4.034932031887207, + "tokens_seen": 380174336 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004468605817452357, + "loss": 2.8974, + "theoretical_loss": 4.0348535559021945, + "tokens_seen": 380239872 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004468505516549649, + "loss": 3.165, + "theoretical_loss": 4.034775097228142, + "tokens_seen": 380305408 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044684052156469405, + "loss": 2.7899, + "theoretical_loss": 4.034696655858246, + "tokens_seen": 380370944 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004468304914744233, + "loss": 2.8711, + "theoretical_loss": 4.0346182317857116, + "tokens_seen": 380436480 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044682046138415246, + "loss": 3.1562, + "theoretical_loss": 4.034539825003746, + "tokens_seen": 380502016 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044681043129388164, + "loss": 2.931, + "theoretical_loss": 4.034461435505557, + "tokens_seen": 380567552 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004468004012036108, + "loss": 2.9581, + "theoretical_loss": 4.034383063284364, + "tokens_seen": 380633088 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044679037111334, + "loss": 2.9538, + "theoretical_loss": 4.034304708333382, + "tokens_seen": 380698624 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004467803410230692, + "loss": 2.7625, + "theoretical_loss": 4.0342263706458334, + "tokens_seen": 380764160 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004467703109327984, + "loss": 2.9775, + "theoretical_loss": 4.034148050214946, + "tokens_seen": 380829696 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044676028084252755, + "loss": 2.9133, + "theoretical_loss": 4.034069747033948, + "tokens_seen": 380895232 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004467502507522568, + "loss": 2.7989, + "theoretical_loss": 4.0339914610960745, + "tokens_seen": 380960768 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044674022066198597, + "loss": 2.5998, + "theoretical_loss": 4.033913192394563, + "tokens_seen": 381026304 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044673019057171515, + "loss": 2.9793, + "theoretical_loss": 4.033834940922653, + "tokens_seen": 381091840 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044672016048144433, + "loss": 3.2243, + "theoretical_loss": 4.033756706673593, + "tokens_seen": 381157376 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004467101303911735, + "loss": 3.2074, + "theoretical_loss": 4.033678489640629, + "tokens_seen": 381222912 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004467001003009027, + "loss": 2.8805, + "theoretical_loss": 4.033600289817015, + "tokens_seen": 381288448 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004466900702106319, + "loss": 2.7748, + "theoretical_loss": 4.033522107196007, + "tokens_seen": 381353984 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004466800401203611, + "loss": 3.0382, + "theoretical_loss": 4.033443941770867, + "tokens_seen": 381419520 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004466700100300903, + "loss": 2.8543, + "theoretical_loss": 4.033365793534857, + "tokens_seen": 381485056 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044665997993981947, + "loss": 2.8942, + "theoretical_loss": 4.033287662481247, + "tokens_seen": 381550592 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044664994984954865, + "loss": 3.0997, + "theoretical_loss": 4.033209548603308, + "tokens_seen": 381616128 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004466399197592779, + "loss": 3.0135, + "theoretical_loss": 4.033131451894317, + "tokens_seen": 381681664 + }, + { + "epoch": 0.12, + "objective/train/docs_used": 227456, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.243720054626465, + "objective/train/theoretical_loss": 4.03305337234755, + "objective/train/tokens_used": 402207200, + "theoretical_loss": 4.03305337234755, + "tokens_seen": 381747200 + }, + { + "epoch": 0.12, + "learning_rate": 0.000446629889669007, + "loss": 3.0505, + "theoretical_loss": 4.03305337234755, + "tokens_seen": 381747200 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044661985957873625, + "loss": 2.8224, + "theoretical_loss": 4.032975309956292, + "tokens_seen": 381812736 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044660982948846543, + "loss": 3.0402, + "theoretical_loss": 4.0328972647138315, + "tokens_seen": 381878272 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004465997993981946, + "loss": 3.1535, + "theoretical_loss": 4.032819236613457, + "tokens_seen": 381943808 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004465897693079238, + "loss": 2.8958, + "theoretical_loss": 4.032741225648463, + "tokens_seen": 382009344 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044657973921765297, + "loss": 2.9952, + "theoretical_loss": 4.03266323181215, + "tokens_seen": 382074880 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044656970912738215, + "loss": 2.95, + "theoretical_loss": 4.0325852550978185, + "tokens_seen": 382140416 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004465596790371114, + "loss": 3.1972, + "theoretical_loss": 4.032507295498774, + "tokens_seen": 382205952 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004465496489468405, + "loss": 2.8314, + "theoretical_loss": 4.032429353008326, + "tokens_seen": 382271488 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044653961885656975, + "loss": 2.9948, + "theoretical_loss": 4.032351427619789, + "tokens_seen": 382337024 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004465295887662989, + "loss": 3.1082, + "theoretical_loss": 4.03227351932648, + "tokens_seen": 382402560 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004465195586760281, + "loss": 3.0092, + "theoretical_loss": 4.032195628121718, + "tokens_seen": 382468096 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004465095285857573, + "loss": 2.8735, + "theoretical_loss": 4.032117753998829, + "tokens_seen": 382533632 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004464994984954865, + "loss": 2.9554, + "theoretical_loss": 4.032039896951142, + "tokens_seen": 382599168 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044648946840521566, + "loss": 2.7243, + "theoretical_loss": 4.031962056971988, + "tokens_seen": 382664704 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044647943831494484, + "loss": 3.1133, + "theoretical_loss": 4.0318842340547025, + "tokens_seen": 382730240 + }, + { + "epoch": 0.12, + "learning_rate": 0.000446469408224674, + "loss": 2.8125, + "theoretical_loss": 4.031806428192627, + "tokens_seen": 382795776 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044645937813440325, + "loss": 2.9852, + "theoretical_loss": 4.031728639379103, + "tokens_seen": 382861312 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004464493480441324, + "loss": 3.0372, + "theoretical_loss": 4.031650867607478, + "tokens_seen": 382926848 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004464393179538616, + "loss": 3.1156, + "theoretical_loss": 4.031573112871103, + "tokens_seen": 382992384 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004464292878635908, + "loss": 3.1483, + "theoretical_loss": 4.031495375163333, + "tokens_seen": 383057920 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044641925777332, + "loss": 3.1227, + "theoretical_loss": 4.031417654477526, + "tokens_seen": 383123456 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044640922768304916, + "loss": 3.035, + "theoretical_loss": 4.031339950807043, + "tokens_seen": 383188992 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044639919759277834, + "loss": 3.1469, + "theoretical_loss": 4.03126226414525, + "tokens_seen": 383254528 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004463891675025075, + "loss": 3.1194, + "theoretical_loss": 4.031184594485517, + "tokens_seen": 383320064 + }, + { + "epoch": 0.12, + "objective/train/docs_used": 228876, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.0608878135681152, + "objective/train/theoretical_loss": 4.031106941821218, + "objective/train/tokens_used": 403845600, + "theoretical_loss": 4.031106941821218, + "tokens_seen": 383385600 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044637913741223676, + "loss": 2.9776, + "theoretical_loss": 4.031106941821218, + "tokens_seen": 383385600 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004463691073219659, + "loss": 2.86, + "theoretical_loss": 4.031029306145728, + "tokens_seen": 383451136 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004463590772316951, + "loss": 2.8785, + "theoretical_loss": 4.030951687452429, + "tokens_seen": 383516672 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044634904714142425, + "loss": 3.1131, + "theoretical_loss": 4.0308740857347045, + "tokens_seen": 383582208 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004463390170511535, + "loss": 2.8338, + "theoretical_loss": 4.030796500985943, + "tokens_seen": 383647744 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044632898696088266, + "loss": 2.9455, + "theoretical_loss": 4.030718933199536, + "tokens_seen": 383713280 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044631895687061184, + "loss": 2.8745, + "theoretical_loss": 4.030641382368878, + "tokens_seen": 383778816 + }, + { + "epoch": 0.12, + "learning_rate": 0.000446308926780341, + "loss": 3.0843, + "theoretical_loss": 4.03056384848737, + "tokens_seen": 383844352 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004462988966900702, + "loss": 3.1465, + "theoretical_loss": 4.030486331548413, + "tokens_seen": 383909888 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004462888665997994, + "loss": 2.9619, + "theoretical_loss": 4.030408831545414, + "tokens_seen": 383975424 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004462788365095286, + "loss": 3.1481, + "theoretical_loss": 4.030331348471784, + "tokens_seen": 384040960 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044626880641925775, + "loss": 2.7997, + "theoretical_loss": 4.030253882320936, + "tokens_seen": 384106496 + }, + { + "epoch": 0.12, + "learning_rate": 0.000446258776328987, + "loss": 2.8678, + "theoretical_loss": 4.030176433086289, + "tokens_seen": 384172032 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044624874623871617, + "loss": 3.0746, + "theoretical_loss": 4.030099000761261, + "tokens_seen": 384237568 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044623871614844535, + "loss": 2.7633, + "theoretical_loss": 4.030021585339281, + "tokens_seen": 384303104 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044622868605817453, + "loss": 2.9408, + "theoretical_loss": 4.029944186813774, + "tokens_seen": 384368640 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004462186559679037, + "loss": 2.9037, + "theoretical_loss": 4.029866805178175, + "tokens_seen": 384434176 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004462086258776329, + "loss": 2.9903, + "theoretical_loss": 4.029789440425917, + "tokens_seen": 384499712 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004461985957873621, + "loss": 2.8878, + "theoretical_loss": 4.029712092550443, + "tokens_seen": 384565248 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044618856569709125, + "loss": 3.042, + "theoretical_loss": 4.029634761545195, + "tokens_seen": 384630784 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004461785356068205, + "loss": 3.1129, + "theoretical_loss": 4.029557447403618, + "tokens_seen": 384696320 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004461685055165496, + "loss": 2.7792, + "theoretical_loss": 4.029480150119165, + "tokens_seen": 384761856 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044615847542627885, + "loss": 2.9519, + "theoretical_loss": 4.029402869685288, + "tokens_seen": 384827392 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044614844533600803, + "loss": 2.7102, + "theoretical_loss": 4.0293256060954485, + "tokens_seen": 384892928 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004461384152457372, + "loss": 2.9142, + "theoretical_loss": 4.029248359343105, + "tokens_seen": 384958464 + }, + { + "epoch": 0.12, + "objective/train/docs_used": 229454, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6346988677978516, + "objective/train/theoretical_loss": 4.029171129421724, + "objective/train/tokens_used": 405484000, + "theoretical_loss": 4.029171129421724, + "tokens_seen": 385024000 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004461283851554664, + "loss": 3.0725, + "theoretical_loss": 4.029171129421724, + "tokens_seen": 385024000 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044611835506519563, + "loss": 2.9605, + "theoretical_loss": 4.029093916324774, + "tokens_seen": 385089536 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044610832497492476, + "loss": 2.8276, + "theoretical_loss": 4.0290167200457265, + "tokens_seen": 385155072 + }, + { + "epoch": 0.12, + "learning_rate": 0.000446098294884654, + "loss": 3.1885, + "theoretical_loss": 4.02893954057806, + "tokens_seen": 385220608 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004460882647943831, + "loss": 2.9628, + "theoretical_loss": 4.028862377915252, + "tokens_seen": 385286144 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044607823470411235, + "loss": 2.9441, + "theoretical_loss": 4.0287852320507875, + "tokens_seen": 385351680 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044606820461384153, + "loss": 3.0888, + "theoretical_loss": 4.028708102978152, + "tokens_seen": 385417216 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004460581745235707, + "loss": 2.9564, + "theoretical_loss": 4.028630990690838, + "tokens_seen": 385482752 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004460481444332999, + "loss": 2.8218, + "theoretical_loss": 4.028553895182339, + "tokens_seen": 385548288 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004460381143430291, + "loss": 3.1587, + "theoretical_loss": 4.028476816446153, + "tokens_seen": 385613824 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044602808425275826, + "loss": 2.9593, + "theoretical_loss": 4.028399754475781, + "tokens_seen": 385679360 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004460180541624875, + "loss": 3.0022, + "theoretical_loss": 4.028322709264729, + "tokens_seen": 385744896 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004460080240722166, + "loss": 3.0352, + "theoretical_loss": 4.0282456808065055, + "tokens_seen": 385810432 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044599799398194586, + "loss": 2.9721, + "theoretical_loss": 4.028168669094623, + "tokens_seen": 385875968 + }, + { + "epoch": 0.12, + "learning_rate": 0.000445987963891675, + "loss": 3.0811, + "theoretical_loss": 4.028091674122598, + "tokens_seen": 385941504 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004459779338014042, + "loss": 3.0054, + "theoretical_loss": 4.0280146958839484, + "tokens_seen": 386007040 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004459679037111334, + "loss": 2.9558, + "theoretical_loss": 4.0279377343722, + "tokens_seen": 386072576 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004459578736208626, + "loss": 2.9158, + "theoretical_loss": 4.027860789580878, + "tokens_seen": 386138112 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044594784353059176, + "loss": 3.0592, + "theoretical_loss": 4.027783861503513, + "tokens_seen": 386203648 + }, + { + "epoch": 0.12, + "learning_rate": 0.000445937813440321, + "loss": 2.9531, + "theoretical_loss": 4.0277069501336396, + "tokens_seen": 386269184 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004459277833500502, + "loss": 2.8179, + "theoretical_loss": 4.027630055464795, + "tokens_seen": 386334720 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044591775325977936, + "loss": 3.0436, + "theoretical_loss": 4.027553177490521, + "tokens_seen": 386400256 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044590772316950854, + "loss": 3.0281, + "theoretical_loss": 4.027476316204362, + "tokens_seen": 386465792 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004458976930792377, + "loss": 3.011, + "theoretical_loss": 4.027399471599867, + "tokens_seen": 386531328 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044588766298896696, + "loss": 2.7141, + "theoretical_loss": 4.027322643670587, + "tokens_seen": 386596864 + }, + { + "epoch": 0.12, + "objective/train/docs_used": 230121, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.2940869331359863, + "objective/train/theoretical_loss": 4.027245832410079, + "objective/train/tokens_used": 407122400, + "theoretical_loss": 4.027245832410079, + "tokens_seen": 386662400 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004458776328986961, + "loss": 3.0625, + "theoretical_loss": 4.027245832410079, + "tokens_seen": 386662400 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004458676028084253, + "loss": 3.0529, + "theoretical_loss": 4.027169037811901, + "tokens_seen": 386727936 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044585757271815445, + "loss": 3.0386, + "theoretical_loss": 4.027092259869616, + "tokens_seen": 386793472 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004458475426278837, + "loss": 2.8267, + "theoretical_loss": 4.027015498576791, + "tokens_seen": 386859008 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044583751253761286, + "loss": 2.6068, + "theoretical_loss": 4.026938753926997, + "tokens_seen": 386924544 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044582748244734204, + "loss": 3.0744, + "theoretical_loss": 4.026862025913804, + "tokens_seen": 386990080 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004458174523570712, + "loss": 2.8501, + "theoretical_loss": 4.026785314530792, + "tokens_seen": 387055616 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004458074222668004, + "loss": 2.8834, + "theoretical_loss": 4.026708619771541, + "tokens_seen": 387121152 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004457973921765296, + "loss": 2.7252, + "theoretical_loss": 4.026631941629635, + "tokens_seen": 387186688 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004457873620862588, + "loss": 3.048, + "theoretical_loss": 4.026555280098661, + "tokens_seen": 387252224 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044577733199598795, + "loss": 2.8975, + "theoretical_loss": 4.026478635172212, + "tokens_seen": 387317760 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004457673019057172, + "loss": 3.0878, + "theoretical_loss": 4.026402006843882, + "tokens_seen": 387383296 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044575727181544637, + "loss": 2.8037, + "theoretical_loss": 4.0263253951072695, + "tokens_seen": 387448832 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044574724172517555, + "loss": 3.0029, + "theoretical_loss": 4.026248799955976, + "tokens_seen": 387514368 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044573721163490473, + "loss": 2.8786, + "theoretical_loss": 4.026172221383607, + "tokens_seen": 387579904 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004457271815446339, + "loss": 2.7987, + "theoretical_loss": 4.026095659383773, + "tokens_seen": 387645440 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004457171514543631, + "loss": 3.1699, + "theoretical_loss": 4.026019113950086, + "tokens_seen": 387710976 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004457071213640923, + "loss": 3.0362, + "theoretical_loss": 4.025942585076161, + "tokens_seen": 387776512 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044569709127382145, + "loss": 2.757, + "theoretical_loss": 4.025866072755619, + "tokens_seen": 387842048 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004456870611835507, + "loss": 2.8963, + "theoretical_loss": 4.025789576982083, + "tokens_seen": 387907584 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004456770310932798, + "loss": 2.8359, + "theoretical_loss": 4.02571309774918, + "tokens_seen": 387973120 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044566700100300905, + "loss": 2.5164, + "theoretical_loss": 4.025636635050539, + "tokens_seen": 388038656 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044565697091273823, + "loss": 2.7412, + "theoretical_loss": 4.025560188879796, + "tokens_seen": 388104192 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004456469408224674, + "loss": 2.8645, + "theoretical_loss": 4.025483759230585, + "tokens_seen": 388169728 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004456369107321966, + "loss": 2.8968, + "theoretical_loss": 4.025407346096551, + "tokens_seen": 388235264 + }, + { + "epoch": 0.12, + "objective/train/docs_used": 231531, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.02329683303833, + "objective/train/theoretical_loss": 4.025330949471336, + "objective/train/tokens_used": 408760800, + "theoretical_loss": 4.025330949471336, + "tokens_seen": 388300800 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044562688064192583, + "loss": 3.1035, + "theoretical_loss": 4.025330949471336, + "tokens_seen": 388300800 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044561685055165496, + "loss": 3.0319, + "theoretical_loss": 4.025254569348588, + "tokens_seen": 388366336 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004456068204613842, + "loss": 3.0267, + "theoretical_loss": 4.025178205721959, + "tokens_seen": 388431872 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004455967903711133, + "loss": 3.2149, + "theoretical_loss": 4.025101858585104, + "tokens_seen": 388497408 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044558676028084255, + "loss": 3.0867, + "theoretical_loss": 4.025025527931681, + "tokens_seen": 388562944 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044557673019057173, + "loss": 2.7795, + "theoretical_loss": 4.0249492137553515, + "tokens_seen": 388628480 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004455667001003009, + "loss": 2.9365, + "theoretical_loss": 4.024872916049782, + "tokens_seen": 388694016 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004455566700100301, + "loss": 2.8175, + "theoretical_loss": 4.0247966348086415, + "tokens_seen": 388759552 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004455466399197593, + "loss": 3.0323, + "theoretical_loss": 4.0247203700256025, + "tokens_seen": 388825088 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044553660982948846, + "loss": 2.7767, + "theoretical_loss": 4.02464412169434, + "tokens_seen": 388890624 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004455265797392177, + "loss": 2.9396, + "theoretical_loss": 4.024567889808534, + "tokens_seen": 388956160 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004455165496489468, + "loss": 3.0072, + "theoretical_loss": 4.024491674361868, + "tokens_seen": 389021696 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044550651955867606, + "loss": 3.0704, + "theoretical_loss": 4.024415475348027, + "tokens_seen": 389087232 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004454964894684052, + "loss": 3.2515, + "theoretical_loss": 4.024339292760703, + "tokens_seen": 389152768 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004454864593781344, + "loss": 3.0315, + "theoretical_loss": 4.024263126593588, + "tokens_seen": 389218304 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004454764292878636, + "loss": 3.0114, + "theoretical_loss": 4.0241869768403795, + "tokens_seen": 389283840 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004454663991975928, + "loss": 2.882, + "theoretical_loss": 4.024110843494777, + "tokens_seen": 389349376 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044545636910732196, + "loss": 2.9268, + "theoretical_loss": 4.024034726550486, + "tokens_seen": 389414912 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004454463390170512, + "loss": 3.2313, + "theoretical_loss": 4.023958626001212, + "tokens_seen": 389480448 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004454363089267803, + "loss": 3.1751, + "theoretical_loss": 4.023882541840667, + "tokens_seen": 389545984 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044542627883650956, + "loss": 3.0054, + "theoretical_loss": 4.023806474062566, + "tokens_seen": 389611520 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004454162487462387, + "loss": 3.0431, + "theoretical_loss": 4.023730422660624, + "tokens_seen": 389677056 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004454062186559679, + "loss": 2.9271, + "theoretical_loss": 4.023654387628565, + "tokens_seen": 389742592 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004453961885656971, + "loss": 3.0666, + "theoretical_loss": 4.023578368960113, + "tokens_seen": 389808128 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004453861584754263, + "loss": 2.9641, + "theoretical_loss": 4.023502366648994, + "tokens_seen": 389873664 + }, + { + "epoch": 0.12, + "objective/train/docs_used": 232298, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.0122013092041016, + "objective/train/theoretical_loss": 4.023426380688943, + "objective/train/tokens_used": 410399200, + "theoretical_loss": 4.023426380688943, + "tokens_seen": 389939200 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044537612838515547, + "loss": 2.9082, + "theoretical_loss": 4.023426380688943, + "tokens_seen": 389939200 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044536609829488465, + "loss": 2.9781, + "theoretical_loss": 4.023350411073693, + "tokens_seen": 390004736 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044535606820461383, + "loss": 3.0196, + "theoretical_loss": 4.023274457796983, + "tokens_seen": 390070272 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044534603811434306, + "loss": 3.0006, + "theoretical_loss": 4.023198520852555, + "tokens_seen": 390135808 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004453360080240722, + "loss": 3.0296, + "theoretical_loss": 4.023122600234155, + "tokens_seen": 390201344 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004453259779338014, + "loss": 3.0131, + "theoretical_loss": 4.0230466959355295, + "tokens_seen": 390266880 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044531594784353055, + "loss": 2.9594, + "theoretical_loss": 4.022970807950434, + "tokens_seen": 390332416 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004453059177532598, + "loss": 2.8384, + "theoretical_loss": 4.022894936272621, + "tokens_seen": 390397952 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044529588766298897, + "loss": 2.8603, + "theoretical_loss": 4.022819080895853, + "tokens_seen": 390463488 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044528585757271815, + "loss": 2.8296, + "theoretical_loss": 4.02274324181389, + "tokens_seen": 390529024 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044527582748244733, + "loss": 3.0741, + "theoretical_loss": 4.022667419020499, + "tokens_seen": 390594560 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044526579739217657, + "loss": 2.7069, + "theoretical_loss": 4.0225916125094505, + "tokens_seen": 390660096 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004452557673019057, + "loss": 2.7453, + "theoretical_loss": 4.022515822274515, + "tokens_seen": 390725632 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044524573721163493, + "loss": 2.871, + "theoretical_loss": 4.0224400483094715, + "tokens_seen": 390791168 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044523570712136406, + "loss": 3.0624, + "theoretical_loss": 4.022364290608098, + "tokens_seen": 390856704 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004452256770310933, + "loss": 2.7658, + "theoretical_loss": 4.022288549164178, + "tokens_seen": 390922240 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044521564694082247, + "loss": 2.8246, + "theoretical_loss": 4.022212823971498, + "tokens_seen": 390987776 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044520561685055165, + "loss": 2.7836, + "theoretical_loss": 4.022137115023848, + "tokens_seen": 391053312 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044519558676028083, + "loss": 3.0888, + "theoretical_loss": 4.022061422315023, + "tokens_seen": 391118848 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044518555667001, + "loss": 2.8614, + "theoretical_loss": 4.021985745838817, + "tokens_seen": 391184384 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044517552657973925, + "loss": 2.917, + "theoretical_loss": 4.021910085589032, + "tokens_seen": 391249920 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044516549648946843, + "loss": 2.9533, + "theoretical_loss": 4.021834441559472, + "tokens_seen": 391315456 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004451554663991976, + "loss": 2.9245, + "theoretical_loss": 4.021758813743942, + "tokens_seen": 391380992 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004451454363089268, + "loss": 2.9226, + "theoretical_loss": 4.021683202136255, + "tokens_seen": 391446528 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044513540621865603, + "loss": 2.9608, + "theoretical_loss": 4.021607606730223, + "tokens_seen": 391512064 + }, + { + "epoch": 0.12, + "objective/train/docs_used": 233436, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.0480480194091797, + "objective/train/theoretical_loss": 4.0215320275196635, + "objective/train/tokens_used": 412037600, + "theoretical_loss": 4.0215320275196635, + "tokens_seen": 391577600 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044512537612838516, + "loss": 3.0297, + "theoretical_loss": 4.0215320275196635, + "tokens_seen": 391577600 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004451153460381144, + "loss": 2.9434, + "theoretical_loss": 4.0214564644983986, + "tokens_seen": 391643136 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004451053159478435, + "loss": 3.0186, + "theoretical_loss": 4.021380917660251, + "tokens_seen": 391708672 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044509528585757275, + "loss": 2.7714, + "theoretical_loss": 4.021305386999048, + "tokens_seen": 391774208 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044508525576730194, + "loss": 2.8185, + "theoretical_loss": 4.021229872508621, + "tokens_seen": 391839744 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004450752256770311, + "loss": 2.8653, + "theoretical_loss": 4.021154374182803, + "tokens_seen": 391905280 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004450651955867603, + "loss": 3.0575, + "theoretical_loss": 4.021078892015433, + "tokens_seen": 391970816 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004450551654964895, + "loss": 2.875, + "theoretical_loss": 4.021003426000353, + "tokens_seen": 392036352 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044504513540621866, + "loss": 2.7092, + "theoretical_loss": 4.020927976131404, + "tokens_seen": 392101888 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004450351053159479, + "loss": 2.9594, + "theoretical_loss": 4.0208525424024355, + "tokens_seen": 392167424 + }, + { + "epoch": 0.12, + "learning_rate": 0.000445025075225677, + "loss": 2.8783, + "theoretical_loss": 4.020777124807299, + "tokens_seen": 392232960 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044501504513540626, + "loss": 2.868, + "theoretical_loss": 4.020701723339848, + "tokens_seen": 392298496 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004450050150451354, + "loss": 2.8882, + "theoretical_loss": 4.020626337993941, + "tokens_seen": 392364032 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004449949849548646, + "loss": 2.7886, + "theoretical_loss": 4.020550968763439, + "tokens_seen": 392429568 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004449849548645938, + "loss": 2.7823, + "theoretical_loss": 4.020475615642207, + "tokens_seen": 392495104 + }, + { + "epoch": 0.12, + "learning_rate": 0.000444974924774323, + "loss": 2.8227, + "theoretical_loss": 4.020400278624112, + "tokens_seen": 392560640 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044496489468405216, + "loss": 2.8585, + "theoretical_loss": 4.020324957703025, + "tokens_seen": 392626176 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004449548645937814, + "loss": 3.0705, + "theoretical_loss": 4.020249652872822, + "tokens_seen": 392691712 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004449448345035105, + "loss": 2.9149, + "theoretical_loss": 4.02017436412738, + "tokens_seen": 392757248 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044493480441323976, + "loss": 3.0324, + "theoretical_loss": 4.02009909146058, + "tokens_seen": 392822784 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004449247743229689, + "loss": 2.8887, + "theoretical_loss": 4.020023834866308, + "tokens_seen": 392888320 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004449147442326981, + "loss": 2.9429, + "theoretical_loss": 4.01994859433845, + "tokens_seen": 392953856 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004449047141424273, + "loss": 2.8394, + "theoretical_loss": 4.019873369870899, + "tokens_seen": 393019392 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004448946840521565, + "loss": 3.0886, + "theoretical_loss": 4.019798161457548, + "tokens_seen": 393084928 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044488465396188567, + "loss": 2.7128, + "theoretical_loss": 4.019722969092298, + "tokens_seen": 393150464 + }, + { + "epoch": 0.12, + "objective/train/docs_used": 234056, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.5951128005981445, + "objective/train/theoretical_loss": 4.019647792769048, + "objective/train/tokens_used": 413676000, + "theoretical_loss": 4.019647792769048, + "tokens_seen": 393216000 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044487462387161485, + "loss": 2.8364, + "theoretical_loss": 4.019647792769048, + "tokens_seen": 393216000 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044486459378134403, + "loss": 2.8241, + "theoretical_loss": 4.019572632481702, + "tokens_seen": 393281536 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044485456369107326, + "loss": 3.0436, + "theoretical_loss": 4.01949748822417, + "tokens_seen": 393347072 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004448445336008024, + "loss": 2.9667, + "theoretical_loss": 4.0194223599903625, + "tokens_seen": 393412608 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004448345035105316, + "loss": 3.0447, + "theoretical_loss": 4.019347247774195, + "tokens_seen": 393478144 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044482447342026075, + "loss": 2.7803, + "theoretical_loss": 4.0192721515695835, + "tokens_seen": 393543680 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044481444332999, + "loss": 2.8078, + "theoretical_loss": 4.019197071370451, + "tokens_seen": 393609216 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044480441323971917, + "loss": 2.9105, + "theoretical_loss": 4.019122007170722, + "tokens_seen": 393674752 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044479438314944835, + "loss": 2.7194, + "theoretical_loss": 4.019046958964324, + "tokens_seen": 393740288 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044478435305917753, + "loss": 2.9395, + "theoretical_loss": 4.018971926745189, + "tokens_seen": 393805824 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044477432296890677, + "loss": 2.9033, + "theoretical_loss": 4.01889691050725, + "tokens_seen": 393871360 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004447642928786359, + "loss": 3.0371, + "theoretical_loss": 4.018821910244446, + "tokens_seen": 393936896 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044475426278836513, + "loss": 2.9617, + "theoretical_loss": 4.018746925950719, + "tokens_seen": 394002432 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044474423269809426, + "loss": 2.8997, + "theoretical_loss": 4.018671957620012, + "tokens_seen": 394067968 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004447342026078235, + "loss": 3.1488, + "theoretical_loss": 4.018597005246273, + "tokens_seen": 394133504 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044472417251755267, + "loss": 3.0434, + "theoretical_loss": 4.018522068823454, + "tokens_seen": 394199040 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044471414242728185, + "loss": 2.8733, + "theoretical_loss": 4.01844714834551, + "tokens_seen": 394264576 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044470411233701103, + "loss": 2.9788, + "theoretical_loss": 4.018372243806397, + "tokens_seen": 394330112 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004446940822467402, + "loss": 3.1968, + "theoretical_loss": 4.018297355200076, + "tokens_seen": 394395648 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004446840521564694, + "loss": 3.1115, + "theoretical_loss": 4.018222482520512, + "tokens_seen": 394461184 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044467402206619863, + "loss": 3.0326, + "theoretical_loss": 4.018147625761673, + "tokens_seen": 394526720 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044466399197592776, + "loss": 2.6834, + "theoretical_loss": 4.018072784917529, + "tokens_seen": 394592256 + }, + { + "epoch": 0.12, + "learning_rate": 0.000444653961885657, + "loss": 2.9545, + "theoretical_loss": 4.017997959982055, + "tokens_seen": 394657792 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004446439317953861, + "loss": 2.6859, + "theoretical_loss": 4.017923150949228, + "tokens_seen": 394723328 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044463390170511536, + "loss": 3.0513, + "theoretical_loss": 4.017848357813028, + "tokens_seen": 394788864 + }, + { + "epoch": 0.12, + "objective/train/docs_used": 235431, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.476510524749756, + "objective/train/theoretical_loss": 4.01777358056744, + "objective/train/tokens_used": 415314400, + "theoretical_loss": 4.01777358056744, + "tokens_seen": 394854400 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044462387161484454, + "loss": 3.0749, + "theoretical_loss": 4.01777358056744, + "tokens_seen": 394854400 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004446138415245737, + "loss": 3.2541, + "theoretical_loss": 4.017698819206451, + "tokens_seen": 394919936 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004446038114343029, + "loss": 2.9952, + "theoretical_loss": 4.017624073724052, + "tokens_seen": 394985472 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044459378134403214, + "loss": 2.8619, + "theoretical_loss": 4.017549344114236, + "tokens_seen": 395051008 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044458375125376126, + "loss": 3.0449, + "theoretical_loss": 4.017474630371, + "tokens_seen": 395116544 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004445737211634905, + "loss": 3.2332, + "theoretical_loss": 4.0173999324883445, + "tokens_seen": 395182080 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004445636910732196, + "loss": 2.9548, + "theoretical_loss": 4.0173252504602734, + "tokens_seen": 395247616 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044455366098294886, + "loss": 2.8745, + "theoretical_loss": 4.017250584280794, + "tokens_seen": 395313152 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044454363089267804, + "loss": 3.2634, + "theoretical_loss": 4.017175933943916, + "tokens_seen": 395378688 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004445336008024072, + "loss": 2.8672, + "theoretical_loss": 4.0171012994436515, + "tokens_seen": 395444224 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004445235707121364, + "loss": 2.7562, + "theoretical_loss": 4.017026680774019, + "tokens_seen": 395509760 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004445135406218656, + "loss": 2.8469, + "theoretical_loss": 4.016952077929037, + "tokens_seen": 395575296 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044450351053159477, + "loss": 3.0102, + "theoretical_loss": 4.016877490902731, + "tokens_seen": 395640832 + }, + { + "epoch": 0.12, + "learning_rate": 0.000444493480441324, + "loss": 3.1428, + "theoretical_loss": 4.016802919689124, + "tokens_seen": 395706368 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044448345035105313, + "loss": 3.0475, + "theoretical_loss": 4.0167283642822476, + "tokens_seen": 395771904 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044447342026078236, + "loss": 2.5586, + "theoretical_loss": 4.016653824676135, + "tokens_seen": 395837440 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004444633901705115, + "loss": 2.9757, + "theoretical_loss": 4.016579300864821, + "tokens_seen": 395902976 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004444533600802407, + "loss": 3.0677, + "theoretical_loss": 4.016504792842346, + "tokens_seen": 395968512 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044444332998996996, + "loss": 2.9622, + "theoretical_loss": 4.016430300602751, + "tokens_seen": 396034048 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004444332998996991, + "loss": 2.8269, + "theoretical_loss": 4.016355824140085, + "tokens_seen": 396099584 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004444232698094283, + "loss": 3.0155, + "theoretical_loss": 4.016281363448394, + "tokens_seen": 396165120 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004444132397191575, + "loss": 2.9248, + "theoretical_loss": 4.016206918521731, + "tokens_seen": 396230656 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004444032096288867, + "loss": 2.9422, + "theoretical_loss": 4.016132489354153, + "tokens_seen": 396296192 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044439317953861587, + "loss": 3.0548, + "theoretical_loss": 4.016058075939716, + "tokens_seen": 396361728 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044438314944834505, + "loss": 2.8783, + "theoretical_loss": 4.015983678272484, + "tokens_seen": 396427264 + }, + { + "epoch": 0.12, + "objective/train/docs_used": 235840, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.3891971111297607, + "objective/train/theoretical_loss": 4.015909296346521, + "objective/train/tokens_used": 416952800, + "theoretical_loss": 4.015909296346521, + "tokens_seen": 396492800 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044437311935807423, + "loss": 3.2357, + "theoretical_loss": 4.015909296346521, + "tokens_seen": 396492800 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044436308926780346, + "loss": 2.9519, + "theoretical_loss": 4.015834930155897, + "tokens_seen": 396558336 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004443530591775326, + "loss": 3.0188, + "theoretical_loss": 4.015760579694683, + "tokens_seen": 396623872 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004443430290872618, + "loss": 2.8152, + "theoretical_loss": 4.015686244956953, + "tokens_seen": 396689408 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044433299899699095, + "loss": 2.9015, + "theoretical_loss": 4.015611925936785, + "tokens_seen": 396754944 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004443229689067202, + "loss": 2.8817, + "theoretical_loss": 4.01553762262826, + "tokens_seen": 396820480 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044431293881644937, + "loss": 3.1108, + "theoretical_loss": 4.015463335025465, + "tokens_seen": 396886016 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044430290872617855, + "loss": 2.9996, + "theoretical_loss": 4.015389063122484, + "tokens_seen": 396951552 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044429287863590773, + "loss": 2.9064, + "theoretical_loss": 4.015314806913411, + "tokens_seen": 397017088 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044428284854563697, + "loss": 3.0985, + "theoretical_loss": 4.015240566392338, + "tokens_seen": 397082624 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004442728184553661, + "loss": 2.9756, + "theoretical_loss": 4.015166341553362, + "tokens_seen": 397148160 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044426278836509533, + "loss": 3.0147, + "theoretical_loss": 4.015092132390585, + "tokens_seen": 397213696 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044425275827482446, + "loss": 3.1101, + "theoretical_loss": 4.01501793889811, + "tokens_seen": 397279232 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004442427281845537, + "loss": 2.86, + "theoretical_loss": 4.014943761070043, + "tokens_seen": 397344768 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044423269809428287, + "loss": 2.9253, + "theoretical_loss": 4.014869598900496, + "tokens_seen": 397410304 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044422266800401205, + "loss": 2.6385, + "theoretical_loss": 4.01479545238358, + "tokens_seen": 397475840 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044421263791374123, + "loss": 2.946, + "theoretical_loss": 4.014721321513412, + "tokens_seen": 397541376 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004442026078234704, + "loss": 2.9925, + "theoretical_loss": 4.0146472062841125, + "tokens_seen": 397606912 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004441925777331996, + "loss": 2.8236, + "theoretical_loss": 4.014573106689803, + "tokens_seen": 397672448 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044418254764292883, + "loss": 2.8805, + "theoretical_loss": 4.0144990227246105, + "tokens_seen": 397737984 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044417251755265796, + "loss": 2.918, + "theoretical_loss": 4.014424954382663, + "tokens_seen": 397803520 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004441624874623872, + "loss": 2.7246, + "theoretical_loss": 4.014350901658093, + "tokens_seen": 397869056 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004441524573721163, + "loss": 2.7663, + "theoretical_loss": 4.014276864545036, + "tokens_seen": 397934592 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044414242728184556, + "loss": 2.893, + "theoretical_loss": 4.014202843037632, + "tokens_seen": 398000128 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044413239719157474, + "loss": 2.9995, + "theoretical_loss": 4.014128837130021, + "tokens_seen": 398065664 + }, + { + "epoch": 0.12, + "objective/train/docs_used": 236468, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.1073646545410156, + "objective/train/theoretical_loss": 4.0140548468163475, + "objective/train/tokens_used": 418591200, + "theoretical_loss": 4.0140548468163475, + "tokens_seen": 398131200 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004441223671013039, + "loss": 2.986, + "theoretical_loss": 4.0140548468163475, + "tokens_seen": 398131200 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004441123370110331, + "loss": 3.1624, + "theoretical_loss": 4.0139808720907615, + "tokens_seen": 398196736 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044410230692076234, + "loss": 2.8174, + "theoretical_loss": 4.013906912947412, + "tokens_seen": 398262272 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044409227683049146, + "loss": 3.0515, + "theoretical_loss": 4.013832969380456, + "tokens_seen": 398327808 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004440822467402207, + "loss": 3.0065, + "theoretical_loss": 4.013759041384049, + "tokens_seen": 398393344 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004440722166499498, + "loss": 2.863, + "theoretical_loss": 4.013685128952353, + "tokens_seen": 398458880 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044406218655967906, + "loss": 3.0951, + "theoretical_loss": 4.013611232079531, + "tokens_seen": 398524416 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044405215646940824, + "loss": 2.7914, + "theoretical_loss": 4.01353735075975, + "tokens_seen": 398589952 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004440421263791374, + "loss": 3.0122, + "theoretical_loss": 4.01346348498718, + "tokens_seen": 398655488 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004440320962888666, + "loss": 3.1171, + "theoretical_loss": 4.013389634755995, + "tokens_seen": 398721024 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004440220661985958, + "loss": 2.8991, + "theoretical_loss": 4.013315800060371, + "tokens_seen": 398786560 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044401203610832497, + "loss": 3.042, + "theoretical_loss": 4.013241980894488, + "tokens_seen": 398852096 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004440020060180542, + "loss": 2.9922, + "theoretical_loss": 4.013168177252528, + "tokens_seen": 398917632 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044399197592778333, + "loss": 3.0204, + "theoretical_loss": 4.013094389128677, + "tokens_seen": 398983168 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044398194583751256, + "loss": 2.9238, + "theoretical_loss": 4.0130206165171245, + "tokens_seen": 399048704 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004439719157472417, + "loss": 2.9763, + "theoretical_loss": 4.012946859412062, + "tokens_seen": 399114240 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004439618856569709, + "loss": 3.0638, + "theoretical_loss": 4.012873117807685, + "tokens_seen": 399179776 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004439518555667001, + "loss": 2.7756, + "theoretical_loss": 4.012799391698191, + "tokens_seen": 399245312 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004439418254764293, + "loss": 3.02, + "theoretical_loss": 4.012725681077783, + "tokens_seen": 399310848 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044393179538615847, + "loss": 3.3233, + "theoretical_loss": 4.012651985940666, + "tokens_seen": 399376384 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004439217652958877, + "loss": 2.9859, + "theoretical_loss": 4.012578306281045, + "tokens_seen": 399441920 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044391173520561683, + "loss": 2.8914, + "theoretical_loss": 4.012504642093133, + "tokens_seen": 399507456 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044390170511534607, + "loss": 2.7176, + "theoretical_loss": 4.012430993371144, + "tokens_seen": 399572992 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004438916750250752, + "loss": 3.0069, + "theoretical_loss": 4.012357360109293, + "tokens_seen": 399638528 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044388164493480443, + "loss": 2.8109, + "theoretical_loss": 4.012283742301802, + "tokens_seen": 399704064 + }, + { + "epoch": 0.12, + "objective/train/docs_used": 237759, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.889021396636963, + "objective/train/theoretical_loss": 4.012210139942894, + "objective/train/tokens_used": 420229600, + "theoretical_loss": 4.012210139942894, + "tokens_seen": 399769600 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004438716148445336, + "loss": 3.0325, + "theoretical_loss": 4.012210139942894, + "tokens_seen": 399769600 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004438615847542628, + "loss": 3.027, + "theoretical_loss": 4.012136553026796, + "tokens_seen": 399835136 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044385155466399197, + "loss": 2.8158, + "theoretical_loss": 4.012062981547736, + "tokens_seen": 399900672 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044384152457372115, + "loss": 3.0035, + "theoretical_loss": 4.011989425499947, + "tokens_seen": 399966208 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044383149448345033, + "loss": 2.836, + "theoretical_loss": 4.0119158848776655, + "tokens_seen": 400031744 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044382146439317957, + "loss": 2.8924, + "theoretical_loss": 4.011842359675129, + "tokens_seen": 400097280 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004438114343029087, + "loss": 3.0137, + "theoretical_loss": 4.011768849886582, + "tokens_seen": 400162816 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044380140421263793, + "loss": 3.1184, + "theoretical_loss": 4.011695355506266, + "tokens_seen": 400228352 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004437913741223671, + "loss": 2.8353, + "theoretical_loss": 4.011621876528431, + "tokens_seen": 400293888 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004437813440320963, + "loss": 3.0332, + "theoretical_loss": 4.011548412947327, + "tokens_seen": 400359424 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004437713139418255, + "loss": 3.0166, + "theoretical_loss": 4.011474964757211, + "tokens_seen": 400424960 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044376128385155466, + "loss": 2.7745, + "theoretical_loss": 4.0114015319523375, + "tokens_seen": 400490496 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044375125376128384, + "loss": 2.9423, + "theoretical_loss": 4.011328114526968, + "tokens_seen": 400556032 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044374122367101307, + "loss": 2.9933, + "theoretical_loss": 4.011254712475365, + "tokens_seen": 400621568 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004437311935807422, + "loss": 2.8447, + "theoretical_loss": 4.011181325791797, + "tokens_seen": 400687104 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044372116349047143, + "loss": 2.9317, + "theoretical_loss": 4.011107954470531, + "tokens_seen": 400752640 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044371113340020056, + "loss": 3.0288, + "theoretical_loss": 4.011034598505843, + "tokens_seen": 400818176 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004437011033099298, + "loss": 2.8681, + "theoretical_loss": 4.010961257892005, + "tokens_seen": 400883712 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044369107321965903, + "loss": 2.9443, + "theoretical_loss": 4.010887932623299, + "tokens_seen": 400949248 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044368104312938816, + "loss": 2.9384, + "theoretical_loss": 4.010814622694006, + "tokens_seen": 401014784 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004436710130391174, + "loss": 2.9839, + "theoretical_loss": 4.01074132809841, + "tokens_seen": 401080320 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004436609829488465, + "loss": 2.9703, + "theoretical_loss": 4.0106680488308, + "tokens_seen": 401145856 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044365095285857576, + "loss": 2.8738, + "theoretical_loss": 4.010594784885466, + "tokens_seen": 401211392 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044364092276830494, + "loss": 2.785, + "theoretical_loss": 4.010521536256705, + "tokens_seen": 401276928 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004436308926780341, + "loss": 3.1535, + "theoretical_loss": 4.01044830293881, + "tokens_seen": 401342464 + }, + { + "epoch": 0.12, + "objective/train/docs_used": 238664, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.713977575302124, + "objective/train/theoretical_loss": 4.0103750849260855, + "objective/train/tokens_used": 421868000, + "theoretical_loss": 4.0103750849260855, + "tokens_seen": 401408000 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004436208625877633, + "loss": 2.8016, + "theoretical_loss": 4.0103750849260855, + "tokens_seen": 401408000 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044361083249749254, + "loss": 3.25, + "theoretical_loss": 4.010301882212831, + "tokens_seen": 401473536 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044360080240722166, + "loss": 2.7826, + "theoretical_loss": 4.010228694793356, + "tokens_seen": 401539072 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004435907723169509, + "loss": 3.2868, + "theoretical_loss": 4.010155522661968, + "tokens_seen": 401604608 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044358074222668, + "loss": 3.0055, + "theoretical_loss": 4.0100823658129805, + "tokens_seen": 401670144 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044357071213640926, + "loss": 2.7937, + "theoretical_loss": 4.010009224240708, + "tokens_seen": 401735680 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044356068204613844, + "loss": 3.134, + "theoretical_loss": 4.0099360979394705, + "tokens_seen": 401801216 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004435506519558676, + "loss": 3.0734, + "theoretical_loss": 4.009862986903588, + "tokens_seen": 401866752 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004435406218655968, + "loss": 2.5019, + "theoretical_loss": 4.009789891127387, + "tokens_seen": 401932288 + }, + { + "epoch": 0.12, + "learning_rate": 0.000443530591775326, + "loss": 2.765, + "theoretical_loss": 4.009716810605193, + "tokens_seen": 401997824 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044352056168505517, + "loss": 3.0761, + "theoretical_loss": 4.009643745331338, + "tokens_seen": 402063360 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004435105315947844, + "loss": 2.8237, + "theoretical_loss": 4.009570695300155, + "tokens_seen": 402128896 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044350050150451353, + "loss": 2.6616, + "theoretical_loss": 4.009497660505982, + "tokens_seen": 402194432 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044349047141424276, + "loss": 2.9323, + "theoretical_loss": 4.009424640943157, + "tokens_seen": 402259968 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004434804413239719, + "loss": 2.9478, + "theoretical_loss": 4.009351636606025, + "tokens_seen": 402325504 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004434704112337011, + "loss": 2.8518, + "theoretical_loss": 4.00927864748893, + "tokens_seen": 402391040 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004434603811434303, + "loss": 2.8679, + "theoretical_loss": 4.009205673586222, + "tokens_seen": 402456576 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004434503510531595, + "loss": 2.8751, + "theoretical_loss": 4.009132714892251, + "tokens_seen": 402522112 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044344032096288867, + "loss": 2.8415, + "theoretical_loss": 4.009059771401375, + "tokens_seen": 402587648 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004434302908726179, + "loss": 2.9001, + "theoretical_loss": 4.008986843107949, + "tokens_seen": 402653184 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044342026078234703, + "loss": 3.196, + "theoretical_loss": 4.008913930006336, + "tokens_seen": 402718720 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044341023069207627, + "loss": 2.8807, + "theoretical_loss": 4.008841032090898, + "tokens_seen": 402784256 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004434002006018054, + "loss": 3.2578, + "theoretical_loss": 4.008768149356003, + "tokens_seen": 402849792 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044339017051153463, + "loss": 3.0425, + "theoretical_loss": 4.0086952817960215, + "tokens_seen": 402915328 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004433801404212638, + "loss": 2.9064, + "theoretical_loss": 4.0086224294053245, + "tokens_seen": 402980864 + }, + { + "epoch": 0.12, + "objective/train/docs_used": 240054, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.820873498916626, + "objective/train/theoretical_loss": 4.008549592178291, + "objective/train/tokens_used": 423506400, + "theoretical_loss": 4.008549592178291, + "tokens_seen": 403046400 + }, + { + "epoch": 0.12, + "learning_rate": 0.000443370110330993, + "loss": 2.8536, + "theoretical_loss": 4.008549592178291, + "tokens_seen": 403046400 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044336008024072217, + "loss": 2.9264, + "theoretical_loss": 4.008476770109296, + "tokens_seen": 403111936 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044335005015045135, + "loss": 3.2399, + "theoretical_loss": 4.008403963192725, + "tokens_seen": 403177472 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044334002006018053, + "loss": 3.1324, + "theoretical_loss": 4.008331171422961, + "tokens_seen": 403243008 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044332998996990977, + "loss": 2.8582, + "theoretical_loss": 4.008258394794392, + "tokens_seen": 403308544 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004433199598796389, + "loss": 3.0519, + "theoretical_loss": 4.0081856333014105, + "tokens_seen": 403374080 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044330992978936813, + "loss": 2.9382, + "theoretical_loss": 4.008112886938409, + "tokens_seen": 403439616 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004432998996990973, + "loss": 2.8629, + "theoretical_loss": 4.008040155699783, + "tokens_seen": 403505152 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004432898696088265, + "loss": 2.7001, + "theoretical_loss": 4.007967439579936, + "tokens_seen": 403570688 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004432798395185557, + "loss": 3.1316, + "theoretical_loss": 4.007894738573267, + "tokens_seen": 403636224 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044326980942828486, + "loss": 2.9832, + "theoretical_loss": 4.007822052674185, + "tokens_seen": 403701760 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044325977933801404, + "loss": 2.841, + "theoretical_loss": 4.007749381877097, + "tokens_seen": 403767296 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044324974924774327, + "loss": 3.1991, + "theoretical_loss": 4.007676726176415, + "tokens_seen": 403832832 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004432397191574724, + "loss": 2.9175, + "theoretical_loss": 4.0076040855665545, + "tokens_seen": 403898368 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044322968906720163, + "loss": 3.133, + "theoretical_loss": 4.007531460041933, + "tokens_seen": 403963904 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044321965897693076, + "loss": 2.9991, + "theoretical_loss": 4.00745884959697, + "tokens_seen": 404029440 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044320962888666, + "loss": 2.9381, + "theoretical_loss": 4.00738625422609, + "tokens_seen": 404094976 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004431995987963892, + "loss": 2.9779, + "theoretical_loss": 4.00731367392372, + "tokens_seen": 404160512 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044318956870611836, + "loss": 2.9138, + "theoretical_loss": 4.0072411086842905, + "tokens_seen": 404226048 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044317953861584754, + "loss": 2.9391, + "theoretical_loss": 4.007168558502232, + "tokens_seen": 404291584 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004431695085255767, + "loss": 3.0409, + "theoretical_loss": 4.007096023371981, + "tokens_seen": 404357120 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004431594784353059, + "loss": 2.9361, + "theoretical_loss": 4.007023503287976, + "tokens_seen": 404422656 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044314944834503514, + "loss": 3.0736, + "theoretical_loss": 4.006950998244658, + "tokens_seen": 404488192 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044313941825476427, + "loss": 3.0139, + "theoretical_loss": 4.006878508236473, + "tokens_seen": 404553728 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004431293881644935, + "loss": 2.6532, + "theoretical_loss": 4.006806033257866, + "tokens_seen": 404619264 + }, + { + "epoch": 0.12, + "objective/train/docs_used": 240788, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.8617284297943115, + "objective/train/theoretical_loss": 4.006733573303289, + "objective/train/tokens_used": 425144800, + "theoretical_loss": 4.006733573303289, + "tokens_seen": 404684800 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004431193580742227, + "loss": 2.9572, + "theoretical_loss": 4.006733573303289, + "tokens_seen": 404684800 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044310932798395186, + "loss": 2.8144, + "theoretical_loss": 4.0066611283671945, + "tokens_seen": 404750336 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044309929789368104, + "loss": 2.932, + "theoretical_loss": 4.0065886984440375, + "tokens_seen": 404815872 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004430892678034102, + "loss": 3.02, + "theoretical_loss": 4.00651628352828, + "tokens_seen": 404881408 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004430792377131394, + "loss": 2.8942, + "theoretical_loss": 4.0064438836143825, + "tokens_seen": 404946944 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044306920762286864, + "loss": 2.7816, + "theoretical_loss": 4.00637149869681, + "tokens_seen": 405012480 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044305917753259777, + "loss": 3.0783, + "theoretical_loss": 4.0062991287700305, + "tokens_seen": 405078016 + }, + { + "epoch": 0.12, + "learning_rate": 0.000443049147442327, + "loss": 3.1303, + "theoretical_loss": 4.006226773828514, + "tokens_seen": 405143552 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044303911735205613, + "loss": 2.8421, + "theoretical_loss": 4.006154433866736, + "tokens_seen": 405209088 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044302908726178537, + "loss": 2.9627, + "theoretical_loss": 4.0060821088791725, + "tokens_seen": 405274624 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044301905717151455, + "loss": 3.0079, + "theoretical_loss": 4.006009798860303, + "tokens_seen": 405340160 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044300902708124373, + "loss": 2.9017, + "theoretical_loss": 4.00593750380461, + "tokens_seen": 405405696 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004429989969909729, + "loss": 3.0713, + "theoretical_loss": 4.005865223706579, + "tokens_seen": 405471232 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004429889669007021, + "loss": 2.9242, + "theoretical_loss": 4.005792958560699, + "tokens_seen": 405536768 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044297893681043127, + "loss": 2.9411, + "theoretical_loss": 4.005720708361461, + "tokens_seen": 405602304 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004429689067201605, + "loss": 2.8948, + "theoretical_loss": 4.00564847310336, + "tokens_seen": 405667840 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044295887662988963, + "loss": 3.1312, + "theoretical_loss": 4.005576252780892, + "tokens_seen": 405733376 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044294884653961887, + "loss": 2.9644, + "theoretical_loss": 4.005504047388558, + "tokens_seen": 405798912 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004429388164493481, + "loss": 2.8347, + "theoretical_loss": 4.00543185692086, + "tokens_seen": 405864448 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044292878635907723, + "loss": 2.9561, + "theoretical_loss": 4.005359681372306, + "tokens_seen": 405929984 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044291875626880647, + "loss": 2.7056, + "theoretical_loss": 4.005287520737403, + "tokens_seen": 405995520 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004429087261785356, + "loss": 3.025, + "theoretical_loss": 4.005215375010662, + "tokens_seen": 406061056 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044289869608826483, + "loss": 3.0738, + "theoretical_loss": 4.0051432441866, + "tokens_seen": 406126592 + }, + { + "epoch": 0.12, + "learning_rate": 0.000442888665997994, + "loss": 2.9981, + "theoretical_loss": 4.005071128259734, + "tokens_seen": 406192128 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004428786359077232, + "loss": 2.9137, + "theoretical_loss": 4.004999027224584, + "tokens_seen": 406257664 + }, + { + "epoch": 0.12, + "objective/train/docs_used": 241796, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.28588604927063, + "objective/train/theoretical_loss": 4.004926941075674, + "objective/train/tokens_used": 426783200, + "theoretical_loss": 4.004926941075674, + "tokens_seen": 406323200 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044286860581745237, + "loss": 3.103, + "theoretical_loss": 4.004926941075674, + "tokens_seen": 406323200 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044285857572718155, + "loss": 2.9223, + "theoretical_loss": 4.004854869807528, + "tokens_seen": 406388736 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044284854563691073, + "loss": 3.2729, + "theoretical_loss": 4.004782813414678, + "tokens_seen": 406454272 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044283851554663997, + "loss": 2.9169, + "theoretical_loss": 4.004710771891654, + "tokens_seen": 406519808 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004428284854563691, + "loss": 2.9267, + "theoretical_loss": 4.0046387452329935, + "tokens_seen": 406585344 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044281845536609833, + "loss": 3.2086, + "theoretical_loss": 4.004566733433231, + "tokens_seen": 406650880 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004428084252758275, + "loss": 2.9732, + "theoretical_loss": 4.00449473648691, + "tokens_seen": 406716416 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004427983951855567, + "loss": 2.7591, + "theoretical_loss": 4.004422754388573, + "tokens_seen": 406781952 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004427883650952859, + "loss": 2.7636, + "theoretical_loss": 4.004350787132766, + "tokens_seen": 406847488 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044277833500501506, + "loss": 2.7561, + "theoretical_loss": 4.004278834714039, + "tokens_seen": 406913024 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044276830491474424, + "loss": 2.7872, + "theoretical_loss": 4.004206897126943, + "tokens_seen": 406978560 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044275827482447347, + "loss": 3.0915, + "theoretical_loss": 4.004134974366036, + "tokens_seen": 407044096 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004427482447342026, + "loss": 2.9789, + "theoretical_loss": 4.004063066425874, + "tokens_seen": 407109632 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044273821464393184, + "loss": 2.8987, + "theoretical_loss": 4.003991173301018, + "tokens_seen": 407175168 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044272818455366096, + "loss": 2.9902, + "theoretical_loss": 4.003919294986032, + "tokens_seen": 407240704 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004427181544633902, + "loss": 2.9191, + "theoretical_loss": 4.003847431475482, + "tokens_seen": 407306240 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004427081243731194, + "loss": 2.787, + "theoretical_loss": 4.003775582763939, + "tokens_seen": 407371776 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044269809428284856, + "loss": 3.041, + "theoretical_loss": 4.0037037488459735, + "tokens_seen": 407437312 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044268806419257774, + "loss": 2.9074, + "theoretical_loss": 4.003631929716162, + "tokens_seen": 407502848 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004426780341023069, + "loss": 3.0442, + "theoretical_loss": 4.0035601253690825, + "tokens_seen": 407568384 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004426680040120361, + "loss": 3.0844, + "theoretical_loss": 4.0034883357993145, + "tokens_seen": 407633920 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044265797392176534, + "loss": 3.1122, + "theoretical_loss": 4.003416561001444, + "tokens_seen": 407699456 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044264794383149447, + "loss": 2.9653, + "theoretical_loss": 4.003344800970056, + "tokens_seen": 407764992 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004426379137412237, + "loss": 2.9027, + "theoretical_loss": 4.003273055699741, + "tokens_seen": 407830528 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004426278836509529, + "loss": 3.0631, + "theoretical_loss": 4.00320132518509, + "tokens_seen": 407896064 + }, + { + "epoch": 0.12, + "objective/train/docs_used": 242638, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.3259775638580322, + "objective/train/theoretical_loss": 4.0031296094207, + "objective/train/tokens_used": 428421600, + "theoretical_loss": 4.0031296094207, + "tokens_seen": 407961600 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044261785356068206, + "loss": 3.0672, + "theoretical_loss": 4.0031296094207, + "tokens_seen": 407961600 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044260782347041124, + "loss": 2.9804, + "theoretical_loss": 4.003057908401168, + "tokens_seen": 408027136 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004425977933801404, + "loss": 2.997, + "theoretical_loss": 4.002986222121095, + "tokens_seen": 408092672 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004425877632898696, + "loss": 2.983, + "theoretical_loss": 4.002914550575085, + "tokens_seen": 408158208 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044257773319959884, + "loss": 3.242, + "theoretical_loss": 4.002842893757744, + "tokens_seen": 408223744 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044256770310932797, + "loss": 3.0042, + "theoretical_loss": 4.002771251663681, + "tokens_seen": 408289280 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004425576730190572, + "loss": 2.6908, + "theoretical_loss": 4.00269962428751, + "tokens_seen": 408354816 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044254764292878633, + "loss": 2.9479, + "theoretical_loss": 4.0026280116238455, + "tokens_seen": 408420352 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044253761283851557, + "loss": 2.9515, + "theoretical_loss": 4.002556413667305, + "tokens_seen": 408485888 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044252758274824475, + "loss": 2.7849, + "theoretical_loss": 4.0024848304125085, + "tokens_seen": 408551424 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044251755265797393, + "loss": 2.7016, + "theoretical_loss": 4.002413261854081, + "tokens_seen": 408616960 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004425075225677031, + "loss": 2.9665, + "theoretical_loss": 4.002341707986648, + "tokens_seen": 408682496 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004424974924774323, + "loss": 3.1005, + "theoretical_loss": 4.00227016880484, + "tokens_seen": 408748032 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044248746238716147, + "loss": 3.079, + "theoretical_loss": 4.002198644303288, + "tokens_seen": 408813568 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004424774322968907, + "loss": 2.7911, + "theoretical_loss": 4.002127134476626, + "tokens_seen": 408879104 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044246740220661983, + "loss": 3.1025, + "theoretical_loss": 4.002055639319494, + "tokens_seen": 408944640 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044245737211634907, + "loss": 3.1551, + "theoretical_loss": 4.00198415882653, + "tokens_seen": 409010176 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044244734202607825, + "loss": 2.7239, + "theoretical_loss": 4.001912692992379, + "tokens_seen": 409075712 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044243731193580743, + "loss": 3.0204, + "theoretical_loss": 4.001841241811688, + "tokens_seen": 409141248 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004424272818455366, + "loss": 3.0167, + "theoretical_loss": 4.001769805279103, + "tokens_seen": 409206784 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004424172517552658, + "loss": 3.0888, + "theoretical_loss": 4.001698383389279, + "tokens_seen": 409272320 + }, + { + "epoch": 0.12, + "learning_rate": 0.000442407221664995, + "loss": 3.0743, + "theoretical_loss": 4.001626976136868, + "tokens_seen": 409337856 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004423971915747242, + "loss": 2.8056, + "theoretical_loss": 4.001555583516529, + "tokens_seen": 409403392 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044238716148445334, + "loss": 2.8722, + "theoretical_loss": 4.001484205522922, + "tokens_seen": 409468928 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044237713139418257, + "loss": 2.79, + "theoretical_loss": 4.00141284215071, + "tokens_seen": 409534464 + }, + { + "epoch": 0.12, + "objective/train/docs_used": 244086, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.0071194171905518, + "objective/train/theoretical_loss": 4.001341493394558, + "objective/train/tokens_used": 430060000, + "theoretical_loss": 4.001341493394558, + "tokens_seen": 409600000 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004423671013039117, + "loss": 3.0407, + "theoretical_loss": 4.001341493394558, + "tokens_seen": 409600000 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044235707121364093, + "loss": 2.9328, + "theoretical_loss": 4.0012701592491355, + "tokens_seen": 409665536 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004423470411233701, + "loss": 3.1627, + "theoretical_loss": 4.001198839709113, + "tokens_seen": 409731072 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004423370110330993, + "loss": 2.9127, + "theoretical_loss": 4.001127534769166, + "tokens_seen": 409796608 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004423269809428285, + "loss": 2.739, + "theoretical_loss": 4.001056244423971, + "tokens_seen": 409862144 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004423169508525577, + "loss": 2.9402, + "theoretical_loss": 4.000984968668206, + "tokens_seen": 409927680 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044230692076228684, + "loss": 2.9893, + "theoretical_loss": 4.000913707496556, + "tokens_seen": 409993216 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004422968906720161, + "loss": 2.9195, + "theoretical_loss": 4.000842460903706, + "tokens_seen": 410058752 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004422868605817452, + "loss": 2.7873, + "theoretical_loss": 4.000771228884344, + "tokens_seen": 410124288 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044227683049147444, + "loss": 2.9708, + "theoretical_loss": 4.0007000114331595, + "tokens_seen": 410189824 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004422668004012036, + "loss": 2.9055, + "theoretical_loss": 4.000628808544848, + "tokens_seen": 410255360 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004422567703109328, + "loss": 2.9924, + "theoretical_loss": 4.000557620214105, + "tokens_seen": 410320896 + }, + { + "epoch": 0.12, + "learning_rate": 0.000442246740220662, + "loss": 2.8924, + "theoretical_loss": 4.000486446435629, + "tokens_seen": 410386432 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044223671013039116, + "loss": 3.0091, + "theoretical_loss": 4.000415287204124, + "tokens_seen": 410451968 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044222668004012034, + "loss": 3.0762, + "theoretical_loss": 4.000344142514294, + "tokens_seen": 410517504 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004422166499498496, + "loss": 2.762, + "theoretical_loss": 4.0002730123608465, + "tokens_seen": 410583040 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004422066198595787, + "loss": 2.9209, + "theoretical_loss": 4.000201896738492, + "tokens_seen": 410648576 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044219658976930794, + "loss": 2.9352, + "theoretical_loss": 4.000130795641944, + "tokens_seen": 410714112 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004421865596790371, + "loss": 3.0744, + "theoretical_loss": 4.0000597090659165, + "tokens_seen": 410779648 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004421765295887663, + "loss": 2.9166, + "theoretical_loss": 3.9999886370051305, + "tokens_seen": 410845184 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044216649949849554, + "loss": 3.0432, + "theoretical_loss": 3.9999175794543063, + "tokens_seen": 410910720 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044215646940822467, + "loss": 3.0351, + "theoretical_loss": 3.9998465364081683, + "tokens_seen": 410976256 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004421464393179539, + "loss": 2.9645, + "theoretical_loss": 3.999775507861443, + "tokens_seen": 411041792 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004421364092276831, + "loss": 2.886, + "theoretical_loss": 3.999704493808861, + "tokens_seen": 411107328 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044212637913741226, + "loss": 3.0806, + "theoretical_loss": 3.9996334942451544, + "tokens_seen": 411172864 + }, + { + "epoch": 0.12, + "objective/train/docs_used": 244630, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.1092371940612793, + "objective/train/theoretical_loss": 3.999562509165058, + "objective/train/tokens_used": 431698400, + "theoretical_loss": 3.999562509165058, + "tokens_seen": 411238400 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044211634904714144, + "loss": 2.9667, + "theoretical_loss": 3.999562509165058, + "tokens_seen": 411238400 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004421063189568706, + "loss": 3.0451, + "theoretical_loss": 3.9994915385633103, + "tokens_seen": 411303936 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004420962888665998, + "loss": 2.9178, + "theoretical_loss": 3.9994205824346514, + "tokens_seen": 411369472 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044208625877632904, + "loss": 2.9462, + "theoretical_loss": 3.9993496407738256, + "tokens_seen": 411435008 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044207622868605817, + "loss": 2.8666, + "theoretical_loss": 3.9992787135755785, + "tokens_seen": 411500544 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004420661985957874, + "loss": 2.9229, + "theoretical_loss": 3.999207800834659, + "tokens_seen": 411566080 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044205616850551653, + "loss": 3.1904, + "theoretical_loss": 3.9991369025458194, + "tokens_seen": 411631616 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044204613841524577, + "loss": 2.874, + "theoretical_loss": 3.999066018703814, + "tokens_seen": 411697152 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044203610832497495, + "loss": 2.9923, + "theoretical_loss": 3.9989951493033997, + "tokens_seen": 411762688 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044202607823470413, + "loss": 3.0227, + "theoretical_loss": 3.9989242943393357, + "tokens_seen": 411828224 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004420160481444333, + "loss": 2.971, + "theoretical_loss": 3.998853453806386, + "tokens_seen": 411893760 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004420060180541625, + "loss": 3.0149, + "theoretical_loss": 3.998782627699315, + "tokens_seen": 411959296 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044199598796389167, + "loss": 3.0236, + "theoretical_loss": 3.998711816012891, + "tokens_seen": 412024832 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004419859578736209, + "loss": 2.7858, + "theoretical_loss": 3.9986410187418855, + "tokens_seen": 412090368 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044197592778335003, + "loss": 2.9733, + "theoretical_loss": 3.9985702358810715, + "tokens_seen": 412155904 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044196589769307927, + "loss": 3.051, + "theoretical_loss": 3.9984994674252254, + "tokens_seen": 412221440 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044195586760280845, + "loss": 3.0019, + "theoretical_loss": 3.998428713369126, + "tokens_seen": 412286976 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044194583751253763, + "loss": 2.7574, + "theoretical_loss": 3.9983579737075554, + "tokens_seen": 412352512 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004419358074222668, + "loss": 2.8937, + "theoretical_loss": 3.9982872484352976, + "tokens_seen": 412418048 + }, + { + "epoch": 0.12, + "learning_rate": 0.000441925777331996, + "loss": 2.9336, + "theoretical_loss": 3.9982165375471403, + "tokens_seen": 412483584 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004419157472417252, + "loss": 2.8124, + "theoretical_loss": 3.998145841037873, + "tokens_seen": 412549120 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004419057171514544, + "loss": 2.8761, + "theoretical_loss": 3.9980751589022887, + "tokens_seen": 412614656 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044189568706118354, + "loss": 2.9633, + "theoretical_loss": 3.9980044911351826, + "tokens_seen": 412680192 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044188565697091277, + "loss": 2.8416, + "theoretical_loss": 3.9979338377313525, + "tokens_seen": 412745728 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004418756268806419, + "loss": 2.7621, + "theoretical_loss": 3.997863198685599, + "tokens_seen": 412811264 + }, + { + "epoch": 0.13, + "objective/train/docs_used": 245807, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.594975709915161, + "objective/train/theoretical_loss": 3.997792573992726, + "objective/train/tokens_used": 433336800, + "theoretical_loss": 3.997792573992726, + "tokens_seen": 412876800 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044186559679037113, + "loss": 2.8874, + "theoretical_loss": 3.997792573992726, + "tokens_seen": 412876800 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004418555667001003, + "loss": 2.7365, + "theoretical_loss": 3.997721963647539, + "tokens_seen": 412942336 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004418455366098295, + "loss": 3.0723, + "theoretical_loss": 3.997651367644848, + "tokens_seen": 413007872 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004418355065195587, + "loss": 3.0726, + "theoretical_loss": 3.997580785979464, + "tokens_seen": 413073408 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004418254764292879, + "loss": 2.9532, + "theoretical_loss": 3.9975102186462004, + "tokens_seen": 413138944 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044181544633901704, + "loss": 2.9759, + "theoretical_loss": 3.997439665639875, + "tokens_seen": 413204480 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004418054162487463, + "loss": 2.9137, + "theoretical_loss": 3.9973691269553076, + "tokens_seen": 413270016 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004417953861584754, + "loss": 2.9483, + "theoretical_loss": 3.9972986025873203, + "tokens_seen": 413335552 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044178535606820464, + "loss": 2.8368, + "theoretical_loss": 3.9972280925307384, + "tokens_seen": 413401088 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004417753259779338, + "loss": 3.0589, + "theoretical_loss": 3.9971575967803887, + "tokens_seen": 413466624 + }, + { + "epoch": 0.13, + "learning_rate": 0.000441765295887663, + "loss": 2.8318, + "theoretical_loss": 3.9970871153311025, + "tokens_seen": 413532160 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004417552657973922, + "loss": 2.9068, + "theoretical_loss": 3.9970166481777127, + "tokens_seen": 413597696 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044174523570712136, + "loss": 2.9722, + "theoretical_loss": 3.996946195315055, + "tokens_seen": 413663232 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044173520561685054, + "loss": 2.838, + "theoretical_loss": 3.996875756737968, + "tokens_seen": 413728768 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004417251755265798, + "loss": 2.9816, + "theoretical_loss": 3.996805332441293, + "tokens_seen": 413794304 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004417151454363089, + "loss": 2.8764, + "theoretical_loss": 3.996734922419873, + "tokens_seen": 413859840 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044170511534603814, + "loss": 2.9898, + "theoretical_loss": 3.996664526668555, + "tokens_seen": 413925376 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044169508525576727, + "loss": 2.727, + "theoretical_loss": 3.9965941451821894, + "tokens_seen": 413990912 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004416850551654965, + "loss": 2.8692, + "theoretical_loss": 3.996523777955626, + "tokens_seen": 414056448 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004416750250752257, + "loss": 2.9088, + "theoretical_loss": 3.996453424983721, + "tokens_seen": 414121984 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044166499498495487, + "loss": 2.8191, + "theoretical_loss": 3.9963830862613303, + "tokens_seen": 414187520 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044165496489468405, + "loss": 3.1524, + "theoretical_loss": 3.9963127617833143, + "tokens_seen": 414253056 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004416449348044133, + "loss": 2.9403, + "theoretical_loss": 3.9962424515445356, + "tokens_seen": 414318592 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004416349047141424, + "loss": 3.0291, + "theoretical_loss": 3.99617215553986, + "tokens_seen": 414384128 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044162487462387164, + "loss": 2.6754, + "theoretical_loss": 3.9961018737641547, + "tokens_seen": 414449664 + }, + { + "epoch": 0.13, + "objective/train/docs_used": 246540, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.9917945861816406, + "objective/train/theoretical_loss": 3.99603160621229, + "objective/train/tokens_used": 434975200, + "theoretical_loss": 3.99603160621229, + "tokens_seen": 414515200 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044161484453360077, + "loss": 2.8112, + "theoretical_loss": 3.99603160621229, + "tokens_seen": 414515200 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044160481444333, + "loss": 2.9529, + "theoretical_loss": 3.9959613528791396, + "tokens_seen": 414580736 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004415947843530592, + "loss": 2.7827, + "theoretical_loss": 3.995891113759579, + "tokens_seen": 414646272 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044158475426278837, + "loss": 2.8413, + "theoretical_loss": 3.9958208888484865, + "tokens_seen": 414711808 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044157472417251755, + "loss": 2.7472, + "theoretical_loss": 3.9957506781407446, + "tokens_seen": 414777344 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044156469408224673, + "loss": 3.0023, + "theoretical_loss": 3.9956804816312355, + "tokens_seen": 414842880 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004415546639919759, + "loss": 2.7607, + "theoretical_loss": 3.995610299314846, + "tokens_seen": 414908416 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044154463390170515, + "loss": 2.7899, + "theoretical_loss": 3.9955401311864662, + "tokens_seen": 414973952 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004415346038114343, + "loss": 2.9712, + "theoretical_loss": 3.9954699772409867, + "tokens_seen": 415039488 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004415245737211635, + "loss": 3.0409, + "theoretical_loss": 3.995399837473302, + "tokens_seen": 415105024 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044151454363089264, + "loss": 2.9101, + "theoretical_loss": 3.9953297118783104, + "tokens_seen": 415170560 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044150451354062187, + "loss": 2.8193, + "theoretical_loss": 3.99525960045091, + "tokens_seen": 415236096 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044149448345035105, + "loss": 3.0032, + "theoretical_loss": 3.9951895031860043, + "tokens_seen": 415301632 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044148445336008023, + "loss": 2.7458, + "theoretical_loss": 3.9951194200784976, + "tokens_seen": 415367168 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004414744232698094, + "loss": 2.7529, + "theoretical_loss": 3.9950493511232974, + "tokens_seen": 415432704 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044146439317953865, + "loss": 2.9451, + "theoretical_loss": 3.9949792963153143, + "tokens_seen": 415498240 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004414543630892678, + "loss": 2.89, + "theoretical_loss": 3.9949092556494614, + "tokens_seen": 415563776 + }, + { + "epoch": 0.13, + "learning_rate": 0.000441444332998997, + "loss": 3.0349, + "theoretical_loss": 3.994839229120654, + "tokens_seen": 415629312 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004414343029087262, + "loss": 2.6928, + "theoretical_loss": 3.99476921672381, + "tokens_seen": 415694848 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004414242728184554, + "loss": 2.8885, + "theoretical_loss": 3.99469921845385, + "tokens_seen": 415760384 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004414142427281846, + "loss": 3.0158, + "theoretical_loss": 3.994629234305698, + "tokens_seen": 415825920 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044140421263791374, + "loss": 2.9806, + "theoretical_loss": 3.9945592642742795, + "tokens_seen": 415891456 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044139418254764297, + "loss": 2.9039, + "theoretical_loss": 3.994489308354523, + "tokens_seen": 415956992 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004413841524573721, + "loss": 2.9566, + "theoretical_loss": 3.9944193665413605, + "tokens_seen": 416022528 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044137412236710133, + "loss": 2.916, + "theoretical_loss": 3.9943494388297256, + "tokens_seen": 416088064 + }, + { + "epoch": 0.13, + "objective/train/docs_used": 247824, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.9209835529327393, + "objective/train/theoretical_loss": 3.994279525214554, + "objective/train/tokens_used": 436613600, + "theoretical_loss": 3.994279525214554, + "tokens_seen": 416153600 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004413640922768305, + "loss": 3.0085, + "theoretical_loss": 3.994279525214554, + "tokens_seen": 416153600 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004413540621865597, + "loss": 2.9901, + "theoretical_loss": 3.9942096256907864, + "tokens_seen": 416219136 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004413440320962889, + "loss": 3.0349, + "theoretical_loss": 3.994139740253363, + "tokens_seen": 416284672 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004413340020060181, + "loss": 2.9968, + "theoretical_loss": 3.9940698688972285, + "tokens_seen": 416350208 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044132397191574724, + "loss": 3.1658, + "theoretical_loss": 3.99400001161733, + "tokens_seen": 416415744 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004413139418254765, + "loss": 2.8617, + "theoretical_loss": 3.9939301684086175, + "tokens_seen": 416481280 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004413039117352056, + "loss": 3.0238, + "theoretical_loss": 3.9938603392660426, + "tokens_seen": 416546816 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044129388164493484, + "loss": 3.1261, + "theoretical_loss": 3.9937905241845595, + "tokens_seen": 416612352 + }, + { + "epoch": 0.13, + "learning_rate": 0.000441283851554664, + "loss": 2.8151, + "theoretical_loss": 3.993720723159127, + "tokens_seen": 416677888 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004412738214643932, + "loss": 3.0868, + "theoretical_loss": 3.9936509361847037, + "tokens_seen": 416743424 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004412637913741224, + "loss": 3.1087, + "theoretical_loss": 3.9935811632562532, + "tokens_seen": 416808960 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044125376128385156, + "loss": 2.9504, + "theoretical_loss": 3.99351140436874, + "tokens_seen": 416874496 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044124373119358074, + "loss": 2.9339, + "theoretical_loss": 3.9934416595171323, + "tokens_seen": 416940032 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044123370110331, + "loss": 3.03, + "theoretical_loss": 3.9933719286963996, + "tokens_seen": 417005568 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004412236710130391, + "loss": 2.9606, + "theoretical_loss": 3.993302211901516, + "tokens_seen": 417071104 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044121364092276834, + "loss": 2.9725, + "theoretical_loss": 3.9932325091274565, + "tokens_seen": 417136640 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044120361083249747, + "loss": 3.0859, + "theoretical_loss": 3.993162820369199, + "tokens_seen": 417202176 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004411935807422267, + "loss": 3.0843, + "theoretical_loss": 3.9930931456217245, + "tokens_seen": 417267712 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004411835506519559, + "loss": 3.1347, + "theoretical_loss": 3.993023484880016, + "tokens_seen": 417333248 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044117352056168507, + "loss": 3.0881, + "theoretical_loss": 3.9929538381390595, + "tokens_seen": 417398784 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044116349047141425, + "loss": 2.8678, + "theoretical_loss": 3.9928842053938443, + "tokens_seen": 417464320 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004411534603811435, + "loss": 2.9665, + "theoretical_loss": 3.9928145866393594, + "tokens_seen": 417529856 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004411434302908726, + "loss": 2.8728, + "theoretical_loss": 3.9927449818706005, + "tokens_seen": 417595392 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044113340020060184, + "loss": 3.0655, + "theoretical_loss": 3.992675391082563, + "tokens_seen": 417660928 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044112337011033097, + "loss": 2.9925, + "theoretical_loss": 3.9926058142702456, + "tokens_seen": 417726464 + }, + { + "epoch": 0.13, + "objective/train/docs_used": 248559, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7170052528381348, + "objective/train/theoretical_loss": 3.9925362514286498, + "objective/train/tokens_used": 438252000, + "theoretical_loss": 3.9925362514286498, + "tokens_seen": 417792000 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004411133400200602, + "loss": 2.7992, + "theoretical_loss": 3.9925362514286498, + "tokens_seen": 417792000 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004411033099297894, + "loss": 3.0964, + "theoretical_loss": 3.992466702552779, + "tokens_seen": 417857536 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044109327983951857, + "loss": 3.028, + "theoretical_loss": 3.99239716763764, + "tokens_seen": 417923072 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044108324974924775, + "loss": 2.6909, + "theoretical_loss": 3.9923276466782425, + "tokens_seen": 417988608 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044107321965897693, + "loss": 2.986, + "theoretical_loss": 3.9922581396695973, + "tokens_seen": 418054144 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004410631895687061, + "loss": 3.061, + "theoretical_loss": 3.992188646606719, + "tokens_seen": 418119680 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044105315947843535, + "loss": 3.0437, + "theoretical_loss": 3.992119167484624, + "tokens_seen": 418185216 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004410431293881645, + "loss": 3.0119, + "theoretical_loss": 3.992049702298332, + "tokens_seen": 418250752 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004410330992978937, + "loss": 2.7768, + "theoretical_loss": 3.9919802510428646, + "tokens_seen": 418316288 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044102306920762284, + "loss": 3.191, + "theoretical_loss": 3.991910813713246, + "tokens_seen": 418381824 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044101303911735207, + "loss": 2.8176, + "theoretical_loss": 3.991841390304504, + "tokens_seen": 418447360 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044100300902708125, + "loss": 2.7464, + "theoretical_loss": 3.991771980811667, + "tokens_seen": 418512896 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044099297893681043, + "loss": 2.8179, + "theoretical_loss": 3.991702585229768, + "tokens_seen": 418578432 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004409829488465396, + "loss": 2.812, + "theoretical_loss": 3.9916332035538415, + "tokens_seen": 418643968 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044097291875626885, + "loss": 2.742, + "theoretical_loss": 3.9915638357789245, + "tokens_seen": 418709504 + }, + { + "epoch": 0.13, + "learning_rate": 0.000440962888665998, + "loss": 2.8186, + "theoretical_loss": 3.991494481900056, + "tokens_seen": 418775040 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004409528585757272, + "loss": 2.9487, + "theoretical_loss": 3.9914251419122797, + "tokens_seen": 418840576 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044094282848545634, + "loss": 2.8501, + "theoretical_loss": 3.99135581581064, + "tokens_seen": 418906112 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004409327983951856, + "loss": 3.2822, + "theoretical_loss": 3.991286503590184, + "tokens_seen": 418971648 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044092276830491476, + "loss": 3.0445, + "theoretical_loss": 3.9912172052459614, + "tokens_seen": 419037184 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044091273821464394, + "loss": 2.8403, + "theoretical_loss": 3.9911479207730247, + "tokens_seen": 419102720 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004409027081243731, + "loss": 2.825, + "theoretical_loss": 3.991078650166429, + "tokens_seen": 419168256 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004408926780341023, + "loss": 3.0838, + "theoretical_loss": 3.9910093934212325, + "tokens_seen": 419233792 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004408826479438315, + "loss": 2.9763, + "theoretical_loss": 3.990940150532494, + "tokens_seen": 419299328 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004408726178535607, + "loss": 2.6652, + "theoretical_loss": 3.990870921495277, + "tokens_seen": 419364864 + }, + { + "epoch": 0.13, + "objective/train/docs_used": 249785, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.660876512527466, + "objective/train/theoretical_loss": 3.990801706304647, + "objective/train/tokens_used": 439890400, + "theoretical_loss": 3.990801706304647, + "tokens_seen": 419430400 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044086258776328984, + "loss": 2.9322, + "theoretical_loss": 3.990801706304647, + "tokens_seen": 419430400 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004408525576730191, + "loss": 2.9812, + "theoretical_loss": 3.99073250495567, + "tokens_seen": 419495936 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004408425275827482, + "loss": 2.8229, + "theoretical_loss": 3.990663317443418, + "tokens_seen": 419561472 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044083249749247744, + "loss": 3.0524, + "theoretical_loss": 3.990594143762962, + "tokens_seen": 419627008 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004408224674022066, + "loss": 3.0486, + "theoretical_loss": 3.9905249839093786, + "tokens_seen": 419692544 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004408124373119358, + "loss": 3.0592, + "theoretical_loss": 3.9904558378777453, + "tokens_seen": 419758080 + }, + { + "epoch": 0.13, + "learning_rate": 0.000440802407221665, + "loss": 2.8402, + "theoretical_loss": 3.990386705663142, + "tokens_seen": 419823616 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004407923771313942, + "loss": 2.6819, + "theoretical_loss": 3.9903175872606518, + "tokens_seen": 419889152 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044078234704112335, + "loss": 2.7856, + "theoretical_loss": 3.990248482665359, + "tokens_seen": 419954688 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004407723169508526, + "loss": 2.9342, + "theoretical_loss": 3.9901793918723527, + "tokens_seen": 420020224 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004407622868605817, + "loss": 2.7905, + "theoretical_loss": 3.9901103148767225, + "tokens_seen": 420085760 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044075225677031094, + "loss": 2.8555, + "theoretical_loss": 3.9900412516735617, + "tokens_seen": 420151296 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004407422266800401, + "loss": 2.7272, + "theoretical_loss": 3.989972202257966, + "tokens_seen": 420216832 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004407321965897693, + "loss": 3.2052, + "theoretical_loss": 3.989903166625032, + "tokens_seen": 420282368 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004407221664994985, + "loss": 3.0387, + "theoretical_loss": 3.989834144769861, + "tokens_seen": 420347904 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044071213640922767, + "loss": 3.0555, + "theoretical_loss": 3.9897651366875557, + "tokens_seen": 420413440 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044070210631895685, + "loss": 3.096, + "theoretical_loss": 3.989696142373221, + "tokens_seen": 420478976 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004406920762286861, + "loss": 2.9167, + "theoretical_loss": 3.9896271618219656, + "tokens_seen": 420544512 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044068204613841527, + "loss": 3.0977, + "theoretical_loss": 3.9895581950288994, + "tokens_seen": 420610048 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044067201604814445, + "loss": 3.0172, + "theoretical_loss": 3.9894892419891352, + "tokens_seen": 420675584 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004406619859578737, + "loss": 2.896, + "theoretical_loss": 3.989420302697789, + "tokens_seen": 420741120 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004406519558676028, + "loss": 2.9206, + "theoretical_loss": 3.989351377149978, + "tokens_seen": 420806656 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044064192577733204, + "loss": 2.9936, + "theoretical_loss": 3.989282465340823, + "tokens_seen": 420872192 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044063189568706117, + "loss": 2.913, + "theoretical_loss": 3.9892135672654465, + "tokens_seen": 420937728 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004406218655967904, + "loss": 3.0088, + "theoretical_loss": 3.9891446829189734, + "tokens_seen": 421003264 + }, + { + "epoch": 0.13, + "objective/train/docs_used": 250392, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.0226376056671143, + "objective/train/theoretical_loss": 3.989075812296533, + "objective/train/tokens_used": 441528800, + "theoretical_loss": 3.989075812296533, + "tokens_seen": 421068800 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004406118355065196, + "loss": 2.7888, + "theoretical_loss": 3.989075812296533, + "tokens_seen": 421068800 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044060180541624877, + "loss": 3.1118, + "theoretical_loss": 3.989006955393254, + "tokens_seen": 421134336 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044059177532597795, + "loss": 3.0743, + "theoretical_loss": 3.98893811220427, + "tokens_seen": 421199872 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044058174523570713, + "loss": 2.9641, + "theoretical_loss": 3.9888692827247167, + "tokens_seen": 421265408 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004405717151454363, + "loss": 3.0078, + "theoretical_loss": 3.9888004669497317, + "tokens_seen": 421330944 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044056168505516555, + "loss": 3.0572, + "theoretical_loss": 3.988731664874454, + "tokens_seen": 421396480 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004405516549648947, + "loss": 2.9755, + "theoretical_loss": 3.9886628764940277, + "tokens_seen": 421462016 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004405416248746239, + "loss": 2.9094, + "theoretical_loss": 3.988594101803598, + "tokens_seen": 421527552 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044053159478435304, + "loss": 3.0133, + "theoretical_loss": 3.9885253407983114, + "tokens_seen": 421593088 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044052156469408227, + "loss": 3.1173, + "theoretical_loss": 3.9884565934733196, + "tokens_seen": 421658624 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044051153460381145, + "loss": 2.9411, + "theoretical_loss": 3.9883878598237743, + "tokens_seen": 421724160 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044050150451354063, + "loss": 2.7918, + "theoretical_loss": 3.9883191398448306, + "tokens_seen": 421789696 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004404914744232698, + "loss": 2.772, + "theoretical_loss": 3.9882504335316464, + "tokens_seen": 421855232 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044048144433299905, + "loss": 2.7503, + "theoretical_loss": 3.988181740879382, + "tokens_seen": 421920768 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004404714142427282, + "loss": 2.9479, + "theoretical_loss": 3.9881130618831984, + "tokens_seen": 421986304 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004404613841524574, + "loss": 2.9715, + "theoretical_loss": 3.988044396538263, + "tokens_seen": 422051840 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044045135406218654, + "loss": 2.8845, + "theoretical_loss": 3.9879757448397406, + "tokens_seen": 422117376 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004404413239719158, + "loss": 3.0707, + "theoretical_loss": 3.9879071067828034, + "tokens_seen": 422182912 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044043129388164496, + "loss": 3.2662, + "theoretical_loss": 3.9878384823626227, + "tokens_seen": 422248448 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044042126379137414, + "loss": 3.1393, + "theoretical_loss": 3.987769871574373, + "tokens_seen": 422313984 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004404112337011033, + "loss": 2.9535, + "theoretical_loss": 3.9877012744132316, + "tokens_seen": 422379520 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004404012036108325, + "loss": 2.67, + "theoretical_loss": 3.987632690874379, + "tokens_seen": 422445056 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004403911735205617, + "loss": 2.9021, + "theoretical_loss": 3.987564120952997, + "tokens_seen": 422510592 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004403811434302909, + "loss": 2.8681, + "theoretical_loss": 3.98749556464427, + "tokens_seen": 422576128 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044037111334002004, + "loss": 2.7834, + "theoretical_loss": 3.987427021943385, + "tokens_seen": 422641664 + }, + { + "epoch": 0.13, + "objective/train/docs_used": 251199, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.432645559310913, + "objective/train/theoretical_loss": 3.987358492845532, + "objective/train/tokens_used": 443167200, + "theoretical_loss": 3.987358492845532, + "tokens_seen": 422707200 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004403610832497493, + "loss": 2.7934, + "theoretical_loss": 3.987358492845532, + "tokens_seen": 422707200 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004403510531594784, + "loss": 2.7831, + "theoretical_loss": 3.9872899773459025, + "tokens_seen": 422772736 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044034102306920764, + "loss": 3.0529, + "theoretical_loss": 3.987221475439691, + "tokens_seen": 422838272 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004403309929789368, + "loss": 2.7232, + "theoretical_loss": 3.9871529871220943, + "tokens_seen": 422903808 + }, + { + "epoch": 0.13, + "learning_rate": 0.000440320962888666, + "loss": 2.9259, + "theoretical_loss": 3.9870845123883116, + "tokens_seen": 422969344 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004403109327983952, + "loss": 2.9452, + "theoretical_loss": 3.987016051233546, + "tokens_seen": 423034880 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004403009027081244, + "loss": 2.8782, + "theoretical_loss": 3.9869476036529994, + "tokens_seen": 423100416 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044029087261785355, + "loss": 2.881, + "theoretical_loss": 3.98687916964188, + "tokens_seen": 423165952 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004402808425275828, + "loss": 2.9461, + "theoretical_loss": 3.986810749195396, + "tokens_seen": 423231488 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004402708124373119, + "loss": 3.0285, + "theoretical_loss": 3.9867423423087596, + "tokens_seen": 423297024 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044026078234704114, + "loss": 2.9852, + "theoretical_loss": 3.9866739489771845, + "tokens_seen": 423362560 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004402507522567703, + "loss": 2.9647, + "theoretical_loss": 3.986605569195887, + "tokens_seen": 423428096 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004402407221664995, + "loss": 3.0778, + "theoretical_loss": 3.986537202960085, + "tokens_seen": 423493632 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004402306920762287, + "loss": 2.9222, + "theoretical_loss": 3.986468850265001, + "tokens_seen": 423559168 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044022066198595787, + "loss": 3.0749, + "theoretical_loss": 3.986400511105858, + "tokens_seen": 423624704 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044021063189568705, + "loss": 2.6807, + "theoretical_loss": 3.9863321854778815, + "tokens_seen": 423690240 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004402006018054163, + "loss": 2.8481, + "theoretical_loss": 3.986263873376301, + "tokens_seen": 423755776 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004401905717151454, + "loss": 3.0253, + "theoretical_loss": 3.986195574796347, + "tokens_seen": 423821312 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044018054162487465, + "loss": 2.8203, + "theoretical_loss": 3.9861272897332527, + "tokens_seen": 423886848 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044017051153460383, + "loss": 2.782, + "theoretical_loss": 3.986059018182254, + "tokens_seen": 423952384 + }, + { + "epoch": 0.13, + "learning_rate": 0.000440160481444333, + "loss": 3.157, + "theoretical_loss": 3.9859907601385887, + "tokens_seen": 424017920 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004401504513540622, + "loss": 2.9779, + "theoretical_loss": 3.9859225155974976, + "tokens_seen": 424083456 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044014042126379137, + "loss": 2.6891, + "theoretical_loss": 3.985854284554223, + "tokens_seen": 424148992 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044013039117352055, + "loss": 2.8815, + "theoretical_loss": 3.9857860670040113, + "tokens_seen": 424214528 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004401203610832498, + "loss": 2.9628, + "theoretical_loss": 3.98571786294211, + "tokens_seen": 424280064 + }, + { + "epoch": 0.13, + "objective/train/docs_used": 252247, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.017324686050415, + "objective/train/theoretical_loss": 3.9856496723637687, + "objective/train/tokens_used": 444805600, + "theoretical_loss": 3.9856496723637687, + "tokens_seen": 424345600 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004401103309929789, + "loss": 3.0005, + "theoretical_loss": 3.9856496723637687, + "tokens_seen": 424345600 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044010030090270815, + "loss": 2.6464, + "theoretical_loss": 3.985581495264241, + "tokens_seen": 424411136 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004400902708124373, + "loss": 3.2376, + "theoretical_loss": 3.9855133316387805, + "tokens_seen": 424476672 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004400802407221665, + "loss": 3.0516, + "theoretical_loss": 3.9854451814826457, + "tokens_seen": 424542208 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004400702106318957, + "loss": 3.0279, + "theoretical_loss": 3.985377044791096, + "tokens_seen": 424607744 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004400601805416249, + "loss": 2.8532, + "theoretical_loss": 3.985308921559394, + "tokens_seen": 424673280 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044005015045135406, + "loss": 3.0635, + "theoretical_loss": 3.9852408117828038, + "tokens_seen": 424738816 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044004012036108324, + "loss": 2.8358, + "theoretical_loss": 3.9851727154565926, + "tokens_seen": 424804352 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004400300902708124, + "loss": 2.9163, + "theoretical_loss": 3.98510463257603, + "tokens_seen": 424869888 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044002006018054165, + "loss": 2.8977, + "theoretical_loss": 3.9850365631363873, + "tokens_seen": 424935424 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004400100300902708, + "loss": 2.7408, + "theoretical_loss": 3.9849685071329386, + "tokens_seen": 425000960 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044, + "loss": 3.0502, + "theoretical_loss": 3.984900464560961, + "tokens_seen": 425066496 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004399899699097292, + "loss": 2.8698, + "theoretical_loss": 3.984832435415733, + "tokens_seen": 425132032 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004399799398194584, + "loss": 2.8615, + "theoretical_loss": 3.984764419692537, + "tokens_seen": 425197568 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043996990972918756, + "loss": 3.0749, + "theoretical_loss": 3.984696417386655, + "tokens_seen": 425263104 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043995987963891674, + "loss": 2.9661, + "theoretical_loss": 3.984628428493374, + "tokens_seen": 425328640 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004399498495486459, + "loss": 3.0412, + "theoretical_loss": 3.9845604530079832, + "tokens_seen": 425394176 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043993981945837516, + "loss": 2.7922, + "theoretical_loss": 3.984492490925772, + "tokens_seen": 425459712 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043992978936810434, + "loss": 2.6942, + "theoretical_loss": 3.984424542242034, + "tokens_seen": 425525248 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004399197592778335, + "loss": 2.7198, + "theoretical_loss": 3.9843566069520664, + "tokens_seen": 425590784 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004399097291875627, + "loss": 2.9271, + "theoretical_loss": 3.9842886850511654, + "tokens_seen": 425656320 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004398996990972919, + "loss": 2.7653, + "theoretical_loss": 3.984220776534632, + "tokens_seen": 425721856 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004398896690070211, + "loss": 3.0602, + "theoretical_loss": 3.984152881397769, + "tokens_seen": 425787392 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043987963891675024, + "loss": 2.8653, + "theoretical_loss": 3.9840849996358814, + "tokens_seen": 425852928 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004398696088264795, + "loss": 3.0701, + "theoretical_loss": 3.9840171312442765, + "tokens_seen": 425918464 + }, + { + "epoch": 0.13, + "objective/train/docs_used": 253004, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.0382609367370605, + "objective/train/theoretical_loss": 3.9839492762182647, + "objective/train/tokens_used": 446444000, + "theoretical_loss": 3.9839492762182647, + "tokens_seen": 425984000 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004398595787362086, + "loss": 3.025, + "theoretical_loss": 3.9839492762182647, + "tokens_seen": 425984000 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043984954864593784, + "loss": 3.1379, + "theoretical_loss": 3.9838814345531577, + "tokens_seen": 426049536 + }, + { + "epoch": 0.13, + "learning_rate": 0.000439839518555667, + "loss": 2.8793, + "theoretical_loss": 3.9838136062442704, + "tokens_seen": 426115072 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004398294884653962, + "loss": 3.0411, + "theoretical_loss": 3.98374579128692, + "tokens_seen": 426180608 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004398194583751254, + "loss": 2.78, + "theoretical_loss": 3.9836779896764254, + "tokens_seen": 426246144 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004398094282848546, + "loss": 2.8897, + "theoretical_loss": 3.9836102014081076, + "tokens_seen": 426311680 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043979939819458375, + "loss": 2.9313, + "theoretical_loss": 3.983542426477292, + "tokens_seen": 426377216 + }, + { + "epoch": 0.13, + "learning_rate": 0.000439789368104313, + "loss": 2.9268, + "theoretical_loss": 3.9834746648793042, + "tokens_seen": 426442752 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004397793380140421, + "loss": 2.9328, + "theoretical_loss": 3.983406916609473, + "tokens_seen": 426508288 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043976930792377134, + "loss": 3.0533, + "theoretical_loss": 3.98333918166313, + "tokens_seen": 426573824 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004397592778335005, + "loss": 3.0209, + "theoretical_loss": 3.983271460035608, + "tokens_seen": 426639360 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004397492477432297, + "loss": 2.9241, + "theoretical_loss": 3.983203751722243, + "tokens_seen": 426704896 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004397392176529589, + "loss": 2.8083, + "theoretical_loss": 3.983136056718373, + "tokens_seen": 426770432 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043972918756268807, + "loss": 2.9585, + "theoretical_loss": 3.9830683750193394, + "tokens_seen": 426835968 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043971915747241725, + "loss": 2.734, + "theoretical_loss": 3.9830007066204836, + "tokens_seen": 426901504 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004397091273821465, + "loss": 2.8001, + "theoretical_loss": 3.9829330515171524, + "tokens_seen": 426967040 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004396990972918756, + "loss": 2.8936, + "theoretical_loss": 3.982865409704692, + "tokens_seen": 427032576 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043968906720160485, + "loss": 2.9378, + "theoretical_loss": 3.9827977811784523, + "tokens_seen": 427098112 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043967903711133403, + "loss": 2.7542, + "theoretical_loss": 3.9827301659337864, + "tokens_seen": 427163648 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004396690070210632, + "loss": 3.0075, + "theoretical_loss": 3.9826625639660485, + "tokens_seen": 427229184 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004396589769307924, + "loss": 2.6937, + "theoretical_loss": 3.982594975270595, + "tokens_seen": 427294720 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043964894684052157, + "loss": 2.8122, + "theoretical_loss": 3.982527399842786, + "tokens_seen": 427360256 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043963891675025075, + "loss": 2.9744, + "theoretical_loss": 3.9824598376779825, + "tokens_seen": 427425792 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043962888665998, + "loss": 2.8555, + "theoretical_loss": 3.982392288771549, + "tokens_seen": 427491328 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004396188565697091, + "loss": 2.9056, + "theoretical_loss": 3.98232475311885, + "tokens_seen": 427556864 + }, + { + "epoch": 0.13, + "objective/train/docs_used": 254260, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.623885154724121, + "objective/train/theoretical_loss": 3.9822572307152564, + "objective/train/tokens_used": 448082400, + "theoretical_loss": 3.9822572307152564, + "tokens_seen": 427622400 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043960882647943835, + "loss": 2.8635, + "theoretical_loss": 3.9822572307152564, + "tokens_seen": 427622400 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004395987963891675, + "loss": 3.0339, + "theoretical_loss": 3.9821897215561375, + "tokens_seen": 427687936 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004395887662988967, + "loss": 3.0097, + "theoretical_loss": 3.982122225636867, + "tokens_seen": 427753472 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004395787362086259, + "loss": 2.67, + "theoretical_loss": 3.98205474295282, + "tokens_seen": 427819008 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004395687061183551, + "loss": 2.9655, + "theoretical_loss": 3.9819872734993753, + "tokens_seen": 427884544 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043955867602808426, + "loss": 2.9709, + "theoretical_loss": 3.9819198172719124, + "tokens_seen": 427950080 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043954864593781344, + "loss": 2.8799, + "theoretical_loss": 3.981852374265814, + "tokens_seen": 428015616 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004395386158475426, + "loss": 2.8985, + "theoretical_loss": 3.9817849444764644, + "tokens_seen": 428081152 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043952858575727185, + "loss": 2.8256, + "theoretical_loss": 3.981717527899252, + "tokens_seen": 428146688 + }, + { + "epoch": 0.13, + "learning_rate": 0.000439518555667001, + "loss": 3.0246, + "theoretical_loss": 3.9816501245295646, + "tokens_seen": 428212224 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004395085255767302, + "loss": 2.9374, + "theoretical_loss": 3.9815827343627954, + "tokens_seen": 428277760 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004394984954864594, + "loss": 2.7449, + "theoretical_loss": 3.981515357394338, + "tokens_seen": 428343296 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004394884653961886, + "loss": 3.0782, + "theoretical_loss": 3.9814479936195886, + "tokens_seen": 428408832 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043947843530591776, + "loss": 2.7616, + "theoretical_loss": 3.981380643033945, + "tokens_seen": 428474368 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043946840521564694, + "loss": 2.7545, + "theoretical_loss": 3.9813133056328103, + "tokens_seen": 428539904 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004394583751253761, + "loss": 2.9056, + "theoretical_loss": 3.981245981411586, + "tokens_seen": 428605440 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043944834503510536, + "loss": 2.9657, + "theoretical_loss": 3.981178670365679, + "tokens_seen": 428670976 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004394383149448345, + "loss": 2.8569, + "theoretical_loss": 3.981111372490496, + "tokens_seen": 428736512 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004394282848545637, + "loss": 2.9878, + "theoretical_loss": 3.9810440877814486, + "tokens_seen": 428802048 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043941825476429285, + "loss": 2.7722, + "theoretical_loss": 3.9809768162339476, + "tokens_seen": 428867584 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004394082246740221, + "loss": 2.9355, + "theoretical_loss": 3.9809095578434093, + "tokens_seen": 428933120 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043939819458375126, + "loss": 3.1091, + "theoretical_loss": 3.9808423126052506, + "tokens_seen": 428998656 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043938816449348044, + "loss": 2.9979, + "theoretical_loss": 3.9807750805148903, + "tokens_seen": 429064192 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004393781344032096, + "loss": 3.093, + "theoretical_loss": 3.9807078615677502, + "tokens_seen": 429129728 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004393681043129388, + "loss": 2.6973, + "theoretical_loss": 3.980640655759255, + "tokens_seen": 429195264 + }, + { + "epoch": 0.13, + "objective/train/docs_used": 255066, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.901578426361084, + "objective/train/theoretical_loss": 3.9805734630848306, + "objective/train/tokens_used": 449720800, + "theoretical_loss": 3.9805734630848306, + "tokens_seen": 429260800 + }, + { + "epoch": 0.13, + "learning_rate": 0.000439358074222668, + "loss": 3.0309, + "theoretical_loss": 3.9805734630848306, + "tokens_seen": 429260800 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004393480441323972, + "loss": 2.7121, + "theoretical_loss": 3.9805062835399054, + "tokens_seen": 429326336 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043933801404212635, + "loss": 2.8877, + "theoretical_loss": 3.9804391171199103, + "tokens_seen": 429391872 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004393279839518556, + "loss": 2.8659, + "theoretical_loss": 3.9803719638202786, + "tokens_seen": 429457408 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043931795386158477, + "loss": 2.8677, + "theoretical_loss": 3.9803048236364456, + "tokens_seen": 429522944 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043930792377131395, + "loss": 3.0073, + "theoretical_loss": 3.9802376965638495, + "tokens_seen": 429588480 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043929789368104313, + "loss": 2.7262, + "theoretical_loss": 3.98017058259793, + "tokens_seen": 429654016 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004392878635907723, + "loss": 2.7498, + "theoretical_loss": 3.980103481734129, + "tokens_seen": 429719552 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004392778335005015, + "loss": 3.0806, + "theoretical_loss": 3.9800363939678918, + "tokens_seen": 429785088 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004392678034102307, + "loss": 2.9495, + "theoretical_loss": 3.9799693192946646, + "tokens_seen": 429850624 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043925777331995985, + "loss": 2.8323, + "theoretical_loss": 3.979902257709897, + "tokens_seen": 429916160 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004392477432296891, + "loss": 2.8573, + "theoretical_loss": 3.97983520920904, + "tokens_seen": 429981696 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004392377131394182, + "loss": 2.7358, + "theoretical_loss": 3.979768173787548, + "tokens_seen": 430047232 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043922768304914745, + "loss": 2.7959, + "theoretical_loss": 3.9797011514408758, + "tokens_seen": 430112768 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043921765295887663, + "loss": 3.1869, + "theoretical_loss": 3.9796341421644827, + "tokens_seen": 430178304 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004392076228686058, + "loss": 2.6937, + "theoretical_loss": 3.979567145953829, + "tokens_seen": 430243840 + }, + { + "epoch": 0.13, + "learning_rate": 0.000439197592778335, + "loss": 3.002, + "theoretical_loss": 3.979500162804377, + "tokens_seen": 430309376 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043918756268806423, + "loss": 2.7068, + "theoretical_loss": 3.979433192711592, + "tokens_seen": 430374912 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004391775325977934, + "loss": 3.0358, + "theoretical_loss": 3.9793662356709407, + "tokens_seen": 430440448 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004391675025075226, + "loss": 2.8857, + "theoretical_loss": 3.979299291677894, + "tokens_seen": 430505984 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043915747241725177, + "loss": 2.834, + "theoretical_loss": 3.9792323607279223, + "tokens_seen": 430571520 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043914744232698095, + "loss": 3.2369, + "theoretical_loss": 3.979165442816501, + "tokens_seen": 430637056 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004391374122367102, + "loss": 2.6797, + "theoretical_loss": 3.9790985379391053, + "tokens_seen": 430702592 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004391273821464393, + "loss": 2.9182, + "theoretical_loss": 3.9790316460912143, + "tokens_seen": 430768128 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043911735205616855, + "loss": 2.9216, + "theoretical_loss": 3.978964767268309, + "tokens_seen": 430833664 + }, + { + "epoch": 0.13, + "objective/train/docs_used": 256338, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.837135076522827, + "objective/train/theoretical_loss": 3.9788979014658716, + "objective/train/tokens_used": 451359200, + "theoretical_loss": 3.9788979014658716, + "tokens_seen": 430899200 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004391073219658977, + "loss": 2.8812, + "theoretical_loss": 3.9788979014658716, + "tokens_seen": 430899200 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004390972918756269, + "loss": 2.833, + "theoretical_loss": 3.978831048679389, + "tokens_seen": 430964736 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004390872617853561, + "loss": 2.8292, + "theoretical_loss": 3.978764208904347, + "tokens_seen": 431030272 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004390772316950853, + "loss": 2.8724, + "theoretical_loss": 3.9786973821362377, + "tokens_seen": 431095808 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043906720160481446, + "loss": 2.7541, + "theoretical_loss": 3.9786305683705505, + "tokens_seen": 431161344 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043905717151454364, + "loss": 2.8703, + "theoretical_loss": 3.978563767602782, + "tokens_seen": 431226880 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004390471414242728, + "loss": 2.7287, + "theoretical_loss": 3.978496979828428, + "tokens_seen": 431292416 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043903711133400205, + "loss": 3.093, + "theoretical_loss": 3.9784302050429874, + "tokens_seen": 431357952 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004390270812437312, + "loss": 3.0969, + "theoretical_loss": 3.978363443241961, + "tokens_seen": 431423488 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004390170511534604, + "loss": 2.8985, + "theoretical_loss": 3.9782966944208518, + "tokens_seen": 431489024 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004390070210631896, + "loss": 2.8515, + "theoretical_loss": 3.9782299585751666, + "tokens_seen": 431554560 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004389969909729188, + "loss": 2.9528, + "theoretical_loss": 3.9781632357004124, + "tokens_seen": 431620096 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043898696088264796, + "loss": 2.9891, + "theoretical_loss": 3.9780965257920995, + "tokens_seen": 431685632 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043897693079237714, + "loss": 2.9879, + "theoretical_loss": 3.9780298288457394, + "tokens_seen": 431751168 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004389669007021063, + "loss": 2.8728, + "theoretical_loss": 3.9779631448568473, + "tokens_seen": 431816704 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043895687061183556, + "loss": 2.9434, + "theoretical_loss": 3.977896473820941, + "tokens_seen": 431882240 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004389468405215647, + "loss": 2.9863, + "theoretical_loss": 3.9778298157335374, + "tokens_seen": 431947776 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004389368104312939, + "loss": 2.607, + "theoretical_loss": 3.977763170590159, + "tokens_seen": 432013312 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043892678034102305, + "loss": 3.031, + "theoretical_loss": 3.9776965383863288, + "tokens_seen": 432078848 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004389167502507523, + "loss": 3.0556, + "theoretical_loss": 3.9776299191175726, + "tokens_seen": 432144384 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043890672016048146, + "loss": 3.1064, + "theoretical_loss": 3.977563312779418, + "tokens_seen": 432209920 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043889669007021064, + "loss": 2.6718, + "theoretical_loss": 3.977496719367396, + "tokens_seen": 432275456 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004388866599799398, + "loss": 3.1611, + "theoretical_loss": 3.9774301388770374, + "tokens_seen": 432340992 + }, + { + "epoch": 0.13, + "learning_rate": 0.000438876629889669, + "loss": 2.8816, + "theoretical_loss": 3.9773635713038784, + "tokens_seen": 432406528 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004388665997993982, + "loss": 2.7733, + "theoretical_loss": 3.9772970166434547, + "tokens_seen": 432472064 + }, + { + "epoch": 0.13, + "objective/train/docs_used": 257001, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.0123116970062256, + "objective/train/theoretical_loss": 3.9772304748913054, + "objective/train/tokens_used": 452997600, + "theoretical_loss": 3.9772304748913054, + "tokens_seen": 432537600 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004388565697091274, + "loss": 3.1152, + "theoretical_loss": 3.9772304748913054, + "tokens_seen": 432537600 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043884653961885655, + "loss": 2.8423, + "theoretical_loss": 3.977163946042973, + "tokens_seen": 432603136 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004388365095285858, + "loss": 2.8273, + "theoretical_loss": 3.977097430093999, + "tokens_seen": 432668672 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043882647943831497, + "loss": 2.8533, + "theoretical_loss": 3.97703092703993, + "tokens_seen": 432734208 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043881644934804415, + "loss": 2.9303, + "theoretical_loss": 3.9769644368763135, + "tokens_seen": 432799744 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043880641925777333, + "loss": 3.0371, + "theoretical_loss": 3.9768979595987, + "tokens_seen": 432865280 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004387963891675025, + "loss": 3.0645, + "theoretical_loss": 3.9768314952026422, + "tokens_seen": 432930816 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004387863590772317, + "loss": 3.1624, + "theoretical_loss": 3.9767650436836934, + "tokens_seen": 432996352 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004387763289869609, + "loss": 3.0907, + "theoretical_loss": 3.976698605037411, + "tokens_seen": 433061888 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043876629889669005, + "loss": 2.9986, + "theoretical_loss": 3.976632179259354, + "tokens_seen": 433127424 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004387562688064193, + "loss": 2.8841, + "theoretical_loss": 3.976565766345083, + "tokens_seen": 433192960 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004387462387161484, + "loss": 2.9911, + "theoretical_loss": 3.9764993662901613, + "tokens_seen": 433258496 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043873620862587765, + "loss": 2.8762, + "theoretical_loss": 3.976432979090155, + "tokens_seen": 433324032 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043872617853560683, + "loss": 3.1392, + "theoretical_loss": 3.9763666047406314, + "tokens_seen": 433389568 + }, + { + "epoch": 0.13, + "learning_rate": 0.000438716148445336, + "loss": 2.8567, + "theoretical_loss": 3.9763002432371604, + "tokens_seen": 433455104 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004387061183550652, + "loss": 2.7282, + "theoretical_loss": 3.9762338945753144, + "tokens_seen": 433520640 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043869608826479443, + "loss": 2.9154, + "theoretical_loss": 3.9761675587506673, + "tokens_seen": 433586176 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043868605817452356, + "loss": 3.0765, + "theoretical_loss": 3.9761012357587955, + "tokens_seen": 433651712 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004386760280842528, + "loss": 2.8579, + "theoretical_loss": 3.976034925595278, + "tokens_seen": 433717248 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004386659979939819, + "loss": 3.1368, + "theoretical_loss": 3.975968628255696, + "tokens_seen": 433782784 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043865596790371115, + "loss": 3.1985, + "theoretical_loss": 3.9759023437356316, + "tokens_seen": 433848320 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043864593781344033, + "loss": 2.9339, + "theoretical_loss": 3.9758360720306705, + "tokens_seen": 433913856 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004386359077231695, + "loss": 2.953, + "theoretical_loss": 3.9757698131364005, + "tokens_seen": 433979392 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004386258776328987, + "loss": 2.8475, + "theoretical_loss": 3.9757035670484107, + "tokens_seen": 434044928 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004386158475426279, + "loss": 2.8882, + "theoretical_loss": 3.9756373337622932, + "tokens_seen": 434110464 + }, + { + "epoch": 0.13, + "objective/train/docs_used": 257982, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.724050998687744, + "objective/train/theoretical_loss": 3.975571113273642, + "objective/train/tokens_used": 454636000, + "theoretical_loss": 3.975571113273642, + "tokens_seen": 434176000 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043860581745235706, + "loss": 2.7431, + "theoretical_loss": 3.975571113273642, + "tokens_seen": 434176000 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004385957873620863, + "loss": 3.0104, + "theoretical_loss": 3.975504905578053, + "tokens_seen": 434241536 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004385857572718154, + "loss": 3.1048, + "theoretical_loss": 3.975438710671125, + "tokens_seen": 434307072 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043857572718154466, + "loss": 2.8299, + "theoretical_loss": 3.9753725285484576, + "tokens_seen": 434372608 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004385656970912738, + "loss": 3.2285, + "theoretical_loss": 3.9753063592056552, + "tokens_seen": 434438144 + }, + { + "epoch": 0.13, + "learning_rate": 0.000438555667001003, + "loss": 2.7789, + "theoretical_loss": 3.9752402026383207, + "tokens_seen": 434503680 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004385456369107322, + "loss": 2.816, + "theoretical_loss": 3.9751740588420623, + "tokens_seen": 434569216 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004385356068204614, + "loss": 2.8701, + "theoretical_loss": 3.9751079278124886, + "tokens_seen": 434634752 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043852557673019056, + "loss": 2.7993, + "theoretical_loss": 3.9750418095452122, + "tokens_seen": 434700288 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004385155466399198, + "loss": 2.8065, + "theoretical_loss": 3.9749757040358453, + "tokens_seen": 434765824 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004385055165496489, + "loss": 2.8499, + "theoretical_loss": 3.9749096112800046, + "tokens_seen": 434831360 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043849548645937816, + "loss": 2.9184, + "theoretical_loss": 3.974843531273307, + "tokens_seen": 434896896 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004384854563691073, + "loss": 2.939, + "theoretical_loss": 3.9747774640113738, + "tokens_seen": 434962432 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004384754262788365, + "loss": 2.95, + "theoretical_loss": 3.974711409489826, + "tokens_seen": 435027968 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004384653961885657, + "loss": 3.0192, + "theoretical_loss": 3.974645367704289, + "tokens_seen": 435093504 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004384553660982949, + "loss": 2.7038, + "theoretical_loss": 3.9745793386503885, + "tokens_seen": 435159040 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043844533600802407, + "loss": 3.0532, + "theoretical_loss": 3.974513322323754, + "tokens_seen": 435224576 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043843530591775325, + "loss": 2.7298, + "theoretical_loss": 3.9744473187200158, + "tokens_seen": 435290112 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004384252758274825, + "loss": 2.8952, + "theoretical_loss": 3.974381327834807, + "tokens_seen": 435355648 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043841524573721166, + "loss": 2.7805, + "theoretical_loss": 3.974315349663763, + "tokens_seen": 435421184 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043840521564694084, + "loss": 3.128, + "theoretical_loss": 3.9742493842025213, + "tokens_seen": 435486720 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043839518555667, + "loss": 2.7731, + "theoretical_loss": 3.974183431446721, + "tokens_seen": 435552256 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004383851554663992, + "loss": 2.766, + "theoretical_loss": 3.974117491392004, + "tokens_seen": 435617792 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004383751253761284, + "loss": 2.8844, + "theoretical_loss": 3.9740515640340135, + "tokens_seen": 435683328 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004383650952858576, + "loss": 3.0195, + "theoretical_loss": 3.9739856493683967, + "tokens_seen": 435748864 + }, + { + "epoch": 0.13, + "objective/train/docs_used": 258654, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.8394222259521484, + "objective/train/theoretical_loss": 3.973919747390801, + "objective/train/tokens_used": 456274400, + "theoretical_loss": 3.973919747390801, + "tokens_seen": 435814400 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043835506519558675, + "loss": 3.0603, + "theoretical_loss": 3.973919747390801, + "tokens_seen": 435814400 + }, + { + "epoch": 0.13, + "learning_rate": 0.000438345035105316, + "loss": 3.1098, + "theoretical_loss": 3.973853858096876, + "tokens_seen": 435879936 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043833500501504517, + "loss": 2.8376, + "theoretical_loss": 3.9737879814822747, + "tokens_seen": 435945472 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043832497492477435, + "loss": 2.7877, + "theoretical_loss": 3.973722117542652, + "tokens_seen": 436011008 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043831494483450353, + "loss": 2.8291, + "theoretical_loss": 3.9736562662736636, + "tokens_seen": 436076544 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004383049147442327, + "loss": 2.8159, + "theoretical_loss": 3.9735904276709695, + "tokens_seen": 436142080 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004382948846539619, + "loss": 2.872, + "theoretical_loss": 3.9735246017302295, + "tokens_seen": 436207616 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004382848545636911, + "loss": 2.8589, + "theoretical_loss": 3.9734587884471075, + "tokens_seen": 436273152 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043827482447342025, + "loss": 3.1836, + "theoretical_loss": 3.9733929878172685, + "tokens_seen": 436338688 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004382647943831495, + "loss": 2.6993, + "theoretical_loss": 3.9733271998363797, + "tokens_seen": 436404224 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004382547642928786, + "loss": 2.9839, + "theoretical_loss": 3.9732614245001106, + "tokens_seen": 436469760 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043824473420260785, + "loss": 2.912, + "theoretical_loss": 3.9731956618041333, + "tokens_seen": 436535296 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043823470411233703, + "loss": 2.9753, + "theoretical_loss": 3.9731299117441203, + "tokens_seen": 436600832 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004382246740220662, + "loss": 2.8748, + "theoretical_loss": 3.9730641743157493, + "tokens_seen": 436666368 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004382146439317954, + "loss": 2.9102, + "theoretical_loss": 3.972998449514697, + "tokens_seen": 436731904 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043820461384152463, + "loss": 2.9307, + "theoretical_loss": 3.9729327373366443, + "tokens_seen": 436797440 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043819458375125376, + "loss": 3.0614, + "theoretical_loss": 3.972867037777273, + "tokens_seen": 436862976 + }, + { + "epoch": 0.13, + "learning_rate": 0.000438184553660983, + "loss": 2.8826, + "theoretical_loss": 3.9728013508322673, + "tokens_seen": 436928512 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004381745235707121, + "loss": 2.8075, + "theoretical_loss": 3.9727356764973143, + "tokens_seen": 436994048 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043816449348044135, + "loss": 3.1614, + "theoretical_loss": 3.972670014768102, + "tokens_seen": 437059584 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043815446339017053, + "loss": 2.6629, + "theoretical_loss": 3.972604365640322, + "tokens_seen": 437125120 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004381444332998997, + "loss": 3.0472, + "theoretical_loss": 3.972538729109666, + "tokens_seen": 437190656 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004381344032096289, + "loss": 2.86, + "theoretical_loss": 3.97247310517183, + "tokens_seen": 437256192 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004381243731193581, + "loss": 2.7282, + "theoretical_loss": 3.972407493822511, + "tokens_seen": 437321728 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043811434302908726, + "loss": 2.741, + "theoretical_loss": 3.972341895057408, + "tokens_seen": 437387264 + }, + { + "epoch": 0.13, + "objective/train/docs_used": 259916, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.993715763092041, + "objective/train/theoretical_loss": 3.972276308872222, + "objective/train/tokens_used": 457912800, + "theoretical_loss": 3.972276308872222, + "tokens_seen": 437452800 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004381043129388165, + "loss": 3.1031, + "theoretical_loss": 3.972276308872222, + "tokens_seen": 437452800 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004380942828485456, + "loss": 2.871, + "theoretical_loss": 3.972210735262657, + "tokens_seen": 437518336 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043808425275827486, + "loss": 2.9324, + "theoretical_loss": 3.9721451742244183, + "tokens_seen": 437583872 + }, + { + "epoch": 0.13, + "learning_rate": 0.000438074222668004, + "loss": 3.0156, + "theoretical_loss": 3.9720796257532136, + "tokens_seen": 437649408 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004380641925777332, + "loss": 2.9294, + "theoretical_loss": 3.9720140898447527, + "tokens_seen": 437714944 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004380541624874624, + "loss": 3.1175, + "theoretical_loss": 3.9719485664947483, + "tokens_seen": 437780480 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004380441323971916, + "loss": 2.9198, + "theoretical_loss": 3.971883055698913, + "tokens_seen": 437846016 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043803410230692076, + "loss": 2.8979, + "theoretical_loss": 3.9718175574529635, + "tokens_seen": 437911552 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043802407221665, + "loss": 2.9282, + "theoretical_loss": 3.9717520717526176, + "tokens_seen": 437977088 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004380140421263791, + "loss": 2.8813, + "theoretical_loss": 3.971686598593596, + "tokens_seen": 438042624 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043800401203610836, + "loss": 2.9149, + "theoretical_loss": 3.971621137971622, + "tokens_seen": 438108160 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004379939819458375, + "loss": 2.9774, + "theoretical_loss": 3.9715556898824182, + "tokens_seen": 438173696 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004379839518555667, + "loss": 2.9117, + "theoretical_loss": 3.971490254321713, + "tokens_seen": 438239232 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004379739217652959, + "loss": 3.1107, + "theoretical_loss": 3.9714248312852343, + "tokens_seen": 438304768 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004379638916750251, + "loss": 3.0212, + "theoretical_loss": 3.971359420768712, + "tokens_seen": 438370304 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043795386158475427, + "loss": 2.9949, + "theoretical_loss": 3.9712940227678803, + "tokens_seen": 438435840 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043794383149448345, + "loss": 2.9347, + "theoretical_loss": 3.9712286372784735, + "tokens_seen": 438501376 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043793380140421263, + "loss": 2.7289, + "theoretical_loss": 3.971163264296229, + "tokens_seen": 438566912 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043792377131394186, + "loss": 3.1885, + "theoretical_loss": 3.971097903816886, + "tokens_seen": 438632448 + }, + { + "epoch": 0.13, + "learning_rate": 0.000437913741223671, + "loss": 2.7088, + "theoretical_loss": 3.9710325558361856, + "tokens_seen": 438697984 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004379037111334002, + "loss": 3.1397, + "theoretical_loss": 3.9709672203498707, + "tokens_seen": 438763520 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043789368104312935, + "loss": 3.1417, + "theoretical_loss": 3.970901897353688, + "tokens_seen": 438829056 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004378836509528586, + "loss": 2.945, + "theoretical_loss": 3.970836586843383, + "tokens_seen": 438894592 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043787362086258777, + "loss": 3.1682, + "theoretical_loss": 3.9707712888147073, + "tokens_seen": 438960128 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043786359077231695, + "loss": 3.0537, + "theoretical_loss": 3.9707060032634107, + "tokens_seen": 439025664 + }, + { + "epoch": 0.13, + "objective/train/docs_used": 260524, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.642634868621826, + "objective/train/theoretical_loss": 3.9706407301852487, + "objective/train/tokens_used": 459551200, + "theoretical_loss": 3.9706407301852487, + "tokens_seen": 439091200 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043785356068204613, + "loss": 2.8124, + "theoretical_loss": 3.9706407301852487, + "tokens_seen": 439091200 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043784353059177537, + "loss": 3.0727, + "theoretical_loss": 3.9705754695759756, + "tokens_seen": 439156736 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004378335005015045, + "loss": 2.8711, + "theoretical_loss": 3.9705102214313506, + "tokens_seen": 439222272 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043782347041123373, + "loss": 2.778, + "theoretical_loss": 3.9704449857471324, + "tokens_seen": 439287808 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043781344032096286, + "loss": 3.0267, + "theoretical_loss": 3.970379762519084, + "tokens_seen": 439353344 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004378034102306921, + "loss": 3.0774, + "theoretical_loss": 3.9703145517429697, + "tokens_seen": 439418880 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043779338014042127, + "loss": 3.1731, + "theoretical_loss": 3.9702493534145544, + "tokens_seen": 439484416 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043778335005015045, + "loss": 2.9973, + "theoretical_loss": 3.9701841675296077, + "tokens_seen": 439549952 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043777331995987963, + "loss": 3.0227, + "theoretical_loss": 3.970118994083899, + "tokens_seen": 439615488 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004377632898696088, + "loss": 2.9803, + "theoretical_loss": 3.9700538330732016, + "tokens_seen": 439681024 + }, + { + "epoch": 0.13, + "learning_rate": 0.000437753259779338, + "loss": 2.8205, + "theoretical_loss": 3.969988684493289, + "tokens_seen": 439746560 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043774322968906723, + "loss": 2.98, + "theoretical_loss": 3.9699235483399384, + "tokens_seen": 439812096 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043773319959879636, + "loss": 2.7891, + "theoretical_loss": 3.969858424608928, + "tokens_seen": 439877632 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004377231695085256, + "loss": 3.0132, + "theoretical_loss": 3.969793313296039, + "tokens_seen": 439943168 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004377131394182547, + "loss": 2.9011, + "theoretical_loss": 3.969728214397053, + "tokens_seen": 440008704 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043770310932798396, + "loss": 2.7895, + "theoretical_loss": 3.969663127907756, + "tokens_seen": 440074240 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043769307923771314, + "loss": 2.9616, + "theoretical_loss": 3.969598053823934, + "tokens_seen": 440139776 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004376830491474423, + "loss": 2.9177, + "theoretical_loss": 3.969532992141377, + "tokens_seen": 440205312 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043767301905717155, + "loss": 2.8528, + "theoretical_loss": 3.969467942855875, + "tokens_seen": 440270848 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043766298896690073, + "loss": 3.0024, + "theoretical_loss": 3.969402905963221, + "tokens_seen": 440336384 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004376529588766299, + "loss": 2.8636, + "theoretical_loss": 3.9693378814592104, + "tokens_seen": 440401920 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004376429287863591, + "loss": 3.0251, + "theoretical_loss": 3.96927286933964, + "tokens_seen": 440467456 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004376328986960883, + "loss": 2.8125, + "theoretical_loss": 3.969207869600309, + "tokens_seen": 440532992 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043762286860581746, + "loss": 2.7728, + "theoretical_loss": 3.969142882237019, + "tokens_seen": 440598528 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004376128385155467, + "loss": 2.8144, + "theoretical_loss": 3.9690779072455733, + "tokens_seen": 440664064 + }, + { + "epoch": 0.13, + "objective/train/docs_used": 261739, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.153048276901245, + "objective/train/theoretical_loss": 3.969012944621776, + "objective/train/tokens_used": 461189600, + "theoretical_loss": 3.969012944621776, + "tokens_seen": 440729600 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004376028084252758, + "loss": 3.0399, + "theoretical_loss": 3.969012944621776, + "tokens_seen": 440729600 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043759277833500506, + "loss": 2.8256, + "theoretical_loss": 3.968947994361436, + "tokens_seen": 440795136 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004375827482447342, + "loss": 3.0384, + "theoretical_loss": 3.9688830564603625, + "tokens_seen": 440860672 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004375727181544634, + "loss": 2.7654, + "theoretical_loss": 3.9688181309143657, + "tokens_seen": 440926208 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004375626880641926, + "loss": 3.0902, + "theoretical_loss": 3.9687532177192604, + "tokens_seen": 440991744 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004375526579739218, + "loss": 2.7889, + "theoretical_loss": 3.9686883168708613, + "tokens_seen": 441057280 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043754262788365096, + "loss": 3.2056, + "theoretical_loss": 3.968623428364986, + "tokens_seen": 441122816 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004375325977933802, + "loss": 2.7294, + "theoretical_loss": 3.9685585521974556, + "tokens_seen": 441188352 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004375225677031093, + "loss": 2.9355, + "theoretical_loss": 3.968493688364089, + "tokens_seen": 441253888 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043751253761283856, + "loss": 2.7611, + "theoretical_loss": 3.968428836860712, + "tokens_seen": 441319424 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004375025075225677, + "loss": 3.0854, + "theoretical_loss": 3.96836399768315, + "tokens_seen": 441384960 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004374924774322969, + "loss": 2.7944, + "theoretical_loss": 3.96829917082723, + "tokens_seen": 441450496 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004374824473420261, + "loss": 2.9939, + "theoretical_loss": 3.9682343562887823, + "tokens_seen": 441516032 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004374724172517553, + "loss": 2.9784, + "theoretical_loss": 3.968169554063639, + "tokens_seen": 441581568 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043746238716148447, + "loss": 3.0248, + "theoretical_loss": 3.9681047641476326, + "tokens_seen": 441647104 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043745235707121365, + "loss": 3.2134, + "theoretical_loss": 3.968039986536601, + "tokens_seen": 441712640 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043744232698094283, + "loss": 3.0607, + "theoretical_loss": 3.9679752212263795, + "tokens_seen": 441778176 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043743229689067206, + "loss": 2.882, + "theoretical_loss": 3.9679104682128106, + "tokens_seen": 441843712 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004374222668004012, + "loss": 3.0015, + "theoretical_loss": 3.967845727491735, + "tokens_seen": 441909248 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004374122367101304, + "loss": 2.7431, + "theoretical_loss": 3.9677809990589967, + "tokens_seen": 441974784 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043740220661985955, + "loss": 3.0387, + "theoretical_loss": 3.967716282910441, + "tokens_seen": 442040320 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004373921765295888, + "loss": 2.9895, + "theoretical_loss": 3.9676515790419176, + "tokens_seen": 442105856 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043738214643931797, + "loss": 2.8428, + "theoretical_loss": 3.967586887449275, + "tokens_seen": 442171392 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043737211634904715, + "loss": 2.8886, + "theoretical_loss": 3.967522208128366, + "tokens_seen": 442236928 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043736208625877633, + "loss": 3.0584, + "theoretical_loss": 3.967457541075044, + "tokens_seen": 442302464 + }, + { + "epoch": 0.13, + "objective/train/docs_used": 262441, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.639289140701294, + "objective/train/theoretical_loss": 3.9673928862851655, + "objective/train/tokens_used": 462828000, + "theoretical_loss": 3.9673928862851655, + "tokens_seen": 442368000 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043735205616850557, + "loss": 2.7949, + "theoretical_loss": 3.9673928862851655, + "tokens_seen": 442368000 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004373420260782347, + "loss": 2.8856, + "theoretical_loss": 3.9673282437545883, + "tokens_seen": 442433536 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043733199598796393, + "loss": 3.1824, + "theoretical_loss": 3.9672636134791737, + "tokens_seen": 442499072 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043732196589769306, + "loss": 3.0573, + "theoretical_loss": 3.967198995454782, + "tokens_seen": 442564608 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004373119358074223, + "loss": 3.1599, + "theoretical_loss": 3.9671343896772786, + "tokens_seen": 442630144 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043730190571715147, + "loss": 2.7257, + "theoretical_loss": 3.9670697961425287, + "tokens_seen": 442695680 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043729187562688065, + "loss": 2.7709, + "theoretical_loss": 3.967005214846401, + "tokens_seen": 442761216 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043728184553660983, + "loss": 2.7809, + "theoretical_loss": 3.9669406457847654, + "tokens_seen": 442826752 + }, + { + "epoch": 0.13, + "learning_rate": 0.000437271815446339, + "loss": 2.9165, + "theoretical_loss": 3.9668760889534944, + "tokens_seen": 442892288 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004372617853560682, + "loss": 2.8465, + "theoretical_loss": 3.9668115443484617, + "tokens_seen": 442957824 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043725175526579743, + "loss": 2.9638, + "theoretical_loss": 3.966747011965544, + "tokens_seen": 443023360 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043724172517552656, + "loss": 2.9446, + "theoretical_loss": 3.9666824918006185, + "tokens_seen": 443088896 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004372316950852558, + "loss": 2.9879, + "theoretical_loss": 3.9666179838495657, + "tokens_seen": 443154432 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004372216649949849, + "loss": 2.9773, + "theoretical_loss": 3.9665534881082687, + "tokens_seen": 443219968 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043721163490471416, + "loss": 2.9571, + "theoretical_loss": 3.96648900457261, + "tokens_seen": 443285504 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043720160481444334, + "loss": 2.851, + "theoretical_loss": 3.966424533238477, + "tokens_seen": 443351040 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004371915747241725, + "loss": 3.029, + "theoretical_loss": 3.9663600741017575, + "tokens_seen": 443416576 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004371815446339017, + "loss": 2.8872, + "theoretical_loss": 3.966295627158341, + "tokens_seen": 443482112 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043717151454363093, + "loss": 3.0154, + "theoretical_loss": 3.9662311924041203, + "tokens_seen": 443547648 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043716148445336006, + "loss": 2.8334, + "theoretical_loss": 3.96616676983499, + "tokens_seen": 443613184 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004371514543630893, + "loss": 2.9599, + "theoretical_loss": 3.966102359446845, + "tokens_seen": 443678720 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004371414242728184, + "loss": 2.9462, + "theoretical_loss": 3.9660379612355836, + "tokens_seen": 443744256 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043713139418254766, + "loss": 2.9191, + "theoretical_loss": 3.9659735751971064, + "tokens_seen": 443809792 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043712136409227684, + "loss": 2.8632, + "theoretical_loss": 3.965909201327315, + "tokens_seen": 443875328 + }, + { + "epoch": 0.13, + "learning_rate": 0.000437111334002006, + "loss": 2.9488, + "theoretical_loss": 3.9658448396221138, + "tokens_seen": 443940864 + }, + { + "epoch": 0.13, + "objective/train/docs_used": 263839, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.5010595321655273, + "objective/train/theoretical_loss": 3.9657804900774085, + "objective/train/tokens_used": 464466400, + "theoretical_loss": 3.9657804900774085, + "tokens_seen": 444006400 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004371013039117352, + "loss": 3.2767, + "theoretical_loss": 3.9657804900774085, + "tokens_seen": 444006400 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004370912738214644, + "loss": 3.0265, + "theoretical_loss": 3.9657161526891076, + "tokens_seen": 444071936 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043708124373119356, + "loss": 2.9417, + "theoretical_loss": 3.9656518274531205, + "tokens_seen": 444137472 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004370712136409228, + "loss": 2.8448, + "theoretical_loss": 3.9655875143653594, + "tokens_seen": 444203008 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004370611835506519, + "loss": 2.7611, + "theoretical_loss": 3.965523213421738, + "tokens_seen": 444268544 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043705115346038116, + "loss": 3.0323, + "theoretical_loss": 3.965458924618173, + "tokens_seen": 444334080 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004370411233701103, + "loss": 2.9254, + "theoretical_loss": 3.965394647950581, + "tokens_seen": 444399616 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004370310932798395, + "loss": 3.0449, + "theoretical_loss": 3.9653303834148828, + "tokens_seen": 444465152 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004370210631895687, + "loss": 3.0189, + "theoretical_loss": 3.9652661310070005, + "tokens_seen": 444530688 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004370110330992979, + "loss": 2.7175, + "theoretical_loss": 3.965201890722857, + "tokens_seen": 444596224 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043700100300902707, + "loss": 2.8998, + "theoretical_loss": 3.9651376625583783, + "tokens_seen": 444661760 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004369909729187563, + "loss": 2.9471, + "theoretical_loss": 3.965073446509493, + "tokens_seen": 444727296 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043698094282848543, + "loss": 2.8046, + "theoretical_loss": 3.9650092425721297, + "tokens_seen": 444792832 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043697091273821467, + "loss": 2.748, + "theoretical_loss": 3.9649450507422204, + "tokens_seen": 444858368 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004369608826479438, + "loss": 3.0717, + "theoretical_loss": 3.9648808710156986, + "tokens_seen": 444923904 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043695085255767303, + "loss": 2.9173, + "theoretical_loss": 3.964816703388501, + "tokens_seen": 444989440 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004369408224674022, + "loss": 2.8848, + "theoretical_loss": 3.9647525478565644, + "tokens_seen": 445054976 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004369307923771314, + "loss": 2.7532, + "theoretical_loss": 3.9646884044158277, + "tokens_seen": 445120512 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004369207622868606, + "loss": 2.9526, + "theoretical_loss": 3.964624273062234, + "tokens_seen": 445186048 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043691073219658975, + "loss": 2.8736, + "theoretical_loss": 3.9645601537917248, + "tokens_seen": 445251584 + }, + { + "epoch": 0.13, + "learning_rate": 0.000436900702106319, + "loss": 2.7773, + "theoretical_loss": 3.964496046600247, + "tokens_seen": 445317120 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043689067201604817, + "loss": 2.95, + "theoretical_loss": 3.9644319514837467, + "tokens_seen": 445382656 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043688064192577735, + "loss": 2.7412, + "theoretical_loss": 3.964367868438175, + "tokens_seen": 445448192 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043687061183550653, + "loss": 2.81, + "theoretical_loss": 3.9643037974594817, + "tokens_seen": 445513728 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043686058174523577, + "loss": 3.0259, + "theoretical_loss": 3.96423973854362, + "tokens_seen": 445579264 + }, + { + "epoch": 0.14, + "objective/train/docs_used": 264540, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.549363136291504, + "objective/train/theoretical_loss": 3.9641756916865463, + "objective/train/tokens_used": 466104800, + "theoretical_loss": 3.9641756916865463, + "tokens_seen": 445644800 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004368505516549649, + "loss": 2.6899, + "theoretical_loss": 3.9641756916865463, + "tokens_seen": 445644800 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043684052156469413, + "loss": 3.0377, + "theoretical_loss": 3.964111656884217, + "tokens_seen": 445710336 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043683049147442326, + "loss": 3.1186, + "theoretical_loss": 3.9640476341325916, + "tokens_seen": 445775872 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004368204613841525, + "loss": 3.0562, + "theoretical_loss": 3.96398362342763, + "tokens_seen": 445841408 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043681043129388167, + "loss": 3.1745, + "theoretical_loss": 3.9639196247652966, + "tokens_seen": 445906944 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043680040120361085, + "loss": 2.9637, + "theoretical_loss": 3.9638556381415553, + "tokens_seen": 445972480 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043679037111334003, + "loss": 2.9261, + "theoretical_loss": 3.9637916635523736, + "tokens_seen": 446038016 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004367803410230692, + "loss": 3.0515, + "theoretical_loss": 3.9637277009937195, + "tokens_seen": 446103552 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004367703109327984, + "loss": 2.8449, + "theoretical_loss": 3.9636637504615653, + "tokens_seen": 446169088 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043676028084252763, + "loss": 2.9893, + "theoretical_loss": 3.963599811951882, + "tokens_seen": 446234624 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043675025075225676, + "loss": 2.9249, + "theoretical_loss": 3.9635358854606455, + "tokens_seen": 446300160 + }, + { + "epoch": 0.14, + "learning_rate": 0.000436740220661986, + "loss": 3.1088, + "theoretical_loss": 3.9634719709838317, + "tokens_seen": 446365696 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004367301905717151, + "loss": 3.164, + "theoretical_loss": 3.963408068517419, + "tokens_seen": 446431232 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043672016048144436, + "loss": 3.1815, + "theoretical_loss": 3.9633441780573886, + "tokens_seen": 446496768 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043671013039117354, + "loss": 2.6739, + "theoretical_loss": 3.963280299599722, + "tokens_seen": 446562304 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004367001003009027, + "loss": 2.9854, + "theoretical_loss": 3.9632164331404045, + "tokens_seen": 446627840 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004366900702106319, + "loss": 2.9633, + "theoretical_loss": 3.9631525786754214, + "tokens_seen": 446693376 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043668004012036113, + "loss": 2.9701, + "theoretical_loss": 3.9630887362007616, + "tokens_seen": 446758912 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043667001003009026, + "loss": 2.9122, + "theoretical_loss": 3.9630249057124147, + "tokens_seen": 446824448 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004366599799398195, + "loss": 2.8864, + "theoretical_loss": 3.9629610872063727, + "tokens_seen": 446889984 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004366499498495486, + "loss": 2.8935, + "theoretical_loss": 3.9628972806786305, + "tokens_seen": 446955520 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043663991975927786, + "loss": 2.9873, + "theoretical_loss": 3.9628334861251826, + "tokens_seen": 447021056 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043662988966900704, + "loss": 2.8976, + "theoretical_loss": 3.9627697035420284, + "tokens_seen": 447086592 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004366198595787362, + "loss": 3.0552, + "theoretical_loss": 3.9627059329251666, + "tokens_seen": 447152128 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004366098294884654, + "loss": 2.9332, + "theoretical_loss": 3.9626421742705986, + "tokens_seen": 447217664 + }, + { + "epoch": 0.14, + "objective/train/docs_used": 265849, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.68418550491333, + "objective/train/theoretical_loss": 3.9625784275743294, + "objective/train/tokens_used": 467743200, + "theoretical_loss": 3.9625784275743294, + "tokens_seen": 447283200 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004365997993981946, + "loss": 2.8171, + "theoretical_loss": 3.9625784275743294, + "tokens_seen": 447283200 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043658976930792376, + "loss": 2.8575, + "theoretical_loss": 3.9625146928323627, + "tokens_seen": 447348736 + }, + { + "epoch": 0.14, + "learning_rate": 0.000436579739217653, + "loss": 2.7542, + "theoretical_loss": 3.962450970040707, + "tokens_seen": 447414272 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043656970912738213, + "loss": 2.9119, + "theoretical_loss": 3.9623872591953715, + "tokens_seen": 447479808 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043655967903711136, + "loss": 2.8427, + "theoretical_loss": 3.9623235602923677, + "tokens_seen": 447545344 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004365496489468405, + "loss": 2.7994, + "theoretical_loss": 3.9622598733277083, + "tokens_seen": 447610880 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004365396188565697, + "loss": 2.7361, + "theoretical_loss": 3.9621961982974088, + "tokens_seen": 447676416 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004365295887662989, + "loss": 2.9639, + "theoretical_loss": 3.962132535197486, + "tokens_seen": 447741952 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004365195586760281, + "loss": 2.8931, + "theoretical_loss": 3.9620688840239593, + "tokens_seen": 447807488 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043650952858575727, + "loss": 3.1691, + "theoretical_loss": 3.9620052447728478, + "tokens_seen": 447873024 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004364994984954865, + "loss": 2.8389, + "theoretical_loss": 3.9619416174401767, + "tokens_seen": 447938560 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043648946840521563, + "loss": 3.1392, + "theoretical_loss": 3.9618780020219693, + "tokens_seen": 448004096 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043647943831494487, + "loss": 3.0089, + "theoretical_loss": 3.961814398514252, + "tokens_seen": 448069632 + }, + { + "epoch": 0.14, + "learning_rate": 0.000436469408224674, + "loss": 2.9484, + "theoretical_loss": 3.961750806913054, + "tokens_seen": 448135168 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043645937813440323, + "loss": 3.0668, + "theoretical_loss": 3.961687227214406, + "tokens_seen": 448200704 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004364493480441324, + "loss": 2.9342, + "theoretical_loss": 3.961623659414338, + "tokens_seen": 448266240 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004364393179538616, + "loss": 2.7881, + "theoretical_loss": 3.961560103508887, + "tokens_seen": 448331776 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043642928786359077, + "loss": 2.7451, + "theoretical_loss": 3.961496559494087, + "tokens_seen": 448397312 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043641925777331995, + "loss": 2.9249, + "theoretical_loss": 3.9614330273659775, + "tokens_seen": 448462848 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043640922768304913, + "loss": 2.8575, + "theoretical_loss": 3.9613695071205974, + "tokens_seen": 448528384 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043639919759277837, + "loss": 2.954, + "theoretical_loss": 3.9613059987539887, + "tokens_seen": 448593920 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004363891675025075, + "loss": 3.0079, + "theoretical_loss": 3.9612425022621953, + "tokens_seen": 448659456 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043637913741223673, + "loss": 2.8508, + "theoretical_loss": 3.9611790176412622, + "tokens_seen": 448724992 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004363691073219659, + "loss": 2.9911, + "theoretical_loss": 3.961115544887238, + "tokens_seen": 448790528 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004363590772316951, + "loss": 2.8011, + "theoretical_loss": 3.961052083996171, + "tokens_seen": 448856064 + }, + { + "epoch": 0.14, + "objective/train/docs_used": 266141, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.220189094543457, + "objective/train/theoretical_loss": 3.960988634964113, + "objective/train/tokens_used": 469381600, + "theoretical_loss": 3.960988634964113, + "tokens_seen": 448921600 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004363490471414243, + "loss": 3.1738, + "theoretical_loss": 3.960988634964113, + "tokens_seen": 448921600 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043633901705115346, + "loss": 2.9528, + "theoretical_loss": 3.960925197787116, + "tokens_seen": 448987136 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043632898696088264, + "loss": 3.1147, + "theoretical_loss": 3.9608617724612367, + "tokens_seen": 449052672 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043631895687061187, + "loss": 2.921, + "theoretical_loss": 3.960798358982531, + "tokens_seen": 449118208 + }, + { + "epoch": 0.14, + "learning_rate": 0.000436308926780341, + "loss": 2.8936, + "theoretical_loss": 3.960734957347058, + "tokens_seen": 449183744 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043629889669007023, + "loss": 3.1296, + "theoretical_loss": 3.960671567550878, + "tokens_seen": 449249280 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043628886659979936, + "loss": 2.9726, + "theoretical_loss": 3.9606081895900545, + "tokens_seen": 449314816 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004362788365095286, + "loss": 2.8201, + "theoretical_loss": 3.9605448234606513, + "tokens_seen": 449380352 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004362688064192578, + "loss": 3.152, + "theoretical_loss": 3.9604814691587347, + "tokens_seen": 449445888 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043625877632898696, + "loss": 2.7238, + "theoretical_loss": 3.9604181266803726, + "tokens_seen": 449511424 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043624874623871614, + "loss": 2.9547, + "theoretical_loss": 3.960354796021636, + "tokens_seen": 449576960 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004362387161484453, + "loss": 3.293, + "theoretical_loss": 3.960291477178596, + "tokens_seen": 449642496 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004362286860581745, + "loss": 2.8418, + "theoretical_loss": 3.960228170147327, + "tokens_seen": 449708032 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043621865596790374, + "loss": 3.1868, + "theoretical_loss": 3.9601648749239047, + "tokens_seen": 449773568 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043620862587763286, + "loss": 2.8602, + "theoretical_loss": 3.960101591504406, + "tokens_seen": 449839104 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004361985957873621, + "loss": 2.8286, + "theoretical_loss": 3.960038319884912, + "tokens_seen": 449904640 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004361885656970913, + "loss": 2.7125, + "theoretical_loss": 3.9599750600615025, + "tokens_seen": 449970176 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043617853560682046, + "loss": 3.1196, + "theoretical_loss": 3.9599118120302608, + "tokens_seen": 450035712 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004361685055165497, + "loss": 3.0553, + "theoretical_loss": 3.959848575787273, + "tokens_seen": 450101248 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004361584754262788, + "loss": 3.0561, + "theoretical_loss": 3.959785351328626, + "tokens_seen": 450166784 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043614844533600806, + "loss": 3.1042, + "theoretical_loss": 3.9597221386504073, + "tokens_seen": 450232320 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043613841524573724, + "loss": 2.7418, + "theoretical_loss": 3.9596589377487086, + "tokens_seen": 450297856 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004361283851554664, + "loss": 2.905, + "theoretical_loss": 3.9595957486196225, + "tokens_seen": 450363392 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004361183550651956, + "loss": 2.9534, + "theoretical_loss": 3.9595325712592433, + "tokens_seen": 450428928 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004361083249749248, + "loss": 2.9211, + "theoretical_loss": 3.9594694056636675, + "tokens_seen": 450494464 + }, + { + "epoch": 0.14, + "objective/train/docs_used": 266764, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.048290967941284, + "objective/train/theoretical_loss": 3.9594062518289923, + "objective/train/tokens_used": 471020000, + "theoretical_loss": 3.9594062518289923, + "tokens_seen": 450560000 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043609829488465397, + "loss": 2.9536, + "theoretical_loss": 3.9594062518289923, + "tokens_seen": 450560000 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004360882647943832, + "loss": 2.9042, + "theoretical_loss": 3.9593431097513188, + "tokens_seen": 450625536 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043607823470411233, + "loss": 2.6775, + "theoretical_loss": 3.9592799794267486, + "tokens_seen": 450691072 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043606820461384156, + "loss": 3.0896, + "theoretical_loss": 3.9592168608513854, + "tokens_seen": 450756608 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004360581745235707, + "loss": 3.05, + "theoretical_loss": 3.959153754021334, + "tokens_seen": 450822144 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004360481444332999, + "loss": 2.8054, + "theoretical_loss": 3.9590906589327037, + "tokens_seen": 450887680 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004360381143430291, + "loss": 2.5939, + "theoretical_loss": 3.9590275755816022, + "tokens_seen": 450953216 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004360280842527583, + "loss": 2.7348, + "theoretical_loss": 3.958964503964141, + "tokens_seen": 451018752 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043601805416248747, + "loss": 3.0194, + "theoretical_loss": 3.9589014440764334, + "tokens_seen": 451084288 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004360080240722167, + "loss": 2.9974, + "theoretical_loss": 3.9588383959145945, + "tokens_seen": 451149824 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043599799398194583, + "loss": 2.8188, + "theoretical_loss": 3.9587753594747404, + "tokens_seen": 451215360 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043598796389167507, + "loss": 3.0854, + "theoretical_loss": 3.95871233475299, + "tokens_seen": 451280896 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004359779338014042, + "loss": 2.9462, + "theoretical_loss": 3.9586493217454635, + "tokens_seen": 451346432 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043596790371113343, + "loss": 2.9017, + "theoretical_loss": 3.9585863204482834, + "tokens_seen": 451411968 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004359578736208626, + "loss": 2.9185, + "theoretical_loss": 3.958523330857574, + "tokens_seen": 451477504 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004359478435305918, + "loss": 2.9132, + "theoretical_loss": 3.9584603529694604, + "tokens_seen": 451543040 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043593781344032097, + "loss": 2.9372, + "theoretical_loss": 3.9583973867800717, + "tokens_seen": 451608576 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043592778335005015, + "loss": 2.7378, + "theoretical_loss": 3.958334432285537, + "tokens_seen": 451674112 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043591775325977933, + "loss": 2.8516, + "theoretical_loss": 3.9582714894819873, + "tokens_seen": 451739648 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043590772316950857, + "loss": 2.9409, + "theoretical_loss": 3.9582085583655564, + "tokens_seen": 451805184 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004358976930792377, + "loss": 2.8468, + "theoretical_loss": 3.958145638932379, + "tokens_seen": 451870720 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043588766298896693, + "loss": 2.8221, + "theoretical_loss": 3.9580827311785924, + "tokens_seen": 451936256 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004358776328986961, + "loss": 3.0365, + "theoretical_loss": 3.9580198351003357, + "tokens_seen": 452001792 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004358676028084253, + "loss": 2.7919, + "theoretical_loss": 3.9579569506937498, + "tokens_seen": 452067328 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004358575727181545, + "loss": 2.8306, + "theoretical_loss": 3.9578940779549763, + "tokens_seen": 452132864 + }, + { + "epoch": 0.14, + "objective/train/docs_used": 268192, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.8374152183532715, + "objective/train/theoretical_loss": 3.9578312168801597, + "objective/train/tokens_used": 472658400, + "theoretical_loss": 3.9578312168801597, + "tokens_seen": 452198400 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043584754262788366, + "loss": 2.9124, + "theoretical_loss": 3.9578312168801597, + "tokens_seen": 452198400 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043583751253761284, + "loss": 3.1365, + "theoretical_loss": 3.957768367465447, + "tokens_seen": 452263936 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043582748244734207, + "loss": 2.9187, + "theoretical_loss": 3.9577055297069856, + "tokens_seen": 452329472 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004358174523570712, + "loss": 2.8891, + "theoretical_loss": 3.957642703600926, + "tokens_seen": 452395008 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043580742226680043, + "loss": 2.9205, + "theoretical_loss": 3.9575798891434184, + "tokens_seen": 452460544 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043579739217652956, + "loss": 3.0621, + "theoretical_loss": 3.957517086330618, + "tokens_seen": 452526080 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004357873620862588, + "loss": 2.9339, + "theoretical_loss": 3.957454295158679, + "tokens_seen": 452591616 + }, + { + "epoch": 0.14, + "learning_rate": 0.000435777331995988, + "loss": 3.061, + "theoretical_loss": 3.9573915156237587, + "tokens_seen": 452657152 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043576730190571716, + "loss": 3.2064, + "theoretical_loss": 3.9573287477220167, + "tokens_seen": 452722688 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043575727181544634, + "loss": 3.0298, + "theoretical_loss": 3.957265991449613, + "tokens_seen": 452788224 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004357472417251755, + "loss": 2.8136, + "theoretical_loss": 3.957203246802711, + "tokens_seen": 452853760 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004357372116349047, + "loss": 2.9707, + "theoretical_loss": 3.9571405137774747, + "tokens_seen": 452919296 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043572718154463394, + "loss": 3.0314, + "theoretical_loss": 3.9570777923700704, + "tokens_seen": 452984832 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043571715145436306, + "loss": 3.0895, + "theoretical_loss": 3.9570150825766657, + "tokens_seen": 453050368 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004357071213640923, + "loss": 2.7052, + "theoretical_loss": 3.956952384393432, + "tokens_seen": 453115904 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004356970912738215, + "loss": 2.8035, + "theoretical_loss": 3.956889697816539, + "tokens_seen": 453181440 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043568706118355066, + "loss": 2.8973, + "theoretical_loss": 3.9568270228421616, + "tokens_seen": 453246976 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043567703109327984, + "loss": 3.0447, + "theoretical_loss": 3.956764359466475, + "tokens_seen": 453312512 + }, + { + "epoch": 0.14, + "learning_rate": 0.000435667001003009, + "loss": 3.0612, + "theoretical_loss": 3.9567017076856565, + "tokens_seen": 453378048 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004356569709127382, + "loss": 2.9588, + "theoretical_loss": 3.9566390674958845, + "tokens_seen": 453443584 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043564694082246744, + "loss": 2.9879, + "theoretical_loss": 3.9565764388933395, + "tokens_seen": 453509120 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043563691073219657, + "loss": 2.7933, + "theoretical_loss": 3.9565138218742053, + "tokens_seen": 453574656 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004356268806419258, + "loss": 3.1831, + "theoretical_loss": 3.9564512164346652, + "tokens_seen": 453640192 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043561685055165493, + "loss": 2.9409, + "theoretical_loss": 3.956388622570906, + "tokens_seen": 453705728 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043560682046138417, + "loss": 2.987, + "theoretical_loss": 3.956326040279116, + "tokens_seen": 453771264 + }, + { + "epoch": 0.14, + "objective/train/docs_used": 268717, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6514406204223633, + "objective/train/theoretical_loss": 3.956263469555484, + "objective/train/tokens_used": 474296800, + "theoretical_loss": 3.956263469555484, + "tokens_seen": 453836800 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043559679037111335, + "loss": 2.8748, + "theoretical_loss": 3.956263469555484, + "tokens_seen": 453836800 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043558676028084253, + "loss": 2.8034, + "theoretical_loss": 3.956200910396202, + "tokens_seen": 453902336 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004355767301905717, + "loss": 3.0274, + "theoretical_loss": 3.9561383627974642, + "tokens_seen": 453967872 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004355667001003009, + "loss": 3.0405, + "theoretical_loss": 3.956075826755465, + "tokens_seen": 454033408 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043555667001003007, + "loss": 2.9237, + "theoretical_loss": 3.9560133022664026, + "tokens_seen": 454098944 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004355466399197593, + "loss": 3.0747, + "theoretical_loss": 3.955950789326474, + "tokens_seen": 454164480 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043553660982948843, + "loss": 3.1102, + "theoretical_loss": 3.955888287931881, + "tokens_seen": 454230016 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043552657973921767, + "loss": 2.8667, + "theoretical_loss": 3.955825798078826, + "tokens_seen": 454295552 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043551654964894685, + "loss": 2.729, + "theoretical_loss": 3.9557633197635127, + "tokens_seen": 454361088 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043550651955867603, + "loss": 2.839, + "theoretical_loss": 3.9557008529821474, + "tokens_seen": 454426624 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004354964894684052, + "loss": 2.9362, + "theoretical_loss": 3.955638397730939, + "tokens_seen": 454492160 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004354864593781344, + "loss": 2.8926, + "theoretical_loss": 3.955575954006095, + "tokens_seen": 454557696 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004354764292878636, + "loss": 2.8919, + "theoretical_loss": 3.9555135218038284, + "tokens_seen": 454623232 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004354663991975928, + "loss": 2.7315, + "theoretical_loss": 3.955451101120352, + "tokens_seen": 454688768 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043545636910732194, + "loss": 2.662, + "theoretical_loss": 3.955388691951881, + "tokens_seen": 454754304 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043544633901705117, + "loss": 2.9167, + "theoretical_loss": 3.9553262942946312, + "tokens_seen": 454819840 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004354363089267803, + "loss": 3.0394, + "theoretical_loss": 3.9552639081448224, + "tokens_seen": 454885376 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043542627883650953, + "loss": 2.8099, + "theoretical_loss": 3.9552015334986743, + "tokens_seen": 454950912 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043541624874623877, + "loss": 3.0192, + "theoretical_loss": 3.9551391703524086, + "tokens_seen": 455016448 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004354062186559679, + "loss": 2.9066, + "theoretical_loss": 3.95507681870225, + "tokens_seen": 455081984 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043539618856569713, + "loss": 2.5164, + "theoretical_loss": 3.955014478544425, + "tokens_seen": 455147520 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004353861584754263, + "loss": 3.023, + "theoretical_loss": 3.954952149875159, + "tokens_seen": 455213056 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004353761283851555, + "loss": 3.0351, + "theoretical_loss": 3.9548898326906823, + "tokens_seen": 455278592 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004353660982948847, + "loss": 2.8113, + "theoretical_loss": 3.954827526987226, + "tokens_seen": 455344128 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043535606820461386, + "loss": 2.7586, + "theoretical_loss": 3.954765232761023, + "tokens_seen": 455409664 + }, + { + "epoch": 0.14, + "objective/train/docs_used": 269967, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.240110397338867, + "objective/train/theoretical_loss": 3.954702950008308, + "objective/train/tokens_used": 475935200, + "theoretical_loss": 3.954702950008308, + "tokens_seen": 455475200 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043534603811434304, + "loss": 2.757, + "theoretical_loss": 3.954702950008308, + "tokens_seen": 455475200 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043533600802407227, + "loss": 3.0691, + "theoretical_loss": 3.954640678725317, + "tokens_seen": 455540736 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004353259779338014, + "loss": 2.9895, + "theoretical_loss": 3.9545784189082887, + "tokens_seen": 455606272 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043531594784353063, + "loss": 2.712, + "theoretical_loss": 3.9545161705534624, + "tokens_seen": 455671808 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043530591775325976, + "loss": 2.9069, + "theoretical_loss": 3.9544539336570796, + "tokens_seen": 455737344 + }, + { + "epoch": 0.14, + "learning_rate": 0.000435295887662989, + "loss": 3.086, + "theoretical_loss": 3.954391708215385, + "tokens_seen": 455802880 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004352858575727182, + "loss": 2.8042, + "theoretical_loss": 3.954329494224623, + "tokens_seen": 455868416 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043527582748244736, + "loss": 2.8207, + "theoretical_loss": 3.9542672916810404, + "tokens_seen": 455933952 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043526579739217654, + "loss": 2.8312, + "theoretical_loss": 3.9542051005808867, + "tokens_seen": 455999488 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004352557673019057, + "loss": 3.0424, + "theoretical_loss": 3.954142920920412, + "tokens_seen": 456065024 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004352457372116349, + "loss": 2.7416, + "theoretical_loss": 3.954080752695868, + "tokens_seen": 456130560 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043523570712136414, + "loss": 3.0586, + "theoretical_loss": 3.95401859590351, + "tokens_seen": 456196096 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043522567703109326, + "loss": 3.0258, + "theoretical_loss": 3.953956450539594, + "tokens_seen": 456261632 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004352156469408225, + "loss": 2.8587, + "theoretical_loss": 3.953894316600376, + "tokens_seen": 456327168 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004352056168505517, + "loss": 2.8673, + "theoretical_loss": 3.953832194082117, + "tokens_seen": 456392704 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043519558676028086, + "loss": 2.9908, + "theoretical_loss": 3.9537700829810767, + "tokens_seen": 456458240 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043518555667001004, + "loss": 2.8675, + "theoretical_loss": 3.95370798329352, + "tokens_seen": 456523776 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004351755265797392, + "loss": 3.1069, + "theoretical_loss": 3.9536458950157094, + "tokens_seen": 456589312 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004351654964894684, + "loss": 2.9164, + "theoretical_loss": 3.9535838181439127, + "tokens_seen": 456654848 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043515546639919764, + "loss": 3.0541, + "theoretical_loss": 3.9535217526743978, + "tokens_seen": 456720384 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043514543630892677, + "loss": 2.879, + "theoretical_loss": 3.953459698603434, + "tokens_seen": 456785920 + }, + { + "epoch": 0.14, + "learning_rate": 0.000435135406218656, + "loss": 2.9333, + "theoretical_loss": 3.9533976559272936, + "tokens_seen": 456851456 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043512537612838513, + "loss": 3.1338, + "theoretical_loss": 3.95333562464225, + "tokens_seen": 456916992 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043511534603811437, + "loss": 2.9149, + "theoretical_loss": 3.9532736047445787, + "tokens_seen": 456982528 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043510531594784355, + "loss": 3.0358, + "theoretical_loss": 3.953211596230556, + "tokens_seen": 457048064 + }, + { + "epoch": 0.14, + "objective/train/docs_used": 270622, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.2910759449005127, + "objective/train/theoretical_loss": 3.953149599096461, + "objective/train/tokens_used": 477573600, + "theoretical_loss": 3.953149599096461, + "tokens_seen": 457113600 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043509528585757273, + "loss": 3.0412, + "theoretical_loss": 3.953149599096461, + "tokens_seen": 457113600 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004350852557673019, + "loss": 2.9573, + "theoretical_loss": 3.9530876133385737, + "tokens_seen": 457179136 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004350752256770311, + "loss": 2.9645, + "theoretical_loss": 3.953025638953177, + "tokens_seen": 457244672 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043506519558676027, + "loss": 2.7628, + "theoretical_loss": 3.9529636759365543, + "tokens_seen": 457310208 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004350551654964895, + "loss": 2.9187, + "theoretical_loss": 3.9529017242849918, + "tokens_seen": 457375744 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043504513540621863, + "loss": 2.6861, + "theoretical_loss": 3.952839783994776, + "tokens_seen": 457441280 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043503510531594787, + "loss": 2.9612, + "theoretical_loss": 3.9527778550621973, + "tokens_seen": 457506816 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043502507522567705, + "loss": 2.9251, + "theoretical_loss": 3.9527159374835463, + "tokens_seen": 457572352 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043501504513540623, + "loss": 2.9556, + "theoretical_loss": 3.952654031255115, + "tokens_seen": 457637888 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004350050150451354, + "loss": 2.8504, + "theoretical_loss": 3.9525921363731977, + "tokens_seen": 457703424 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004349949849548646, + "loss": 2.8919, + "theoretical_loss": 3.9525302528340918, + "tokens_seen": 457768960 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004349849548645938, + "loss": 2.8991, + "theoretical_loss": 3.9524683806340937, + "tokens_seen": 457834496 + }, + { + "epoch": 0.14, + "learning_rate": 0.000434974924774323, + "loss": 2.8702, + "theoretical_loss": 3.9524065197695046, + "tokens_seen": 457900032 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043496489468405214, + "loss": 2.9465, + "theoretical_loss": 3.9523446702366245, + "tokens_seen": 457965568 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043495486459378137, + "loss": 2.8566, + "theoretical_loss": 3.9522828320317576, + "tokens_seen": 458031104 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004349448345035105, + "loss": 2.9084, + "theoretical_loss": 3.9522210051512077, + "tokens_seen": 458096640 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043493480441323973, + "loss": 3.0886, + "theoretical_loss": 3.9521591895912818, + "tokens_seen": 458162176 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004349247743229689, + "loss": 3.0459, + "theoretical_loss": 3.9520973853482886, + "tokens_seen": 458227712 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004349147442326981, + "loss": 3.0259, + "theoretical_loss": 3.9520355924185377, + "tokens_seen": 458293248 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004349047141424273, + "loss": 2.8961, + "theoretical_loss": 3.951973810798341, + "tokens_seen": 458358784 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004348946840521565, + "loss": 2.8867, + "theoretical_loss": 3.951912040484012, + "tokens_seen": 458424320 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043488465396188564, + "loss": 3.0054, + "theoretical_loss": 3.9518502814718652, + "tokens_seen": 458489856 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004348746238716149, + "loss": 2.738, + "theoretical_loss": 3.9517885337582186, + "tokens_seen": 458555392 + }, + { + "epoch": 0.14, + "learning_rate": 0.000434864593781344, + "loss": 2.9471, + "theoretical_loss": 3.951726797339391, + "tokens_seen": 458620928 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043485456369107324, + "loss": 2.9291, + "theoretical_loss": 3.9516650722117017, + "tokens_seen": 458686464 + }, + { + "epoch": 0.14, + "objective/train/docs_used": 271952, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.8424344062805176, + "objective/train/theoretical_loss": 3.9516033583714734, + "objective/train/tokens_used": 479212000, + "theoretical_loss": 3.9516033583714734, + "tokens_seen": 458752000 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004348445336008024, + "loss": 3.0455, + "theoretical_loss": 3.9516033583714734, + "tokens_seen": 458752000 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004348345035105316, + "loss": 2.7559, + "theoretical_loss": 3.9515416558150305, + "tokens_seen": 458817536 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004348244734202608, + "loss": 2.8139, + "theoretical_loss": 3.9514799645386973, + "tokens_seen": 458883072 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043481444332998996, + "loss": 3.0102, + "theoretical_loss": 3.9514182845388026, + "tokens_seen": 458948608 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043480441323971914, + "loss": 3.0064, + "theoretical_loss": 3.951356615811674, + "tokens_seen": 459014144 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004347943831494484, + "loss": 2.9433, + "theoretical_loss": 3.9512949583536434, + "tokens_seen": 459079680 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004347843530591775, + "loss": 3.0566, + "theoretical_loss": 3.951233312161042, + "tokens_seen": 459145216 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043477432296890674, + "loss": 2.8804, + "theoretical_loss": 3.9511716772302057, + "tokens_seen": 459210752 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043476429287863587, + "loss": 2.9976, + "theoretical_loss": 3.9511100535574686, + "tokens_seen": 459276288 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004347542627883651, + "loss": 2.8573, + "theoretical_loss": 3.9510484411391698, + "tokens_seen": 459341824 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004347442326980943, + "loss": 2.653, + "theoretical_loss": 3.9509868399716472, + "tokens_seen": 459407360 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043473420260782346, + "loss": 2.9946, + "theoretical_loss": 3.9509252500512426, + "tokens_seen": 459472896 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043472417251755265, + "loss": 3.0649, + "theoretical_loss": 3.950863671374299, + "tokens_seen": 459538432 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004347141424272819, + "loss": 3.0308, + "theoretical_loss": 3.9508021039371597, + "tokens_seen": 459603968 + }, + { + "epoch": 0.14, + "learning_rate": 0.000434704112337011, + "loss": 3.1554, + "theoretical_loss": 3.9507405477361726, + "tokens_seen": 459669504 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043469408224674024, + "loss": 3.2222, + "theoretical_loss": 3.9506790027676844, + "tokens_seen": 459735040 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004346840521564694, + "loss": 3.0198, + "theoretical_loss": 3.9506174690280442, + "tokens_seen": 459800576 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004346740220661986, + "loss": 2.6551, + "theoretical_loss": 3.950555946513605, + "tokens_seen": 459866112 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043466399197592784, + "loss": 2.8597, + "theoretical_loss": 3.9504944352207176, + "tokens_seen": 459931648 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043465396188565697, + "loss": 3.0194, + "theoretical_loss": 3.9504329351457383, + "tokens_seen": 459997184 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004346439317953862, + "loss": 2.9285, + "theoretical_loss": 3.9503714462850237, + "tokens_seen": 460062720 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043463390170511533, + "loss": 3.2117, + "theoretical_loss": 3.9503099686349303, + "tokens_seen": 460128256 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043462387161484457, + "loss": 2.8888, + "theoretical_loss": 3.950248502191819, + "tokens_seen": 460193792 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043461384152457375, + "loss": 2.8049, + "theoretical_loss": 3.950187046952051, + "tokens_seen": 460259328 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043460381143430293, + "loss": 3.0334, + "theoretical_loss": 3.9501256029119896, + "tokens_seen": 460324864 + }, + { + "epoch": 0.14, + "objective/train/docs_used": 272724, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.9320149421691895, + "objective/train/theoretical_loss": 3.950064170068, + "objective/train/tokens_used": 480850400, + "theoretical_loss": 3.950064170068, + "tokens_seen": 460390400 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004345937813440321, + "loss": 2.9787, + "theoretical_loss": 3.950064170068, + "tokens_seen": 460390400 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004345837512537613, + "loss": 3.1212, + "theoretical_loss": 3.9500027484164484, + "tokens_seen": 460455936 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043457372116349047, + "loss": 2.784, + "theoretical_loss": 3.9499413379537027, + "tokens_seen": 460521472 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004345636910732197, + "loss": 2.8545, + "theoretical_loss": 3.9498799386761334, + "tokens_seen": 460587008 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043455366098294883, + "loss": 2.91, + "theoretical_loss": 3.949818550580112, + "tokens_seen": 460652544 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043454363089267807, + "loss": 2.9668, + "theoretical_loss": 3.9497571736620127, + "tokens_seen": 460718080 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043453360080240725, + "loss": 2.7516, + "theoretical_loss": 3.949695807918209, + "tokens_seen": 460783616 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043452357071213643, + "loss": 2.8102, + "theoretical_loss": 3.9496344533450793, + "tokens_seen": 460849152 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004345135406218656, + "loss": 3.0728, + "theoretical_loss": 3.949573109939001, + "tokens_seen": 460914688 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004345035105315948, + "loss": 2.9752, + "theoretical_loss": 3.9495117776963538, + "tokens_seen": 460980224 + }, + { + "epoch": 0.14, + "learning_rate": 0.000434493480441324, + "loss": 3.0792, + "theoretical_loss": 3.949450456613521, + "tokens_seen": 461045760 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004344834503510532, + "loss": 2.8097, + "theoretical_loss": 3.9493891466868845, + "tokens_seen": 461111296 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043447342026078234, + "loss": 2.9605, + "theoretical_loss": 3.949327847912831, + "tokens_seen": 461176832 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043446339017051157, + "loss": 2.9155, + "theoretical_loss": 3.9492665602877457, + "tokens_seen": 461242368 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004344533600802407, + "loss": 2.8054, + "theoretical_loss": 3.9492052838080185, + "tokens_seen": 461307904 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043444332998996993, + "loss": 2.9457, + "theoretical_loss": 3.9491440184700393, + "tokens_seen": 461373440 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004344332998996991, + "loss": 2.7742, + "theoretical_loss": 3.9490827642701998, + "tokens_seen": 461438976 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004344232698094283, + "loss": 2.8908, + "theoretical_loss": 3.9490215212048936, + "tokens_seen": 461504512 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004344132397191575, + "loss": 3.0281, + "theoretical_loss": 3.9489602892705165, + "tokens_seen": 461570048 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004344032096288867, + "loss": 2.8196, + "theoretical_loss": 3.948899068463465, + "tokens_seen": 461635584 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043439317953861584, + "loss": 3.1254, + "theoretical_loss": 3.948837858780138, + "tokens_seen": 461701120 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004343831494483451, + "loss": 3.0572, + "theoretical_loss": 3.948776660216935, + "tokens_seen": 461766656 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004343731193580742, + "loss": 3.1401, + "theoretical_loss": 3.948715472770259, + "tokens_seen": 461832192 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043436308926780344, + "loss": 3.0277, + "theoretical_loss": 3.948654296436513, + "tokens_seen": 461897728 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004343530591775326, + "loss": 3.0919, + "theoretical_loss": 3.9485931312121028, + "tokens_seen": 461963264 + }, + { + "epoch": 0.14, + "objective/train/docs_used": 273842, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.260068655014038, + "objective/train/theoretical_loss": 3.9485319770934355, + "objective/train/tokens_used": 482488800, + "theoretical_loss": 3.9485319770934355, + "tokens_seen": 462028800 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004343430290872618, + "loss": 2.8893, + "theoretical_loss": 3.9485319770934355, + "tokens_seen": 462028800 + }, + { + "epoch": 0.14, + "learning_rate": 0.000434332998996991, + "loss": 2.7668, + "theoretical_loss": 3.948470834076919, + "tokens_seen": 462094336 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043432296890672016, + "loss": 2.9771, + "theoretical_loss": 3.9484097021589646, + "tokens_seen": 462159872 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043431293881644934, + "loss": 2.8363, + "theoretical_loss": 3.948348581335984, + "tokens_seen": 462225408 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004343029087261786, + "loss": 2.8293, + "theoretical_loss": 3.94828747160439, + "tokens_seen": 462290944 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004342928786359077, + "loss": 2.8199, + "theoretical_loss": 3.9482263729605993, + "tokens_seen": 462356480 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043428284854563694, + "loss": 2.7446, + "theoretical_loss": 3.9481652854010285, + "tokens_seen": 462422016 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043427281845536607, + "loss": 3.032, + "theoretical_loss": 3.948104208922096, + "tokens_seen": 462487552 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004342627883650953, + "loss": 2.7982, + "theoretical_loss": 3.9480431435202226, + "tokens_seen": 462553088 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004342527582748245, + "loss": 3.1648, + "theoretical_loss": 3.947982089191829, + "tokens_seen": 462618624 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043424272818455367, + "loss": 2.9575, + "theoretical_loss": 3.9479210459333407, + "tokens_seen": 462684160 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043423269809428285, + "loss": 2.9104, + "theoretical_loss": 3.9478600137411823, + "tokens_seen": 462749696 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004342226680040121, + "loss": 2.8028, + "theoretical_loss": 3.94779899261178, + "tokens_seen": 462815232 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004342126379137412, + "loss": 2.7424, + "theoretical_loss": 3.9477379825415637, + "tokens_seen": 462880768 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043420260782347044, + "loss": 2.6088, + "theoretical_loss": 3.9476769835269634, + "tokens_seen": 462946304 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043419257773319957, + "loss": 2.8134, + "theoretical_loss": 3.9476159955644112, + "tokens_seen": 463011840 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004341825476429288, + "loss": 2.7405, + "theoretical_loss": 3.94755501865034, + "tokens_seen": 463077376 + }, + { + "epoch": 0.14, + "learning_rate": 0.000434172517552658, + "loss": 2.848, + "theoretical_loss": 3.947494052781185, + "tokens_seen": 463142912 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043416248746238717, + "loss": 2.7543, + "theoretical_loss": 3.9474330979533843, + "tokens_seen": 463208448 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043415245737211635, + "loss": 2.8824, + "theoretical_loss": 3.947372154163375, + "tokens_seen": 463273984 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043414242728184553, + "loss": 3.0944, + "theoretical_loss": 3.947311221407599, + "tokens_seen": 463339520 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004341323971915747, + "loss": 2.7683, + "theoretical_loss": 3.947250299682498, + "tokens_seen": 463405056 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043412236710130395, + "loss": 2.7817, + "theoretical_loss": 3.9471893889845138, + "tokens_seen": 463470592 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004341123370110331, + "loss": 2.8079, + "theoretical_loss": 3.947128489310093, + "tokens_seen": 463536128 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004341023069207623, + "loss": 2.7468, + "theoretical_loss": 3.947067600655682, + "tokens_seen": 463601664 + }, + { + "epoch": 0.14, + "objective/train/docs_used": 274430, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.837568759918213, + "objective/train/theoretical_loss": 3.94700672301773, + "objective/train/tokens_used": 484127200, + "theoretical_loss": 3.94700672301773, + "tokens_seen": 463667200 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043409227683049144, + "loss": 2.8531, + "theoretical_loss": 3.94700672301773, + "tokens_seen": 463667200 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043408224674022067, + "loss": 2.6696, + "theoretical_loss": 3.946945856392687, + "tokens_seen": 463732736 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043407221664994985, + "loss": 2.847, + "theoretical_loss": 3.9468850007770033, + "tokens_seen": 463798272 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043406218655967903, + "loss": 2.7664, + "theoretical_loss": 3.9468241561671342, + "tokens_seen": 463863808 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004340521564694082, + "loss": 3.0612, + "theoretical_loss": 3.946763322559534, + "tokens_seen": 463929344 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043404212637913745, + "loss": 2.7908, + "theoretical_loss": 3.946702499950659, + "tokens_seen": 463994880 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004340320962888666, + "loss": 2.9674, + "theoretical_loss": 3.9466416883369684, + "tokens_seen": 464060416 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004340220661985958, + "loss": 2.8774, + "theoretical_loss": 3.946580887714921, + "tokens_seen": 464125952 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043401203610832494, + "loss": 3.0304, + "theoretical_loss": 3.9465200980809803, + "tokens_seen": 464191488 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004340020060180542, + "loss": 2.8004, + "theoretical_loss": 3.946459319431608, + "tokens_seen": 464257024 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043399197592778336, + "loss": 2.6148, + "theoretical_loss": 3.9463985517632696, + "tokens_seen": 464322560 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043398194583751254, + "loss": 2.8749, + "theoretical_loss": 3.9463377950724317, + "tokens_seen": 464388096 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004339719157472417, + "loss": 3.1042, + "theoretical_loss": 3.9462770493555617, + "tokens_seen": 464453632 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004339618856569709, + "loss": 2.6774, + "theoretical_loss": 3.9462163146091305, + "tokens_seen": 464519168 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004339518555667001, + "loss": 2.9142, + "theoretical_loss": 3.9461555908296093, + "tokens_seen": 464584704 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004339418254764293, + "loss": 2.7762, + "theoretical_loss": 3.9460948780134713, + "tokens_seen": 464650240 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004339317953861585, + "loss": 2.9453, + "theoretical_loss": 3.94603417615719, + "tokens_seen": 464715776 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004339217652958877, + "loss": 2.9644, + "theoretical_loss": 3.9459734852572437, + "tokens_seen": 464781312 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004339117352056169, + "loss": 2.8071, + "theoretical_loss": 3.9459128053101082, + "tokens_seen": 464846848 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043390170511534604, + "loss": 2.9659, + "theoretical_loss": 3.945852136312265, + "tokens_seen": 464912384 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004338916750250753, + "loss": 2.9349, + "theoretical_loss": 3.945791478260195, + "tokens_seen": 464977920 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004338816449348044, + "loss": 2.6885, + "theoretical_loss": 3.94573083115038, + "tokens_seen": 465043456 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043387161484453364, + "loss": 2.8725, + "theoretical_loss": 3.9456701949793054, + "tokens_seen": 465108992 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004338615847542628, + "loss": 2.9323, + "theoretical_loss": 3.9456095697434566, + "tokens_seen": 465174528 + }, + { + "epoch": 0.14, + "learning_rate": 0.000433851554663992, + "loss": 2.8075, + "theoretical_loss": 3.945548955439322, + "tokens_seen": 465240064 + }, + { + "epoch": 0.14, + "objective/train/docs_used": 275809, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7746970653533936, + "objective/train/theoretical_loss": 3.945488352063391, + "objective/train/tokens_used": 485765600, + "theoretical_loss": 3.945488352063391, + "tokens_seen": 465305600 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004338415245737212, + "loss": 2.7726, + "theoretical_loss": 3.945488352063391, + "tokens_seen": 465305600 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043383149448345036, + "loss": 2.9782, + "theoretical_loss": 3.945427759612154, + "tokens_seen": 465371136 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043382146439317954, + "loss": 2.7619, + "theoretical_loss": 3.945367178082104, + "tokens_seen": 465436672 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004338114343029088, + "loss": 3.0327, + "theoretical_loss": 3.945306607469735, + "tokens_seen": 465502208 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004338014042126379, + "loss": 3.0114, + "theoretical_loss": 3.945246047771543, + "tokens_seen": 465567744 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043379137412236714, + "loss": 2.8958, + "theoretical_loss": 3.9451854989840256, + "tokens_seen": 465633280 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043378134403209627, + "loss": 2.8474, + "theoretical_loss": 3.945124961103682, + "tokens_seen": 465698816 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004337713139418255, + "loss": 2.6359, + "theoretical_loss": 3.9450644341270116, + "tokens_seen": 465764352 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004337612838515547, + "loss": 2.6819, + "theoretical_loss": 3.945003918050518, + "tokens_seen": 465829888 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043375125376128387, + "loss": 2.9443, + "theoretical_loss": 3.9449434128707055, + "tokens_seen": 465895424 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043374122367101305, + "loss": 2.9283, + "theoretical_loss": 3.9448829185840784, + "tokens_seen": 465960960 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004337311935807423, + "loss": 2.8687, + "theoretical_loss": 3.9448224351871444, + "tokens_seen": 466026496 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004337211634904714, + "loss": 2.7588, + "theoretical_loss": 3.944761962676412, + "tokens_seen": 466092032 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043371113340020064, + "loss": 2.7858, + "theoretical_loss": 3.9447015010483923, + "tokens_seen": 466157568 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043370110330992977, + "loss": 2.8465, + "theoretical_loss": 3.9446410502995963, + "tokens_seen": 466223104 + }, + { + "epoch": 0.14, + "learning_rate": 0.000433691073219659, + "loss": 3.211, + "theoretical_loss": 3.9445806104265384, + "tokens_seen": 466288640 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004336810431293882, + "loss": 2.8194, + "theoretical_loss": 3.9445201814257334, + "tokens_seen": 466354176 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043367101303911737, + "loss": 2.8983, + "theoretical_loss": 3.9444597632936977, + "tokens_seen": 466419712 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043366098294884655, + "loss": 2.7465, + "theoretical_loss": 3.9443993560269504, + "tokens_seen": 466485248 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043365095285857573, + "loss": 3.1156, + "theoretical_loss": 3.9443389596220113, + "tokens_seen": 466550784 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004336409227683049, + "loss": 2.9699, + "theoretical_loss": 3.944278574075402, + "tokens_seen": 466616320 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043363089267803415, + "loss": 2.812, + "theoretical_loss": 3.944218199383645, + "tokens_seen": 466681856 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004336208625877633, + "loss": 2.9681, + "theoretical_loss": 3.9441578355432663, + "tokens_seen": 466747392 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004336108324974925, + "loss": 2.8162, + "theoretical_loss": 3.9440974825507915, + "tokens_seen": 466812928 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043360080240722164, + "loss": 2.917, + "theoretical_loss": 3.9440371404027488, + "tokens_seen": 466878464 + }, + { + "epoch": 0.14, + "objective/train/docs_used": 276320, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.3916354179382324, + "objective/train/theoretical_loss": 3.943976809095668, + "objective/train/tokens_used": 487404000, + "theoretical_loss": 3.943976809095668, + "tokens_seen": 466944000 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043359077231695087, + "loss": 3.0526, + "theoretical_loss": 3.943976809095668, + "tokens_seen": 466944000 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043358074222668005, + "loss": 2.7699, + "theoretical_loss": 3.94391648862608, + "tokens_seen": 467009536 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043357071213640923, + "loss": 2.8517, + "theoretical_loss": 3.9438561789905178, + "tokens_seen": 467075072 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004335606820461384, + "loss": 2.9844, + "theoretical_loss": 3.9437958801855157, + "tokens_seen": 467140608 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043355065195586765, + "loss": 3.0042, + "theoretical_loss": 3.9437355922076094, + "tokens_seen": 467206144 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004335406218655968, + "loss": 2.9703, + "theoretical_loss": 3.9436753150533375, + "tokens_seen": 467271680 + }, + { + "epoch": 0.14, + "learning_rate": 0.000433530591775326, + "loss": 2.8942, + "theoretical_loss": 3.943615048719238, + "tokens_seen": 467337216 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043352056168505514, + "loss": 3.0964, + "theoretical_loss": 3.9435547932018524, + "tokens_seen": 467402752 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004335105315947844, + "loss": 2.9685, + "theoretical_loss": 3.943494548497723, + "tokens_seen": 467468288 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043350050150451356, + "loss": 2.8605, + "theoretical_loss": 3.943434314603394, + "tokens_seen": 467533824 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043349047141424274, + "loss": 2.8521, + "theoretical_loss": 3.9433740915154094, + "tokens_seen": 467599360 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004334804413239719, + "loss": 2.8114, + "theoretical_loss": 3.9433138792303177, + "tokens_seen": 467664896 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004334704112337011, + "loss": 2.9503, + "theoretical_loss": 3.943253677744668, + "tokens_seen": 467730432 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004334603811434303, + "loss": 2.9241, + "theoretical_loss": 3.943193487055009, + "tokens_seen": 467795968 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004334503510531595, + "loss": 2.9977, + "theoretical_loss": 3.9431333071578942, + "tokens_seen": 467861504 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043344032096288864, + "loss": 2.9703, + "theoretical_loss": 3.9430731380498765, + "tokens_seen": 467927040 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004334302908726179, + "loss": 2.9968, + "theoretical_loss": 3.94301297972751, + "tokens_seen": 467992576 + }, + { + "epoch": 0.14, + "learning_rate": 0.000433420260782347, + "loss": 2.7806, + "theoretical_loss": 3.9429528321873537, + "tokens_seen": 468058112 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043341023069207624, + "loss": 3.0084, + "theoretical_loss": 3.942892695425963, + "tokens_seen": 468123648 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004334002006018054, + "loss": 2.8026, + "theoretical_loss": 3.9428325694398993, + "tokens_seen": 468189184 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004333901705115346, + "loss": 2.7822, + "theoretical_loss": 3.942772454225724, + "tokens_seen": 468254720 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004333801404212638, + "loss": 3.0415, + "theoretical_loss": 3.94271234978, + "tokens_seen": 468320256 + }, + { + "epoch": 0.14, + "learning_rate": 0.000433370110330993, + "loss": 2.842, + "theoretical_loss": 3.9426522560992905, + "tokens_seen": 468385792 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043336008024072215, + "loss": 3.0318, + "theoretical_loss": 3.942592173180163, + "tokens_seen": 468451328 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004333500501504514, + "loss": 3.1644, + "theoretical_loss": 3.9425321010191854, + "tokens_seen": 468516864 + }, + { + "epoch": 0.14, + "objective/train/docs_used": 277290, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7773685455322266, + "objective/train/theoretical_loss": 3.942472039612926, + "objective/train/tokens_used": 489042400, + "theoretical_loss": 3.942472039612926, + "tokens_seen": 468582400 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004333400200601805, + "loss": 2.8786, + "theoretical_loss": 3.942472039612926, + "tokens_seen": 468582400 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043332998996990974, + "loss": 3.0105, + "theoretical_loss": 3.9424119889579563, + "tokens_seen": 468647936 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004333199598796389, + "loss": 2.9234, + "theoretical_loss": 3.9423519490508476, + "tokens_seen": 468713472 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004333099297893681, + "loss": 3.0443, + "theoretical_loss": 3.9422919198881754, + "tokens_seen": 468779008 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004332998996990973, + "loss": 2.7391, + "theoretical_loss": 3.9422319014665144, + "tokens_seen": 468844544 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043328986960882647, + "loss": 2.9718, + "theoretical_loss": 3.942171893782442, + "tokens_seen": 468910080 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043327983951855565, + "loss": 2.9722, + "theoretical_loss": 3.942111896832537, + "tokens_seen": 468975616 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004332698094282849, + "loss": 3.2219, + "theoretical_loss": 3.9420519106133787, + "tokens_seen": 469041152 + }, + { + "epoch": 0.14, + "learning_rate": 0.000433259779338014, + "loss": 2.863, + "theoretical_loss": 3.9419919351215498, + "tokens_seen": 469106688 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043324974924774325, + "loss": 2.9886, + "theoretical_loss": 3.9419319703536337, + "tokens_seen": 469172224 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043323971915747243, + "loss": 2.7929, + "theoretical_loss": 3.941872016306215, + "tokens_seen": 469237760 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004332296890672016, + "loss": 2.8495, + "theoretical_loss": 3.941812072975881, + "tokens_seen": 469303296 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004332196589769308, + "loss": 2.8848, + "theoretical_loss": 3.9417521403592186, + "tokens_seen": 469368832 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043320962888665997, + "loss": 3.0686, + "theoretical_loss": 3.9416922184528183, + "tokens_seen": 469434368 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043319959879638915, + "loss": 3.0079, + "theoretical_loss": 3.94163230725327, + "tokens_seen": 469499904 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004331895687061184, + "loss": 2.9217, + "theoretical_loss": 3.9415724067571687, + "tokens_seen": 469565440 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043317953861584757, + "loss": 3.0409, + "theoretical_loss": 3.941512516961107, + "tokens_seen": 469630976 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043316950852557675, + "loss": 2.9687, + "theoretical_loss": 3.941452637861681, + "tokens_seen": 469696512 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043315947843530593, + "loss": 2.667, + "theoretical_loss": 3.9413927694554887, + "tokens_seen": 469762048 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004331494483450351, + "loss": 2.9462, + "theoretical_loss": 3.9413329117391287, + "tokens_seen": 469827584 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043313941825476435, + "loss": 2.8807, + "theoretical_loss": 3.9412730647092022, + "tokens_seen": 469893120 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004331293881644935, + "loss": 2.8462, + "theoretical_loss": 3.94121322836231, + "tokens_seen": 469958656 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004331193580742227, + "loss": 2.8125, + "theoretical_loss": 3.9411534026950568, + "tokens_seen": 470024192 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043310932798395184, + "loss": 3.021, + "theoretical_loss": 3.9410935877040476, + "tokens_seen": 470089728 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043309929789368107, + "loss": 2.7756, + "theoretical_loss": 3.9410337833858886, + "tokens_seen": 470155264 + }, + { + "epoch": 0.14, + "objective/train/docs_used": 277851, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.0070855617523193, + "objective/train/theoretical_loss": 3.9409739897371896, + "objective/train/tokens_used": 490680800, + "theoretical_loss": 3.9409739897371896, + "tokens_seen": 470220800 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043308926780341025, + "loss": 2.9121, + "theoretical_loss": 3.9409739897371896, + "tokens_seen": 470220800 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043307923771313943, + "loss": 2.8422, + "theoretical_loss": 3.9409142067545586, + "tokens_seen": 470286336 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004330692076228686, + "loss": 2.997, + "theoretical_loss": 3.940854434434608, + "tokens_seen": 470351872 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043305917753259785, + "loss": 2.8227, + "theoretical_loss": 3.940794672773951, + "tokens_seen": 470417408 + }, + { + "epoch": 0.14, + "learning_rate": 0.000433049147442327, + "loss": 2.7279, + "theoretical_loss": 3.9407349217692023, + "tokens_seen": 470482944 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004330391173520562, + "loss": 2.8458, + "theoretical_loss": 3.9406751814169763, + "tokens_seen": 470548480 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043302908726178534, + "loss": 2.9565, + "theoretical_loss": 3.9406154517138923, + "tokens_seen": 470614016 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004330190571715146, + "loss": 2.736, + "theoretical_loss": 3.940555732656569, + "tokens_seen": 470679552 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043300902708124376, + "loss": 2.9886, + "theoretical_loss": 3.940496024241627, + "tokens_seen": 470745088 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043299899699097294, + "loss": 2.9796, + "theoretical_loss": 3.9404363264656883, + "tokens_seen": 470810624 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004329889669007021, + "loss": 2.8368, + "theoretical_loss": 3.9403766393253767, + "tokens_seen": 470876160 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004329789368104313, + "loss": 2.6814, + "theoretical_loss": 3.9403169628173185, + "tokens_seen": 470941696 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004329689067201605, + "loss": 2.9172, + "theoretical_loss": 3.9402572969381393, + "tokens_seen": 471007232 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004329588766298897, + "loss": 2.8832, + "theoretical_loss": 3.9401976416844677, + "tokens_seen": 471072768 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043294884653961884, + "loss": 2.7385, + "theoretical_loss": 3.9401379970529344, + "tokens_seen": 471138304 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004329388164493481, + "loss": 2.6295, + "theoretical_loss": 3.94007836304017, + "tokens_seen": 471203840 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004329287863590772, + "loss": 2.7271, + "theoretical_loss": 3.9400187396428086, + "tokens_seen": 471269376 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043291875626880644, + "loss": 2.9083, + "theoretical_loss": 3.9399591268574836, + "tokens_seen": 471334912 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004329087261785356, + "loss": 2.639, + "theoretical_loss": 3.9398995246808317, + "tokens_seen": 471400448 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004328986960882648, + "loss": 2.5891, + "theoretical_loss": 3.93983993310949, + "tokens_seen": 471465984 + }, + { + "epoch": 0.14, + "learning_rate": 0.000432888665997994, + "loss": 3.0669, + "theoretical_loss": 3.9397803521400983, + "tokens_seen": 471531520 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004328786359077232, + "loss": 3.0142, + "theoretical_loss": 3.9397207817692967, + "tokens_seen": 471597056 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043286860581745235, + "loss": 2.8592, + "theoretical_loss": 3.9396612219937284, + "tokens_seen": 471662592 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004328585757271816, + "loss": 2.9532, + "theoretical_loss": 3.9396016728100363, + "tokens_seen": 471728128 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004328485456369107, + "loss": 2.8753, + "theoretical_loss": 3.9395421342148653, + "tokens_seen": 471793664 + }, + { + "epoch": 0.14, + "objective/train/docs_used": 279270, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7227942943573, + "objective/train/theoretical_loss": 3.939482606204863, + "objective/train/tokens_used": 492319200, + "theoretical_loss": 3.939482606204863, + "tokens_seen": 471859200 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043283851554663994, + "loss": 2.8261, + "theoretical_loss": 3.939482606204863, + "tokens_seen": 471859200 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004328284854563691, + "loss": 2.8851, + "theoretical_loss": 3.9394230887766772, + "tokens_seen": 471924736 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004328184553660983, + "loss": 3.1478, + "theoretical_loss": 3.939363581926959, + "tokens_seen": 471990272 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004328084252758275, + "loss": 2.9901, + "theoretical_loss": 3.939304085652358, + "tokens_seen": 472055808 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043279839518555667, + "loss": 2.8518, + "theoretical_loss": 3.939244599949528, + "tokens_seen": 472121344 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043278836509528585, + "loss": 2.9039, + "theoretical_loss": 3.939185124815123, + "tokens_seen": 472186880 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004327783350050151, + "loss": 2.7766, + "theoretical_loss": 3.9391256602457996, + "tokens_seen": 472252416 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004327683049147442, + "loss": 2.854, + "theoretical_loss": 3.9390662062382145, + "tokens_seen": 472317952 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043275827482447345, + "loss": 2.8696, + "theoretical_loss": 3.9390067627890275, + "tokens_seen": 472383488 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043274824473420263, + "loss": 2.9184, + "theoretical_loss": 3.938947329894899, + "tokens_seen": 472449024 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004327382146439318, + "loss": 2.8024, + "theoretical_loss": 3.9388879075524907, + "tokens_seen": 472514560 + }, + { + "epoch": 0.14, + "learning_rate": 0.000432728184553661, + "loss": 2.9424, + "theoretical_loss": 3.9388284957584663, + "tokens_seen": 472580096 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043271815446339017, + "loss": 2.9769, + "theoretical_loss": 3.9387690945094906, + "tokens_seen": 472645632 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043270812437311935, + "loss": 3.04, + "theoretical_loss": 3.93870970380223, + "tokens_seen": 472711168 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004326980942828486, + "loss": 2.9603, + "theoretical_loss": 3.938650323633354, + "tokens_seen": 472776704 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004326880641925777, + "loss": 2.9029, + "theoretical_loss": 3.9385909539995305, + "tokens_seen": 472842240 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043267803410230695, + "loss": 2.972, + "theoretical_loss": 3.9385315948974315, + "tokens_seen": 472907776 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004326680040120361, + "loss": 2.9711, + "theoretical_loss": 3.9384722463237294, + "tokens_seen": 472973312 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004326579739217653, + "loss": 2.7436, + "theoretical_loss": 3.938412908275099, + "tokens_seen": 473038848 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004326479438314945, + "loss": 3.0124, + "theoretical_loss": 3.938353580748215, + "tokens_seen": 473104384 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004326379137412237, + "loss": 2.7606, + "theoretical_loss": 3.938294263739755, + "tokens_seen": 473169920 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043262788365095286, + "loss": 2.9038, + "theoretical_loss": 3.9382349572463973, + "tokens_seen": 473235456 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043261785356068204, + "loss": 2.8369, + "theoretical_loss": 3.938175661264823, + "tokens_seen": 473300992 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004326078234704112, + "loss": 3.0219, + "theoretical_loss": 3.9381163757917133, + "tokens_seen": 473366528 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043259779338014045, + "loss": 2.7548, + "theoretical_loss": 3.938057100823751, + "tokens_seen": 473432064 + }, + { + "epoch": 0.14, + "objective/train/docs_used": 279967, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.657824754714966, + "objective/train/theoretical_loss": 3.9379978363576216, + "objective/train/tokens_used": 493957600, + "theoretical_loss": 3.9379978363576216, + "tokens_seen": 473497600 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004325877632898696, + "loss": 2.9267, + "theoretical_loss": 3.9379978363576216, + "tokens_seen": 473497600 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004325777331995988, + "loss": 2.7148, + "theoretical_loss": 3.937938582390011, + "tokens_seen": 473563136 + }, + { + "epoch": 0.14, + "learning_rate": 0.000432567703109328, + "loss": 3.1084, + "theoretical_loss": 3.9378793389176066, + "tokens_seen": 473628672 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004325576730190572, + "loss": 2.9651, + "theoretical_loss": 3.9378201059370976, + "tokens_seen": 473694208 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043254764292878636, + "loss": 2.7828, + "theoretical_loss": 3.9377608834451756, + "tokens_seen": 473759744 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043253761283851554, + "loss": 2.8254, + "theoretical_loss": 3.9377016714385316, + "tokens_seen": 473825280 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004325275827482447, + "loss": 2.9816, + "theoretical_loss": 3.9376424699138606, + "tokens_seen": 473890816 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043251755265797396, + "loss": 2.8706, + "theoretical_loss": 3.937583278867857, + "tokens_seen": 473956352 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004325075225677031, + "loss": 2.9757, + "theoretical_loss": 3.9375240982972173, + "tokens_seen": 474021888 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004324974924774323, + "loss": 2.8231, + "theoretical_loss": 3.9374649281986405, + "tokens_seen": 474087424 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043248746238716145, + "loss": 2.8118, + "theoretical_loss": 3.937405768568826, + "tokens_seen": 474152960 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004324774322968907, + "loss": 2.7456, + "theoretical_loss": 3.937346619404475, + "tokens_seen": 474218496 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043246740220661986, + "loss": 2.9256, + "theoretical_loss": 3.93728748070229, + "tokens_seen": 474284032 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043245737211634904, + "loss": 2.5714, + "theoretical_loss": 3.937228352458976, + "tokens_seen": 474349568 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004324473420260782, + "loss": 2.9909, + "theoretical_loss": 3.9371692346712375, + "tokens_seen": 474415104 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004324373119358074, + "loss": 2.9631, + "theoretical_loss": 3.9371101273357825, + "tokens_seen": 474480640 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043242728184553664, + "loss": 3.0121, + "theoretical_loss": 3.93705103044932, + "tokens_seen": 474546176 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004324172517552658, + "loss": 2.8121, + "theoretical_loss": 3.9369919440085592, + "tokens_seen": 474611712 + }, + { + "epoch": 0.14, + "learning_rate": 0.000432407221664995, + "loss": 3.0241, + "theoretical_loss": 3.9369328680102122, + "tokens_seen": 474677248 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004323971915747242, + "loss": 2.9241, + "theoretical_loss": 3.9368738024509926, + "tokens_seen": 474742784 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004323871614844534, + "loss": 2.8599, + "theoretical_loss": 3.9368147473276145, + "tokens_seen": 474808320 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043237713139418255, + "loss": 2.9685, + "theoretical_loss": 3.936755702636794, + "tokens_seen": 474873856 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004323671013039118, + "loss": 3.0489, + "theoretical_loss": 3.9366966683752493, + "tokens_seen": 474939392 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004323570712136409, + "loss": 2.7665, + "theoretical_loss": 3.936637644539699, + "tokens_seen": 475004928 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043234704112337014, + "loss": 2.8861, + "theoretical_loss": 3.9365786311268636, + "tokens_seen": 475070464 + }, + { + "epoch": 0.14, + "objective/train/docs_used": 280356, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6364424228668213, + "objective/train/theoretical_loss": 3.936519628133466, + "objective/train/tokens_used": 495596000, + "theoretical_loss": 3.936519628133466, + "tokens_seen": 475136000 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004323370110330993, + "loss": 2.8943, + "theoretical_loss": 3.936519628133466, + "tokens_seen": 475136000 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004323269809428285, + "loss": 2.7973, + "theoretical_loss": 3.9364606355562284, + "tokens_seen": 475201536 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004323169508525577, + "loss": 2.991, + "theoretical_loss": 3.9364016533918766, + "tokens_seen": 475267072 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043230692076228687, + "loss": 2.9916, + "theoretical_loss": 3.9363426816371376, + "tokens_seen": 475332608 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043229689067201605, + "loss": 3.2034, + "theoretical_loss": 3.9362837202887384, + "tokens_seen": 475398144 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004322868605817453, + "loss": 2.8219, + "theoretical_loss": 3.9362247693434096, + "tokens_seen": 475463680 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004322768304914744, + "loss": 2.974, + "theoretical_loss": 3.9361658287978813, + "tokens_seen": 475529216 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043226680040120365, + "loss": 2.614, + "theoretical_loss": 3.9361068986488856, + "tokens_seen": 475594752 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043225677031093283, + "loss": 2.7481, + "theoretical_loss": 3.936047978893158, + "tokens_seen": 475660288 + }, + { + "epoch": 0.14, + "learning_rate": 0.000432246740220662, + "loss": 2.741, + "theoretical_loss": 3.935989069527432, + "tokens_seen": 475725824 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004322367101303912, + "loss": 2.9539, + "theoretical_loss": 3.9359301705484455, + "tokens_seen": 475791360 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043222668004012037, + "loss": 2.8519, + "theoretical_loss": 3.935871281952937, + "tokens_seen": 475856896 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043221664994984955, + "loss": 2.9118, + "theoretical_loss": 3.935812403737646, + "tokens_seen": 475922432 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004322066198595788, + "loss": 2.7821, + "theoretical_loss": 3.9357535358993134, + "tokens_seen": 475987968 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004321965897693079, + "loss": 3.0796, + "theoretical_loss": 3.9356946784346825, + "tokens_seen": 476053504 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043218655967903715, + "loss": 2.815, + "theoretical_loss": 3.9356358313404973, + "tokens_seen": 476119040 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004321765295887663, + "loss": 2.7652, + "theoretical_loss": 3.9355769946135037, + "tokens_seen": 476184576 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004321664994984955, + "loss": 2.6323, + "theoretical_loss": 3.9355181682504483, + "tokens_seen": 476250112 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004321564694082247, + "loss": 2.9008, + "theoretical_loss": 3.9354593522480803, + "tokens_seen": 476315648 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004321464393179539, + "loss": 2.8904, + "theoretical_loss": 3.93540054660315, + "tokens_seen": 476381184 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043213640922768306, + "loss": 2.9004, + "theoretical_loss": 3.935341751312408, + "tokens_seen": 476446720 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043212637913741224, + "loss": 3.0093, + "theoretical_loss": 3.9352829663726085, + "tokens_seen": 476512256 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004321163490471414, + "loss": 2.9371, + "theoretical_loss": 3.935224191780505, + "tokens_seen": 476577792 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043210631895687065, + "loss": 3.0159, + "theoretical_loss": 3.9351654275328536, + "tokens_seen": 476643328 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004320962888665998, + "loss": 2.9568, + "theoretical_loss": 3.9351066736264126, + "tokens_seen": 476708864 + }, + { + "epoch": 0.14, + "objective/train/docs_used": 281398, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.84214448928833, + "objective/train/theoretical_loss": 3.93504793005794, + "objective/train/tokens_used": 497234400, + "theoretical_loss": 3.93504793005794, + "tokens_seen": 476774400 + }, + { + "epoch": 0.14, + "learning_rate": 0.000432086258776329, + "loss": 2.9255, + "theoretical_loss": 3.93504793005794, + "tokens_seen": 476774400 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004320762286860582, + "loss": 2.9232, + "theoretical_loss": 3.9349891968241963, + "tokens_seen": 476839936 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004320661985957874, + "loss": 2.9091, + "theoretical_loss": 3.934930473921944, + "tokens_seen": 476905472 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043205616850551656, + "loss": 2.7975, + "theoretical_loss": 3.934871761347945, + "tokens_seen": 476971008 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043204613841524574, + "loss": 2.6699, + "theoretical_loss": 3.9348130590989654, + "tokens_seen": 477036544 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004320361083249749, + "loss": 3.1476, + "theoretical_loss": 3.93475436717177, + "tokens_seen": 477102080 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043202607823470416, + "loss": 3.0251, + "theoretical_loss": 3.9346956855631285, + "tokens_seen": 477167616 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004320160481444333, + "loss": 2.9126, + "theoretical_loss": 3.934637014269808, + "tokens_seen": 477233152 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004320060180541625, + "loss": 2.9272, + "theoretical_loss": 3.93457835328858, + "tokens_seen": 477298688 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043199598796389165, + "loss": 2.604, + "theoretical_loss": 3.9345197026162158, + "tokens_seen": 477364224 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004319859578736209, + "loss": 3.236, + "theoretical_loss": 3.9344610622494893, + "tokens_seen": 477429760 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043197592778335006, + "loss": 2.8634, + "theoretical_loss": 3.934402432185176, + "tokens_seen": 477495296 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043196589769307924, + "loss": 3.0631, + "theoretical_loss": 3.9343438124200514, + "tokens_seen": 477560832 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004319558676028084, + "loss": 3.1505, + "theoretical_loss": 3.9342852029508935, + "tokens_seen": 477626368 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004319458375125376, + "loss": 2.8762, + "theoretical_loss": 3.9342266037744817, + "tokens_seen": 477691904 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004319358074222668, + "loss": 2.8732, + "theoretical_loss": 3.934168014887597, + "tokens_seen": 477757440 + }, + { + "epoch": 0.14, + "learning_rate": 0.000431925777331996, + "loss": 2.8298, + "theoretical_loss": 3.934109436287021, + "tokens_seen": 477822976 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043191574724172515, + "loss": 2.9661, + "theoretical_loss": 3.934050867969537, + "tokens_seen": 477888512 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004319057171514544, + "loss": 2.9302, + "theoretical_loss": 3.9339923099319316, + "tokens_seen": 477954048 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043189568706118357, + "loss": 2.8435, + "theoretical_loss": 3.93393376217099, + "tokens_seen": 478019584 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043188565697091275, + "loss": 2.7831, + "theoretical_loss": 3.9338752246835, + "tokens_seen": 478085120 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043187562688064193, + "loss": 2.7874, + "theoretical_loss": 3.933816697466252, + "tokens_seen": 478150656 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004318655967903711, + "loss": 2.6769, + "theoretical_loss": 3.933758180516036, + "tokens_seen": 478216192 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004318555667001003, + "loss": 3.1009, + "theoretical_loss": 3.9336996738296444, + "tokens_seen": 478281728 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004318455366098295, + "loss": 2.9826, + "theoretical_loss": 3.9336411774038713, + "tokens_seen": 478347264 + }, + { + "epoch": 0.14, + "objective/train/docs_used": 281924, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.9448225498199463, + "objective/train/theoretical_loss": 3.9335826912355114, + "objective/train/tokens_used": 498872800, + "theoretical_loss": 3.9335826912355114, + "tokens_seen": 478412800 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043183550651955865, + "loss": 3.0112, + "theoretical_loss": 3.9335826912355114, + "tokens_seen": 478412800 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004318254764292879, + "loss": 2.8666, + "theoretical_loss": 3.9335242153213614, + "tokens_seen": 478478336 + }, + { + "epoch": 0.15, + "learning_rate": 0.000431815446339017, + "loss": 2.8611, + "theoretical_loss": 3.9334657496582195, + "tokens_seen": 478543872 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043180541624874625, + "loss": 2.7421, + "theoretical_loss": 3.933407294242885, + "tokens_seen": 478609408 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043179538615847543, + "loss": 3.1235, + "theoretical_loss": 3.9333488490721598, + "tokens_seen": 478674944 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004317853560682046, + "loss": 2.9725, + "theoretical_loss": 3.933290414142845, + "tokens_seen": 478740480 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004317753259779338, + "loss": 2.9723, + "theoretical_loss": 3.933231989451744, + "tokens_seen": 478806016 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043176529588766303, + "loss": 2.8758, + "theoretical_loss": 3.9331735749956627, + "tokens_seen": 478871552 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043175526579739215, + "loss": 3.1388, + "theoretical_loss": 3.933115170771409, + "tokens_seen": 478937088 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004317452357071214, + "loss": 3.0295, + "theoretical_loss": 3.9330567767757887, + "tokens_seen": 479002624 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004317352056168505, + "loss": 2.9062, + "theoretical_loss": 3.932998393005613, + "tokens_seen": 479068160 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043172517552657975, + "loss": 3.1517, + "theoretical_loss": 3.932940019457692, + "tokens_seen": 479133696 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043171514543630893, + "loss": 3.0873, + "theoretical_loss": 3.9328816561288384, + "tokens_seen": 479199232 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004317051153460381, + "loss": 2.5959, + "theoretical_loss": 3.932823303015866, + "tokens_seen": 479264768 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004316950852557673, + "loss": 2.9965, + "theoretical_loss": 3.93276496011559, + "tokens_seen": 479330304 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004316850551654965, + "loss": 3.1326, + "theoretical_loss": 3.9327066274248272, + "tokens_seen": 479395840 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004316750250752257, + "loss": 2.8017, + "theoretical_loss": 3.932648304940395, + "tokens_seen": 479461376 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004316649949849549, + "loss": 2.8858, + "theoretical_loss": 3.932589992659114, + "tokens_seen": 479526912 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004316549648946841, + "loss": 2.9178, + "theoretical_loss": 3.9325316905778047, + "tokens_seen": 479592448 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043164493480441326, + "loss": 2.6871, + "theoretical_loss": 3.932473398693289, + "tokens_seen": 479657984 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043163490471414244, + "loss": 2.7482, + "theoretical_loss": 3.932415117002391, + "tokens_seen": 479723520 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004316248746238716, + "loss": 2.937, + "theoretical_loss": 3.9323568455019364, + "tokens_seen": 479789056 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043161484453360085, + "loss": 2.9414, + "theoretical_loss": 3.932298584188751, + "tokens_seen": 479854592 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043160481444333, + "loss": 2.9829, + "theoretical_loss": 3.9322403330596636, + "tokens_seen": 479920128 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004315947843530592, + "loss": 2.8144, + "theoretical_loss": 3.9321820921115034, + "tokens_seen": 479985664 + }, + { + "epoch": 0.15, + "objective/train/docs_used": 283309, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.1536002159118652, + "objective/train/theoretical_loss": 3.932123861341101, + "objective/train/tokens_used": 500511200, + "theoretical_loss": 3.932123861341101, + "tokens_seen": 480051200 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004315847542627884, + "loss": 3.084, + "theoretical_loss": 3.932123861341101, + "tokens_seen": 480051200 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004315747241725176, + "loss": 3.1238, + "theoretical_loss": 3.932065640745289, + "tokens_seen": 480116736 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043156469408224676, + "loss": 2.7977, + "theoretical_loss": 3.932007430320901, + "tokens_seen": 480182272 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043155466399197594, + "loss": 2.8473, + "theoretical_loss": 3.9319492300647725, + "tokens_seen": 480247808 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004315446339017051, + "loss": 2.9037, + "theoretical_loss": 3.93189103997374, + "tokens_seen": 480313344 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043153460381143436, + "loss": 3.1715, + "theoretical_loss": 3.9318328600446413, + "tokens_seen": 480378880 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004315245737211635, + "loss": 2.7681, + "theoretical_loss": 3.931774690274316, + "tokens_seen": 480444416 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004315145436308927, + "loss": 3.0102, + "theoretical_loss": 3.931716530659604, + "tokens_seen": 480509952 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043150451354062185, + "loss": 2.9581, + "theoretical_loss": 3.931658381197349, + "tokens_seen": 480575488 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004314944834503511, + "loss": 2.8529, + "theoretical_loss": 3.9316002418843934, + "tokens_seen": 480641024 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043148445336008026, + "loss": 3.1412, + "theoretical_loss": 3.931542112717583, + "tokens_seen": 480706560 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043147442326980944, + "loss": 2.9265, + "theoretical_loss": 3.931483993693764, + "tokens_seen": 480772096 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004314643931795386, + "loss": 2.9135, + "theoretical_loss": 3.9314258848097845, + "tokens_seen": 480837632 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004314543630892678, + "loss": 3.0839, + "theoretical_loss": 3.9313677860624936, + "tokens_seen": 480903168 + }, + { + "epoch": 0.15, + "learning_rate": 0.000431444332998997, + "loss": 2.6448, + "theoretical_loss": 3.931309697448742, + "tokens_seen": 480968704 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004314343029087262, + "loss": 2.8397, + "theoretical_loss": 3.9312516189653817, + "tokens_seen": 481034240 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043142427281845535, + "loss": 2.8338, + "theoretical_loss": 3.931193550609266, + "tokens_seen": 481099776 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004314142427281846, + "loss": 2.876, + "theoretical_loss": 3.9311354923772504, + "tokens_seen": 481165312 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043140421263791377, + "loss": 2.7882, + "theoretical_loss": 3.9310774442661907, + "tokens_seen": 481230848 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043139418254764295, + "loss": 2.9475, + "theoretical_loss": 3.931019406272945, + "tokens_seen": 481296384 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043138415245737213, + "loss": 2.9823, + "theoretical_loss": 3.930961378394372, + "tokens_seen": 481361920 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004313741223671013, + "loss": 2.7761, + "theoretical_loss": 3.9309033606273327, + "tokens_seen": 481427456 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004313640922768305, + "loss": 2.9472, + "theoretical_loss": 3.9308453529686886, + "tokens_seen": 481492992 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004313540621865597, + "loss": 3.1145, + "theoretical_loss": 3.930787355415304, + "tokens_seen": 481558528 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043134403209628885, + "loss": 3.212, + "theoretical_loss": 3.930729367964042, + "tokens_seen": 481624064 + }, + { + "epoch": 0.15, + "objective/train/docs_used": 283520, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.8117966651916504, + "objective/train/theoretical_loss": 3.93067139061177, + "objective/train/tokens_used": 502149600, + "theoretical_loss": 3.93067139061177, + "tokens_seen": 481689600 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004313340020060181, + "loss": 2.996, + "theoretical_loss": 3.93067139061177, + "tokens_seen": 481689600 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004313239719157472, + "loss": 3.053, + "theoretical_loss": 3.930613423355355, + "tokens_seen": 481755136 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043131394182547645, + "loss": 2.7716, + "theoretical_loss": 3.9305554661916657, + "tokens_seen": 481820672 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043130391173520563, + "loss": 2.7984, + "theoretical_loss": 3.9304975191175733, + "tokens_seen": 481886208 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004312938816449348, + "loss": 3.0125, + "theoretical_loss": 3.9304395821299485, + "tokens_seen": 481951744 + }, + { + "epoch": 0.15, + "learning_rate": 0.000431283851554664, + "loss": 2.7483, + "theoretical_loss": 3.9303816552256654, + "tokens_seen": 482017280 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043127382146439323, + "loss": 3.135, + "theoretical_loss": 3.930323738401597, + "tokens_seen": 482082816 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043126379137412236, + "loss": 2.9922, + "theoretical_loss": 3.930265831654621, + "tokens_seen": 482148352 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004312537612838516, + "loss": 2.9674, + "theoretical_loss": 3.930207934981614, + "tokens_seen": 482213888 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004312437311935807, + "loss": 3.064, + "theoretical_loss": 3.930150048379454, + "tokens_seen": 482279424 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043123370110330995, + "loss": 2.8894, + "theoretical_loss": 3.930092171845022, + "tokens_seen": 482344960 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043122367101303913, + "loss": 2.7913, + "theoretical_loss": 3.930034305375199, + "tokens_seen": 482410496 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004312136409227683, + "loss": 2.7843, + "theoretical_loss": 3.929976448966868, + "tokens_seen": 482476032 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004312036108324975, + "loss": 3.1469, + "theoretical_loss": 3.929918602616913, + "tokens_seen": 482541568 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004311935807422267, + "loss": 3.0558, + "theoretical_loss": 3.92986076632222, + "tokens_seen": 482607104 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043118355065195586, + "loss": 2.9427, + "theoretical_loss": 3.929802940079676, + "tokens_seen": 482672640 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004311735205616851, + "loss": 2.7694, + "theoretical_loss": 3.9297451238861685, + "tokens_seen": 482738176 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004311634904714142, + "loss": 2.9408, + "theoretical_loss": 3.9296873177385887, + "tokens_seen": 482803712 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043115346038114346, + "loss": 2.8998, + "theoretical_loss": 3.929629521633827, + "tokens_seen": 482869248 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004311434302908726, + "loss": 2.9549, + "theoretical_loss": 3.929571735568776, + "tokens_seen": 482934784 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004311334002006018, + "loss": 2.8668, + "theoretical_loss": 3.92951395954033, + "tokens_seen": 483000320 + }, + { + "epoch": 0.15, + "learning_rate": 0.000431123370110331, + "loss": 2.8782, + "theoretical_loss": 3.9294561935453833, + "tokens_seen": 483065856 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004311133400200602, + "loss": 2.9373, + "theoretical_loss": 3.9293984375808337, + "tokens_seen": 483131392 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043110330992978936, + "loss": 2.7618, + "theoretical_loss": 3.929340691643579, + "tokens_seen": 483196928 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004310932798395186, + "loss": 2.8704, + "theoretical_loss": 3.929282955730519, + "tokens_seen": 483262464 + }, + { + "epoch": 0.15, + "objective/train/docs_used": 284734, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.56854510307312, + "objective/train/theoretical_loss": 3.9292252298385533, + "objective/train/tokens_used": 503788000, + "theoretical_loss": 3.9292252298385533, + "tokens_seen": 483328000 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004310832497492477, + "loss": 3.108, + "theoretical_loss": 3.9292252298385533, + "tokens_seen": 483328000 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043107321965897696, + "loss": 3.3435, + "theoretical_loss": 3.929167513964585, + "tokens_seen": 483393536 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004310631895687061, + "loss": 2.7865, + "theoretical_loss": 3.929109808105518, + "tokens_seen": 483459072 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004310531594784353, + "loss": 2.9898, + "theoretical_loss": 3.9290521122582573, + "tokens_seen": 483524608 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004310431293881645, + "loss": 2.8893, + "theoretical_loss": 3.928994426419708, + "tokens_seen": 483590144 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004310330992978937, + "loss": 2.85, + "theoretical_loss": 3.928936750586779, + "tokens_seen": 483655680 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043102306920762286, + "loss": 2.9866, + "theoretical_loss": 3.9288790847563795, + "tokens_seen": 483721216 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043101303911735205, + "loss": 3.1277, + "theoretical_loss": 3.9288214289254193, + "tokens_seen": 483786752 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004310030090270812, + "loss": 3.0371, + "theoretical_loss": 3.9287637830908104, + "tokens_seen": 483852288 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043099297893681046, + "loss": 3.0523, + "theoretical_loss": 3.928706147249466, + "tokens_seen": 483917824 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004309829488465396, + "loss": 3.0268, + "theoretical_loss": 3.9286485213983013, + "tokens_seen": 483983360 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004309729187562688, + "loss": 3.1136, + "theoretical_loss": 3.9285909055342314, + "tokens_seen": 484048896 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043096288866599795, + "loss": 2.9115, + "theoretical_loss": 3.928533299654174, + "tokens_seen": 484114432 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004309528585757272, + "loss": 2.9282, + "theoretical_loss": 3.928475703755048, + "tokens_seen": 484179968 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043094282848545637, + "loss": 3.0863, + "theoretical_loss": 3.9284181178337736, + "tokens_seen": 484245504 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043093279839518555, + "loss": 2.975, + "theoretical_loss": 3.9283605418872716, + "tokens_seen": 484311040 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004309227683049148, + "loss": 2.8693, + "theoretical_loss": 3.9283029759124646, + "tokens_seen": 484376576 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043091273821464397, + "loss": 2.9222, + "theoretical_loss": 3.9282454199062777, + "tokens_seen": 484442112 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043090270812437315, + "loss": 2.828, + "theoretical_loss": 3.928187873865636, + "tokens_seen": 484507648 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043089267803410233, + "loss": 2.984, + "theoretical_loss": 3.928130337787467, + "tokens_seen": 484573184 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004308826479438315, + "loss": 2.9425, + "theoretical_loss": 3.9280728116686974, + "tokens_seen": 484638720 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004308726178535607, + "loss": 3.0575, + "theoretical_loss": 3.9280152955062584, + "tokens_seen": 484704256 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004308625877632899, + "loss": 2.9109, + "theoretical_loss": 3.92795778929708, + "tokens_seen": 484769792 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043085255767301905, + "loss": 3.0603, + "theoretical_loss": 3.9279002930380953, + "tokens_seen": 484835328 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004308425275827483, + "loss": 2.9368, + "theoretical_loss": 3.927842806726237, + "tokens_seen": 484900864 + }, + { + "epoch": 0.15, + "objective/train/docs_used": 285372, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.320526123046875, + "objective/train/theoretical_loss": 3.927785330358441, + "objective/train/tokens_used": 505426400, + "theoretical_loss": 3.927785330358441, + "tokens_seen": 484966400 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004308324974924774, + "loss": 3.0306, + "theoretical_loss": 3.927785330358441, + "tokens_seen": 484966400 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043082246740220665, + "loss": 3.1124, + "theoretical_loss": 3.9277278639316435, + "tokens_seen": 485031936 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043081243731193583, + "loss": 2.6666, + "theoretical_loss": 3.9276704074427826, + "tokens_seen": 485097472 + }, + { + "epoch": 0.15, + "learning_rate": 0.000430802407221665, + "loss": 3.0938, + "theoretical_loss": 3.927612960888797, + "tokens_seen": 485163008 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004307923771313942, + "loss": 2.91, + "theoretical_loss": 3.9275555242666265, + "tokens_seen": 485228544 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043078234704112343, + "loss": 2.6939, + "theoretical_loss": 3.9274980975732143, + "tokens_seen": 485294080 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043077231695085256, + "loss": 2.9326, + "theoretical_loss": 3.927440680805503, + "tokens_seen": 485359616 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004307622868605818, + "loss": 2.749, + "theoretical_loss": 3.9273832739604373, + "tokens_seen": 485425152 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004307522567703109, + "loss": 2.8792, + "theoretical_loss": 3.9273258770349626, + "tokens_seen": 485490688 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043074222668004015, + "loss": 2.8887, + "theoretical_loss": 3.9272684900260266, + "tokens_seen": 485556224 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043073219658976933, + "loss": 2.8105, + "theoretical_loss": 3.9272111129305785, + "tokens_seen": 485621760 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004307221664994985, + "loss": 3.0073, + "theoretical_loss": 3.927153745745567, + "tokens_seen": 485687296 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004307121364092277, + "loss": 2.8398, + "theoretical_loss": 3.9270963884679437, + "tokens_seen": 485752832 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004307021063189569, + "loss": 3.0086, + "theoretical_loss": 3.9270390410946625, + "tokens_seen": 485818368 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043069207622868606, + "loss": 2.9538, + "theoretical_loss": 3.9269817036226757, + "tokens_seen": 485883904 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004306820461384153, + "loss": 2.7207, + "theoretical_loss": 3.92692437604894, + "tokens_seen": 485949440 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004306720160481444, + "loss": 3.1528, + "theoretical_loss": 3.9268670583704113, + "tokens_seen": 486014976 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043066198595787366, + "loss": 2.9905, + "theoretical_loss": 3.926809750584048, + "tokens_seen": 486080512 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004306519558676028, + "loss": 2.7692, + "theoretical_loss": 3.9267524526868085, + "tokens_seen": 486146048 + }, + { + "epoch": 0.15, + "learning_rate": 0.000430641925777332, + "loss": 2.8217, + "theoretical_loss": 3.9266951646756554, + "tokens_seen": 486211584 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004306318956870612, + "loss": 2.9899, + "theoretical_loss": 3.9266378865475495, + "tokens_seen": 486277120 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004306218655967904, + "loss": 2.852, + "theoretical_loss": 3.9265806182994543, + "tokens_seen": 486342656 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043061183550651956, + "loss": 2.6956, + "theoretical_loss": 3.926523359928335, + "tokens_seen": 486408192 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004306018054162488, + "loss": 2.8655, + "theoretical_loss": 3.9264661114311576, + "tokens_seen": 486473728 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004305917753259779, + "loss": 2.8693, + "theoretical_loss": 3.926408872804889, + "tokens_seen": 486539264 + }, + { + "epoch": 0.15, + "objective/train/docs_used": 285999, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.2663328647613525, + "objective/train/theoretical_loss": 3.9263516440464987, + "objective/train/tokens_used": 507064800, + "theoretical_loss": 3.9263516440464987, + "tokens_seen": 486604800 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043058174523570716, + "loss": 3.0994, + "theoretical_loss": 3.9263516440464987, + "tokens_seen": 486604800 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004305717151454363, + "loss": 3.1576, + "theoretical_loss": 3.926294425152957, + "tokens_seen": 486670336 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004305616850551655, + "loss": 2.9948, + "theoretical_loss": 3.926237216121234, + "tokens_seen": 486735872 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004305516549648947, + "loss": 2.9742, + "theoretical_loss": 3.9261800169483037, + "tokens_seen": 486801408 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004305416248746239, + "loss": 2.634, + "theoretical_loss": 3.9261228276311404, + "tokens_seen": 486866944 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043053159478435306, + "loss": 2.875, + "theoretical_loss": 3.926065648166719, + "tokens_seen": 486932480 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043052156469408225, + "loss": 2.8287, + "theoretical_loss": 3.926008478552016, + "tokens_seen": 486998016 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004305115346038114, + "loss": 2.907, + "theoretical_loss": 3.9259513187840103, + "tokens_seen": 487063552 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043050150451354066, + "loss": 2.9674, + "theoretical_loss": 3.9258941688596805, + "tokens_seen": 487129088 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004304914744232698, + "loss": 3.0808, + "theoretical_loss": 3.9258370287760087, + "tokens_seen": 487194624 + }, + { + "epoch": 0.15, + "learning_rate": 0.000430481444332999, + "loss": 2.9677, + "theoretical_loss": 3.9257798985299757, + "tokens_seen": 487260160 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043047141424272815, + "loss": 2.8697, + "theoretical_loss": 3.9257227781185655, + "tokens_seen": 487325696 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004304613841524574, + "loss": 2.6114, + "theoretical_loss": 3.9256656675387633, + "tokens_seen": 487391232 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043045135406218657, + "loss": 2.8936, + "theoretical_loss": 3.925608566787555, + "tokens_seen": 487456768 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043044132397191575, + "loss": 2.9393, + "theoretical_loss": 3.925551475861928, + "tokens_seen": 487522304 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043043129388164493, + "loss": 2.8641, + "theoretical_loss": 3.9254943947588705, + "tokens_seen": 487587840 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043042126379137417, + "loss": 2.7563, + "theoretical_loss": 3.9254373234753737, + "tokens_seen": 487653376 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004304112337011033, + "loss": 2.9566, + "theoretical_loss": 3.925380262008428, + "tokens_seen": 487718912 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043040120361083253, + "loss": 2.773, + "theoretical_loss": 3.9253232103550273, + "tokens_seen": 487784448 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043039117352056165, + "loss": 2.934, + "theoretical_loss": 3.925266168512165, + "tokens_seen": 487849984 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004303811434302909, + "loss": 2.9198, + "theoretical_loss": 3.925209136476836, + "tokens_seen": 487915520 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043037111334002007, + "loss": 3.0079, + "theoretical_loss": 3.9251521142460377, + "tokens_seen": 487981056 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043036108324974925, + "loss": 2.8801, + "theoretical_loss": 3.925095101816768, + "tokens_seen": 488046592 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043035105315947843, + "loss": 2.8975, + "theoretical_loss": 3.925038099186027, + "tokens_seen": 488112128 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004303410230692076, + "loss": 2.9001, + "theoretical_loss": 3.924981106350815, + "tokens_seen": 488177664 + }, + { + "epoch": 0.15, + "objective/train/docs_used": 287428, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.3198938369750977, + "objective/train/theoretical_loss": 3.9249241233081333, + "objective/train/tokens_used": 508703200, + "theoretical_loss": 3.9249241233081333, + "tokens_seen": 488243200 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004303309929789368, + "loss": 3.1042, + "theoretical_loss": 3.9249241233081333, + "tokens_seen": 488243200 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043032096288866603, + "loss": 2.8614, + "theoretical_loss": 3.924867150054985, + "tokens_seen": 488308736 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043031093279839516, + "loss": 2.8117, + "theoretical_loss": 3.9248101865883767, + "tokens_seen": 488374272 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004303009027081244, + "loss": 2.9518, + "theoretical_loss": 3.9247532329053127, + "tokens_seen": 488439808 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004302908726178535, + "loss": 3.0532, + "theoretical_loss": 3.924696289002801, + "tokens_seen": 488505344 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043028084252758276, + "loss": 3.0014, + "theoretical_loss": 3.9246393548778498, + "tokens_seen": 488570880 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043027081243731194, + "loss": 2.8698, + "theoretical_loss": 3.9245824305274697, + "tokens_seen": 488636416 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004302607823470411, + "loss": 2.7369, + "theoretical_loss": 3.9245255159486714, + "tokens_seen": 488701952 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004302507522567703, + "loss": 2.7464, + "theoretical_loss": 3.924468611138467, + "tokens_seen": 488767488 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043024072216649953, + "loss": 2.876, + "theoretical_loss": 3.9244117160938714, + "tokens_seen": 488833024 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043023069207622866, + "loss": 2.988, + "theoretical_loss": 3.9243548308118994, + "tokens_seen": 488898560 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004302206619859579, + "loss": 2.8025, + "theoretical_loss": 3.9242979552895676, + "tokens_seen": 488964096 + }, + { + "epoch": 0.15, + "learning_rate": 0.000430210631895687, + "loss": 3.0412, + "theoretical_loss": 3.9242410895238935, + "tokens_seen": 489029632 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043020060180541626, + "loss": 3.0487, + "theoretical_loss": 3.9241842335118964, + "tokens_seen": 489095168 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043019057171514544, + "loss": 2.8995, + "theoretical_loss": 3.9241273872505973, + "tokens_seen": 489160704 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004301805416248746, + "loss": 2.8383, + "theoretical_loss": 3.924070550737017, + "tokens_seen": 489226240 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043017051153460386, + "loss": 2.8536, + "theoretical_loss": 3.9240137239681787, + "tokens_seen": 489291776 + }, + { + "epoch": 0.15, + "learning_rate": 0.000430160481444333, + "loss": 2.9207, + "theoretical_loss": 3.9239569069411067, + "tokens_seen": 489357312 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004301504513540622, + "loss": 2.8677, + "theoretical_loss": 3.923900099652828, + "tokens_seen": 489422848 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004301404212637914, + "loss": 3.0554, + "theoretical_loss": 3.923843302100368, + "tokens_seen": 489488384 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004301303911735206, + "loss": 2.9006, + "theoretical_loss": 3.9237865142807555, + "tokens_seen": 489553920 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043012036108324976, + "loss": 2.6803, + "theoretical_loss": 3.92372973619102, + "tokens_seen": 489619456 + }, + { + "epoch": 0.15, + "learning_rate": 0.000430110330992979, + "loss": 2.8591, + "theoretical_loss": 3.9236729678281925, + "tokens_seen": 489684992 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004301003009027081, + "loss": 2.9975, + "theoretical_loss": 3.923616209189306, + "tokens_seen": 489750528 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043009027081243736, + "loss": 3.0802, + "theoretical_loss": 3.9235594602713926, + "tokens_seen": 489816064 + }, + { + "epoch": 0.15, + "objective/train/docs_used": 288116, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.310366153717041, + "objective/train/theoretical_loss": 3.923502721071488, + "objective/train/tokens_used": 510341600, + "theoretical_loss": 3.923502721071488, + "tokens_seen": 489881600 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004300802407221665, + "loss": 2.9753, + "theoretical_loss": 3.923502721071488, + "tokens_seen": 489881600 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004300702106318957, + "loss": 3.0156, + "theoretical_loss": 3.9234459915866275, + "tokens_seen": 489947136 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004300601805416249, + "loss": 3.2263, + "theoretical_loss": 3.923389271813849, + "tokens_seen": 490012672 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004300501504513541, + "loss": 3.0369, + "theoretical_loss": 3.923332561750192, + "tokens_seen": 490078208 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043004012036108326, + "loss": 2.9112, + "theoretical_loss": 3.9232758613926952, + "tokens_seen": 490143744 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043003009027081245, + "loss": 2.8623, + "theoretical_loss": 3.9232191707384, + "tokens_seen": 490209280 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043002006018054163, + "loss": 2.9178, + "theoretical_loss": 3.9231624897843496, + "tokens_seen": 490274816 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043001003009027086, + "loss": 2.819, + "theoretical_loss": 3.9231058185275884, + "tokens_seen": 490340352 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043, + "loss": 2.7452, + "theoretical_loss": 3.9230491569651598, + "tokens_seen": 490405888 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004299899699097292, + "loss": 2.8238, + "theoretical_loss": 3.922992505094112, + "tokens_seen": 490471424 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042997993981945835, + "loss": 2.9465, + "theoretical_loss": 3.9229358629114923, + "tokens_seen": 490536960 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004299699097291876, + "loss": 2.846, + "theoretical_loss": 3.922879230414349, + "tokens_seen": 490602496 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042995987963891677, + "loss": 2.9373, + "theoretical_loss": 3.9228226075997332, + "tokens_seen": 490668032 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042994984954864595, + "loss": 2.7309, + "theoretical_loss": 3.9227659944646964, + "tokens_seen": 490733568 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042993981945837513, + "loss": 2.9189, + "theoretical_loss": 3.9227093910062916, + "tokens_seen": 490799104 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042992978936810437, + "loss": 3.1859, + "theoretical_loss": 3.922652797221573, + "tokens_seen": 490864640 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004299197592778335, + "loss": 2.7944, + "theoretical_loss": 3.922596213107596, + "tokens_seen": 490930176 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042990972918756273, + "loss": 2.8067, + "theoretical_loss": 3.922539638661418, + "tokens_seen": 490995712 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042989969909729185, + "loss": 2.991, + "theoretical_loss": 3.9224830738800964, + "tokens_seen": 491061248 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004298896690070211, + "loss": 2.9528, + "theoretical_loss": 3.92242651876069, + "tokens_seen": 491126784 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042987963891675027, + "loss": 2.7902, + "theoretical_loss": 3.9223699733002615, + "tokens_seen": 491192320 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042986960882647945, + "loss": 3.1173, + "theoretical_loss": 3.922313437495871, + "tokens_seen": 491257856 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042985957873620863, + "loss": 2.8256, + "theoretical_loss": 3.9222569113445824, + "tokens_seen": 491323392 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004298495486459378, + "loss": 2.9143, + "theoretical_loss": 3.9222003948434603, + "tokens_seen": 491388928 + }, + { + "epoch": 0.15, + "learning_rate": 0.000429839518555667, + "loss": 2.9891, + "theoretical_loss": 3.92214388798957, + "tokens_seen": 491454464 + }, + { + "debugging/Self-BLEU-5": 0.29933558328520904, + "debugging/distinct-1-grams": 0.7812461879009056, + "debugging/distinct-2-grams": 0.9481517711182267, + "debugging/entropy-1-grams": 5.457543907702052, + "debugging/entropy-2-grams": 6.107665944528154, + "debugging/length": 478.57142857142856, + "debugging/num_segments": 7, + "debugging/score": 0.004207094925093749, + "debugging/score_std": 0.0033741757902465514, + "epoch": 0.15, + "objective/train/docs_used": 289197, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.0408923625946045, + "objective/train/theoretical_loss": 3.92208739077998, + "objective/train/tokens_used": 511980000, + "theoretical_loss": 3.92208739077998, + "tokens_seen": 491520000 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042982948846539623, + "loss": 3.0432, + "theoretical_loss": 3.92208739077998, + "tokens_seen": 491520000 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042981945837512536, + "loss": 2.8882, + "theoretical_loss": 3.9220309032117573, + "tokens_seen": 491585536 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004298094282848546, + "loss": 2.856, + "theoretical_loss": 3.921974425281972, + "tokens_seen": 491651072 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004297993981945837, + "loss": 2.8119, + "theoretical_loss": 3.921917956987695, + "tokens_seen": 491716608 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042978936810431296, + "loss": 3.0812, + "theoretical_loss": 3.921861498325998, + "tokens_seen": 491782144 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042977933801404214, + "loss": 3.0534, + "theoretical_loss": 3.921805049293956, + "tokens_seen": 491847680 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004297693079237713, + "loss": 2.889, + "theoretical_loss": 3.9217486098886427, + "tokens_seen": 491913216 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004297592778335005, + "loss": 2.8318, + "theoretical_loss": 3.921692180107134, + "tokens_seen": 491978752 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042974924774322973, + "loss": 2.8851, + "theoretical_loss": 3.9216357599465073, + "tokens_seen": 492044288 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042973921765295886, + "loss": 2.8892, + "theoretical_loss": 3.921579349403842, + "tokens_seen": 492109824 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004297291875626881, + "loss": 3.0539, + "theoretical_loss": 3.921522948476217, + "tokens_seen": 492175360 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004297191574724172, + "loss": 3.1051, + "theoretical_loss": 3.9214665571607146, + "tokens_seen": 492240896 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042970912738214646, + "loss": 3.0621, + "theoretical_loss": 3.921410175454416, + "tokens_seen": 492306432 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042969909729187564, + "loss": 2.6807, + "theoretical_loss": 3.921353803354406, + "tokens_seen": 492371968 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004296890672016048, + "loss": 2.8774, + "theoretical_loss": 3.9212974408577685, + "tokens_seen": 492437504 + }, + { + "epoch": 0.15, + "learning_rate": 0.000429679037111334, + "loss": 2.6384, + "theoretical_loss": 3.921241087961591, + "tokens_seen": 492503040 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004296690070210632, + "loss": 2.9234, + "theoretical_loss": 3.92118474466296, + "tokens_seen": 492568576 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042965897693079236, + "loss": 2.8968, + "theoretical_loss": 3.9211284109589646, + "tokens_seen": 492634112 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004296489468405216, + "loss": 2.7982, + "theoretical_loss": 3.9210720868466953, + "tokens_seen": 492699648 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004296389167502507, + "loss": 2.7853, + "theoretical_loss": 3.9210157723232424, + "tokens_seen": 492765184 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042962888665997996, + "loss": 2.6334, + "theoretical_loss": 3.9209594673857, + "tokens_seen": 492830720 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004296188565697091, + "loss": 3.0944, + "theoretical_loss": 3.920903172031161, + "tokens_seen": 492896256 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004296088264794383, + "loss": 2.8478, + "theoretical_loss": 3.9208468862567205, + "tokens_seen": 492961792 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004295987963891675, + "loss": 2.8111, + "theoretical_loss": 3.9207906100594756, + "tokens_seen": 493027328 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004295887662988967, + "loss": 2.9091, + "theoretical_loss": 3.920734343436523, + "tokens_seen": 493092864 + }, + { + "epoch": 0.15, + "objective/train/docs_used": 289918, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.1965503692626953, + "objective/train/theoretical_loss": 3.920678086384963, + "objective/train/tokens_used": 513618400, + "theoretical_loss": 3.920678086384963, + "tokens_seen": 493158400 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042957873620862587, + "loss": 2.6993, + "theoretical_loss": 3.920678086384963, + "tokens_seen": 493158400 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004295687061183551, + "loss": 2.8643, + "theoretical_loss": 3.9206218389018943, + "tokens_seen": 493223936 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042955867602808423, + "loss": 2.7212, + "theoretical_loss": 3.9205656009844194, + "tokens_seen": 493289472 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042954864593781347, + "loss": 2.9698, + "theoretical_loss": 3.9205093726296405, + "tokens_seen": 493355008 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004295386158475426, + "loss": 2.9752, + "theoretical_loss": 3.920453153834662, + "tokens_seen": 493420544 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042952858575727183, + "loss": 3.0748, + "theoretical_loss": 3.9203969445965887, + "tokens_seen": 493486080 + }, + { + "epoch": 0.15, + "learning_rate": 0.000429518555667001, + "loss": 2.8983, + "theoretical_loss": 3.9203407449125276, + "tokens_seen": 493551616 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004295085255767302, + "loss": 2.8256, + "theoretical_loss": 3.9202845547795864, + "tokens_seen": 493617152 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042949849548645937, + "loss": 2.5989, + "theoretical_loss": 3.920228374194874, + "tokens_seen": 493682688 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042948846539618855, + "loss": 3.0458, + "theoretical_loss": 3.9201722031555004, + "tokens_seen": 493748224 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042947843530591773, + "loss": 2.8063, + "theoretical_loss": 3.920116041658578, + "tokens_seen": 493813760 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042946840521564697, + "loss": 2.8478, + "theoretical_loss": 3.9200598897012187, + "tokens_seen": 493879296 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004294583751253761, + "loss": 2.9737, + "theoretical_loss": 3.9200037472805374, + "tokens_seen": 493944832 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042944834503510533, + "loss": 2.9114, + "theoretical_loss": 3.9199476143936485, + "tokens_seen": 494010368 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004294383149448345, + "loss": 3.0112, + "theoretical_loss": 3.919891491037669, + "tokens_seen": 494075904 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004294282848545637, + "loss": 3.218, + "theoretical_loss": 3.9198353772097168, + "tokens_seen": 494141440 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042941825476429293, + "loss": 3.1216, + "theoretical_loss": 3.9197792729069114, + "tokens_seen": 494206976 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042940822467402206, + "loss": 2.8219, + "theoretical_loss": 3.9197231781263726, + "tokens_seen": 494272512 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004293981945837513, + "loss": 2.6738, + "theoretical_loss": 3.919667092865222, + "tokens_seen": 494338048 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042938816449348047, + "loss": 3.0089, + "theoretical_loss": 3.919611017120583, + "tokens_seen": 494403584 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042937813440320965, + "loss": 2.9393, + "theoretical_loss": 3.919554950889579, + "tokens_seen": 494469120 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042936810431293883, + "loss": 2.897, + "theoretical_loss": 3.9194988941693354, + "tokens_seen": 494534656 + }, + { + "epoch": 0.15, + "learning_rate": 0.000429358074222668, + "loss": 3.0068, + "theoretical_loss": 3.9194428469569793, + "tokens_seen": 494600192 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004293480441323972, + "loss": 2.8919, + "theoretical_loss": 3.9193868092496382, + "tokens_seen": 494665728 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042933801404212643, + "loss": 2.8938, + "theoretical_loss": 3.919330781044441, + "tokens_seen": 494731264 + }, + { + "epoch": 0.15, + "objective/train/docs_used": 291263, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.851949453353882, + "objective/train/theoretical_loss": 3.919274762338519, + "objective/train/tokens_used": 515256800, + "theoretical_loss": 3.919274762338519, + "tokens_seen": 494796800 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042932798395185556, + "loss": 2.8708, + "theoretical_loss": 3.919274762338519, + "tokens_seen": 494796800 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004293179538615848, + "loss": 2.9587, + "theoretical_loss": 3.9192187531290026, + "tokens_seen": 494862336 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004293079237713139, + "loss": 2.8492, + "theoretical_loss": 3.9191627534130253, + "tokens_seen": 494927872 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042929789368104316, + "loss": 3.0361, + "theoretical_loss": 3.9191067631877208, + "tokens_seen": 494993408 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042928786359077234, + "loss": 2.8502, + "theoretical_loss": 3.919050782450225, + "tokens_seen": 495058944 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004292778335005015, + "loss": 2.7377, + "theoretical_loss": 3.9189948111976736, + "tokens_seen": 495124480 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004292678034102307, + "loss": 2.8325, + "theoretical_loss": 3.9189388494272057, + "tokens_seen": 495190016 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042925777331995993, + "loss": 2.8738, + "theoretical_loss": 3.9188828971359584, + "tokens_seen": 495255552 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042924774322968906, + "loss": 2.9533, + "theoretical_loss": 3.918826954321074, + "tokens_seen": 495321088 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004292377131394183, + "loss": 2.9083, + "theoretical_loss": 3.918771020979693, + "tokens_seen": 495386624 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004292276830491474, + "loss": 3.1463, + "theoretical_loss": 3.9187150971089584, + "tokens_seen": 495452160 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042921765295887666, + "loss": 2.9702, + "theoretical_loss": 3.918659182706014, + "tokens_seen": 495517696 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042920762286860584, + "loss": 2.929, + "theoretical_loss": 3.9186032777680055, + "tokens_seen": 495583232 + }, + { + "epoch": 0.15, + "learning_rate": 0.000429197592778335, + "loss": 2.9069, + "theoretical_loss": 3.918547382292079, + "tokens_seen": 495648768 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004291875626880642, + "loss": 2.6513, + "theoretical_loss": 3.918491496275383, + "tokens_seen": 495714304 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004291775325977934, + "loss": 2.9242, + "theoretical_loss": 3.9184356197150647, + "tokens_seen": 495779840 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042916750250752256, + "loss": 2.7778, + "theoretical_loss": 3.918379752608276, + "tokens_seen": 495845376 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004291574724172518, + "loss": 2.859, + "theoretical_loss": 3.918323894952168, + "tokens_seen": 495910912 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004291474423269809, + "loss": 3.0265, + "theoretical_loss": 3.9182680467438926, + "tokens_seen": 495976448 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042913741223671016, + "loss": 3.0242, + "theoretical_loss": 3.9182122079806048, + "tokens_seen": 496041984 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004291273821464393, + "loss": 3.0625, + "theoretical_loss": 3.918156378659459, + "tokens_seen": 496107520 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004291173520561685, + "loss": 3.0897, + "theoretical_loss": 3.9181005587776117, + "tokens_seen": 496173056 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004291073219658977, + "loss": 2.7418, + "theoretical_loss": 3.9180447483322203, + "tokens_seen": 496238592 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004290972918756269, + "loss": 2.8765, + "theoretical_loss": 3.9179889473204446, + "tokens_seen": 496304128 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042908726178535607, + "loss": 2.9394, + "theoretical_loss": 3.9179331557394432, + "tokens_seen": 496369664 + }, + { + "epoch": 0.15, + "objective/train/docs_used": 291574, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.8119137287139893, + "objective/train/theoretical_loss": 3.9178773735863786, + "objective/train/tokens_used": 516895200, + "theoretical_loss": 3.9178773735863786, + "tokens_seen": 496435200 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004290772316950853, + "loss": 2.8349, + "theoretical_loss": 3.9178773735863786, + "tokens_seen": 496435200 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042906720160481443, + "loss": 2.7471, + "theoretical_loss": 3.917821600858413, + "tokens_seen": 496500736 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042905717151454367, + "loss": 2.7906, + "theoretical_loss": 3.917765837552709, + "tokens_seen": 496566272 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004290471414242728, + "loss": 2.9434, + "theoretical_loss": 3.917710083666434, + "tokens_seen": 496631808 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042903711133400203, + "loss": 3.0404, + "theoretical_loss": 3.917654339196752, + "tokens_seen": 496697344 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004290270812437312, + "loss": 2.8492, + "theoretical_loss": 3.9175986041408315, + "tokens_seen": 496762880 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004290170511534604, + "loss": 3.0537, + "theoretical_loss": 3.9175428784958406, + "tokens_seen": 496828416 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042900702106318957, + "loss": 3.0132, + "theoretical_loss": 3.9174871622589498, + "tokens_seen": 496893952 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042899699097291875, + "loss": 3.0025, + "theoretical_loss": 3.91743145542733, + "tokens_seen": 496959488 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042898696088264793, + "loss": 3.0339, + "theoretical_loss": 3.9173757579981534, + "tokens_seen": 497025024 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042897693079237717, + "loss": 2.9427, + "theoretical_loss": 3.9173200699685937, + "tokens_seen": 497090560 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004289669007021063, + "loss": 3.0043, + "theoretical_loss": 3.9172643913358254, + "tokens_seen": 497156096 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042895687061183553, + "loss": 3.1313, + "theoretical_loss": 3.9172087220970244, + "tokens_seen": 497221632 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004289468405215647, + "loss": 3.1154, + "theoretical_loss": 3.917153062249368, + "tokens_seen": 497287168 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004289368104312939, + "loss": 2.8462, + "theoretical_loss": 3.917097411790036, + "tokens_seen": 497352704 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004289267803410231, + "loss": 2.9908, + "theoretical_loss": 3.9170417707162057, + "tokens_seen": 497418240 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042891675025075226, + "loss": 2.9634, + "theoretical_loss": 3.9169861390250595, + "tokens_seen": 497483776 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042890672016048144, + "loss": 2.8281, + "theoretical_loss": 3.9169305167137796, + "tokens_seen": 497549312 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042889669007021067, + "loss": 2.9926, + "theoretical_loss": 3.916874903779549, + "tokens_seen": 497614848 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004288866599799398, + "loss": 2.8312, + "theoretical_loss": 3.9168193002195513, + "tokens_seen": 497680384 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042887662988966903, + "loss": 3.1638, + "theoretical_loss": 3.9167637060309732, + "tokens_seen": 497745920 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042886659979939816, + "loss": 2.96, + "theoretical_loss": 3.916708121211002, + "tokens_seen": 497811456 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004288565697091274, + "loss": 2.8966, + "theoretical_loss": 3.9166525457568246, + "tokens_seen": 497876992 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004288465396188566, + "loss": 2.8574, + "theoretical_loss": 3.916596979665632, + "tokens_seen": 497942528 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042883650952858576, + "loss": 2.933, + "theoretical_loss": 3.9165414229346136, + "tokens_seen": 498008064 + }, + { + "epoch": 0.15, + "objective/train/docs_used": 293024, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.908184051513672, + "objective/train/theoretical_loss": 3.9164858755609613, + "objective/train/tokens_used": 518533600, + "theoretical_loss": 3.9164858755609613, + "tokens_seen": 498073600 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042882647943831494, + "loss": 2.8466, + "theoretical_loss": 3.9164858755609613, + "tokens_seen": 498073600 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004288164493480441, + "loss": 2.7834, + "theoretical_loss": 3.916430337541869, + "tokens_seen": 498139136 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004288064192577733, + "loss": 2.8424, + "theoretical_loss": 3.9163748088745303, + "tokens_seen": 498204672 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042879638916750254, + "loss": 2.9795, + "theoretical_loss": 3.9163192895561405, + "tokens_seen": 498270208 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042878635907723166, + "loss": 3.1154, + "theoretical_loss": 3.916263779583896, + "tokens_seen": 498335744 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004287763289869609, + "loss": 2.7956, + "theoretical_loss": 3.9162082789549952, + "tokens_seen": 498401280 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004287662988966901, + "loss": 2.7543, + "theoretical_loss": 3.916152787666638, + "tokens_seen": 498466816 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042875626880641926, + "loss": 2.9317, + "theoretical_loss": 3.916097305716023, + "tokens_seen": 498532352 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042874623871614844, + "loss": 2.7106, + "theoretical_loss": 3.9160418331003526, + "tokens_seen": 498597888 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004287362086258776, + "loss": 2.7216, + "theoretical_loss": 3.91598636981683, + "tokens_seen": 498663424 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004287261785356068, + "loss": 2.86, + "theoretical_loss": 3.9159309158626576, + "tokens_seen": 498728960 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042871614844533604, + "loss": 3.0506, + "theoretical_loss": 3.9158754712350414, + "tokens_seen": 498794496 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042870611835506517, + "loss": 3.0257, + "theoretical_loss": 3.915820035931188, + "tokens_seen": 498860032 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004286960882647944, + "loss": 2.8683, + "theoretical_loss": 3.9157646099483046, + "tokens_seen": 498925568 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042868605817452353, + "loss": 2.9048, + "theoretical_loss": 3.9157091932836, + "tokens_seen": 498991104 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042867602808425276, + "loss": 2.8773, + "theoretical_loss": 3.9156537859342837, + "tokens_seen": 499056640 + }, + { + "epoch": 0.15, + "learning_rate": 0.000428665997993982, + "loss": 3.0567, + "theoretical_loss": 3.9155983878975675, + "tokens_seen": 499122176 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004286559679037111, + "loss": 2.9745, + "theoretical_loss": 3.9155429991706634, + "tokens_seen": 499187712 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042864593781344036, + "loss": 3.0627, + "theoretical_loss": 3.9154876197507846, + "tokens_seen": 499253248 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004286359077231695, + "loss": 2.8533, + "theoretical_loss": 3.915432249635147, + "tokens_seen": 499318784 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004286258776328987, + "loss": 2.8514, + "theoretical_loss": 3.9153768888209646, + "tokens_seen": 499384320 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004286158475426279, + "loss": 2.9884, + "theoretical_loss": 3.915321537305456, + "tokens_seen": 499449856 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004286058174523571, + "loss": 2.8498, + "theoretical_loss": 3.915266195085839, + "tokens_seen": 499515392 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042859578736208627, + "loss": 2.9201, + "theoretical_loss": 3.9152108621593333, + "tokens_seen": 499580928 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004285857572718155, + "loss": 3.092, + "theoretical_loss": 3.91515553852316, + "tokens_seen": 499646464 + }, + { + "epoch": 0.15, + "objective/train/docs_used": 293514, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.9235665798187256, + "objective/train/theoretical_loss": 3.91510022417454, + "objective/train/tokens_used": 520172000, + "theoretical_loss": 3.91510022417454, + "tokens_seen": 499712000 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042857572718154463, + "loss": 2.8539, + "theoretical_loss": 3.91510022417454, + "tokens_seen": 499712000 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042856569709127387, + "loss": 3.0331, + "theoretical_loss": 3.915044919110697, + "tokens_seen": 499777536 + }, + { + "epoch": 0.15, + "learning_rate": 0.000428555667001003, + "loss": 2.8622, + "theoretical_loss": 3.9149896233288555, + "tokens_seen": 499843072 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042854563691073223, + "loss": 3.0036, + "theoretical_loss": 3.9149343368262404, + "tokens_seen": 499908608 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004285356068204614, + "loss": 3.0279, + "theoretical_loss": 3.9148790596000786, + "tokens_seen": 499974144 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004285255767301906, + "loss": 2.6873, + "theoretical_loss": 3.9148237916475983, + "tokens_seen": 500039680 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042851554663991977, + "loss": 3.0611, + "theoretical_loss": 3.9147685329660282, + "tokens_seen": 500105216 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042850551654964895, + "loss": 3.0403, + "theoretical_loss": 3.914713283552599, + "tokens_seen": 500170752 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042849548645937813, + "loss": 2.89, + "theoretical_loss": 3.9146580434045415, + "tokens_seen": 500236288 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042848545636910737, + "loss": 3.0, + "theoretical_loss": 3.914602812519089, + "tokens_seen": 500301824 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004284754262788365, + "loss": 3.0307, + "theoretical_loss": 3.9145475908934753, + "tokens_seen": 500367360 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042846539618856573, + "loss": 2.9831, + "theoretical_loss": 3.914492378524934, + "tokens_seen": 500432896 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004284553660982949, + "loss": 3.2041, + "theoretical_loss": 3.9144371754107032, + "tokens_seen": 500498432 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004284453360080241, + "loss": 2.7832, + "theoretical_loss": 3.9143819815480194, + "tokens_seen": 500563968 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004284353059177533, + "loss": 3.2082, + "theoretical_loss": 3.9143267969341218, + "tokens_seen": 500629504 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042842527582748246, + "loss": 2.9567, + "theoretical_loss": 3.9142716215662494, + "tokens_seen": 500695040 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042841524573721164, + "loss": 3.0619, + "theoretical_loss": 3.9142164554416428, + "tokens_seen": 500760576 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042840521564694087, + "loss": 2.7947, + "theoretical_loss": 3.9141612985575454, + "tokens_seen": 500826112 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042839518555667, + "loss": 2.9372, + "theoretical_loss": 3.9141061509111994, + "tokens_seen": 500891648 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042838515546639923, + "loss": 2.9884, + "theoretical_loss": 3.91405101249985, + "tokens_seen": 500957184 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042837512537612836, + "loss": 2.8291, + "theoretical_loss": 3.9139958833207427, + "tokens_seen": 501022720 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004283650952858576, + "loss": 2.8736, + "theoretical_loss": 3.9139407633711243, + "tokens_seen": 501088256 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004283550651955868, + "loss": 3.0983, + "theoretical_loss": 3.9138856526482426, + "tokens_seen": 501153792 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042834503510531596, + "loss": 3.0688, + "theoretical_loss": 3.9138305511493474, + "tokens_seen": 501219328 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042833500501504514, + "loss": 2.7766, + "theoretical_loss": 3.9137754588716884, + "tokens_seen": 501284864 + }, + { + "epoch": 0.15, + "objective/train/docs_used": 294722, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.217176675796509, + "objective/train/theoretical_loss": 3.9137203758125176, + "objective/train/tokens_used": 521810400, + "theoretical_loss": 3.9137203758125176, + "tokens_seen": 501350400 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004283249749247743, + "loss": 3.0202, + "theoretical_loss": 3.9137203758125176, + "tokens_seen": 501350400 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004283149448345035, + "loss": 2.7547, + "theoretical_loss": 3.913665301969088, + "tokens_seen": 501415936 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042830491474423274, + "loss": 2.9357, + "theoretical_loss": 3.913610237338653, + "tokens_seen": 501481472 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042829488465396186, + "loss": 2.9646, + "theoretical_loss": 3.9135551819184675, + "tokens_seen": 501547008 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004282848545636911, + "loss": 3.133, + "theoretical_loss": 3.913500135705789, + "tokens_seen": 501612544 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004282748244734203, + "loss": 3.099, + "theoretical_loss": 3.9134450986978737, + "tokens_seen": 501678080 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042826479438314946, + "loss": 2.8801, + "theoretical_loss": 3.913390070891981, + "tokens_seen": 501743616 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042825476429287864, + "loss": 2.9018, + "theoretical_loss": 3.9133350522853703, + "tokens_seen": 501809152 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004282447342026078, + "loss": 2.9876, + "theoretical_loss": 3.9132800428753027, + "tokens_seen": 501874688 + }, + { + "epoch": 0.15, + "learning_rate": 0.000428234704112337, + "loss": 2.9594, + "theoretical_loss": 3.9132250426590405, + "tokens_seen": 501940224 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042822467402206624, + "loss": 2.6909, + "theoretical_loss": 3.9131700516338475, + "tokens_seen": 502005760 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042821464393179537, + "loss": 2.7774, + "theoretical_loss": 3.9131150697969868, + "tokens_seen": 502071296 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004282046138415246, + "loss": 2.8814, + "theoretical_loss": 3.913060097145725, + "tokens_seen": 502136832 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042819458375125373, + "loss": 3.1048, + "theoretical_loss": 3.913005133677329, + "tokens_seen": 502202368 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042818455366098296, + "loss": 2.9774, + "theoretical_loss": 3.9129501793890666, + "tokens_seen": 502267904 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042817452357071215, + "loss": 2.9506, + "theoretical_loss": 3.9128952342782073, + "tokens_seen": 502333440 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004281644934804413, + "loss": 2.9012, + "theoretical_loss": 3.9128402983420214, + "tokens_seen": 502398976 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004281544633901705, + "loss": 3.0771, + "theoretical_loss": 3.91278537157778, + "tokens_seen": 502464512 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004281444332998997, + "loss": 2.8678, + "theoretical_loss": 3.912730453982756, + "tokens_seen": 502530048 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042813440320962887, + "loss": 2.9035, + "theoretical_loss": 3.9126755455542233, + "tokens_seen": 502595584 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004281243731193581, + "loss": 2.9445, + "theoretical_loss": 3.912620646289457, + "tokens_seen": 502661120 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042811434302908723, + "loss": 2.878, + "theoretical_loss": 3.9125657561857334, + "tokens_seen": 502726656 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042810431293881647, + "loss": 3.1892, + "theoretical_loss": 3.9125108752403293, + "tokens_seen": 502792192 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042809428284854565, + "loss": 2.8769, + "theoretical_loss": 3.9124560034505236, + "tokens_seen": 502857728 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042808425275827483, + "loss": 2.9809, + "theoretical_loss": 3.9124011408135964, + "tokens_seen": 502923264 + }, + { + "epoch": 0.15, + "objective/train/docs_used": 295112, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.51133131980896, + "objective/train/theoretical_loss": 3.9123462873268275, + "objective/train/tokens_used": 523448800, + "theoretical_loss": 3.9123462873268275, + "tokens_seen": 502988800 + }, + { + "epoch": 0.15, + "learning_rate": 0.000428074222668004, + "loss": 2.5333, + "theoretical_loss": 3.9123462873268275, + "tokens_seen": 502988800 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004280641925777332, + "loss": 2.7801, + "theoretical_loss": 3.9122914429874998, + "tokens_seen": 503054336 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004280541624874624, + "loss": 3.061, + "theoretical_loss": 3.912236607792896, + "tokens_seen": 503119872 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004280441323971916, + "loss": 2.8056, + "theoretical_loss": 3.9121817817403004, + "tokens_seen": 503185408 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042803410230692074, + "loss": 3.165, + "theoretical_loss": 3.912126964826999, + "tokens_seen": 503250944 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042802407221664997, + "loss": 2.7886, + "theoretical_loss": 3.912072157050278, + "tokens_seen": 503316480 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004280140421263791, + "loss": 3.125, + "theoretical_loss": 3.9120173584074256, + "tokens_seen": 503382016 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042800401203610833, + "loss": 3.2098, + "theoretical_loss": 3.91196256889573, + "tokens_seen": 503447552 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004279939819458375, + "loss": 2.8893, + "theoretical_loss": 3.911907788512482, + "tokens_seen": 503513088 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004279839518555667, + "loss": 3.0083, + "theoretical_loss": 3.911853017254973, + "tokens_seen": 503578624 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004279739217652959, + "loss": 2.9316, + "theoretical_loss": 3.9117982551204946, + "tokens_seen": 503644160 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004279638916750251, + "loss": 3.0345, + "theoretical_loss": 3.911743502106341, + "tokens_seen": 503709696 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042795386158475424, + "loss": 3.0886, + "theoretical_loss": 3.911688758209807, + "tokens_seen": 503775232 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004279438314944835, + "loss": 2.7835, + "theoretical_loss": 3.9116340234281886, + "tokens_seen": 503840768 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004279338014042126, + "loss": 2.7769, + "theoretical_loss": 3.911579297758782, + "tokens_seen": 503906304 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042792377131394184, + "loss": 2.9644, + "theoretical_loss": 3.9115245811988864, + "tokens_seen": 503971840 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042791374122367107, + "loss": 2.9864, + "theoretical_loss": 3.9114698737458005, + "tokens_seen": 504037376 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004279037111334002, + "loss": 2.8318, + "theoretical_loss": 3.911415175396825, + "tokens_seen": 504102912 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042789368104312943, + "loss": 3.0419, + "theoretical_loss": 3.911360486149262, + "tokens_seen": 504168448 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042788365095285856, + "loss": 2.9707, + "theoretical_loss": 3.9113058060004136, + "tokens_seen": 504233984 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004278736208625878, + "loss": 2.7938, + "theoretical_loss": 3.9112511349475843, + "tokens_seen": 504299520 + }, + { + "epoch": 0.15, + "learning_rate": 0.000427863590772317, + "loss": 2.9589, + "theoretical_loss": 3.911196472988079, + "tokens_seen": 504365056 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042785356068204616, + "loss": 2.998, + "theoretical_loss": 3.9111418201192034, + "tokens_seen": 504430592 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042784353059177534, + "loss": 2.9114, + "theoretical_loss": 3.911087176338266, + "tokens_seen": 504496128 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004278335005015045, + "loss": 2.7702, + "theoretical_loss": 3.911032541642575, + "tokens_seen": 504561664 + }, + { + "epoch": 0.15, + "objective/train/docs_used": 296352, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.9434258937835693, + "objective/train/theoretical_loss": 3.910977916029439, + "objective/train/tokens_used": 525087200, + "theoretical_loss": 3.910977916029439, + "tokens_seen": 504627200 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004278234704112337, + "loss": 2.9923, + "theoretical_loss": 3.910977916029439, + "tokens_seen": 504627200 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042781344032096294, + "loss": 2.651, + "theoretical_loss": 3.9109232994961705, + "tokens_seen": 504692736 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042780341023069206, + "loss": 3.0268, + "theoretical_loss": 3.91086869204008, + "tokens_seen": 504758272 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004277933801404213, + "loss": 2.7599, + "theoretical_loss": 3.910814093658482, + "tokens_seen": 504823808 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004277833500501505, + "loss": 2.8698, + "theoretical_loss": 3.9107595043486905, + "tokens_seen": 504889344 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042777331995987966, + "loss": 3.0686, + "theoretical_loss": 3.91070492410802, + "tokens_seen": 504954880 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042776328986960884, + "loss": 2.9831, + "theoretical_loss": 3.9106503529337875, + "tokens_seen": 505020416 + }, + { + "epoch": 0.15, + "learning_rate": 0.000427753259779338, + "loss": 2.8274, + "theoretical_loss": 3.9105957908233107, + "tokens_seen": 505085952 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004277432296890672, + "loss": 2.9748, + "theoretical_loss": 3.910541237773909, + "tokens_seen": 505151488 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042773319959879644, + "loss": 2.9065, + "theoretical_loss": 3.910486693782902, + "tokens_seen": 505217024 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042772316950852557, + "loss": 3.0698, + "theoretical_loss": 3.9104321588476108, + "tokens_seen": 505282560 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004277131394182548, + "loss": 2.9682, + "theoretical_loss": 3.910377632965357, + "tokens_seen": 505348096 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042770310932798393, + "loss": 2.8996, + "theoretical_loss": 3.9103231161334655, + "tokens_seen": 505413632 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042769307923771316, + "loss": 2.9008, + "theoretical_loss": 3.9102686083492593, + "tokens_seen": 505479168 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042768304914744235, + "loss": 3.1123, + "theoretical_loss": 3.910214109610065, + "tokens_seen": 505544704 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042767301905717153, + "loss": 2.9594, + "theoretical_loss": 3.910159619913209, + "tokens_seen": 505610240 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004276629889669007, + "loss": 2.8751, + "theoretical_loss": 3.9101051392560198, + "tokens_seen": 505675776 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004276529588766299, + "loss": 2.9747, + "theoretical_loss": 3.9100506676358266, + "tokens_seen": 505741312 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042764292878635907, + "loss": 3.072, + "theoretical_loss": 3.9099962050499584, + "tokens_seen": 505806848 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004276328986960883, + "loss": 2.8964, + "theoretical_loss": 3.9099417514957473, + "tokens_seen": 505872384 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042762286860581743, + "loss": 3.062, + "theoretical_loss": 3.9098873069705267, + "tokens_seen": 505937920 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042761283851554667, + "loss": 2.9964, + "theoretical_loss": 3.9098328714716284, + "tokens_seen": 506003456 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042760280842527585, + "loss": 2.896, + "theoretical_loss": 3.909778444996389, + "tokens_seen": 506068992 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042759277833500503, + "loss": 2.8993, + "theoretical_loss": 3.9097240275421425, + "tokens_seen": 506134528 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004275827482447342, + "loss": 2.8902, + "theoretical_loss": 3.9096696191062277, + "tokens_seen": 506200064 + }, + { + "epoch": 0.15, + "objective/train/docs_used": 297118, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.651106357574463, + "objective/train/theoretical_loss": 3.909615219685982, + "objective/train/tokens_used": 526725600, + "theoretical_loss": 3.909615219685982, + "tokens_seen": 506265600 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004275727181544634, + "loss": 3.338, + "theoretical_loss": 3.909615219685982, + "tokens_seen": 506265600 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004275626880641926, + "loss": 2.9699, + "theoretical_loss": 3.9095608292787443, + "tokens_seen": 506331136 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004275526579739218, + "loss": 2.6794, + "theoretical_loss": 3.9095064478818555, + "tokens_seen": 506396672 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042754262788365094, + "loss": 3.0646, + "theoretical_loss": 3.9094520754926574, + "tokens_seen": 506462208 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042753259779338017, + "loss": 3.2271, + "theoretical_loss": 3.9093977121084915, + "tokens_seen": 506527744 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004275225677031093, + "loss": 2.8417, + "theoretical_loss": 3.909343357726703, + "tokens_seen": 506593280 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042751253761283853, + "loss": 3.158, + "theoretical_loss": 3.9092890123446367, + "tokens_seen": 506658816 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004275025075225677, + "loss": 2.8694, + "theoretical_loss": 3.9092346759596373, + "tokens_seen": 506724352 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004274924774322969, + "loss": 3.0276, + "theoretical_loss": 3.9091803485690537, + "tokens_seen": 506789888 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004274824473420261, + "loss": 2.9536, + "theoretical_loss": 3.9091260301702326, + "tokens_seen": 506855424 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004274724172517553, + "loss": 2.8008, + "theoretical_loss": 3.9090717207605246, + "tokens_seen": 506920960 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042746238716148444, + "loss": 2.8363, + "theoretical_loss": 3.90901742033728, + "tokens_seen": 506986496 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004274523570712137, + "loss": 2.6921, + "theoretical_loss": 3.9089631288978497, + "tokens_seen": 507052032 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004274423269809428, + "loss": 3.0912, + "theoretical_loss": 3.9089088464395876, + "tokens_seen": 507117568 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042743229689067204, + "loss": 3.2292, + "theoretical_loss": 3.908854572959847, + "tokens_seen": 507183104 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004274222668004012, + "loss": 2.9903, + "theoretical_loss": 3.9088003084559833, + "tokens_seen": 507248640 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004274122367101304, + "loss": 2.7156, + "theoretical_loss": 3.908746052925352, + "tokens_seen": 507314176 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004274022066198596, + "loss": 2.6405, + "theoretical_loss": 3.9086918063653115, + "tokens_seen": 507379712 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042739217652958876, + "loss": 2.944, + "theoretical_loss": 3.908637568773219, + "tokens_seen": 507445248 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042738214643931794, + "loss": 3.0759, + "theoretical_loss": 3.9085833401464347, + "tokens_seen": 507510784 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004273721163490472, + "loss": 2.972, + "theoretical_loss": 3.908529120482319, + "tokens_seen": 507576320 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004273620862587763, + "loss": 3.0427, + "theoretical_loss": 3.908474909778234, + "tokens_seen": 507641856 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042735205616850554, + "loss": 2.815, + "theoretical_loss": 3.908420708031542, + "tokens_seen": 507707392 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042734202607823467, + "loss": 3.0261, + "theoretical_loss": 3.9083665152396074, + "tokens_seen": 507772928 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004273319959879639, + "loss": 2.8889, + "theoretical_loss": 3.908312331399795, + "tokens_seen": 507838464 + }, + { + "epoch": 0.15, + "objective/train/docs_used": 298304, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.1829707622528076, + "objective/train/theoretical_loss": 3.908258156509472, + "objective/train/tokens_used": 528364000, + "theoretical_loss": 3.908258156509472, + "tokens_seen": 507904000 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004273219658976931, + "loss": 2.8852, + "theoretical_loss": 3.908258156509472, + "tokens_seen": 507904000 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042731193580742226, + "loss": 2.9553, + "theoretical_loss": 3.9082039905660046, + "tokens_seen": 507969536 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042730190571715145, + "loss": 2.9469, + "theoretical_loss": 3.9081498335667613, + "tokens_seen": 508035072 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004272918756268807, + "loss": 2.9631, + "theoretical_loss": 3.908095685509112, + "tokens_seen": 508100608 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004272818455366098, + "loss": 2.818, + "theoretical_loss": 3.908041546390428, + "tokens_seen": 508166144 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042727181544633904, + "loss": 2.857, + "theoretical_loss": 3.9079874162080794, + "tokens_seen": 508231680 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042726178535606817, + "loss": 2.9251, + "theoretical_loss": 3.9079332949594408, + "tokens_seen": 508297216 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004272517552657974, + "loss": 3.035, + "theoretical_loss": 3.907879182641886, + "tokens_seen": 508362752 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004272417251755266, + "loss": 3.1266, + "theoretical_loss": 3.907825079252789, + "tokens_seen": 508428288 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042723169508525577, + "loss": 2.8149, + "theoretical_loss": 3.9077709847895266, + "tokens_seen": 508493824 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042722166499498495, + "loss": 2.8729, + "theoretical_loss": 3.907716899249477, + "tokens_seen": 508559360 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042721163490471413, + "loss": 2.6458, + "theoretical_loss": 3.9076628226300167, + "tokens_seen": 508624896 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004272016048144433, + "loss": 3.0502, + "theoretical_loss": 3.9076087549285274, + "tokens_seen": 508690432 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042719157472417255, + "loss": 3.0218, + "theoretical_loss": 3.907554696142389, + "tokens_seen": 508755968 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004271815446339017, + "loss": 3.1713, + "theoretical_loss": 3.9075006462689825, + "tokens_seen": 508821504 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004271715145436309, + "loss": 3.1987, + "theoretical_loss": 3.9074466053056915, + "tokens_seen": 508887040 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004271614844533601, + "loss": 2.9845, + "theoretical_loss": 3.9073925732498997, + "tokens_seen": 508952576 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042715145436308927, + "loss": 2.9345, + "theoretical_loss": 3.907338550098993, + "tokens_seen": 509018112 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004271414242728185, + "loss": 2.9731, + "theoretical_loss": 3.907284535850356, + "tokens_seen": 509083648 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042713139418254763, + "loss": 2.8541, + "theoretical_loss": 3.9072305305013773, + "tokens_seen": 509149184 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042712136409227687, + "loss": 3.1979, + "theoretical_loss": 3.9071765340494453, + "tokens_seen": 509214720 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042711133400200605, + "loss": 3.0489, + "theoretical_loss": 3.9071225464919483, + "tokens_seen": 509280256 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042710130391173523, + "loss": 2.7799, + "theoretical_loss": 3.907068567826278, + "tokens_seen": 509345792 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004270912738214644, + "loss": 2.9427, + "theoretical_loss": 3.9070145980498263, + "tokens_seen": 509411328 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004270812437311936, + "loss": 3.0491, + "theoretical_loss": 3.906960637159985, + "tokens_seen": 509476864 + }, + { + "epoch": 0.15, + "objective/train/docs_used": 298986, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.338592767715454, + "objective/train/theoretical_loss": 3.9069066851541487, + "objective/train/tokens_used": 530002400, + "theoretical_loss": 3.9069066851541487, + "tokens_seen": 509542400 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004270712136409228, + "loss": 3.12, + "theoretical_loss": 3.9069066851541487, + "tokens_seen": 509542400 + }, + { + "epoch": 0.15, + "learning_rate": 0.000427061183550652, + "loss": 2.9199, + "theoretical_loss": 3.906852742029712, + "tokens_seen": 509607936 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042705115346038114, + "loss": 2.8314, + "theoretical_loss": 3.906798807784072, + "tokens_seen": 509673472 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042704112337011037, + "loss": 2.9591, + "theoretical_loss": 3.9067448824146243, + "tokens_seen": 509739008 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004270310932798395, + "loss": 3.0428, + "theoretical_loss": 3.9066909659187683, + "tokens_seen": 509804544 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042702106318956873, + "loss": 2.8248, + "theoretical_loss": 3.906637058293903, + "tokens_seen": 509870080 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004270110330992979, + "loss": 2.9371, + "theoretical_loss": 3.906583159537429, + "tokens_seen": 509935616 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004270010030090271, + "loss": 3.1524, + "theoretical_loss": 3.9065292696467475, + "tokens_seen": 510001152 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004269909729187563, + "loss": 2.8554, + "theoretical_loss": 3.906475388619262, + "tokens_seen": 510066688 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004269809428284855, + "loss": 3.0133, + "theoretical_loss": 3.906421516452376, + "tokens_seen": 510132224 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042697091273821464, + "loss": 2.9912, + "theoretical_loss": 3.9063676531434934, + "tokens_seen": 510197760 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004269608826479439, + "loss": 3.0673, + "theoretical_loss": 3.9063137986900216, + "tokens_seen": 510263296 + }, + { + "epoch": 0.15, + "learning_rate": 0.000426950852557673, + "loss": 2.8742, + "theoretical_loss": 3.906259953089367, + "tokens_seen": 510328832 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042694082246740224, + "loss": 3.0843, + "theoretical_loss": 3.9062061163389377, + "tokens_seen": 510394368 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004269307923771314, + "loss": 2.7839, + "theoretical_loss": 3.906152288436143, + "tokens_seen": 510459904 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004269207622868606, + "loss": 2.8636, + "theoretical_loss": 3.906098469378393, + "tokens_seen": 510525440 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004269107321965898, + "loss": 2.8408, + "theoretical_loss": 3.9060446591630997, + "tokens_seen": 510590976 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042690070210631896, + "loss": 2.8919, + "theoretical_loss": 3.905990857787675, + "tokens_seen": 510656512 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042689067201604814, + "loss": 2.9582, + "theoretical_loss": 3.9059370652495327, + "tokens_seen": 510722048 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004268806419257774, + "loss": 3.1478, + "theoretical_loss": 3.905883281546088, + "tokens_seen": 510787584 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004268706118355065, + "loss": 2.698, + "theoretical_loss": 3.9058295066747553, + "tokens_seen": 510853120 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042686058174523574, + "loss": 2.8973, + "theoretical_loss": 3.9057757406329534, + "tokens_seen": 510918656 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042685055165496487, + "loss": 2.9555, + "theoretical_loss": 3.9057219834180987, + "tokens_seen": 510984192 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004268405215646941, + "loss": 2.9659, + "theoretical_loss": 3.9056682350276106, + "tokens_seen": 511049728 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004268304914744233, + "loss": 3.0446, + "theoretical_loss": 3.90561449545891, + "tokens_seen": 511115264 + }, + { + "epoch": 0.15, + "objective/train/docs_used": 299996, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7854602336883545, + "objective/train/theoretical_loss": 3.905560764709417, + "objective/train/tokens_used": 531640800, + "theoretical_loss": 3.905560764709417, + "tokens_seen": 511180800 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042682046138415246, + "loss": 2.9953, + "theoretical_loss": 3.905560764709417, + "tokens_seen": 511180800 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042681043129388165, + "loss": 2.9675, + "theoretical_loss": 3.905507042776554, + "tokens_seen": 511246336 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004268004012036109, + "loss": 3.2021, + "theoretical_loss": 3.905453329657745, + "tokens_seen": 511311872 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042679037111334, + "loss": 3.2077, + "theoretical_loss": 3.905399625350414, + "tokens_seen": 511377408 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042678034102306924, + "loss": 2.7907, + "theoretical_loss": 3.9053459298519875, + "tokens_seen": 511442944 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042677031093279837, + "loss": 3.058, + "theoretical_loss": 3.905292243159891, + "tokens_seen": 511508480 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004267602808425276, + "loss": 2.9276, + "theoretical_loss": 3.905238565271553, + "tokens_seen": 511574016 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004267502507522568, + "loss": 2.7992, + "theoretical_loss": 3.9051848961844007, + "tokens_seen": 511639552 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042674022066198597, + "loss": 3.067, + "theoretical_loss": 3.9051312358958663, + "tokens_seen": 511705088 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042673019057171515, + "loss": 2.6203, + "theoretical_loss": 3.9050775844033785, + "tokens_seen": 511770624 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042672016048144433, + "loss": 3.0024, + "theoretical_loss": 3.905023941704371, + "tokens_seen": 511836160 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004267101303911735, + "loss": 3.1049, + "theoretical_loss": 3.904970307796276, + "tokens_seen": 511901696 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042670010030090275, + "loss": 2.7871, + "theoretical_loss": 3.9049166826765283, + "tokens_seen": 511967232 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004266900702106319, + "loss": 3.0709, + "theoretical_loss": 3.9048630663425623, + "tokens_seen": 512032768 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004266800401203611, + "loss": 2.9742, + "theoretical_loss": 3.9048094587918154, + "tokens_seen": 512098304 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042667001003009024, + "loss": 2.674, + "theoretical_loss": 3.904755860021724, + "tokens_seen": 512163840 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042665997993981947, + "loss": 3.0139, + "theoretical_loss": 3.904702270029727, + "tokens_seen": 512229376 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042664994984954865, + "loss": 2.7943, + "theoretical_loss": 3.904648688813264, + "tokens_seen": 512294912 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042663991975927783, + "loss": 2.8329, + "theoretical_loss": 3.9045951163697756, + "tokens_seen": 512360448 + }, + { + "epoch": 0.16, + "learning_rate": 0.000426629889669007, + "loss": 2.9783, + "theoretical_loss": 3.9045415526967036, + "tokens_seen": 512425984 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042661985957873625, + "loss": 3.0398, + "theoretical_loss": 3.904487997791491, + "tokens_seen": 512491520 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004266098294884654, + "loss": 3.0462, + "theoretical_loss": 3.904434451651581, + "tokens_seen": 512557056 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004265997993981946, + "loss": 3.1083, + "theoretical_loss": 3.904380914274419, + "tokens_seen": 512622592 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042658976930792374, + "loss": 2.9584, + "theoretical_loss": 3.9043273856574507, + "tokens_seen": 512688128 + }, + { + "epoch": 0.16, + "learning_rate": 0.000426579739217653, + "loss": 3.0369, + "theoretical_loss": 3.9042738657981237, + "tokens_seen": 512753664 + }, + { + "epoch": 0.16, + "objective/train/docs_used": 300643, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.8312289714813232, + "objective/train/theoretical_loss": 3.9042203546938854, + "objective/train/tokens_used": 533279200, + "theoretical_loss": 3.9042203546938854, + "tokens_seen": 512819200 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042656970912738216, + "loss": 2.9682, + "theoretical_loss": 3.9042203546938854, + "tokens_seen": 512819200 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042655967903711134, + "loss": 2.9802, + "theoretical_loss": 3.9041668523421853, + "tokens_seen": 512884736 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004265496489468405, + "loss": 3.2464, + "theoretical_loss": 3.9041133587404744, + "tokens_seen": 512950272 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004265396188565697, + "loss": 3.0224, + "theoretical_loss": 3.9040598738862027, + "tokens_seen": 513015808 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004265295887662989, + "loss": 3.1773, + "theoretical_loss": 3.904006397776823, + "tokens_seen": 513081344 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004265195586760281, + "loss": 3.0895, + "theoretical_loss": 3.90395293040979, + "tokens_seen": 513146880 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042650952858575724, + "loss": 3.0235, + "theoretical_loss": 3.903899471782557, + "tokens_seen": 513212416 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004264994984954865, + "loss": 3.0339, + "theoretical_loss": 3.90384602189258, + "tokens_seen": 513277952 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004264894684052156, + "loss": 3.1271, + "theoretical_loss": 3.9037925807373153, + "tokens_seen": 513343488 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042647943831494484, + "loss": 2.7846, + "theoretical_loss": 3.9037391483142208, + "tokens_seen": 513409024 + }, + { + "epoch": 0.16, + "learning_rate": 0.000426469408224674, + "loss": 2.9764, + "theoretical_loss": 3.9036857246207557, + "tokens_seen": 513474560 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004264593781344032, + "loss": 2.9497, + "theoretical_loss": 3.9036323096543795, + "tokens_seen": 513540096 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004264493480441324, + "loss": 3.0947, + "theoretical_loss": 3.903578903412553, + "tokens_seen": 513605632 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004264393179538616, + "loss": 2.8976, + "theoretical_loss": 3.9035255058927394, + "tokens_seen": 513671168 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042642928786359075, + "loss": 2.9121, + "theoretical_loss": 3.9034721170924005, + "tokens_seen": 513736704 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042641925777332, + "loss": 2.7269, + "theoretical_loss": 3.903418737009, + "tokens_seen": 513802240 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042640922768304916, + "loss": 2.8165, + "theoretical_loss": 3.9033653656400045, + "tokens_seen": 513867776 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042639919759277834, + "loss": 3.0661, + "theoretical_loss": 3.903312002982879, + "tokens_seen": 513933312 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004263891675025076, + "loss": 3.1111, + "theoretical_loss": 3.9032586490350916, + "tokens_seen": 513998848 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004263791374122367, + "loss": 3.0253, + "theoretical_loss": 3.9032053037941106, + "tokens_seen": 514064384 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042636910732196594, + "loss": 3.0854, + "theoretical_loss": 3.9031519672574047, + "tokens_seen": 514129920 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042635907723169507, + "loss": 3.0627, + "theoretical_loss": 3.903098639422445, + "tokens_seen": 514195456 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004263490471414243, + "loss": 3.0863, + "theoretical_loss": 3.9030453202867035, + "tokens_seen": 514260992 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004263390170511535, + "loss": 2.8966, + "theoretical_loss": 3.902992009847652, + "tokens_seen": 514326528 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042632898696088266, + "loss": 3.0579, + "theoretical_loss": 3.9029387081027638, + "tokens_seen": 514392064 + }, + { + "epoch": 0.16, + "objective/train/docs_used": 301818, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.63741135597229, + "objective/train/theoretical_loss": 3.9028854150495143, + "objective/train/tokens_used": 534917600, + "theoretical_loss": 3.9028854150495143, + "tokens_seen": 514457600 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042631895687061185, + "loss": 2.8324, + "theoretical_loss": 3.9028854150495143, + "tokens_seen": 514457600 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004263089267803411, + "loss": 2.9547, + "theoretical_loss": 3.9028321306853795, + "tokens_seen": 514523136 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004262988966900702, + "loss": 3.0593, + "theoretical_loss": 3.9027788550078357, + "tokens_seen": 514588672 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042628886659979944, + "loss": 2.7927, + "theoretical_loss": 3.902725588014361, + "tokens_seen": 514654208 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042627883650952857, + "loss": 3.037, + "theoretical_loss": 3.9026723297024337, + "tokens_seen": 514719744 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004262688064192578, + "loss": 2.7432, + "theoretical_loss": 3.9026190800695346, + "tokens_seen": 514785280 + }, + { + "epoch": 0.16, + "learning_rate": 0.000426258776328987, + "loss": 2.7779, + "theoretical_loss": 3.9025658391131444, + "tokens_seen": 514850816 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042624874623871617, + "loss": 2.9256, + "theoretical_loss": 3.902512606830745, + "tokens_seen": 514916352 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042623871614844535, + "loss": 2.7962, + "theoretical_loss": 3.9024593832198198, + "tokens_seen": 514981888 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042622868605817453, + "loss": 2.8452, + "theoretical_loss": 3.902406168277853, + "tokens_seen": 515047424 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004262186559679037, + "loss": 2.9985, + "theoretical_loss": 3.9023529620023294, + "tokens_seen": 515112960 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042620862587763295, + "loss": 3.1085, + "theoretical_loss": 3.902299764390736, + "tokens_seen": 515178496 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004261985957873621, + "loss": 2.9562, + "theoretical_loss": 3.902246575440559, + "tokens_seen": 515244032 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004261885656970913, + "loss": 3.0855, + "theoretical_loss": 3.902193395149288, + "tokens_seen": 515309568 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042617853560682044, + "loss": 3.0318, + "theoretical_loss": 3.9021402235144116, + "tokens_seen": 515375104 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042616850551654967, + "loss": 2.7043, + "theoretical_loss": 3.9020870605334204, + "tokens_seen": 515440640 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042615847542627885, + "loss": 3.167, + "theoretical_loss": 3.9020339062038065, + "tokens_seen": 515506176 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042614844533600803, + "loss": 2.9298, + "theoretical_loss": 3.9019807605230614, + "tokens_seen": 515571712 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004261384152457372, + "loss": 3.0367, + "theoretical_loss": 3.90192762348868, + "tokens_seen": 515637248 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042612838515546645, + "loss": 2.9388, + "theoretical_loss": 3.9018744950981556, + "tokens_seen": 515702784 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004261183550651956, + "loss": 2.7624, + "theoretical_loss": 3.901821375348985, + "tokens_seen": 515768320 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004261083249749248, + "loss": 2.8855, + "theoretical_loss": 3.9017682642386644, + "tokens_seen": 515833856 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042609829488465394, + "loss": 2.9966, + "theoretical_loss": 3.9017151617646917, + "tokens_seen": 515899392 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004260882647943832, + "loss": 2.7181, + "theoretical_loss": 3.901662067924566, + "tokens_seen": 515964928 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042607823470411236, + "loss": 3.1247, + "theoretical_loss": 3.9016089827157865, + "tokens_seen": 516030464 + }, + { + "epoch": 0.16, + "objective/train/docs_used": 302480, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.203036308288574, + "objective/train/theoretical_loss": 3.9015559061358545, + "objective/train/tokens_used": 536556000, + "theoretical_loss": 3.9015559061358545, + "tokens_seen": 516096000 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042606820461384154, + "loss": 2.757, + "theoretical_loss": 3.9015559061358545, + "tokens_seen": 516096000 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004260581745235707, + "loss": 2.9554, + "theoretical_loss": 3.901502838182272, + "tokens_seen": 516161536 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004260481444332999, + "loss": 3.0517, + "theoretical_loss": 3.901449778852542, + "tokens_seen": 516227072 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004260381143430291, + "loss": 3.0671, + "theoretical_loss": 3.9013967281441677, + "tokens_seen": 516292608 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004260280842527583, + "loss": 3.1749, + "theoretical_loss": 3.9013436860546555, + "tokens_seen": 516358144 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042601805416248744, + "loss": 3.0206, + "theoretical_loss": 3.901290652581511, + "tokens_seen": 516423680 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004260080240722167, + "loss": 2.925, + "theoretical_loss": 3.901237627722241, + "tokens_seen": 516489216 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004259979939819458, + "loss": 3.0778, + "theoretical_loss": 3.9011846114743536, + "tokens_seen": 516554752 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042598796389167504, + "loss": 2.7559, + "theoretical_loss": 3.901131603835359, + "tokens_seen": 516620288 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004259779338014042, + "loss": 3.0448, + "theoretical_loss": 3.9010786048027666, + "tokens_seen": 516685824 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004259679037111334, + "loss": 2.8123, + "theoretical_loss": 3.9010256143740873, + "tokens_seen": 516751360 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004259578736208626, + "loss": 2.7357, + "theoretical_loss": 3.900972632546835, + "tokens_seen": 516816896 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004259478435305918, + "loss": 2.9926, + "theoretical_loss": 3.900919659318521, + "tokens_seen": 516882432 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042593781344032095, + "loss": 2.8616, + "theoretical_loss": 3.9008666946866613, + "tokens_seen": 516947968 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004259277833500502, + "loss": 2.8152, + "theoretical_loss": 3.900813738648771, + "tokens_seen": 517013504 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004259177532597793, + "loss": 2.9415, + "theoretical_loss": 3.900760791202366, + "tokens_seen": 517079040 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042590772316950854, + "loss": 2.8222, + "theoretical_loss": 3.900707852344964, + "tokens_seen": 517144576 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004258976930792377, + "loss": 3.0943, + "theoretical_loss": 3.9006549220740836, + "tokens_seen": 517210112 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004258876629889669, + "loss": 2.7673, + "theoretical_loss": 3.9006020003872446, + "tokens_seen": 517275648 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004258776328986961, + "loss": 2.9953, + "theoretical_loss": 3.9005490872819673, + "tokens_seen": 517341184 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042586760280842527, + "loss": 3.0722, + "theoretical_loss": 3.900496182755773, + "tokens_seen": 517406720 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042585757271815445, + "loss": 3.0237, + "theoretical_loss": 3.9004432868061856, + "tokens_seen": 517472256 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004258475426278837, + "loss": 3.174, + "theoretical_loss": 3.9003903994307274, + "tokens_seen": 517537792 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004258375125376128, + "loss": 3.0428, + "theoretical_loss": 3.9003375206269233, + "tokens_seen": 517603328 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042582748244734205, + "loss": 2.9034, + "theoretical_loss": 3.9002846503923, + "tokens_seen": 517668864 + }, + { + "epoch": 0.16, + "objective/train/docs_used": 303675, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.4355645179748535, + "objective/train/theoretical_loss": 3.9002317887243834, + "objective/train/tokens_used": 538194400, + "theoretical_loss": 3.9002317887243834, + "tokens_seen": 517734400 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004258174523570712, + "loss": 3.1518, + "theoretical_loss": 3.9002317887243834, + "tokens_seen": 517734400 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004258074222668004, + "loss": 3.0084, + "theoretical_loss": 3.9001789356207013, + "tokens_seen": 517799936 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004257973921765296, + "loss": 3.02, + "theoretical_loss": 3.900126091078783, + "tokens_seen": 517865472 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042578736208625877, + "loss": 2.8624, + "theoretical_loss": 3.9000732550961574, + "tokens_seen": 517931008 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042577733199598795, + "loss": 2.8849, + "theoretical_loss": 3.9000204276703565, + "tokens_seen": 517996544 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004257673019057172, + "loss": 2.8102, + "theoretical_loss": 3.8999676087989115, + "tokens_seen": 518062080 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004257572718154463, + "loss": 3.0644, + "theoretical_loss": 3.8999147984793554, + "tokens_seen": 518127616 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042574724172517555, + "loss": 3.0067, + "theoretical_loss": 3.8998619967092223, + "tokens_seen": 518193152 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004257372116349047, + "loss": 2.8062, + "theoretical_loss": 3.8998092034860474, + "tokens_seen": 518258688 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004257271815446339, + "loss": 2.8895, + "theoretical_loss": 3.8997564188073657, + "tokens_seen": 518324224 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004257171514543631, + "loss": 2.894, + "theoretical_loss": 3.8997036426707155, + "tokens_seen": 518389760 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004257071213640923, + "loss": 2.9759, + "theoretical_loss": 3.8996508750736343, + "tokens_seen": 518455296 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042569709127382145, + "loss": 2.8975, + "theoretical_loss": 3.89959811601366, + "tokens_seen": 518520832 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042568706118355064, + "loss": 2.8918, + "theoretical_loss": 3.8995453654883345, + "tokens_seen": 518586368 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004256770310932798, + "loss": 2.8332, + "theoretical_loss": 3.899492623495198, + "tokens_seen": 518651904 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042566700100300905, + "loss": 2.9059, + "theoretical_loss": 3.8994398900317924, + "tokens_seen": 518717440 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042565697091273823, + "loss": 3.0019, + "theoretical_loss": 3.8993871650956606, + "tokens_seen": 518782976 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004256469408224674, + "loss": 2.9708, + "theoretical_loss": 3.899334448684348, + "tokens_seen": 518848512 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042563691073219665, + "loss": 2.9748, + "theoretical_loss": 3.8992817407953986, + "tokens_seen": 518914048 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004256268806419258, + "loss": 3.0244, + "theoretical_loss": 3.8992290414263593, + "tokens_seen": 518979584 + }, + { + "epoch": 0.16, + "learning_rate": 0.000425616850551655, + "loss": 2.8855, + "theoretical_loss": 3.8991763505747765, + "tokens_seen": 519045120 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042560682046138414, + "loss": 2.8383, + "theoretical_loss": 3.8991236682381993, + "tokens_seen": 519110656 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004255967903711134, + "loss": 2.9275, + "theoretical_loss": 3.899070994414176, + "tokens_seen": 519176192 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042558676028084256, + "loss": 2.5734, + "theoretical_loss": 3.899018329100257, + "tokens_seen": 519241728 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042557673019057174, + "loss": 2.9448, + "theoretical_loss": 3.8989656722939943, + "tokens_seen": 519307264 + }, + { + "epoch": 0.16, + "objective/train/docs_used": 304298, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.8509061336517334, + "objective/train/theoretical_loss": 3.8989130239929395, + "objective/train/tokens_used": 539832800, + "theoretical_loss": 3.8989130239929395, + "tokens_seen": 519372800 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004255667001003009, + "loss": 2.9777, + "theoretical_loss": 3.8989130239929395, + "tokens_seen": 519372800 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004255566700100301, + "loss": 3.0034, + "theoretical_loss": 3.8988603841946463, + "tokens_seen": 519438336 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004255466399197593, + "loss": 2.8929, + "theoretical_loss": 3.8988077528966687, + "tokens_seen": 519503872 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004255366098294885, + "loss": 3.0907, + "theoretical_loss": 3.898755130096562, + "tokens_seen": 519569408 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042552657973921764, + "loss": 2.6244, + "theoretical_loss": 3.898702515791882, + "tokens_seen": 519634944 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004255165496489469, + "loss": 2.7489, + "theoretical_loss": 3.8986499099801875, + "tokens_seen": 519700480 + }, + { + "epoch": 0.16, + "learning_rate": 0.000425506519558676, + "loss": 2.9308, + "theoretical_loss": 3.898597312659035, + "tokens_seen": 519766016 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042549648946840524, + "loss": 2.8944, + "theoretical_loss": 3.8985447238259856, + "tokens_seen": 519831552 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004254864593781344, + "loss": 3.0895, + "theoretical_loss": 3.8984921434785984, + "tokens_seen": 519897088 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004254764292878636, + "loss": 2.8078, + "theoretical_loss": 3.8984395716144347, + "tokens_seen": 519962624 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004254663991975928, + "loss": 2.8201, + "theoretical_loss": 3.898387008231058, + "tokens_seen": 520028160 + }, + { + "epoch": 0.16, + "learning_rate": 0.000425456369107322, + "loss": 3.0088, + "theoretical_loss": 3.8983344533260307, + "tokens_seen": 520093696 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042544633901705115, + "loss": 2.9415, + "theoretical_loss": 3.8982819068969174, + "tokens_seen": 520159232 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004254363089267804, + "loss": 2.9224, + "theoretical_loss": 3.8982293689412835, + "tokens_seen": 520224768 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004254262788365095, + "loss": 2.8736, + "theoretical_loss": 3.8981768394566956, + "tokens_seen": 520290304 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042541624874623874, + "loss": 2.9549, + "theoretical_loss": 3.898124318440721, + "tokens_seen": 520355840 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004254062186559679, + "loss": 2.988, + "theoretical_loss": 3.8980718058909276, + "tokens_seen": 520421376 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004253961885656971, + "loss": 3.0078, + "theoretical_loss": 3.8980193018048857, + "tokens_seen": 520486912 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004253861584754263, + "loss": 2.8788, + "theoretical_loss": 3.8979668061801656, + "tokens_seen": 520552448 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042537612838515547, + "loss": 2.8787, + "theoretical_loss": 3.8979143190143377, + "tokens_seen": 520617984 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042536609829488465, + "loss": 3.2018, + "theoretical_loss": 3.8978618403049747, + "tokens_seen": 520683520 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004253560682046139, + "loss": 2.493, + "theoretical_loss": 3.8978093700496514, + "tokens_seen": 520749056 + }, + { + "epoch": 0.16, + "learning_rate": 0.000425346038114343, + "loss": 3.0099, + "theoretical_loss": 3.897756908245941, + "tokens_seen": 520814592 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042533600802407225, + "loss": 2.9934, + "theoretical_loss": 3.897704454891419, + "tokens_seen": 520880128 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004253259779338014, + "loss": 2.8865, + "theoretical_loss": 3.897652009983662, + "tokens_seen": 520945664 + }, + { + "epoch": 0.16, + "objective/train/docs_used": 305033, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.1320838928222656, + "objective/train/theoretical_loss": 3.897599573520247, + "objective/train/tokens_used": 541471200, + "theoretical_loss": 3.897599573520247, + "tokens_seen": 521011200 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004253159478435306, + "loss": 3.1803, + "theoretical_loss": 3.897599573520247, + "tokens_seen": 521011200 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004253059177532598, + "loss": 2.8785, + "theoretical_loss": 3.897547145498753, + "tokens_seen": 521076736 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042529588766298897, + "loss": 3.0068, + "theoretical_loss": 3.8974947259167596, + "tokens_seen": 521142272 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042528585757271815, + "loss": 2.9494, + "theoretical_loss": 3.8974423147718467, + "tokens_seen": 521207808 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004252758274824474, + "loss": 2.7137, + "theoretical_loss": 3.8973899120615956, + "tokens_seen": 521273344 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004252657973921765, + "loss": 2.9046, + "theoretical_loss": 3.8973375177835896, + "tokens_seen": 521338880 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042525576730190575, + "loss": 3.0154, + "theoretical_loss": 3.897285131935411, + "tokens_seen": 521404416 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004252457372116349, + "loss": 2.8266, + "theoretical_loss": 3.8972327545146443, + "tokens_seen": 521469952 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004252357071213641, + "loss": 2.7908, + "theoretical_loss": 3.897180385518876, + "tokens_seen": 521535488 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004252256770310933, + "loss": 2.7944, + "theoretical_loss": 3.897128024945692, + "tokens_seen": 521601024 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004252156469408225, + "loss": 3.0129, + "theoretical_loss": 3.8970756727926785, + "tokens_seen": 521666560 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042520561685055165, + "loss": 2.6956, + "theoretical_loss": 3.8970233290574257, + "tokens_seen": 521732096 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042519558676028084, + "loss": 3.1755, + "theoretical_loss": 3.896970993737522, + "tokens_seen": 521797632 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042518555667001, + "loss": 3.072, + "theoretical_loss": 3.896918666830558, + "tokens_seen": 521863168 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042517552657973925, + "loss": 3.0098, + "theoretical_loss": 3.8968663483341253, + "tokens_seen": 521928704 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004251654964894684, + "loss": 2.8251, + "theoretical_loss": 3.8968140382458154, + "tokens_seen": 521994240 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004251554663991976, + "loss": 2.9584, + "theoretical_loss": 3.8967617365632217, + "tokens_seen": 522059776 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004251454363089268, + "loss": 2.7749, + "theoretical_loss": 3.8967094432839398, + "tokens_seen": 522125312 + }, + { + "epoch": 0.16, + "learning_rate": 0.000425135406218656, + "loss": 2.9198, + "theoretical_loss": 3.8966571584055645, + "tokens_seen": 522190848 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042512537612838516, + "loss": 2.6691, + "theoretical_loss": 3.896604881925691, + "tokens_seen": 522256384 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042511534603811434, + "loss": 2.7376, + "theoretical_loss": 3.8965526138419184, + "tokens_seen": 522321920 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004251053159478435, + "loss": 3.0931, + "theoretical_loss": 3.8965003541518435, + "tokens_seen": 522387456 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042509528585757276, + "loss": 3.0997, + "theoretical_loss": 3.896448102853067, + "tokens_seen": 522452992 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004250852557673019, + "loss": 2.8555, + "theoretical_loss": 3.8963958599431874, + "tokens_seen": 522518528 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004250752256770311, + "loss": 2.8124, + "theoretical_loss": 3.8963436254198074, + "tokens_seen": 522584064 + }, + { + "epoch": 0.16, + "objective/train/docs_used": 306243, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.116842746734619, + "objective/train/theoretical_loss": 3.8962913992805284, + "objective/train/tokens_used": 543109600, + "theoretical_loss": 3.8962913992805284, + "tokens_seen": 522649600 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042506519558676024, + "loss": 3.0328, + "theoretical_loss": 3.8962913992805284, + "tokens_seen": 522649600 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004250551654964895, + "loss": 2.9249, + "theoretical_loss": 3.896239181522954, + "tokens_seen": 522715136 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042504513540621866, + "loss": 3.0191, + "theoretical_loss": 3.8961869721446885, + "tokens_seen": 522780672 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042503510531594784, + "loss": 2.9222, + "theoretical_loss": 3.8961347711433376, + "tokens_seen": 522846208 + }, + { + "epoch": 0.16, + "learning_rate": 0.000425025075225677, + "loss": 2.6449, + "theoretical_loss": 3.896082578516506, + "tokens_seen": 522911744 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004250150451354062, + "loss": 2.847, + "theoretical_loss": 3.8960303942618024, + "tokens_seen": 522977280 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004250050150451354, + "loss": 2.933, + "theoretical_loss": 3.8959782183768343, + "tokens_seen": 523042816 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004249949849548646, + "loss": 2.9474, + "theoretical_loss": 3.8959260508592104, + "tokens_seen": 523108352 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042498495486459375, + "loss": 2.6623, + "theoretical_loss": 3.895873891706542, + "tokens_seen": 523173888 + }, + { + "epoch": 0.16, + "learning_rate": 0.000424974924774323, + "loss": 3.0693, + "theoretical_loss": 3.895821740916439, + "tokens_seen": 523239424 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042496489468405216, + "loss": 2.9283, + "theoretical_loss": 3.8957695984865137, + "tokens_seen": 523304960 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042495486459378135, + "loss": 2.9361, + "theoretical_loss": 3.89571746441438, + "tokens_seen": 523370496 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004249448345035105, + "loss": 3.1131, + "theoretical_loss": 3.8956653386976514, + "tokens_seen": 523436032 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004249348044132397, + "loss": 3.0092, + "theoretical_loss": 3.8956132213339423, + "tokens_seen": 523501568 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042492477432296894, + "loss": 2.9121, + "theoretical_loss": 3.8955611123208698, + "tokens_seen": 523567104 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004249147442326981, + "loss": 2.8729, + "theoretical_loss": 3.8955090116560505, + "tokens_seen": 523632640 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004249047141424273, + "loss": 2.7129, + "theoretical_loss": 3.895456919337102, + "tokens_seen": 523698176 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004248946840521565, + "loss": 3.0303, + "theoretical_loss": 3.8954048353616435, + "tokens_seen": 523763712 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042488465396188567, + "loss": 2.759, + "theoretical_loss": 3.8953527597272952, + "tokens_seen": 523829248 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042487462387161485, + "loss": 2.7957, + "theoretical_loss": 3.895300692431677, + "tokens_seen": 523894784 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004248645937813441, + "loss": 2.856, + "theoretical_loss": 3.8952486334724123, + "tokens_seen": 523960320 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004248545636910732, + "loss": 2.8704, + "theoretical_loss": 3.895196582847123, + "tokens_seen": 524025856 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042484453360080245, + "loss": 2.9073, + "theoretical_loss": 3.8951445405534324, + "tokens_seen": 524091392 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004248345035105316, + "loss": 2.8645, + "theoretical_loss": 3.8950925065889663, + "tokens_seen": 524156928 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004248244734202608, + "loss": 2.9463, + "theoretical_loss": 3.8950404809513506, + "tokens_seen": 524222464 + }, + { + "epoch": 0.16, + "objective/train/docs_used": 306972, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.0469322204589844, + "objective/train/theoretical_loss": 3.8949884636382106, + "objective/train/tokens_used": 544748000, + "theoretical_loss": 3.8949884636382106, + "tokens_seen": 524288000 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042481444332999, + "loss": 2.9347, + "theoretical_loss": 3.8949884636382106, + "tokens_seen": 524288000 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042480441323971917, + "loss": 2.9757, + "theoretical_loss": 3.8949364546471754, + "tokens_seen": 524353536 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042479438314944835, + "loss": 2.8584, + "theoretical_loss": 3.894884453975873, + "tokens_seen": 524419072 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004247843530591776, + "loss": 3.0884, + "theoretical_loss": 3.894832461621934, + "tokens_seen": 524484608 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004247743229689067, + "loss": 2.933, + "theoretical_loss": 3.894780477582988, + "tokens_seen": 524550144 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042476429287863595, + "loss": 2.8411, + "theoretical_loss": 3.894728501856667, + "tokens_seen": 524615680 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004247542627883651, + "loss": 2.9042, + "theoretical_loss": 3.8946765344406034, + "tokens_seen": 524681216 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004247442326980943, + "loss": 3.1263, + "theoretical_loss": 3.8946245753324304, + "tokens_seen": 524746752 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004247342026078235, + "loss": 2.8407, + "theoretical_loss": 3.8945726245297836, + "tokens_seen": 524812288 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004247241725175527, + "loss": 3.0941, + "theoretical_loss": 3.894520682030297, + "tokens_seen": 524877824 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042471414242728186, + "loss": 2.8913, + "theoretical_loss": 3.894468747831609, + "tokens_seen": 524943360 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042470411233701104, + "loss": 2.8406, + "theoretical_loss": 3.8944168219313546, + "tokens_seen": 525008896 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004246940822467402, + "loss": 3.0931, + "theoretical_loss": 3.8943649043271744, + "tokens_seen": 525074432 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042468405215646945, + "loss": 2.6704, + "theoretical_loss": 3.8943129950167066, + "tokens_seen": 525139968 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004246740220661986, + "loss": 2.8924, + "theoretical_loss": 3.894261093997592, + "tokens_seen": 525205504 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004246639919759278, + "loss": 2.7959, + "theoretical_loss": 3.894209201267471, + "tokens_seen": 525271040 + }, + { + "epoch": 0.16, + "learning_rate": 0.000424653961885657, + "loss": 3.1657, + "theoretical_loss": 3.8941573168239874, + "tokens_seen": 525336576 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004246439317953862, + "loss": 2.8126, + "theoretical_loss": 3.8941054406647826, + "tokens_seen": 525402112 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042463390170511536, + "loss": 2.7601, + "theoretical_loss": 3.894053572787502, + "tokens_seen": 525467648 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042462387161484454, + "loss": 2.8674, + "theoretical_loss": 3.89400171318979, + "tokens_seen": 525533184 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004246138415245737, + "loss": 2.9239, + "theoretical_loss": 3.8939498618692934, + "tokens_seen": 525598720 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042460381143430296, + "loss": 2.847, + "theoretical_loss": 3.893898018823659, + "tokens_seen": 525664256 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004245937813440321, + "loss": 2.8299, + "theoretical_loss": 3.8938461840505347, + "tokens_seen": 525729792 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004245837512537613, + "loss": 2.9468, + "theoretical_loss": 3.8937943575475695, + "tokens_seen": 525795328 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042457372116349044, + "loss": 2.8589, + "theoretical_loss": 3.893742539312414, + "tokens_seen": 525860864 + }, + { + "epoch": 0.16, + "objective/train/docs_used": 308263, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.9339330196380615, + "objective/train/theoretical_loss": 3.8936907293427176, + "objective/train/tokens_used": 546386400, + "theoretical_loss": 3.8936907293427176, + "tokens_seen": 525926400 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004245636910732197, + "loss": 2.6238, + "theoretical_loss": 3.8936907293427176, + "tokens_seen": 525926400 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042455366098294886, + "loss": 2.7164, + "theoretical_loss": 3.893638927636134, + "tokens_seen": 525991936 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042454363089267804, + "loss": 2.9197, + "theoretical_loss": 3.893587134190314, + "tokens_seen": 526057472 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004245336008024072, + "loss": 2.9524, + "theoretical_loss": 3.8935353490029136, + "tokens_seen": 526123008 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004245235707121364, + "loss": 2.6753, + "theoretical_loss": 3.893483572071586, + "tokens_seen": 526188544 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004245135406218656, + "loss": 3.0177, + "theoretical_loss": 3.8934318033939874, + "tokens_seen": 526254080 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004245035105315948, + "loss": 2.8069, + "theoretical_loss": 3.8933800429677743, + "tokens_seen": 526319616 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042449348044132395, + "loss": 3.0825, + "theoretical_loss": 3.8933282907906044, + "tokens_seen": 526385152 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004244834503510532, + "loss": 2.7798, + "theoretical_loss": 3.8932765468601365, + "tokens_seen": 526450688 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042447342026078236, + "loss": 3.0972, + "theoretical_loss": 3.89322481117403, + "tokens_seen": 526516224 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042446339017051155, + "loss": 2.8734, + "theoretical_loss": 3.8931730837299456, + "tokens_seen": 526581760 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004244533600802407, + "loss": 2.8566, + "theoretical_loss": 3.893121364525544, + "tokens_seen": 526647296 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004244433299899699, + "loss": 2.8304, + "theoretical_loss": 3.893069653558488, + "tokens_seen": 526712832 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004244332998996991, + "loss": 2.9442, + "theoretical_loss": 3.893017950826441, + "tokens_seen": 526778368 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004244232698094283, + "loss": 2.8081, + "theoretical_loss": 3.892966256327068, + "tokens_seen": 526843904 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042441323971915745, + "loss": 3.0143, + "theoretical_loss": 3.892914570058033, + "tokens_seen": 526909440 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004244032096288867, + "loss": 2.7673, + "theoretical_loss": 3.8928628920170034, + "tokens_seen": 526974976 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004243931795386158, + "loss": 2.7712, + "theoretical_loss": 3.8928112222016455, + "tokens_seen": 527040512 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042438314944834505, + "loss": 2.804, + "theoretical_loss": 3.892759560609627, + "tokens_seen": 527106048 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042437311935807423, + "loss": 3.0228, + "theoretical_loss": 3.8927079072386186, + "tokens_seen": 527171584 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004243630892678034, + "loss": 2.8812, + "theoretical_loss": 3.892656262086289, + "tokens_seen": 527237120 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004243530591775326, + "loss": 2.9267, + "theoretical_loss": 3.8926046251503092, + "tokens_seen": 527302656 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004243430290872618, + "loss": 3.0808, + "theoretical_loss": 3.8925529964283516, + "tokens_seen": 527368192 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042433299899699095, + "loss": 2.7072, + "theoretical_loss": 3.892501375918089, + "tokens_seen": 527433728 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004243229689067202, + "loss": 2.8456, + "theoretical_loss": 3.8924497636171953, + "tokens_seen": 527499264 + }, + { + "epoch": 0.16, + "objective/train/docs_used": 309026, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.9527432918548584, + "objective/train/theoretical_loss": 3.892398159523345, + "objective/train/tokens_used": 548024800, + "theoretical_loss": 3.892398159523345, + "tokens_seen": 527564800 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004243129388164493, + "loss": 2.9524, + "theoretical_loss": 3.892398159523345, + "tokens_seen": 527564800 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042430290872617855, + "loss": 2.9366, + "theoretical_loss": 3.892346563634213, + "tokens_seen": 527630336 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042429287863590773, + "loss": 3.0411, + "theoretical_loss": 3.892294975947477, + "tokens_seen": 527695872 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004242828485456369, + "loss": 2.7857, + "theoretical_loss": 3.892243396460815, + "tokens_seen": 527761408 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004242728184553661, + "loss": 2.7106, + "theoretical_loss": 3.8921918251719054, + "tokens_seen": 527826944 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004242627883650953, + "loss": 2.8524, + "theoretical_loss": 3.892140262078427, + "tokens_seen": 527892480 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042425275827482446, + "loss": 2.931, + "theoretical_loss": 3.8920887071780594, + "tokens_seen": 527958016 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004242427281845537, + "loss": 2.9897, + "theoretical_loss": 3.8920371604684862, + "tokens_seen": 528023552 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004242326980942828, + "loss": 3.1178, + "theoretical_loss": 3.891985621947388, + "tokens_seen": 528089088 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042422266800401206, + "loss": 2.7958, + "theoretical_loss": 3.891934091612449, + "tokens_seen": 528154624 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004242126379137412, + "loss": 2.9019, + "theoretical_loss": 3.891882569461353, + "tokens_seen": 528220160 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004242026078234704, + "loss": 2.9834, + "theoretical_loss": 3.891831055491785, + "tokens_seen": 528285696 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004241925777331996, + "loss": 3.1314, + "theoretical_loss": 3.8917795497014316, + "tokens_seen": 528351232 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004241825476429288, + "loss": 2.8231, + "theoretical_loss": 3.891728052087979, + "tokens_seen": 528416768 + }, + { + "epoch": 0.16, + "learning_rate": 0.000424172517552658, + "loss": 2.9678, + "theoretical_loss": 3.891676562649116, + "tokens_seen": 528482304 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004241624874623872, + "loss": 2.8343, + "theoretical_loss": 3.8916250813825313, + "tokens_seen": 528547840 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004241524573721164, + "loss": 2.8924, + "theoretical_loss": 3.891573608285915, + "tokens_seen": 528613376 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042414242728184556, + "loss": 2.9965, + "theoretical_loss": 3.891522143356957, + "tokens_seen": 528678912 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042413239719157474, + "loss": 3.0864, + "theoretical_loss": 3.89147068659335, + "tokens_seen": 528744448 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004241223671013039, + "loss": 3.0405, + "theoretical_loss": 3.8914192379927863, + "tokens_seen": 528809984 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042411233701103316, + "loss": 3.0661, + "theoretical_loss": 3.891367797552959, + "tokens_seen": 528875520 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004241023069207623, + "loss": 2.6476, + "theoretical_loss": 3.891316365271564, + "tokens_seen": 528941056 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004240922768304915, + "loss": 2.7898, + "theoretical_loss": 3.891264941146295, + "tokens_seen": 529006592 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042408224674022065, + "loss": 2.8571, + "theoretical_loss": 3.89121352517485, + "tokens_seen": 529072128 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004240722166499499, + "loss": 2.8856, + "theoretical_loss": 3.8911621173549253, + "tokens_seen": 529137664 + }, + { + "epoch": 0.16, + "objective/train/docs_used": 310459, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.4450857639312744, + "objective/train/theoretical_loss": 3.89111071768422, + "objective/train/tokens_used": 549663200, + "theoretical_loss": 3.89111071768422, + "tokens_seen": 529203200 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042406218655967906, + "loss": 2.6917, + "theoretical_loss": 3.89111071768422, + "tokens_seen": 529203200 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042405215646940824, + "loss": 2.8561, + "theoretical_loss": 3.8910593261604323, + "tokens_seen": 529268736 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004240421263791374, + "loss": 2.9365, + "theoretical_loss": 3.891007942781264, + "tokens_seen": 529334272 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004240320962888666, + "loss": 2.9435, + "theoretical_loss": 3.8909565675444138, + "tokens_seen": 529399808 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004240220661985958, + "loss": 2.7801, + "theoretical_loss": 3.8909052004475857, + "tokens_seen": 529465344 + }, + { + "epoch": 0.16, + "learning_rate": 0.000424012036108325, + "loss": 2.9773, + "theoretical_loss": 3.8908538414884823, + "tokens_seen": 529530880 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042400200601805415, + "loss": 2.7113, + "theoretical_loss": 3.8908024906648073, + "tokens_seen": 529596416 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004239919759277834, + "loss": 2.9899, + "theoretical_loss": 3.890751147974265, + "tokens_seen": 529661952 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042398194583751256, + "loss": 2.8365, + "theoretical_loss": 3.890699813414562, + "tokens_seen": 529727488 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042397191574724175, + "loss": 2.851, + "theoretical_loss": 3.890648486983405, + "tokens_seen": 529793024 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004239618856569709, + "loss": 2.6801, + "theoretical_loss": 3.8905971686785006, + "tokens_seen": 529858560 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004239518555667001, + "loss": 2.8486, + "theoretical_loss": 3.8905458584975583, + "tokens_seen": 529924096 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004239418254764293, + "loss": 2.777, + "theoretical_loss": 3.890494556438287, + "tokens_seen": 529989632 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004239317953861585, + "loss": 2.7167, + "theoretical_loss": 3.8904432624983976, + "tokens_seen": 530055168 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042392176529588765, + "loss": 2.8318, + "theoretical_loss": 3.890391976675601, + "tokens_seen": 530120704 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004239117352056169, + "loss": 2.9663, + "theoretical_loss": 3.89034069896761, + "tokens_seen": 530186240 + }, + { + "epoch": 0.16, + "learning_rate": 0.000423901705115346, + "loss": 2.8264, + "theoretical_loss": 3.890289429372137, + "tokens_seen": 530251776 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042389167502507525, + "loss": 2.8953, + "theoretical_loss": 3.8902381678868974, + "tokens_seen": 530317312 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042388164493480443, + "loss": 2.8357, + "theoretical_loss": 3.890186914509605, + "tokens_seen": 530382848 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004238716148445336, + "loss": 2.7754, + "theoretical_loss": 3.890135669237976, + "tokens_seen": 530448384 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004238615847542628, + "loss": 2.7546, + "theoretical_loss": 3.8900844320697274, + "tokens_seen": 530513920 + }, + { + "epoch": 0.16, + "learning_rate": 0.000423851554663992, + "loss": 2.8953, + "theoretical_loss": 3.8900332030025773, + "tokens_seen": 530579456 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042384152457372115, + "loss": 2.921, + "theoretical_loss": 3.8899819820342443, + "tokens_seen": 530644992 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004238314944834504, + "loss": 3.0062, + "theoretical_loss": 3.889930769162448, + "tokens_seen": 530710528 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004238214643931795, + "loss": 2.8075, + "theoretical_loss": 3.8898795643849087, + "tokens_seen": 530776064 + }, + { + "epoch": 0.16, + "objective/train/docs_used": 311126, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.1827123165130615, + "objective/train/theoretical_loss": 3.889828367699349, + "objective/train/tokens_used": 551301600, + "theoretical_loss": 3.889828367699349, + "tokens_seen": 530841600 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042381143430290875, + "loss": 2.8086, + "theoretical_loss": 3.889828367699349, + "tokens_seen": 530841600 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042380140421263793, + "loss": 2.7818, + "theoretical_loss": 3.88977717910349, + "tokens_seen": 530907136 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004237913741223671, + "loss": 2.8432, + "theoretical_loss": 3.8897259985950554, + "tokens_seen": 530972672 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004237813440320963, + "loss": 2.8869, + "theoretical_loss": 3.8896748261717695, + "tokens_seen": 531038208 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004237713139418255, + "loss": 2.617, + "theoretical_loss": 3.889623661831359, + "tokens_seen": 531103744 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042376128385155466, + "loss": 2.8901, + "theoretical_loss": 3.8895725055715475, + "tokens_seen": 531169280 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004237512537612839, + "loss": 2.9752, + "theoretical_loss": 3.8895213573900635, + "tokens_seen": 531234816 + }, + { + "epoch": 0.16, + "learning_rate": 0.000423741223671013, + "loss": 2.8583, + "theoretical_loss": 3.889470217284635, + "tokens_seen": 531300352 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042373119358074226, + "loss": 2.8615, + "theoretical_loss": 3.8894190852529906, + "tokens_seen": 531365888 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004237211634904714, + "loss": 2.9967, + "theoretical_loss": 3.8893679612928604, + "tokens_seen": 531431424 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004237111334002006, + "loss": 2.8052, + "theoretical_loss": 3.889316845401974, + "tokens_seen": 531496960 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004237011033099298, + "loss": 2.8371, + "theoretical_loss": 3.8892657375780644, + "tokens_seen": 531562496 + }, + { + "epoch": 0.16, + "learning_rate": 0.000423691073219659, + "loss": 2.8738, + "theoretical_loss": 3.8892146378188643, + "tokens_seen": 531628032 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042368104312938816, + "loss": 3.1103, + "theoretical_loss": 3.8891635461221057, + "tokens_seen": 531693568 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004236710130391174, + "loss": 3.0551, + "theoretical_loss": 3.8891124624855236, + "tokens_seen": 531759104 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004236609829488465, + "loss": 2.6987, + "theoretical_loss": 3.889061386906854, + "tokens_seen": 531824640 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042365095285857576, + "loss": 2.9752, + "theoretical_loss": 3.889010319383832, + "tokens_seen": 531890176 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004236409227683049, + "loss": 2.4988, + "theoretical_loss": 3.8889592599141958, + "tokens_seen": 531955712 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004236308926780341, + "loss": 2.9229, + "theoretical_loss": 3.888908208495683, + "tokens_seen": 532021248 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004236208625877633, + "loss": 3.1384, + "theoretical_loss": 3.8888571651260326, + "tokens_seen": 532086784 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004236108324974925, + "loss": 2.9061, + "theoretical_loss": 3.888806129802984, + "tokens_seen": 532152320 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042360080240722166, + "loss": 2.9164, + "theoretical_loss": 3.888755102524279, + "tokens_seen": 532217856 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042359077231695085, + "loss": 2.8019, + "theoretical_loss": 3.8887040832876587, + "tokens_seen": 532283392 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042358074222668, + "loss": 2.843, + "theoretical_loss": 3.8886530720908654, + "tokens_seen": 532348928 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042357071213640926, + "loss": 3.0357, + "theoretical_loss": 3.888602068931643, + "tokens_seen": 532414464 + }, + { + "epoch": 0.16, + "objective/train/docs_used": 312276, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.676740884780884, + "objective/train/theoretical_loss": 3.888551073807736, + "objective/train/tokens_used": 552940000, + "theoretical_loss": 3.888551073807736, + "tokens_seen": 532480000 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004235606820461384, + "loss": 2.7638, + "theoretical_loss": 3.888551073807736, + "tokens_seen": 532480000 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004235506519558676, + "loss": 2.9984, + "theoretical_loss": 3.8885000867168893, + "tokens_seen": 532545536 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042354062186559675, + "loss": 3.0746, + "theoretical_loss": 3.88844910765685, + "tokens_seen": 532611072 + }, + { + "epoch": 0.16, + "learning_rate": 0.000423530591775326, + "loss": 2.8577, + "theoretical_loss": 3.888398136625364, + "tokens_seen": 532676608 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042352056168505517, + "loss": 2.6773, + "theoretical_loss": 3.888347173620181, + "tokens_seen": 532742144 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042351053159478435, + "loss": 2.9021, + "theoretical_loss": 3.8882962186390486, + "tokens_seen": 532807680 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042350050150451353, + "loss": 2.978, + "theoretical_loss": 3.8882452716797173, + "tokens_seen": 532873216 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042349047141424276, + "loss": 3.107, + "theoretical_loss": 3.8881943327399378, + "tokens_seen": 532938752 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004234804413239719, + "loss": 2.9604, + "theoretical_loss": 3.8881434018174614, + "tokens_seen": 533004288 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042347041123370113, + "loss": 2.9037, + "theoretical_loss": 3.8880924789100417, + "tokens_seen": 533069824 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042346038114343025, + "loss": 2.8473, + "theoretical_loss": 3.888041564015431, + "tokens_seen": 533135360 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004234503510531595, + "loss": 2.8208, + "theoretical_loss": 3.8879906571313847, + "tokens_seen": 533200896 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042344032096288867, + "loss": 2.8238, + "theoretical_loss": 3.887939758255658, + "tokens_seen": 533266432 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042343029087261785, + "loss": 2.9435, + "theoretical_loss": 3.887888867386007, + "tokens_seen": 533331968 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004234202607823471, + "loss": 2.9686, + "theoretical_loss": 3.8878379845201883, + "tokens_seen": 533397504 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004234102306920762, + "loss": 3.0119, + "theoretical_loss": 3.88778710965596, + "tokens_seen": 533463040 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042340020060180545, + "loss": 2.5169, + "theoretical_loss": 3.887736242791082, + "tokens_seen": 533528576 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042339017051153463, + "loss": 2.8188, + "theoretical_loss": 3.887685383923314, + "tokens_seen": 533594112 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004233801404212638, + "loss": 3.1537, + "theoretical_loss": 3.8876345330504156, + "tokens_seen": 533659648 + }, + { + "epoch": 0.16, + "learning_rate": 0.000423370110330993, + "loss": 3.0247, + "theoretical_loss": 3.887583690170149, + "tokens_seen": 533725184 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004233600802407222, + "loss": 3.065, + "theoretical_loss": 3.887532855280277, + "tokens_seen": 533790720 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042335005015045135, + "loss": 2.7605, + "theoretical_loss": 3.887482028378564, + "tokens_seen": 533856256 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004233400200601806, + "loss": 2.7755, + "theoretical_loss": 3.8874312094627723, + "tokens_seen": 533921792 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004233299899699097, + "loss": 2.8278, + "theoretical_loss": 3.8873803985306683, + "tokens_seen": 533987328 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042331995987963895, + "loss": 2.8705, + "theoretical_loss": 3.8873295955800184, + "tokens_seen": 534052864 + }, + { + "epoch": 0.16, + "objective/train/docs_used": 312792, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.836430549621582, + "objective/train/theoretical_loss": 3.8872788006085894, + "objective/train/tokens_used": 554578400, + "theoretical_loss": 3.8872788006085894, + "tokens_seen": 534118400 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042330992978936813, + "loss": 2.7082, + "theoretical_loss": 3.8872788006085894, + "tokens_seen": 534118400 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004232998996990973, + "loss": 3.1645, + "theoretical_loss": 3.8872280136141484, + "tokens_seen": 534183936 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004232898696088265, + "loss": 2.8199, + "theoretical_loss": 3.887177234594466, + "tokens_seen": 534249472 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004232798395185557, + "loss": 2.8115, + "theoretical_loss": 3.88712646354731, + "tokens_seen": 534315008 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042326980942828486, + "loss": 2.8924, + "theoretical_loss": 3.8870757004704526, + "tokens_seen": 534380544 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004232597793380141, + "loss": 2.9678, + "theoretical_loss": 3.8870249453616648, + "tokens_seen": 534446080 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004232497492477432, + "loss": 2.9354, + "theoretical_loss": 3.8869741982187183, + "tokens_seen": 534511616 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042323971915747246, + "loss": 2.8208, + "theoretical_loss": 3.8869234590393873, + "tokens_seen": 534577152 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004232296890672016, + "loss": 2.8999, + "theoretical_loss": 3.886872727821446, + "tokens_seen": 534642688 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004232196589769308, + "loss": 2.8578, + "theoretical_loss": 3.88682200456267, + "tokens_seen": 534708224 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042320962888666, + "loss": 2.888, + "theoretical_loss": 3.886771289260834, + "tokens_seen": 534773760 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004231995987963892, + "loss": 2.8298, + "theoretical_loss": 3.886720581913716, + "tokens_seen": 534839296 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042318956870611836, + "loss": 3.1318, + "theoretical_loss": 3.886669882519093, + "tokens_seen": 534904832 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004231795386158476, + "loss": 2.7523, + "theoretical_loss": 3.886619191074744, + "tokens_seen": 534970368 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004231695085255767, + "loss": 2.8671, + "theoretical_loss": 3.886568507578448, + "tokens_seen": 535035904 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042315947843530596, + "loss": 2.8762, + "theoretical_loss": 3.8865178320279874, + "tokens_seen": 535101440 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004231494483450351, + "loss": 2.9422, + "theoretical_loss": 3.886467164421142, + "tokens_seen": 535166976 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004231394182547643, + "loss": 2.8903, + "theoretical_loss": 3.8864165047556942, + "tokens_seen": 535232512 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004231293881644935, + "loss": 3.0428, + "theoretical_loss": 3.886365853029427, + "tokens_seen": 535298048 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004231193580742227, + "loss": 2.7576, + "theoretical_loss": 3.8863152092401254, + "tokens_seen": 535363584 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042310932798395186, + "loss": 2.9614, + "theoretical_loss": 3.886264573385574, + "tokens_seen": 535429120 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042309929789368105, + "loss": 2.9026, + "theoretical_loss": 3.886213945463558, + "tokens_seen": 535494656 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004230892678034102, + "loss": 3.048, + "theoretical_loss": 3.886163325471864, + "tokens_seen": 535560192 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042307923771313946, + "loss": 3.0195, + "theoretical_loss": 3.8861127134082807, + "tokens_seen": 535625728 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004230692076228686, + "loss": 2.8061, + "theoretical_loss": 3.8860621092705956, + "tokens_seen": 535691264 + }, + { + "epoch": 0.16, + "objective/train/docs_used": 314102, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.960132122039795, + "objective/train/theoretical_loss": 3.8860115130565984, + "objective/train/tokens_used": 556216800, + "theoretical_loss": 3.8860115130565984, + "tokens_seen": 535756800 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004230591775325978, + "loss": 2.9978, + "theoretical_loss": 3.8860115130565984, + "tokens_seen": 535756800 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042304914744232695, + "loss": 2.8643, + "theoretical_loss": 3.8859609247640794, + "tokens_seen": 535822336 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004230391173520562, + "loss": 3.0183, + "theoretical_loss": 3.88591034439083, + "tokens_seen": 535887872 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042302908726178537, + "loss": 2.8329, + "theoretical_loss": 3.885859771934642, + "tokens_seen": 535953408 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042301905717151455, + "loss": 3.0875, + "theoretical_loss": 3.885809207393308, + "tokens_seen": 536018944 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042300902708124373, + "loss": 2.8032, + "theoretical_loss": 3.8857586507646227, + "tokens_seen": 536084480 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042299899699097297, + "loss": 2.6994, + "theoretical_loss": 3.88570810204638, + "tokens_seen": 536150016 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004229889669007021, + "loss": 3.1056, + "theoretical_loss": 3.8856575612363753, + "tokens_seen": 536215552 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042297893681043133, + "loss": 3.0487, + "theoretical_loss": 3.8856070283324056, + "tokens_seen": 536281088 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042296890672016045, + "loss": 2.6521, + "theoretical_loss": 3.885556503332268, + "tokens_seen": 536346624 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004229588766298897, + "loss": 2.8809, + "theoretical_loss": 3.8855059862337606, + "tokens_seen": 536412160 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042294884653961887, + "loss": 2.7366, + "theoretical_loss": 3.885455477034683, + "tokens_seen": 536477696 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042293881644934805, + "loss": 2.7611, + "theoretical_loss": 3.8854049757328353, + "tokens_seen": 536543232 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042292878635907723, + "loss": 2.761, + "theoretical_loss": 3.8853544823260178, + "tokens_seen": 536608768 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004229187562688064, + "loss": 2.7029, + "theoretical_loss": 3.8853039968120324, + "tokens_seen": 536674304 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004229087261785356, + "loss": 2.6343, + "theoretical_loss": 3.885253519188682, + "tokens_seen": 536739840 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042289869608826483, + "loss": 2.9906, + "theoretical_loss": 3.885203049453769, + "tokens_seen": 536805376 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042288866599799396, + "loss": 2.9234, + "theoretical_loss": 3.8851525876050994, + "tokens_seen": 536870912 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004228786359077232, + "loss": 2.7906, + "theoretical_loss": 3.885102133640477, + "tokens_seen": 536936448 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004228686058174523, + "loss": 2.8607, + "theoretical_loss": 3.8850516875577092, + "tokens_seen": 537001984 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042285857572718155, + "loss": 2.8776, + "theoretical_loss": 3.8850012493546027, + "tokens_seen": 537067520 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042284854563691074, + "loss": 3.1319, + "theoretical_loss": 3.884950819028965, + "tokens_seen": 537133056 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004228385155466399, + "loss": 2.9636, + "theoretical_loss": 3.884900396578605, + "tokens_seen": 537198592 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004228284854563691, + "loss": 2.8746, + "theoretical_loss": 3.8848499820013327, + "tokens_seen": 537264128 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042281845536609833, + "loss": 2.8853, + "theoretical_loss": 3.884799575294958, + "tokens_seen": 537329664 + }, + { + "epoch": 0.16, + "objective/train/docs_used": 314596, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.935256242752075, + "objective/train/theoretical_loss": 3.8847491764572926, + "objective/train/tokens_used": 557855200, + "theoretical_loss": 3.8847491764572926, + "tokens_seen": 537395200 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042280842527582746, + "loss": 2.928, + "theoretical_loss": 3.8847491764572926, + "tokens_seen": 537395200 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004227983951855567, + "loss": 2.9151, + "theoretical_loss": 3.884698785486149, + "tokens_seen": 537460736 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004227883650952858, + "loss": 2.6575, + "theoretical_loss": 3.884648402379341, + "tokens_seen": 537526272 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042277833500501506, + "loss": 2.7136, + "theoretical_loss": 3.8845980271346816, + "tokens_seen": 537591808 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042276830491474424, + "loss": 2.7524, + "theoretical_loss": 3.8845476597499857, + "tokens_seen": 537657344 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004227582748244734, + "loss": 2.9858, + "theoretical_loss": 3.884497300223069, + "tokens_seen": 537722880 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004227482447342026, + "loss": 2.8558, + "theoretical_loss": 3.8844469485517488, + "tokens_seen": 537788416 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004227382146439318, + "loss": 3.0815, + "theoretical_loss": 3.8843966047338423, + "tokens_seen": 537853952 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042272818455366096, + "loss": 2.7769, + "theoretical_loss": 3.8843462687671684, + "tokens_seen": 537919488 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004227181544633902, + "loss": 2.8422, + "theoretical_loss": 3.8842959406495456, + "tokens_seen": 537985024 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004227081243731193, + "loss": 2.7772, + "theoretical_loss": 3.884245620378794, + "tokens_seen": 538050560 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042269809428284856, + "loss": 3.0184, + "theoretical_loss": 3.8841953079527354, + "tokens_seen": 538116096 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004226880641925777, + "loss": 3.1093, + "theoretical_loss": 3.8841450033691913, + "tokens_seen": 538181632 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004226780341023069, + "loss": 2.902, + "theoretical_loss": 3.8840947066259845, + "tokens_seen": 538247168 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042266800401203616, + "loss": 2.9983, + "theoretical_loss": 3.884044417720938, + "tokens_seen": 538312704 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004226579739217653, + "loss": 2.8264, + "theoretical_loss": 3.8839941366518778, + "tokens_seen": 538378240 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004226479438314945, + "loss": 2.8081, + "theoretical_loss": 3.8839438634166275, + "tokens_seen": 538443776 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004226379137412237, + "loss": 2.7086, + "theoretical_loss": 3.883893598013014, + "tokens_seen": 538509312 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004226278836509529, + "loss": 2.8416, + "theoretical_loss": 3.883843340438865, + "tokens_seen": 538574848 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042261785356068206, + "loss": 2.8464, + "theoretical_loss": 3.8837930906920075, + "tokens_seen": 538640384 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042260782347041125, + "loss": 2.5902, + "theoretical_loss": 3.8837428487702716, + "tokens_seen": 538705920 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004225977933801404, + "loss": 3.0017, + "theoretical_loss": 3.8836926146714856, + "tokens_seen": 538771456 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042258776328986966, + "loss": 2.9867, + "theoretical_loss": 3.883642388393481, + "tokens_seen": 538836992 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004225777331995988, + "loss": 2.9134, + "theoretical_loss": 3.8835921699340887, + "tokens_seen": 538902528 + }, + { + "epoch": 0.16, + "learning_rate": 0.000422567703109328, + "loss": 2.7619, + "theoretical_loss": 3.8835419592911413, + "tokens_seen": 538968064 + }, + { + "epoch": 0.16, + "objective/train/docs_used": 315427, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.4939568042755127, + "objective/train/theoretical_loss": 3.8834917564624716, + "objective/train/tokens_used": 559493600, + "theoretical_loss": 3.8834917564624716, + "tokens_seen": 539033600 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042255767301905715, + "loss": 2.7711, + "theoretical_loss": 3.8834917564624716, + "tokens_seen": 539033600 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004225476429287864, + "loss": 2.8169, + "theoretical_loss": 3.883441561445914, + "tokens_seen": 539099136 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042253761283851557, + "loss": 2.8529, + "theoretical_loss": 3.8833913742393036, + "tokens_seen": 539164672 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042252758274824475, + "loss": 2.6078, + "theoretical_loss": 3.8833411948404755, + "tokens_seen": 539230208 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042251755265797393, + "loss": 2.8866, + "theoretical_loss": 3.883291023247267, + "tokens_seen": 539295744 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042250752256770317, + "loss": 2.743, + "theoretical_loss": 3.8832408594575147, + "tokens_seen": 539361280 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004224974924774323, + "loss": 3.0086, + "theoretical_loss": 3.883190703469058, + "tokens_seen": 539426816 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042248746238716153, + "loss": 2.7537, + "theoretical_loss": 3.883140555279735, + "tokens_seen": 539492352 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042247743229689065, + "loss": 2.9777, + "theoretical_loss": 3.883090414887387, + "tokens_seen": 539557888 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004224674022066199, + "loss": 2.738, + "theoretical_loss": 3.8830402822898535, + "tokens_seen": 539623424 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042245737211634907, + "loss": 2.9687, + "theoretical_loss": 3.8829901574849774, + "tokens_seen": 539688960 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042244734202607825, + "loss": 2.7361, + "theoretical_loss": 3.8829400404706003, + "tokens_seen": 539754496 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042243731193580743, + "loss": 2.8223, + "theoretical_loss": 3.8828899312445673, + "tokens_seen": 539820032 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004224272818455366, + "loss": 2.9411, + "theoretical_loss": 3.8828398298047206, + "tokens_seen": 539885568 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004224172517552658, + "loss": 2.9454, + "theoretical_loss": 3.8827897361489074, + "tokens_seen": 539951104 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042240722166499503, + "loss": 2.8357, + "theoretical_loss": 3.8827396502749734, + "tokens_seen": 540016640 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042239719157472416, + "loss": 2.9541, + "theoretical_loss": 3.882689572180764, + "tokens_seen": 540082176 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004223871614844534, + "loss": 3.066, + "theoretical_loss": 3.882639501864129, + "tokens_seen": 540147712 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004223771313941825, + "loss": 2.9765, + "theoretical_loss": 3.8825894393229152, + "tokens_seen": 540213248 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042236710130391176, + "loss": 2.9042, + "theoretical_loss": 3.882539384554974, + "tokens_seen": 540278784 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042235707121364094, + "loss": 2.9909, + "theoretical_loss": 3.8824893375581544, + "tokens_seen": 540344320 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004223470411233701, + "loss": 2.8827, + "theoretical_loss": 3.8824392983303078, + "tokens_seen": 540409856 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004223370110330993, + "loss": 2.7419, + "theoretical_loss": 3.8823892668692865, + "tokens_seen": 540475392 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042232698094282853, + "loss": 2.9962, + "theoretical_loss": 3.8823392431729435, + "tokens_seen": 540540928 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042231695085255766, + "loss": 3.0481, + "theoretical_loss": 3.8822892272391325, + "tokens_seen": 540606464 + }, + { + "epoch": 0.16, + "objective/train/docs_used": 316530, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.0311472415924072, + "objective/train/theoretical_loss": 3.882239219065708, + "objective/train/tokens_used": 561132000, + "theoretical_loss": 3.882239219065708, + "tokens_seen": 540672000 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004223069207622869, + "loss": 2.9144, + "theoretical_loss": 3.882239219065708, + "tokens_seen": 540672000 + }, + { + "epoch": 0.16, + "learning_rate": 0.000422296890672016, + "loss": 2.6777, + "theoretical_loss": 3.882189218650525, + "tokens_seen": 540737536 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042228686058174526, + "loss": 2.8608, + "theoretical_loss": 3.882139225991441, + "tokens_seen": 540803072 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042227683049147444, + "loss": 2.8357, + "theoretical_loss": 3.8820892410863124, + "tokens_seen": 540868608 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004222668004012036, + "loss": 2.841, + "theoretical_loss": 3.882039263932997, + "tokens_seen": 540934144 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004222567703109328, + "loss": 2.6845, + "theoretical_loss": 3.8819892945293546, + "tokens_seen": 540999680 + }, + { + "epoch": 0.16, + "learning_rate": 0.000422246740220662, + "loss": 2.6987, + "theoretical_loss": 3.8819393328732446, + "tokens_seen": 541065216 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042223671013039116, + "loss": 2.9168, + "theoretical_loss": 3.8818893789625273, + "tokens_seen": 541130752 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004222266800401204, + "loss": 2.9064, + "theoretical_loss": 3.881839432795064, + "tokens_seen": 541196288 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004222166499498495, + "loss": 2.7624, + "theoretical_loss": 3.881789494368717, + "tokens_seen": 541261824 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042220661985957876, + "loss": 3.0751, + "theoretical_loss": 3.88173956368135, + "tokens_seen": 541327360 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004221965897693079, + "loss": 2.7497, + "theoretical_loss": 3.8816896407308263, + "tokens_seen": 541392896 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004221865596790371, + "loss": 2.8959, + "theoretical_loss": 3.8816397255150115, + "tokens_seen": 541458432 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004221765295887663, + "loss": 2.6955, + "theoretical_loss": 3.8815898180317703, + "tokens_seen": 541523968 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004221664994984955, + "loss": 3.2167, + "theoretical_loss": 3.8815399182789703, + "tokens_seen": 541589504 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042215646940822467, + "loss": 3.0417, + "theoretical_loss": 3.881490026254478, + "tokens_seen": 541655040 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004221464393179539, + "loss": 3.0272, + "theoretical_loss": 3.881440141956162, + "tokens_seen": 541720576 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042213640922768303, + "loss": 2.9219, + "theoretical_loss": 3.8813902653818912, + "tokens_seen": 541786112 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042212637913741226, + "loss": 3.0161, + "theoretical_loss": 3.8813403965295357, + "tokens_seen": 541851648 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004221163490471414, + "loss": 2.6266, + "theoretical_loss": 3.8812905353969667, + "tokens_seen": 541917184 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004221063189568706, + "loss": 3.0422, + "theoretical_loss": 3.881240681982054, + "tokens_seen": 541982720 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004220962888665998, + "loss": 2.7677, + "theoretical_loss": 3.8811908362826726, + "tokens_seen": 542048256 + }, + { + "epoch": 0.16, + "learning_rate": 0.000422086258776329, + "loss": 2.7841, + "theoretical_loss": 3.881140998296694, + "tokens_seen": 542113792 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042207622868605817, + "loss": 3.0576, + "theoretical_loss": 3.8810911680219933, + "tokens_seen": 542179328 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042206619859578735, + "loss": 2.749, + "theoretical_loss": 3.881041345456444, + "tokens_seen": 542244864 + }, + { + "epoch": 0.16, + "objective/train/docs_used": 317270, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.866762638092041, + "objective/train/theoretical_loss": 3.8809915305979237, + "objective/train/tokens_used": 562770400, + "theoretical_loss": 3.8809915305979237, + "tokens_seen": 542310400 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042205616850551653, + "loss": 2.8391, + "theoretical_loss": 3.8809915305979237, + "tokens_seen": 542310400 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042204613841524577, + "loss": 2.8479, + "theoretical_loss": 3.880941723444308, + "tokens_seen": 542375936 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004220361083249749, + "loss": 2.7996, + "theoretical_loss": 3.8808919239934747, + "tokens_seen": 542441472 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042202607823470413, + "loss": 2.9353, + "theoretical_loss": 3.8808421322433024, + "tokens_seen": 542507008 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004220160481444333, + "loss": 3.1562, + "theoretical_loss": 3.8807923481916697, + "tokens_seen": 542572544 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004220060180541625, + "loss": 3.0191, + "theoretical_loss": 3.880742571836457, + "tokens_seen": 542638080 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004219959879638917, + "loss": 2.9505, + "theoretical_loss": 3.8806928031755454, + "tokens_seen": 542703616 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042198595787362085, + "loss": 2.9146, + "theoretical_loss": 3.880643042206816, + "tokens_seen": 542769152 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042197592778335004, + "loss": 2.6524, + "theoretical_loss": 3.880593288928152, + "tokens_seen": 542834688 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042196589769307927, + "loss": 3.0142, + "theoretical_loss": 3.8805435433374353, + "tokens_seen": 542900224 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004219558676028084, + "loss": 3.0136, + "theoretical_loss": 3.880493805432552, + "tokens_seen": 542965760 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042194583751253763, + "loss": 2.855, + "theoretical_loss": 3.8804440752113862, + "tokens_seen": 543031296 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042193580742226676, + "loss": 3.0735, + "theoretical_loss": 3.880394352671824, + "tokens_seen": 543096832 + }, + { + "epoch": 0.16, + "learning_rate": 0.000421925777331996, + "loss": 2.7591, + "theoretical_loss": 3.880344637811752, + "tokens_seen": 543162368 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042191574724172523, + "loss": 2.5691, + "theoretical_loss": 3.880294930629058, + "tokens_seen": 543227904 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042190571715145436, + "loss": 2.7601, + "theoretical_loss": 3.8802452311216302, + "tokens_seen": 543293440 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004218956870611836, + "loss": 2.991, + "theoretical_loss": 3.880195539287358, + "tokens_seen": 543358976 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004218856569709127, + "loss": 2.9317, + "theoretical_loss": 3.8801458551241312, + "tokens_seen": 543424512 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042187562688064196, + "loss": 2.8661, + "theoretical_loss": 3.8800961786298407, + "tokens_seen": 543490048 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042186559679037114, + "loss": 2.9067, + "theoretical_loss": 3.880046509802378, + "tokens_seen": 543555584 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004218555667001003, + "loss": 2.8519, + "theoretical_loss": 3.879996848639637, + "tokens_seen": 543621120 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004218455366098295, + "loss": 2.9094, + "theoretical_loss": 3.8799471951395095, + "tokens_seen": 543686656 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042183550651955873, + "loss": 2.9546, + "theoretical_loss": 3.8798975492998906, + "tokens_seen": 543752192 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042182547642928786, + "loss": 3.03, + "theoretical_loss": 3.8798479111186754, + "tokens_seen": 543817728 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004218154463390171, + "loss": 2.8642, + "theoretical_loss": 3.879798280593759, + "tokens_seen": 543883264 + }, + { + "epoch": 0.16, + "objective/train/docs_used": 318285, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.0952112674713135, + "objective/train/theoretical_loss": 3.879748657723039, + "objective/train/tokens_used": 564408800, + "theoretical_loss": 3.879748657723039, + "tokens_seen": 543948800 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004218054162487462, + "loss": 2.8699, + "theoretical_loss": 3.879748657723039, + "tokens_seen": 543948800 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042179538615847546, + "loss": 2.8675, + "theoretical_loss": 3.8796990425044124, + "tokens_seen": 544014336 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042178535606820464, + "loss": 2.8958, + "theoretical_loss": 3.879649434935778, + "tokens_seen": 544079872 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004217753259779338, + "loss": 2.9045, + "theoretical_loss": 3.8795998350150347, + "tokens_seen": 544145408 + }, + { + "epoch": 0.16, + "learning_rate": 0.000421765295887663, + "loss": 2.6795, + "theoretical_loss": 3.879550242740083, + "tokens_seen": 544210944 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004217552657973922, + "loss": 3.0074, + "theoretical_loss": 3.8795006581088236, + "tokens_seen": 544276480 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042174523570712136, + "loss": 2.8439, + "theoretical_loss": 3.879451081119158, + "tokens_seen": 544342016 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004217352056168506, + "loss": 2.8512, + "theoretical_loss": 3.8794015117689886, + "tokens_seen": 544407552 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004217251755265797, + "loss": 2.9305, + "theoretical_loss": 3.8793519500562192, + "tokens_seen": 544473088 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042171514543630896, + "loss": 2.7384, + "theoretical_loss": 3.879302395978754, + "tokens_seen": 544538624 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004217051153460381, + "loss": 2.9918, + "theoretical_loss": 3.879252849534498, + "tokens_seen": 544604160 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004216950852557673, + "loss": 3.0998, + "theoretical_loss": 3.8792033107213566, + "tokens_seen": 544669696 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004216850551654965, + "loss": 2.802, + "theoretical_loss": 3.8791537795372366, + "tokens_seen": 544735232 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004216750250752257, + "loss": 2.9623, + "theoretical_loss": 3.879104255980046, + "tokens_seen": 544800768 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042166499498495487, + "loss": 2.9675, + "theoretical_loss": 3.8790547400476925, + "tokens_seen": 544866304 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004216549648946841, + "loss": 3.1056, + "theoretical_loss": 3.879005231738086, + "tokens_seen": 544931840 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042164493480441323, + "loss": 2.9588, + "theoretical_loss": 3.878955731049136, + "tokens_seen": 544997376 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042163490471414246, + "loss": 2.9141, + "theoretical_loss": 3.878906237978753, + "tokens_seen": 545062912 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004216248746238716, + "loss": 2.9245, + "theoretical_loss": 3.8788567525248494, + "tokens_seen": 545128448 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004216148445336008, + "loss": 2.9224, + "theoretical_loss": 3.878807274685337, + "tokens_seen": 545193984 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042160481444333, + "loss": 2.9945, + "theoretical_loss": 3.8787578044581292, + "tokens_seen": 545259520 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004215947843530592, + "loss": 3.0882, + "theoretical_loss": 3.87870834184114, + "tokens_seen": 545325056 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042158475426278837, + "loss": 2.9014, + "theoretical_loss": 3.8786588868322847, + "tokens_seen": 545390592 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042157472417251755, + "loss": 2.9761, + "theoretical_loss": 3.8786094394294794, + "tokens_seen": 545456128 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042156469408224673, + "loss": 3.0424, + "theoretical_loss": 3.8785599996306397, + "tokens_seen": 545521664 + }, + { + "epoch": 0.17, + "objective/train/docs_used": 318961, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.2215077877044678, + "objective/train/theoretical_loss": 3.8785105674336835, + "objective/train/tokens_used": 566047200, + "theoretical_loss": 3.8785105674336835, + "tokens_seen": 545587200 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042155466399197597, + "loss": 2.836, + "theoretical_loss": 3.8785105674336835, + "tokens_seen": 545587200 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004215446339017051, + "loss": 2.8326, + "theoretical_loss": 3.8784611428365285, + "tokens_seen": 545652736 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042153460381143433, + "loss": 2.9618, + "theoretical_loss": 3.8784117258370943, + "tokens_seen": 545718272 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004215245737211635, + "loss": 2.7413, + "theoretical_loss": 3.878362316433301, + "tokens_seen": 545783808 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004215145436308927, + "loss": 2.9772, + "theoretical_loss": 3.878312914623068, + "tokens_seen": 545849344 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004215045135406219, + "loss": 3.0972, + "theoretical_loss": 3.8782635204043188, + "tokens_seen": 545914880 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042149448345035105, + "loss": 2.751, + "theoretical_loss": 3.8782141337749736, + "tokens_seen": 545980416 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042148445336008024, + "loss": 2.8092, + "theoretical_loss": 3.8781647547329565, + "tokens_seen": 546045952 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042147442326980947, + "loss": 2.9074, + "theoretical_loss": 3.8781153832761914, + "tokens_seen": 546111488 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004214643931795386, + "loss": 2.6035, + "theoretical_loss": 3.8780660194026035, + "tokens_seen": 546177024 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042145436308926783, + "loss": 3.0213, + "theoretical_loss": 3.8780166631101176, + "tokens_seen": 546242560 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042144433299899696, + "loss": 2.8903, + "theoretical_loss": 3.8779673143966606, + "tokens_seen": 546308096 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004214343029087262, + "loss": 2.883, + "theoretical_loss": 3.8779179732601587, + "tokens_seen": 546373632 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004214242728184554, + "loss": 2.964, + "theoretical_loss": 3.8778686396985416, + "tokens_seen": 546439168 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042141424272818456, + "loss": 3.0887, + "theoretical_loss": 3.877819313709736, + "tokens_seen": 546504704 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042140421263791374, + "loss": 2.6989, + "theoretical_loss": 3.8777699952916738, + "tokens_seen": 546570240 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004213941825476429, + "loss": 2.7352, + "theoretical_loss": 3.8777206844422842, + "tokens_seen": 546635776 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004213841524573721, + "loss": 2.7296, + "theoretical_loss": 3.877671381159499, + "tokens_seen": 546701312 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042137412236710134, + "loss": 2.9016, + "theoretical_loss": 3.8776220854412493, + "tokens_seen": 546766848 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042136409227683046, + "loss": 2.97, + "theoretical_loss": 3.877572797285469, + "tokens_seen": 546832384 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004213540621865597, + "loss": 2.8595, + "theoretical_loss": 3.8775235166900917, + "tokens_seen": 546897920 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004213440320962889, + "loss": 2.7882, + "theoretical_loss": 3.877474243653051, + "tokens_seen": 546963456 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042133400200601806, + "loss": 2.8407, + "theoretical_loss": 3.8774249781722836, + "tokens_seen": 547028992 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042132397191574724, + "loss": 3.0436, + "theoretical_loss": 3.877375720245725, + "tokens_seen": 547094528 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004213139418254764, + "loss": 2.9176, + "theoretical_loss": 3.8773264698713117, + "tokens_seen": 547160064 + }, + { + "epoch": 0.17, + "objective/train/docs_used": 320353, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.773749351501465, + "objective/train/theoretical_loss": 3.8772772270469824, + "objective/train/tokens_used": 567685600, + "theoretical_loss": 3.8772772270469824, + "tokens_seen": 547225600 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004213039117352056, + "loss": 2.8091, + "theoretical_loss": 3.8772772270469824, + "tokens_seen": 547225600 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042129388164493484, + "loss": 3.0391, + "theoretical_loss": 3.877227991770675, + "tokens_seen": 547291136 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042128385155466397, + "loss": 2.9231, + "theoretical_loss": 3.877178764040329, + "tokens_seen": 547356672 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004212738214643932, + "loss": 2.9136, + "theoretical_loss": 3.8771295438538846, + "tokens_seen": 547422208 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042126379137412233, + "loss": 2.8658, + "theoretical_loss": 3.8770803312092834, + "tokens_seen": 547487744 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042125376128385156, + "loss": 2.6838, + "theoretical_loss": 3.877031126104466, + "tokens_seen": 547553280 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042124373119358075, + "loss": 2.7451, + "theoretical_loss": 3.8769819285373766, + "tokens_seen": 547618816 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004212337011033099, + "loss": 2.8937, + "theoretical_loss": 3.876932738505957, + "tokens_seen": 547684352 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004212236710130391, + "loss": 2.8746, + "theoretical_loss": 3.8768835560081527, + "tokens_seen": 547749888 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004212136409227683, + "loss": 2.9699, + "theoretical_loss": 3.876834381041908, + "tokens_seen": 547815424 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042120361083249747, + "loss": 2.8668, + "theoretical_loss": 3.8767852136051686, + "tokens_seen": 547880960 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004211935807422267, + "loss": 2.7507, + "theoretical_loss": 3.8767360536958817, + "tokens_seen": 547946496 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042118355065195583, + "loss": 2.9112, + "theoretical_loss": 3.876686901311995, + "tokens_seen": 548012032 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042117352056168507, + "loss": 2.7109, + "theoretical_loss": 3.8766377564514563, + "tokens_seen": 548077568 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004211634904714143, + "loss": 3.0159, + "theoretical_loss": 3.8765886191122148, + "tokens_seen": 548143104 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042115346038114343, + "loss": 2.9348, + "theoretical_loss": 3.87653948929222, + "tokens_seen": 548208640 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042114343029087266, + "loss": 2.7605, + "theoretical_loss": 3.876490366989423, + "tokens_seen": 548274176 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004211334002006018, + "loss": 2.6026, + "theoretical_loss": 3.876441252201775, + "tokens_seen": 548339712 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042112337011033103, + "loss": 2.6092, + "theoretical_loss": 3.876392144927229, + "tokens_seen": 548405248 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004211133400200602, + "loss": 2.8352, + "theoretical_loss": 3.876343045163737, + "tokens_seen": 548470784 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004211033099297894, + "loss": 2.9303, + "theoretical_loss": 3.8762939529092537, + "tokens_seen": 548536320 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042109327983951857, + "loss": 2.7173, + "theoretical_loss": 3.8762448681617334, + "tokens_seen": 548601856 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042108324974924775, + "loss": 3.0451, + "theoretical_loss": 3.876195790919132, + "tokens_seen": 548667392 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042107321965897693, + "loss": 2.5712, + "theoretical_loss": 3.876146721179405, + "tokens_seen": 548732928 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042106318956870617, + "loss": 2.7421, + "theoretical_loss": 3.8760976589405107, + "tokens_seen": 548798464 + }, + { + "epoch": 0.17, + "objective/train/docs_used": 320989, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.595377206802368, + "objective/train/theoretical_loss": 3.8760486042004056, + "objective/train/tokens_used": 569324000, + "theoretical_loss": 3.8760486042004056, + "tokens_seen": 548864000 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004210531594784353, + "loss": 2.605, + "theoretical_loss": 3.8760486042004056, + "tokens_seen": 548864000 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042104312938816453, + "loss": 3.0457, + "theoretical_loss": 3.8759995569570496, + "tokens_seen": 548929536 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004210330992978937, + "loss": 2.9721, + "theoretical_loss": 3.8759505172084014, + "tokens_seen": 548995072 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004210230692076229, + "loss": 2.9312, + "theoretical_loss": 3.875901484952422, + "tokens_seen": 549060608 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004210130391173521, + "loss": 2.8009, + "theoretical_loss": 3.8758524601870716, + "tokens_seen": 549126144 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042100300902708125, + "loss": 2.8216, + "theoretical_loss": 3.8758034429103128, + "tokens_seen": 549191680 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042099297893681044, + "loss": 2.8882, + "theoretical_loss": 3.8757544331201084, + "tokens_seen": 549257216 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042098294884653967, + "loss": 2.7153, + "theoretical_loss": 3.875705430814421, + "tokens_seen": 549322752 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004209729187562688, + "loss": 2.896, + "theoretical_loss": 3.8756564359912153, + "tokens_seen": 549388288 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042096288866599803, + "loss": 2.9037, + "theoretical_loss": 3.8756074486484566, + "tokens_seen": 549453824 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042095285857572716, + "loss": 2.8598, + "theoretical_loss": 3.8755584687841114, + "tokens_seen": 549519360 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004209428284854564, + "loss": 2.6226, + "theoretical_loss": 3.8755094963961447, + "tokens_seen": 549584896 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004209327983951856, + "loss": 2.9382, + "theoretical_loss": 3.875460531482526, + "tokens_seen": 549650432 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042092276830491476, + "loss": 2.8071, + "theoretical_loss": 3.8754115740412214, + "tokens_seen": 549715968 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042091273821464394, + "loss": 2.6774, + "theoretical_loss": 3.8753626240702013, + "tokens_seen": 549781504 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004209027081243731, + "loss": 2.8182, + "theoretical_loss": 3.8753136815674356, + "tokens_seen": 549847040 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004208926780341023, + "loss": 3.1658, + "theoretical_loss": 3.875264746530894, + "tokens_seen": 549912576 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042088264794383154, + "loss": 2.7419, + "theoretical_loss": 3.875215818958549, + "tokens_seen": 549978112 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042087261785356066, + "loss": 2.7855, + "theoretical_loss": 3.875166898848372, + "tokens_seen": 550043648 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004208625877632899, + "loss": 2.9926, + "theoretical_loss": 3.8751179861983367, + "tokens_seen": 550109184 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004208525576730191, + "loss": 3.0088, + "theoretical_loss": 3.8750690810064166, + "tokens_seen": 550174720 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042084252758274826, + "loss": 2.5929, + "theoretical_loss": 3.8750201832705864, + "tokens_seen": 550240256 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042083249749247744, + "loss": 2.9118, + "theoretical_loss": 3.874971292988821, + "tokens_seen": 550305792 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004208224674022066, + "loss": 3.1091, + "theoretical_loss": 3.8749224101590976, + "tokens_seen": 550371328 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004208124373119358, + "loss": 2.8516, + "theoretical_loss": 3.8748735347793923, + "tokens_seen": 550436864 + }, + { + "epoch": 0.17, + "objective/train/docs_used": 322374, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.728480339050293, + "objective/train/theoretical_loss": 3.8748246668476827, + "objective/train/tokens_used": 570962400, + "theoretical_loss": 3.8748246668476827, + "tokens_seen": 550502400 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042080240722166504, + "loss": 2.8204, + "theoretical_loss": 3.8748246668476827, + "tokens_seen": 550502400 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042079237713139417, + "loss": 2.9299, + "theoretical_loss": 3.874775806361948, + "tokens_seen": 550567936 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004207823470411234, + "loss": 2.8736, + "theoretical_loss": 3.8747269533201676, + "tokens_seen": 550633472 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042077231695085253, + "loss": 2.9479, + "theoretical_loss": 3.874678107720321, + "tokens_seen": 550699008 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042076228686058176, + "loss": 2.8594, + "theoretical_loss": 3.8746292695603897, + "tokens_seen": 550764544 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042075225677031095, + "loss": 2.906, + "theoretical_loss": 3.874580438838355, + "tokens_seen": 550830080 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004207422266800401, + "loss": 2.6158, + "theoretical_loss": 3.8745316155521996, + "tokens_seen": 550895616 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004207321965897693, + "loss": 2.6722, + "theoretical_loss": 3.8744827996999067, + "tokens_seen": 550961152 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004207221664994985, + "loss": 2.7958, + "theoretical_loss": 3.874433991279461, + "tokens_seen": 551026688 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042071213640922767, + "loss": 2.9235, + "theoretical_loss": 3.874385190288846, + "tokens_seen": 551092224 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004207021063189569, + "loss": 2.9381, + "theoretical_loss": 3.874336396726049, + "tokens_seen": 551157760 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042069207622868603, + "loss": 3.1463, + "theoretical_loss": 3.8742876105890547, + "tokens_seen": 551223296 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042068204613841527, + "loss": 2.8491, + "theoretical_loss": 3.874238831875852, + "tokens_seen": 551288832 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042067201604814445, + "loss": 2.7954, + "theoretical_loss": 3.8741900605844273, + "tokens_seen": 551354368 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042066198595787363, + "loss": 2.861, + "theoretical_loss": 3.8741412967127706, + "tokens_seen": 551419904 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004206519558676028, + "loss": 3.0586, + "theoretical_loss": 3.8740925402588706, + "tokens_seen": 551485440 + }, + { + "epoch": 0.17, + "learning_rate": 0.000420641925777332, + "loss": 2.8358, + "theoretical_loss": 3.8740437912207186, + "tokens_seen": 551550976 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004206318956870612, + "loss": 2.9814, + "theoretical_loss": 3.873995049596305, + "tokens_seen": 551616512 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004206218655967904, + "loss": 2.9671, + "theoretical_loss": 3.873946315383622, + "tokens_seen": 551682048 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042061183550651954, + "loss": 2.8929, + "theoretical_loss": 3.873897588580662, + "tokens_seen": 551747584 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042060180541624877, + "loss": 2.8012, + "theoretical_loss": 3.8738488691854194, + "tokens_seen": 551813120 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004205917753259779, + "loss": 2.9326, + "theoretical_loss": 3.873800157195887, + "tokens_seen": 551878656 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042058174523570713, + "loss": 2.9168, + "theoretical_loss": 3.873751452610061, + "tokens_seen": 551944192 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004205717151454363, + "loss": 2.7729, + "theoretical_loss": 3.8737027554259367, + "tokens_seen": 552009728 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004205616850551655, + "loss": 2.6827, + "theoretical_loss": 3.873654065641511, + "tokens_seen": 552075264 + }, + { + "epoch": 0.17, + "objective/train/docs_used": 322945, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.925690174102783, + "objective/train/theoretical_loss": 3.8736053832547808, + "objective/train/tokens_used": 572600800, + "theoretical_loss": 3.8736053832547808, + "tokens_seen": 552140800 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004205516549648947, + "loss": 2.9947, + "theoretical_loss": 3.8736053832547808, + "tokens_seen": 552140800 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004205416248746239, + "loss": 2.965, + "theoretical_loss": 3.873556708263745, + "tokens_seen": 552206336 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042053159478435304, + "loss": 3.0093, + "theoretical_loss": 3.873508040666402, + "tokens_seen": 552271872 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004205215646940823, + "loss": 2.9645, + "theoretical_loss": 3.8734593804607513, + "tokens_seen": 552337408 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004205115346038114, + "loss": 2.6992, + "theoretical_loss": 3.873410727644794, + "tokens_seen": 552402944 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042050150451354064, + "loss": 3.0297, + "theoretical_loss": 3.8733620822165316, + "tokens_seen": 552468480 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004204914744232698, + "loss": 3.0783, + "theoretical_loss": 3.8733134441739647, + "tokens_seen": 552534016 + }, + { + "epoch": 0.17, + "learning_rate": 0.000420481444332999, + "loss": 2.709, + "theoretical_loss": 3.8732648135150978, + "tokens_seen": 552599552 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004204714142427282, + "loss": 2.727, + "theoretical_loss": 3.8732161902379336, + "tokens_seen": 552665088 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042046138415245736, + "loss": 2.6835, + "theoretical_loss": 3.873167574340477, + "tokens_seen": 552730624 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042045135406218654, + "loss": 2.9523, + "theoretical_loss": 3.873118965820733, + "tokens_seen": 552796160 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004204413239719158, + "loss": 3.021, + "theoretical_loss": 3.873070364676707, + "tokens_seen": 552861696 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004204312938816449, + "loss": 2.8156, + "theoretical_loss": 3.8730217709064063, + "tokens_seen": 552927232 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042042126379137414, + "loss": 2.7867, + "theoretical_loss": 3.8729731845078383, + "tokens_seen": 552992768 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004204112337011033, + "loss": 2.68, + "theoretical_loss": 3.872924605479011, + "tokens_seen": 553058304 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004204012036108325, + "loss": 2.5461, + "theoretical_loss": 3.872876033817934, + "tokens_seen": 553123840 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042039117352056174, + "loss": 2.8528, + "theoretical_loss": 3.8728274695226164, + "tokens_seen": 553189376 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042038114343029086, + "loss": 2.8084, + "theoretical_loss": 3.872778912591069, + "tokens_seen": 553254912 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004203711133400201, + "loss": 2.7178, + "theoretical_loss": 3.872730363021304, + "tokens_seen": 553320448 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004203610832497493, + "loss": 2.6789, + "theoretical_loss": 3.8726818208113323, + "tokens_seen": 553385984 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042035105315947846, + "loss": 2.8584, + "theoretical_loss": 3.872633285959167, + "tokens_seen": 553451520 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042034102306920764, + "loss": 2.9067, + "theoretical_loss": 3.872584758462823, + "tokens_seen": 553517056 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004203309929789368, + "loss": 2.8583, + "theoretical_loss": 3.872536238320313, + "tokens_seen": 553582592 + }, + { + "epoch": 0.17, + "learning_rate": 0.000420320962888666, + "loss": 2.9402, + "theoretical_loss": 3.8724877255296533, + "tokens_seen": 553648128 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042031093279839524, + "loss": 2.6756, + "theoretical_loss": 3.872439220088859, + "tokens_seen": 553713664 + }, + { + "epoch": 0.17, + "objective/train/docs_used": 324208, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.4875364303588867, + "objective/train/theoretical_loss": 3.8723907219959486, + "objective/train/tokens_used": 574239200, + "theoretical_loss": 3.8723907219959486, + "tokens_seen": 553779200 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042030090270812437, + "loss": 2.7754, + "theoretical_loss": 3.8723907219959486, + "tokens_seen": 553779200 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004202908726178536, + "loss": 2.9256, + "theoretical_loss": 3.872342231248938, + "tokens_seen": 553844736 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042028084252758273, + "loss": 2.7574, + "theoretical_loss": 3.872293747845846, + "tokens_seen": 553910272 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042027081243731196, + "loss": 2.6954, + "theoretical_loss": 3.872245271784692, + "tokens_seen": 553975808 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042026078234704115, + "loss": 2.9261, + "theoretical_loss": 3.872196803063495, + "tokens_seen": 554041344 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004202507522567703, + "loss": 2.8764, + "theoretical_loss": 3.872148341680276, + "tokens_seen": 554106880 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004202407221664995, + "loss": 2.9142, + "theoretical_loss": 3.8720998876330572, + "tokens_seen": 554172416 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004202306920762287, + "loss": 2.636, + "theoretical_loss": 3.8720514409198596, + "tokens_seen": 554237952 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042022066198595787, + "loss": 2.7876, + "theoretical_loss": 3.8720030015387064, + "tokens_seen": 554303488 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004202106318956871, + "loss": 2.8895, + "theoretical_loss": 3.871954569487622, + "tokens_seen": 554369024 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042020060180541623, + "loss": 2.6619, + "theoretical_loss": 3.8719061447646297, + "tokens_seen": 554434560 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042019057171514547, + "loss": 2.8202, + "theoretical_loss": 3.8718577273677557, + "tokens_seen": 554500096 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042018054162487465, + "loss": 2.6734, + "theoretical_loss": 3.8718093172950248, + "tokens_seen": 554565632 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042017051153460383, + "loss": 2.6691, + "theoretical_loss": 3.8717609145444647, + "tokens_seen": 554631168 + }, + { + "epoch": 0.17, + "learning_rate": 0.000420160481444333, + "loss": 2.7255, + "theoretical_loss": 3.8717125191141024, + "tokens_seen": 554696704 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004201504513540622, + "loss": 3.064, + "theoretical_loss": 3.871664131001967, + "tokens_seen": 554762240 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004201404212637914, + "loss": 2.966, + "theoretical_loss": 3.8716157502060864, + "tokens_seen": 554827776 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004201303911735206, + "loss": 2.5495, + "theoretical_loss": 3.8715673767244914, + "tokens_seen": 554893312 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042012036108324974, + "loss": 2.9044, + "theoretical_loss": 3.871519010555211, + "tokens_seen": 554958848 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042011033099297897, + "loss": 2.7584, + "theoretical_loss": 3.8714706516962787, + "tokens_seen": 555024384 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004201003009027081, + "loss": 2.8037, + "theoretical_loss": 3.871422300145725, + "tokens_seen": 555089920 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042009027081243733, + "loss": 3.1062, + "theoretical_loss": 3.8713739559015834, + "tokens_seen": 555155456 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004200802407221665, + "loss": 2.6873, + "theoretical_loss": 3.871325618961887, + "tokens_seen": 555220992 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004200702106318957, + "loss": 2.4009, + "theoretical_loss": 3.871277289324671, + "tokens_seen": 555286528 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004200601805416249, + "loss": 2.7423, + "theoretical_loss": 3.871228966987969, + "tokens_seen": 555352064 + }, + { + "epoch": 0.17, + "objective/train/docs_used": 324930, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.965287208557129, + "objective/train/theoretical_loss": 3.871180651949819, + "objective/train/tokens_used": 575877600, + "theoretical_loss": 3.871180651949819, + "tokens_seen": 555417600 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004200501504513541, + "loss": 2.8369, + "theoretical_loss": 3.871180651949819, + "tokens_seen": 555417600 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042004012036108324, + "loss": 2.7022, + "theoretical_loss": 3.871132344208256, + "tokens_seen": 555483136 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004200300902708125, + "loss": 3.1754, + "theoretical_loss": 3.871084043761318, + "tokens_seen": 555548672 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004200200601805416, + "loss": 2.7519, + "theoretical_loss": 3.8710357506070427, + "tokens_seen": 555614208 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042001003009027084, + "loss": 2.9232, + "theoretical_loss": 3.87098746474347, + "tokens_seen": 555679744 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042, + "loss": 2.9495, + "theoretical_loss": 3.870939186168638, + "tokens_seen": 555745280 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004199899699097292, + "loss": 2.8986, + "theoretical_loss": 3.8708909148805892, + "tokens_seen": 555810816 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004199799398194584, + "loss": 2.6948, + "theoretical_loss": 3.8708426508773632, + "tokens_seen": 555876352 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041996990972918756, + "loss": 2.6195, + "theoretical_loss": 3.870794394157002, + "tokens_seen": 555941888 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041995987963891674, + "loss": 2.6001, + "theoretical_loss": 3.8707461447175495, + "tokens_seen": 556007424 + }, + { + "epoch": 0.17, + "learning_rate": 0.000419949849548646, + "loss": 2.8978, + "theoretical_loss": 3.8706979025570485, + "tokens_seen": 556072960 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004199398194583751, + "loss": 2.6534, + "theoretical_loss": 3.870649667673543, + "tokens_seen": 556138496 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041992978936810434, + "loss": 2.8225, + "theoretical_loss": 3.870601440065078, + "tokens_seen": 556204032 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041991975927783347, + "loss": 2.8896, + "theoretical_loss": 3.8705532197296995, + "tokens_seen": 556269568 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004199097291875627, + "loss": 2.6052, + "theoretical_loss": 3.8705050066654536, + "tokens_seen": 556335104 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004198996990972919, + "loss": 2.7476, + "theoretical_loss": 3.870456800870388, + "tokens_seen": 556400640 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041988966900702106, + "loss": 2.6026, + "theoretical_loss": 3.8704086023425512, + "tokens_seen": 556466176 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041987963891675025, + "loss": 2.889, + "theoretical_loss": 3.8703604110799903, + "tokens_seen": 556531712 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004198696088264795, + "loss": 3.0944, + "theoretical_loss": 3.870312227080756, + "tokens_seen": 556597248 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004198595787362086, + "loss": 2.6756, + "theoretical_loss": 3.8702640503428984, + "tokens_seen": 556662784 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041984954864593784, + "loss": 2.9698, + "theoretical_loss": 3.870215880864469, + "tokens_seen": 556728320 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041983951855566697, + "loss": 2.8574, + "theoretical_loss": 3.8701677186435184, + "tokens_seen": 556793856 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004198294884653962, + "loss": 2.9415, + "theoretical_loss": 3.8701195636781, + "tokens_seen": 556859392 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004198194583751254, + "loss": 3.1886, + "theoretical_loss": 3.8700714159662666, + "tokens_seen": 556924928 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041980942828485457, + "loss": 2.9209, + "theoretical_loss": 3.870023275506073, + "tokens_seen": 556990464 + }, + { + "epoch": 0.17, + "objective/train/docs_used": 326252, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7125627994537354, + "objective/train/theoretical_loss": 3.869975142295573, + "objective/train/tokens_used": 577516000, + "theoretical_loss": 3.869975142295573, + "tokens_seen": 557056000 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041979939819458375, + "loss": 2.6543, + "theoretical_loss": 3.869975142295573, + "tokens_seen": 557056000 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041978936810431293, + "loss": 2.7298, + "theoretical_loss": 3.8699270163328223, + "tokens_seen": 557121536 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004197793380140421, + "loss": 2.997, + "theoretical_loss": 3.8698788976158776, + "tokens_seen": 557187072 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041976930792377135, + "loss": 2.7324, + "theoretical_loss": 3.8698307861427956, + "tokens_seen": 557252608 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041975927783350047, + "loss": 2.8657, + "theoretical_loss": 3.8697826819116345, + "tokens_seen": 557318144 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004197492477432297, + "loss": 2.6564, + "theoretical_loss": 3.8697345849204523, + "tokens_seen": 557383680 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041973921765295883, + "loss": 3.0381, + "theoretical_loss": 3.8696864951673087, + "tokens_seen": 557449216 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041972918756268807, + "loss": 2.9743, + "theoretical_loss": 3.8696384126502634, + "tokens_seen": 557514752 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041971915747241725, + "loss": 2.868, + "theoretical_loss": 3.8695903373673772, + "tokens_seen": 557580288 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041970912738214643, + "loss": 3.0219, + "theoretical_loss": 3.869542269316711, + "tokens_seen": 557645824 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004196990972918756, + "loss": 2.6485, + "theoretical_loss": 3.869494208496329, + "tokens_seen": 557711360 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041968906720160485, + "loss": 2.7259, + "theoretical_loss": 3.8694461549042924, + "tokens_seen": 557776896 + }, + { + "epoch": 0.17, + "learning_rate": 0.000419679037111334, + "loss": 2.8034, + "theoretical_loss": 3.8693981085386655, + "tokens_seen": 557842432 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004196690070210632, + "loss": 2.8072, + "theoretical_loss": 3.8693500693975125, + "tokens_seen": 557907968 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004196589769307924, + "loss": 2.7459, + "theoretical_loss": 3.8693020374788993, + "tokens_seen": 557973504 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004196489468405216, + "loss": 2.9362, + "theoretical_loss": 3.8692540127808916, + "tokens_seen": 558039040 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004196389167502508, + "loss": 2.6047, + "theoretical_loss": 3.8692059953015563, + "tokens_seen": 558104576 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041962888665997994, + "loss": 2.9256, + "theoretical_loss": 3.86915798503896, + "tokens_seen": 558170112 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041961885656970917, + "loss": 3.1276, + "theoretical_loss": 3.8691099819911723, + "tokens_seen": 558235648 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004196088264794383, + "loss": 2.7668, + "theoretical_loss": 3.869061986156261, + "tokens_seen": 558301184 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041959879638916753, + "loss": 3.0299, + "theoretical_loss": 3.8690139975322966, + "tokens_seen": 558366720 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004195887662988967, + "loss": 2.9317, + "theoretical_loss": 3.8689660161173487, + "tokens_seen": 558432256 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004195787362086259, + "loss": 2.7285, + "theoretical_loss": 3.8689180419094895, + "tokens_seen": 558497792 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004195687061183551, + "loss": 2.8405, + "theoretical_loss": 3.8688700749067904, + "tokens_seen": 558563328 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004195586760280843, + "loss": 2.8236, + "theoretical_loss": 3.868822115107324, + "tokens_seen": 558628864 + }, + { + "epoch": 0.17, + "objective/train/docs_used": 326886, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.981873035430908, + "objective/train/theoretical_loss": 3.8687741625091636, + "objective/train/tokens_used": 579154400, + "theoretical_loss": 3.8687741625091636, + "tokens_seen": 558694400 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041954864593781344, + "loss": 2.8344, + "theoretical_loss": 3.8687741625091636, + "tokens_seen": 558694400 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004195386158475427, + "loss": 2.8208, + "theoretical_loss": 3.8687262171103836, + "tokens_seen": 558759936 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004195285857572718, + "loss": 2.8084, + "theoretical_loss": 3.8686782789090595, + "tokens_seen": 558825472 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041951855566700104, + "loss": 2.8628, + "theoretical_loss": 3.868630347903266, + "tokens_seen": 558891008 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004195085255767302, + "loss": 2.7393, + "theoretical_loss": 3.8685824240910804, + "tokens_seen": 558956544 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004194984954864594, + "loss": 2.6413, + "theoretical_loss": 3.868534507470579, + "tokens_seen": 559022080 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004194884653961886, + "loss": 2.759, + "theoretical_loss": 3.8684865980398393, + "tokens_seen": 559087616 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041947843530591776, + "loss": 2.7109, + "theoretical_loss": 3.868438695796941, + "tokens_seen": 559153152 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041946840521564694, + "loss": 2.9011, + "theoretical_loss": 3.8683908007399634, + "tokens_seen": 559218688 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004194583751253762, + "loss": 2.9508, + "theoretical_loss": 3.8683429128669857, + "tokens_seen": 559284224 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004194483450351053, + "loss": 3.0686, + "theoretical_loss": 3.8682950321760896, + "tokens_seen": 559349760 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041943831494483454, + "loss": 2.7994, + "theoretical_loss": 3.868247158665356, + "tokens_seen": 559415296 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041942828485456367, + "loss": 3.0897, + "theoretical_loss": 3.8681992923328674, + "tokens_seen": 559480832 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004194182547642929, + "loss": 2.8764, + "theoretical_loss": 3.868151433176707, + "tokens_seen": 559546368 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004194082246740221, + "loss": 2.7726, + "theoretical_loss": 3.8681035811949585, + "tokens_seen": 559611904 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041939819458375126, + "loss": 3.0407, + "theoretical_loss": 3.868055736385706, + "tokens_seen": 559677440 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041938816449348045, + "loss": 2.7728, + "theoretical_loss": 3.868007898747036, + "tokens_seen": 559742976 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004193781344032097, + "loss": 2.9241, + "theoretical_loss": 3.867960068277033, + "tokens_seen": 559808512 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004193681043129388, + "loss": 2.9265, + "theoretical_loss": 3.867912244973784, + "tokens_seen": 559874048 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041935807422266804, + "loss": 2.6691, + "theoretical_loss": 3.867864428835377, + "tokens_seen": 559939584 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041934804413239717, + "loss": 3.079, + "theoretical_loss": 3.8678166198598998, + "tokens_seen": 560005120 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004193380140421264, + "loss": 3.1257, + "theoretical_loss": 3.8677688180454415, + "tokens_seen": 560070656 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004193279839518556, + "loss": 2.9229, + "theoretical_loss": 3.867721023390091, + "tokens_seen": 560136192 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041931795386158477, + "loss": 3.0877, + "theoretical_loss": 3.8676732358919406, + "tokens_seen": 560201728 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041930792377131395, + "loss": 2.7265, + "theoretical_loss": 3.8676254555490788, + "tokens_seen": 560267264 + }, + { + "epoch": 0.17, + "objective/train/docs_used": 327930, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.987152576446533, + "objective/train/theoretical_loss": 3.8675776823595998, + "objective/train/tokens_used": 580792800, + "theoretical_loss": 3.8675776823595998, + "tokens_seen": 560332800 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041929789368104313, + "loss": 2.8072, + "theoretical_loss": 3.8675776823595998, + "tokens_seen": 560332800 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004192878635907723, + "loss": 2.7142, + "theoretical_loss": 3.867529916321595, + "tokens_seen": 560398336 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041927783350050155, + "loss": 2.9421, + "theoretical_loss": 3.867482157433157, + "tokens_seen": 560463872 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041926780341023067, + "loss": 3.0434, + "theoretical_loss": 3.8674344056923813, + "tokens_seen": 560529408 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004192577733199599, + "loss": 2.9565, + "theoretical_loss": 3.8673866610973624, + "tokens_seen": 560594944 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041924774322968904, + "loss": 2.8994, + "theoretical_loss": 3.8673389236461952, + "tokens_seen": 560660480 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041923771313941827, + "loss": 2.7528, + "theoretical_loss": 3.867291193336976, + "tokens_seen": 560726016 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041922768304914745, + "loss": 2.806, + "theoretical_loss": 3.8672434701678022, + "tokens_seen": 560791552 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041921765295887663, + "loss": 2.7821, + "theoretical_loss": 3.867195754136771, + "tokens_seen": 560857088 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004192076228686058, + "loss": 3.0044, + "theoretical_loss": 3.867148045241981, + "tokens_seen": 560922624 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041919759277833505, + "loss": 2.821, + "theoretical_loss": 3.867100343481532, + "tokens_seen": 560988160 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004191875626880642, + "loss": 2.8286, + "theoretical_loss": 3.8670526488535226, + "tokens_seen": 561053696 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004191775325977934, + "loss": 2.8672, + "theoretical_loss": 3.867004961356054, + "tokens_seen": 561119232 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041916750250752254, + "loss": 2.9248, + "theoretical_loss": 3.8669572809872284, + "tokens_seen": 561184768 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004191574724172518, + "loss": 3.0027, + "theoretical_loss": 3.8669096077451464, + "tokens_seen": 561250304 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041914744232698095, + "loss": 2.9043, + "theoretical_loss": 3.8668619416279117, + "tokens_seen": 561315840 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041913741223671014, + "loss": 2.9646, + "theoretical_loss": 3.8668142826336274, + "tokens_seen": 561381376 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004191273821464393, + "loss": 2.6701, + "theoretical_loss": 3.866766630760398, + "tokens_seen": 561446912 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004191173520561685, + "loss": 2.7239, + "theoretical_loss": 3.8667189860063282, + "tokens_seen": 561512448 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004191073219658977, + "loss": 2.8731, + "theoretical_loss": 3.866671348369524, + "tokens_seen": 561577984 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004190972918756269, + "loss": 2.5886, + "theoretical_loss": 3.866623717848092, + "tokens_seen": 561643520 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041908726178535604, + "loss": 2.6842, + "theoretical_loss": 3.8665760944401377, + "tokens_seen": 561709056 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004190772316950853, + "loss": 2.9271, + "theoretical_loss": 3.8665284781437714, + "tokens_seen": 561774592 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004190672016048144, + "loss": 2.8644, + "theoretical_loss": 3.8664808689571, + "tokens_seen": 561840128 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041905717151454364, + "loss": 2.8095, + "theoretical_loss": 3.866433266878234, + "tokens_seen": 561905664 + }, + { + "epoch": 0.17, + "objective/train/docs_used": 328746, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.3812899589538574, + "objective/train/theoretical_loss": 3.8663856719052827, + "objective/train/tokens_used": 582431200, + "theoretical_loss": 3.8663856719052827, + "tokens_seen": 561971200 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004190471414242728, + "loss": 2.7966, + "theoretical_loss": 3.8663856719052827, + "tokens_seen": 561971200 + }, + { + "epoch": 0.17, + "learning_rate": 0.000419037111334002, + "loss": 2.9188, + "theoretical_loss": 3.8663380840363564, + "tokens_seen": 562036736 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004190270812437312, + "loss": 3.0222, + "theoretical_loss": 3.866290503269567, + "tokens_seen": 562102272 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004190170511534604, + "loss": 3.1056, + "theoretical_loss": 3.866242929603027, + "tokens_seen": 562167808 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041900702106318954, + "loss": 2.7517, + "theoretical_loss": 3.8661953630348496, + "tokens_seen": 562233344 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004189969909729188, + "loss": 2.8535, + "theoretical_loss": 3.866147803563148, + "tokens_seen": 562298880 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004189869608826479, + "loss": 3.0347, + "theoretical_loss": 3.866100251186036, + "tokens_seen": 562364416 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041897693079237714, + "loss": 2.774, + "theoretical_loss": 3.8660527059016294, + "tokens_seen": 562429952 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004189669007021063, + "loss": 2.7268, + "theoretical_loss": 3.8660051677080434, + "tokens_seen": 562495488 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004189568706118355, + "loss": 2.8606, + "theoretical_loss": 3.8659576366033956, + "tokens_seen": 562561024 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004189468405215647, + "loss": 2.8223, + "theoretical_loss": 3.865910112585802, + "tokens_seen": 562626560 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041893681043129387, + "loss": 3.0343, + "theoretical_loss": 3.8658625956533816, + "tokens_seen": 562692096 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041892678034102305, + "loss": 2.7921, + "theoretical_loss": 3.8658150858042526, + "tokens_seen": 562757632 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004189167502507523, + "loss": 2.9943, + "theoretical_loss": 3.8657675830365346, + "tokens_seen": 562823168 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041890672016048146, + "loss": 2.7147, + "theoretical_loss": 3.865720087348347, + "tokens_seen": 562888704 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041889669007021065, + "loss": 2.595, + "theoretical_loss": 3.8656725987378118, + "tokens_seen": 562954240 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004188866599799399, + "loss": 2.7918, + "theoretical_loss": 3.8656251172030496, + "tokens_seen": 563019776 + }, + { + "epoch": 0.17, + "learning_rate": 0.000418876629889669, + "loss": 2.8475, + "theoretical_loss": 3.865577642742183, + "tokens_seen": 563085312 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041886659979939824, + "loss": 2.7182, + "theoretical_loss": 3.865530175353335, + "tokens_seen": 563150848 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041885656970912737, + "loss": 2.7433, + "theoretical_loss": 3.86548271503463, + "tokens_seen": 563216384 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004188465396188566, + "loss": 2.7497, + "theoretical_loss": 3.865435261784191, + "tokens_seen": 563281920 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004188365095285858, + "loss": 2.9154, + "theoretical_loss": 3.865387815600144, + "tokens_seen": 563347456 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041882647943831497, + "loss": 2.9431, + "theoretical_loss": 3.8653403764806145, + "tokens_seen": 563412992 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041881644934804415, + "loss": 3.0352, + "theoretical_loss": 3.86529294442373, + "tokens_seen": 563478528 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041880641925777333, + "loss": 2.9686, + "theoretical_loss": 3.8652455194276163, + "tokens_seen": 563544064 + }, + { + "epoch": 0.17, + "objective/train/docs_used": 329425, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7898786067962646, + "objective/train/theoretical_loss": 3.8651981014904027, + "objective/train/tokens_used": 584069600, + "theoretical_loss": 3.8651981014904027, + "tokens_seen": 563609600 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004187963891675025, + "loss": 2.7493, + "theoretical_loss": 3.8651981014904027, + "tokens_seen": 563609600 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041878635907723175, + "loss": 2.9443, + "theoretical_loss": 3.8651506906102173, + "tokens_seen": 563675136 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004187763289869609, + "loss": 3.0628, + "theoretical_loss": 3.8651032867851893, + "tokens_seen": 563740672 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004187662988966901, + "loss": 2.7984, + "theoretical_loss": 3.8650558900134495, + "tokens_seen": 563806208 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041875626880641924, + "loss": 2.7272, + "theoretical_loss": 3.8650085002931283, + "tokens_seen": 563871744 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041874623871614847, + "loss": 2.909, + "theoretical_loss": 3.8649611176223573, + "tokens_seen": 563937280 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041873620862587765, + "loss": 2.6739, + "theoretical_loss": 3.8649137419992687, + "tokens_seen": 564002816 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041872617853560683, + "loss": 2.953, + "theoretical_loss": 3.864866373421996, + "tokens_seen": 564068352 + }, + { + "epoch": 0.17, + "learning_rate": 0.000418716148445336, + "loss": 2.9904, + "theoretical_loss": 3.8648190118886725, + "tokens_seen": 564133888 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041870611835506525, + "loss": 2.7935, + "theoretical_loss": 3.8647716573974327, + "tokens_seen": 564199424 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004186960882647944, + "loss": 2.9973, + "theoretical_loss": 3.8647243099464106, + "tokens_seen": 564264960 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004186860581745236, + "loss": 3.1176, + "theoretical_loss": 3.864676969533744, + "tokens_seen": 564330496 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041867602808425274, + "loss": 2.9712, + "theoretical_loss": 3.8646296361575687, + "tokens_seen": 564396032 + }, + { + "epoch": 0.17, + "learning_rate": 0.000418665997993982, + "loss": 3.079, + "theoretical_loss": 3.8645823098160212, + "tokens_seen": 564461568 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041865596790371115, + "loss": 2.8108, + "theoretical_loss": 3.8645349905072406, + "tokens_seen": 564527104 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041864593781344034, + "loss": 2.8196, + "theoretical_loss": 3.8644876782293647, + "tokens_seen": 564592640 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004186359077231695, + "loss": 2.6867, + "theoretical_loss": 3.864440372980533, + "tokens_seen": 564658176 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004186258776328987, + "loss": 2.9149, + "theoretical_loss": 3.8643930747588855, + "tokens_seen": 564723712 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004186158475426279, + "loss": 2.8609, + "theoretical_loss": 3.8643457835625634, + "tokens_seen": 564789248 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004186058174523571, + "loss": 3.0104, + "theoretical_loss": 3.8642984993897085, + "tokens_seen": 564854784 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041859578736208624, + "loss": 2.9591, + "theoretical_loss": 3.864251222238462, + "tokens_seen": 564920320 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004185857572718155, + "loss": 3.0257, + "theoretical_loss": 3.8642039521069678, + "tokens_seen": 564985856 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004185757271815446, + "loss": 2.8971, + "theoretical_loss": 3.864156688993369, + "tokens_seen": 565051392 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041856569709127384, + "loss": 2.9153, + "theoretical_loss": 3.86410943289581, + "tokens_seen": 565116928 + }, + { + "epoch": 0.17, + "learning_rate": 0.000418555667001003, + "loss": 2.8213, + "theoretical_loss": 3.864062183812436, + "tokens_seen": 565182464 + }, + { + "epoch": 0.17, + "objective/train/docs_used": 329612, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.669862985610962, + "objective/train/theoretical_loss": 3.864014941741392, + "objective/train/tokens_used": 585708000, + "theoretical_loss": 3.864014941741392, + "tokens_seen": 565248000 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004185456369107322, + "loss": 3.0311, + "theoretical_loss": 3.864014941741392, + "tokens_seen": 565248000 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004185356068204614, + "loss": 2.871, + "theoretical_loss": 3.8639677066808256, + "tokens_seen": 565313536 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004185255767301906, + "loss": 3.063, + "theoretical_loss": 3.863920478628883, + "tokens_seen": 565379072 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041851554663991974, + "loss": 3.0061, + "theoretical_loss": 3.8638732575837125, + "tokens_seen": 565444608 + }, + { + "epoch": 0.17, + "learning_rate": 0.000418505516549649, + "loss": 3.1087, + "theoretical_loss": 3.8638260435434626, + "tokens_seen": 565510144 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004184954864593781, + "loss": 2.8405, + "theoretical_loss": 3.863778836506283, + "tokens_seen": 565575680 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041848545636910734, + "loss": 2.9144, + "theoretical_loss": 3.8637316364703227, + "tokens_seen": 565641216 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004184754262788365, + "loss": 2.9361, + "theoretical_loss": 3.8636844434337325, + "tokens_seen": 565706752 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004184653961885657, + "loss": 2.9155, + "theoretical_loss": 3.863637257394665, + "tokens_seen": 565772288 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004184553660982949, + "loss": 3.0121, + "theoretical_loss": 3.863590078351271, + "tokens_seen": 565837824 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041844533600802407, + "loss": 2.7726, + "theoretical_loss": 3.8635429063017037, + "tokens_seen": 565903360 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041843530591775325, + "loss": 2.7738, + "theoretical_loss": 3.8634957412441167, + "tokens_seen": 565968896 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004184252758274825, + "loss": 3.0622, + "theoretical_loss": 3.8634485831766634, + "tokens_seen": 566034432 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004184152457372116, + "loss": 3.0123, + "theoretical_loss": 3.8634014320975005, + "tokens_seen": 566099968 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041840521564694085, + "loss": 2.9769, + "theoretical_loss": 3.8633542880047815, + "tokens_seen": 566165504 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041839518555666997, + "loss": 2.929, + "theoretical_loss": 3.863307150896664, + "tokens_seen": 566231040 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004183851554663992, + "loss": 3.0376, + "theoretical_loss": 3.863260020771304, + "tokens_seen": 566296576 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004183751253761284, + "loss": 2.8272, + "theoretical_loss": 3.8632128976268603, + "tokens_seen": 566362112 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041836509528585757, + "loss": 2.7962, + "theoretical_loss": 3.8631657814614906, + "tokens_seen": 566427648 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041835506519558675, + "loss": 3.0316, + "theoretical_loss": 3.8631186722733535, + "tokens_seen": 566493184 + }, + { + "epoch": 0.17, + "learning_rate": 0.000418345035105316, + "loss": 2.9764, + "theoretical_loss": 3.86307157006061, + "tokens_seen": 566558720 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004183350050150451, + "loss": 3.0297, + "theoretical_loss": 3.8630244748214198, + "tokens_seen": 566624256 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041832497492477435, + "loss": 2.8021, + "theoretical_loss": 3.862977386553944, + "tokens_seen": 566689792 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004183149448345035, + "loss": 3.1444, + "theoretical_loss": 3.862930305256345, + "tokens_seen": 566755328 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004183049147442327, + "loss": 2.7447, + "theoretical_loss": 3.8628832309267844, + "tokens_seen": 566820864 + }, + { + "epoch": 0.17, + "objective/train/docs_used": 331071, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7089223861694336, + "objective/train/theoretical_loss": 3.8628361635634265, + "objective/train/tokens_used": 587346400, + "theoretical_loss": 3.8628361635634265, + "tokens_seen": 566886400 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004182948846539619, + "loss": 3.0193, + "theoretical_loss": 3.8628361635634265, + "tokens_seen": 566886400 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004182848545636911, + "loss": 2.9216, + "theoretical_loss": 3.862789103164435, + "tokens_seen": 566951936 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041827482447342025, + "loss": 2.9421, + "theoretical_loss": 3.8627420497279745, + "tokens_seen": 567017472 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041826479438314944, + "loss": 2.952, + "theoretical_loss": 3.8626950032522096, + "tokens_seen": 567083008 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004182547642928786, + "loss": 2.8801, + "theoretical_loss": 3.8626479637353075, + "tokens_seen": 567148544 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041824473420260785, + "loss": 2.916, + "theoretical_loss": 3.862600931175434, + "tokens_seen": 567214080 + }, + { + "epoch": 0.17, + "learning_rate": 0.000418234704112337, + "loss": 2.8864, + "theoretical_loss": 3.8625539055707576, + "tokens_seen": 567279616 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004182246740220662, + "loss": 2.8347, + "theoretical_loss": 3.862506886919445, + "tokens_seen": 567345152 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004182146439317954, + "loss": 2.906, + "theoretical_loss": 3.862459875219667, + "tokens_seen": 567410688 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004182046138415246, + "loss": 2.93, + "theoretical_loss": 3.862412870469591, + "tokens_seen": 567476224 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041819458375125376, + "loss": 2.992, + "theoretical_loss": 3.8623658726673886, + "tokens_seen": 567541760 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041818455366098294, + "loss": 2.7413, + "theoretical_loss": 3.86231888181123, + "tokens_seen": 567607296 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004181745235707121, + "loss": 3.1447, + "theoretical_loss": 3.862271897899287, + "tokens_seen": 567672832 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041816449348044136, + "loss": 3.0228, + "theoretical_loss": 3.862224920929732, + "tokens_seen": 567738368 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041815446339017054, + "loss": 2.8566, + "theoretical_loss": 3.8621779509007377, + "tokens_seen": 567803904 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004181444332998997, + "loss": 2.9452, + "theoretical_loss": 3.8621309878104784, + "tokens_seen": 567869440 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004181344032096289, + "loss": 2.8373, + "theoretical_loss": 3.862084031657128, + "tokens_seen": 567934976 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004181243731193581, + "loss": 2.9396, + "theoretical_loss": 3.862037082438861, + "tokens_seen": 568000512 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004181143430290873, + "loss": 2.8415, + "theoretical_loss": 3.8619901401538543, + "tokens_seen": 568066048 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041810431293881644, + "loss": 2.8336, + "theoretical_loss": 3.8619432048002835, + "tokens_seen": 568131584 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004180942828485457, + "loss": 2.9697, + "theoretical_loss": 3.861896276376326, + "tokens_seen": 568197120 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004180842527582748, + "loss": 2.7627, + "theoretical_loss": 3.8618493548801602, + "tokens_seen": 568262656 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041807422266800404, + "loss": 3.1173, + "theoretical_loss": 3.8618024403099636, + "tokens_seen": 568328192 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004180641925777332, + "loss": 2.8712, + "theoretical_loss": 3.8617555326639157, + "tokens_seen": 568393728 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004180541624874624, + "loss": 3.0508, + "theoretical_loss": 3.8617086319401968, + "tokens_seen": 568459264 + }, + { + "epoch": 0.17, + "objective/train/docs_used": 331416, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.217491388320923, + "objective/train/theoretical_loss": 3.861661738136987, + "objective/train/tokens_used": 588984800, + "theoretical_loss": 3.861661738136987, + "tokens_seen": 568524800 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004180441323971916, + "loss": 2.9246, + "theoretical_loss": 3.861661738136987, + "tokens_seen": 568524800 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004180341023069208, + "loss": 2.5508, + "theoretical_loss": 3.8616148512524675, + "tokens_seen": 568590336 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041802407221664994, + "loss": 2.9353, + "theoretical_loss": 3.8615679712848205, + "tokens_seen": 568655872 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004180140421263792, + "loss": 3.0046, + "theoretical_loss": 3.8615210982322288, + "tokens_seen": 568721408 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004180040120361083, + "loss": 3.1391, + "theoretical_loss": 3.861474232092876, + "tokens_seen": 568786944 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041799398194583754, + "loss": 2.8942, + "theoretical_loss": 3.8614273728649446, + "tokens_seen": 568852480 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004179839518555667, + "loss": 3.0, + "theoretical_loss": 3.8613805205466205, + "tokens_seen": 568918016 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004179739217652959, + "loss": 2.8118, + "theoretical_loss": 3.8613336751360894, + "tokens_seen": 568983552 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004179638916750251, + "loss": 3.0289, + "theoretical_loss": 3.861286836631537, + "tokens_seen": 569049088 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041795386158475427, + "loss": 2.6248, + "theoretical_loss": 3.8612400050311493, + "tokens_seen": 569114624 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041794383149448345, + "loss": 2.9753, + "theoretical_loss": 3.8611931803331148, + "tokens_seen": 569180160 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004179338014042127, + "loss": 2.7124, + "theoretical_loss": 3.861146362535621, + "tokens_seen": 569245696 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004179237713139418, + "loss": 3.1015, + "theoretical_loss": 3.8610995516368565, + "tokens_seen": 569311232 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041791374122367105, + "loss": 2.6766, + "theoretical_loss": 3.8610527476350116, + "tokens_seen": 569376768 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041790371113340017, + "loss": 2.8523, + "theoretical_loss": 3.8610059505282757, + "tokens_seen": 569442304 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004178936810431294, + "loss": 3.1629, + "theoretical_loss": 3.86095916031484, + "tokens_seen": 569507840 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004178836509528586, + "loss": 2.9532, + "theoretical_loss": 3.860912376992896, + "tokens_seen": 569573376 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041787362086258777, + "loss": 2.9674, + "theoretical_loss": 3.8608656005606354, + "tokens_seen": 569638912 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041786359077231695, + "loss": 3.0518, + "theoretical_loss": 3.8608188310162523, + "tokens_seen": 569704448 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004178535606820462, + "loss": 2.5653, + "theoretical_loss": 3.8607720683579396, + "tokens_seen": 569769984 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004178435305917753, + "loss": 2.8754, + "theoretical_loss": 3.8607253125838916, + "tokens_seen": 569835520 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041783350050150455, + "loss": 2.5972, + "theoretical_loss": 3.8606785636923027, + "tokens_seen": 569901056 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004178234704112337, + "loss": 2.8451, + "theoretical_loss": 3.8606318216813693, + "tokens_seen": 569966592 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004178134403209629, + "loss": 2.8104, + "theoretical_loss": 3.8605850865492872, + "tokens_seen": 570032128 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004178034102306921, + "loss": 2.8106, + "theoretical_loss": 3.860538358294254, + "tokens_seen": 570097664 + }, + { + "epoch": 0.17, + "objective/train/docs_used": 332819, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.9043915271759033, + "objective/train/theoretical_loss": 3.8604916369144666, + "objective/train/tokens_used": 590623200, + "theoretical_loss": 3.8604916369144666, + "tokens_seen": 570163200 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004177933801404213, + "loss": 3.0457, + "theoretical_loss": 3.8604916369144666, + "tokens_seen": 570163200 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041778335005015045, + "loss": 2.948, + "theoretical_loss": 3.8604449224081234, + "tokens_seen": 570228736 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041777331995987964, + "loss": 2.8539, + "theoretical_loss": 3.860398214773425, + "tokens_seen": 570294272 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004177632898696088, + "loss": 2.9868, + "theoretical_loss": 3.8603515140085687, + "tokens_seen": 570359808 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041775325977933805, + "loss": 2.989, + "theoretical_loss": 3.8603048201117565, + "tokens_seen": 570425344 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004177432296890672, + "loss": 2.7859, + "theoretical_loss": 3.8602581330811887, + "tokens_seen": 570490880 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004177331995987964, + "loss": 2.8167, + "theoretical_loss": 3.8602114529150673, + "tokens_seen": 570556416 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004177231695085256, + "loss": 2.6652, + "theoretical_loss": 3.860164779611595, + "tokens_seen": 570621952 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004177131394182548, + "loss": 2.6566, + "theoretical_loss": 3.8601181131689737, + "tokens_seen": 570687488 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041770310932798396, + "loss": 3.0036, + "theoretical_loss": 3.8600714535854093, + "tokens_seen": 570753024 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041769307923771314, + "loss": 2.8123, + "theoretical_loss": 3.8600248008591045, + "tokens_seen": 570818560 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004176830491474423, + "loss": 3.0504, + "theoretical_loss": 3.859978154988265, + "tokens_seen": 570884096 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041767301905717156, + "loss": 2.9857, + "theoretical_loss": 3.8599315159710965, + "tokens_seen": 570949632 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004176629889669007, + "loss": 3.0748, + "theoretical_loss": 3.8598848838058055, + "tokens_seen": 571015168 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004176529588766299, + "loss": 2.7449, + "theoretical_loss": 3.8598382584905995, + "tokens_seen": 571080704 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041764292878635904, + "loss": 2.7664, + "theoretical_loss": 3.859791640023685, + "tokens_seen": 571146240 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004176328986960883, + "loss": 2.9228, + "theoretical_loss": 3.859745028403273, + "tokens_seen": 571211776 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041762286860581746, + "loss": 3.1111, + "theoretical_loss": 3.85969842362757, + "tokens_seen": 571277312 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041761283851554664, + "loss": 2.8858, + "theoretical_loss": 3.859651825694787, + "tokens_seen": 571342848 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004176028084252758, + "loss": 2.7068, + "theoretical_loss": 3.859605234603135, + "tokens_seen": 571408384 + }, + { + "epoch": 0.17, + "learning_rate": 0.000417592778335005, + "loss": 2.9055, + "theoretical_loss": 3.8595586503508246, + "tokens_seen": 571473920 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004175827482447342, + "loss": 2.7867, + "theoretical_loss": 3.8595120729360675, + "tokens_seen": 571539456 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004175727181544634, + "loss": 2.9183, + "theoretical_loss": 3.8594655023570765, + "tokens_seen": 571604992 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041756268806419255, + "loss": 3.0039, + "theoretical_loss": 3.859418938612065, + "tokens_seen": 571670528 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004175526579739218, + "loss": 2.9079, + "theoretical_loss": 3.859372381699247, + "tokens_seen": 571736064 + }, + { + "epoch": 0.17, + "objective/train/docs_used": 333556, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.4566822052001953, + "objective/train/theoretical_loss": 3.859325831616836, + "objective/train/tokens_used": 592261600, + "theoretical_loss": 3.859325831616836, + "tokens_seen": 571801600 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041754262788365096, + "loss": 2.4581, + "theoretical_loss": 3.859325831616836, + "tokens_seen": 571801600 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041753259779338015, + "loss": 2.4392, + "theoretical_loss": 3.859279288363048, + "tokens_seen": 571867136 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004175225677031093, + "loss": 3.1507, + "theoretical_loss": 3.8592327519360996, + "tokens_seen": 571932672 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004175125376128385, + "loss": 2.6826, + "theoretical_loss": 3.8591862223342064, + "tokens_seen": 571998208 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004175025075225677, + "loss": 2.8582, + "theoretical_loss": 3.8591396995555853, + "tokens_seen": 572063744 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004174924774322969, + "loss": 2.9655, + "theoretical_loss": 3.859093183598455, + "tokens_seen": 572129280 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041748244734202605, + "loss": 2.9763, + "theoretical_loss": 3.859046674461034, + "tokens_seen": 572194816 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004174724172517553, + "loss": 3.1296, + "theoretical_loss": 3.859000172141541, + "tokens_seen": 572260352 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004174623871614844, + "loss": 2.4627, + "theoretical_loss": 3.8589536766381967, + "tokens_seen": 572325888 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041745235707121365, + "loss": 2.9227, + "theoretical_loss": 3.858907187949221, + "tokens_seen": 572391424 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041744232698094283, + "loss": 3.0929, + "theoretical_loss": 3.8588607060728357, + "tokens_seen": 572456960 + }, + { + "epoch": 0.17, + "learning_rate": 0.000417432296890672, + "loss": 2.7831, + "theoretical_loss": 3.858814231007262, + "tokens_seen": 572522496 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004174222668004012, + "loss": 2.9829, + "theoretical_loss": 3.8587677627507233, + "tokens_seen": 572588032 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041741223671013037, + "loss": 3.0598, + "theoretical_loss": 3.858721301301442, + "tokens_seen": 572653568 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004174022066198596, + "loss": 2.9374, + "theoretical_loss": 3.8586748466576424, + "tokens_seen": 572719104 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004173921765295888, + "loss": 3.0136, + "theoretical_loss": 3.8586283988175496, + "tokens_seen": 572784640 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041738214643931797, + "loss": 2.8211, + "theoretical_loss": 3.8585819577793883, + "tokens_seen": 572850176 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041737211634904715, + "loss": 3.0281, + "theoretical_loss": 3.8585355235413843, + "tokens_seen": 572915712 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004173620862587764, + "loss": 2.7019, + "theoretical_loss": 3.858489096101765, + "tokens_seen": 572981248 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004173520561685055, + "loss": 3.1147, + "theoretical_loss": 3.858442675458756, + "tokens_seen": 573046784 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041734202607823475, + "loss": 2.8118, + "theoretical_loss": 3.8583962616105874, + "tokens_seen": 573112320 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004173319959879639, + "loss": 2.9836, + "theoretical_loss": 3.858349854555486, + "tokens_seen": 573177856 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004173219658976931, + "loss": 2.8185, + "theoretical_loss": 3.858303454291682, + "tokens_seen": 573243392 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004173119358074223, + "loss": 3.0078, + "theoretical_loss": 3.8582570608174045, + "tokens_seen": 573308928 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004173019057171515, + "loss": 3.0161, + "theoretical_loss": 3.858210674130885, + "tokens_seen": 573374464 + }, + { + "epoch": 0.17, + "objective/train/docs_used": 334721, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.8636512756347656, + "objective/train/theoretical_loss": 3.858164294230354, + "objective/train/tokens_used": 593900000, + "theoretical_loss": 3.858164294230354, + "tokens_seen": 573440000 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041729187562688065, + "loss": 3.0377, + "theoretical_loss": 3.858164294230354, + "tokens_seen": 573440000 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041728184553660984, + "loss": 2.7509, + "theoretical_loss": 3.8581179211140437, + "tokens_seen": 573505536 + }, + { + "epoch": 0.17, + "learning_rate": 0.000417271815446339, + "loss": 2.6503, + "theoretical_loss": 3.8580715547801865, + "tokens_seen": 573571072 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041726178535606825, + "loss": 2.7489, + "theoretical_loss": 3.8580251952270164, + "tokens_seen": 573636608 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004172517552657974, + "loss": 2.763, + "theoretical_loss": 3.8579788424527655, + "tokens_seen": 573702144 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004172417251755266, + "loss": 2.832, + "theoretical_loss": 3.8579324964556703, + "tokens_seen": 573767680 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004172316950852558, + "loss": 2.766, + "theoretical_loss": 3.857886157233965, + "tokens_seen": 573833216 + }, + { + "epoch": 0.17, + "learning_rate": 0.000417221664994985, + "loss": 2.8561, + "theoretical_loss": 3.8578398247858856, + "tokens_seen": 573898752 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041721163490471416, + "loss": 2.9427, + "theoretical_loss": 3.857793499109669, + "tokens_seen": 573964288 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041720160481444334, + "loss": 2.795, + "theoretical_loss": 3.8577471802035515, + "tokens_seen": 574029824 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004171915747241725, + "loss": 3.0053, + "theoretical_loss": 3.8577008680657716, + "tokens_seen": 574095360 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041718154463390176, + "loss": 2.8852, + "theoretical_loss": 3.8576545626945684, + "tokens_seen": 574160896 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004171715145436309, + "loss": 2.8531, + "theoretical_loss": 3.8576082640881797, + "tokens_seen": 574226432 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004171614844533601, + "loss": 2.8861, + "theoretical_loss": 3.857561972244846, + "tokens_seen": 574291968 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041715145436308924, + "loss": 3.1175, + "theoretical_loss": 3.8575156871628082, + "tokens_seen": 574357504 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004171414242728185, + "loss": 2.802, + "theoretical_loss": 3.8574694088403065, + "tokens_seen": 574423040 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041713139418254766, + "loss": 2.6776, + "theoretical_loss": 3.8574231372755836, + "tokens_seen": 574488576 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041712136409227684, + "loss": 2.709, + "theoretical_loss": 3.8573768724668813, + "tokens_seen": 574554112 + }, + { + "epoch": 0.17, + "learning_rate": 0.000417111334002006, + "loss": 2.7818, + "theoretical_loss": 3.8573306144124437, + "tokens_seen": 574619648 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004171013039117352, + "loss": 3.023, + "theoretical_loss": 3.857284363110513, + "tokens_seen": 574685184 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004170912738214644, + "loss": 2.9759, + "theoretical_loss": 3.8572381185593354, + "tokens_seen": 574750720 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004170812437311936, + "loss": 2.9899, + "theoretical_loss": 3.8571918807571546, + "tokens_seen": 574816256 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041707121364092275, + "loss": 2.867, + "theoretical_loss": 3.8571456497022165, + "tokens_seen": 574881792 + }, + { + "epoch": 0.17, + "learning_rate": 0.000417061183550652, + "loss": 2.9031, + "theoretical_loss": 3.857099425392768, + "tokens_seen": 574947328 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041705115346038116, + "loss": 2.7439, + "theoretical_loss": 3.8570532078270565, + "tokens_seen": 575012864 + }, + { + "epoch": 0.17, + "objective/train/docs_used": 335233, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.3804471492767334, + "objective/train/theoretical_loss": 3.857006997003329, + "objective/train/tokens_used": 595538400, + "theoretical_loss": 3.857006997003329, + "tokens_seen": 575078400 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041704112337011035, + "loss": 2.6027, + "theoretical_loss": 3.857006997003329, + "tokens_seen": 575078400 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041704112337011035, + "loss": 2.6623, + "theoretical_loss": 3.8569607929198337, + "tokens_seen": 575143936 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004170310932798395, + "loss": 2.8412, + "theoretical_loss": 3.85691459557482, + "tokens_seen": 575209472 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004170210631895687, + "loss": 3.2392, + "theoretical_loss": 3.856868404966538, + "tokens_seen": 575275008 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004170110330992979, + "loss": 3.0332, + "theoretical_loss": 3.8568222210932372, + "tokens_seen": 575340544 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004170010030090271, + "loss": 2.7566, + "theoretical_loss": 3.856776043953169, + "tokens_seen": 575406080 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041699097291875625, + "loss": 2.8564, + "theoretical_loss": 3.8567298735445847, + "tokens_seen": 575471616 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004169809428284855, + "loss": 2.8774, + "theoretical_loss": 3.856683709865737, + "tokens_seen": 575537152 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004169709127382146, + "loss": 2.7313, + "theoretical_loss": 3.8566375529148793, + "tokens_seen": 575602688 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041696088264794385, + "loss": 2.7615, + "theoretical_loss": 3.856591402690264, + "tokens_seen": 575668224 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041695085255767303, + "loss": 2.9461, + "theoretical_loss": 3.8565452591901463, + "tokens_seen": 575733760 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004169408224674022, + "loss": 3.0049, + "theoretical_loss": 3.856499122412781, + "tokens_seen": 575799296 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004169307923771314, + "loss": 3.0039, + "theoretical_loss": 3.8564529923564232, + "tokens_seen": 575864832 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004169207622868606, + "loss": 2.8256, + "theoretical_loss": 3.856406869019329, + "tokens_seen": 575930368 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041691073219658975, + "loss": 2.7089, + "theoretical_loss": 3.8563607523997563, + "tokens_seen": 575995904 + }, + { + "epoch": 0.17, + "learning_rate": 0.000416900702106319, + "loss": 2.6502, + "theoretical_loss": 3.856314642495961, + "tokens_seen": 576061440 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004168906720160481, + "loss": 2.7483, + "theoretical_loss": 3.8562685393062024, + "tokens_seen": 576126976 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041688064192577735, + "loss": 2.5723, + "theoretical_loss": 3.8562224428287393, + "tokens_seen": 576192512 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041687061183550653, + "loss": 2.839, + "theoretical_loss": 3.85617635306183, + "tokens_seen": 576258048 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004168605817452357, + "loss": 2.7057, + "theoretical_loss": 3.856130270003736, + "tokens_seen": 576323584 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004168505516549649, + "loss": 2.9297, + "theoretical_loss": 3.856084193652718, + "tokens_seen": 576389120 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004168405215646941, + "loss": 2.9692, + "theoretical_loss": 3.8560381240070365, + "tokens_seen": 576454656 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041683049147442326, + "loss": 2.7601, + "theoretical_loss": 3.8559920610649536, + "tokens_seen": 576520192 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004168204613841525, + "loss": 2.9518, + "theoretical_loss": 3.855946004824733, + "tokens_seen": 576585728 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004168104312938816, + "loss": 2.9652, + "theoretical_loss": 3.8558999552846367, + "tokens_seen": 576651264 + }, + { + "epoch": 0.17, + "objective/train/docs_used": 335838, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.517998456954956, + "objective/train/theoretical_loss": 3.85585391244293, + "objective/train/tokens_used": 597176800, + "theoretical_loss": 3.85585391244293, + "tokens_seen": 576716800 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041680040120361085, + "loss": 2.5517, + "theoretical_loss": 3.85585391244293, + "tokens_seen": 576716800 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041679037111334, + "loss": 2.9633, + "theoretical_loss": 3.855807876297876, + "tokens_seen": 576782336 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004167803410230692, + "loss": 2.8053, + "theoretical_loss": 3.8557618468477415, + "tokens_seen": 576847872 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004167703109327984, + "loss": 2.8028, + "theoretical_loss": 3.8557158240907916, + "tokens_seen": 576913408 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004167602808425276, + "loss": 2.9866, + "theoretical_loss": 3.8556698080252936, + "tokens_seen": 576978944 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041675025075225676, + "loss": 2.7691, + "theoretical_loss": 3.8556237986495137, + "tokens_seen": 577044480 + }, + { + "epoch": 0.17, + "learning_rate": 0.000416740220661986, + "loss": 2.9592, + "theoretical_loss": 3.8555777959617203, + "tokens_seen": 577110016 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004167301905717151, + "loss": 2.7571, + "theoretical_loss": 3.8555317999601817, + "tokens_seen": 577175552 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041672016048144436, + "loss": 2.8994, + "theoretical_loss": 3.8554858106431675, + "tokens_seen": 577241088 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004167101303911735, + "loss": 2.7119, + "theoretical_loss": 3.8554398280089472, + "tokens_seen": 577306624 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004167001003009027, + "loss": 2.9314, + "theoretical_loss": 3.8553938520557915, + "tokens_seen": 577372160 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004166900702106319, + "loss": 2.9253, + "theoretical_loss": 3.8553478827819707, + "tokens_seen": 577437696 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004166800401203611, + "loss": 2.6856, + "theoretical_loss": 3.8553019201857577, + "tokens_seen": 577503232 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041667001003009026, + "loss": 2.86, + "theoretical_loss": 3.8552559642654236, + "tokens_seen": 577568768 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041665997993981944, + "loss": 2.8018, + "theoretical_loss": 3.8552100150192423, + "tokens_seen": 577634304 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004166499498495487, + "loss": 2.9712, + "theoretical_loss": 3.855164072445487, + "tokens_seen": 577699840 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041663991975927786, + "loss": 2.9165, + "theoretical_loss": 3.855118136542433, + "tokens_seen": 577765376 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041662988966900704, + "loss": 3.0342, + "theoretical_loss": 3.8550722073083543, + "tokens_seen": 577830912 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004166198595787362, + "loss": 3.0278, + "theoretical_loss": 3.855026284741526, + "tokens_seen": 577896448 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004166098294884654, + "loss": 3.028, + "theoretical_loss": 3.8549803688402253, + "tokens_seen": 577961984 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004165997993981946, + "loss": 2.9817, + "theoretical_loss": 3.8549344596027293, + "tokens_seen": 578027520 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004165897693079238, + "loss": 2.8341, + "theoretical_loss": 3.854888557027315, + "tokens_seen": 578093056 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041657973921765295, + "loss": 2.8691, + "theoretical_loss": 3.8548426611122597, + "tokens_seen": 578158592 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004165697091273822, + "loss": 2.9255, + "theoretical_loss": 3.8547967718558436, + "tokens_seen": 578224128 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041655967903711136, + "loss": 2.854, + "theoretical_loss": 3.854750889256345, + "tokens_seen": 578289664 + }, + { + "epoch": 0.18, + "objective/train/docs_used": 337192, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.379192352294922, + "objective/train/theoretical_loss": 3.854705013312045, + "objective/train/tokens_used": 598815200, + "theoretical_loss": 3.854705013312045, + "tokens_seen": 578355200 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041654964894684055, + "loss": 2.6264, + "theoretical_loss": 3.854705013312045, + "tokens_seen": 578355200 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004165396188565697, + "loss": 2.7966, + "theoretical_loss": 3.8546591440212237, + "tokens_seen": 578420736 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004165295887662989, + "loss": 2.7849, + "theoretical_loss": 3.8546132813821625, + "tokens_seen": 578486272 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004165195586760281, + "loss": 2.784, + "theoretical_loss": 3.8545674253931432, + "tokens_seen": 578551808 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004165095285857573, + "loss": 3.0275, + "theoretical_loss": 3.8545215760524494, + "tokens_seen": 578617344 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041649949849548645, + "loss": 2.7683, + "theoretical_loss": 3.8544757333583632, + "tokens_seen": 578682880 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004164894684052157, + "loss": 2.8309, + "theoretical_loss": 3.854429897309169, + "tokens_seen": 578748416 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004164794383149448, + "loss": 2.797, + "theoretical_loss": 3.854384067903151, + "tokens_seen": 578813952 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041646940822467405, + "loss": 2.8531, + "theoretical_loss": 3.854338245138595, + "tokens_seen": 578879488 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041645937813440323, + "loss": 2.6649, + "theoretical_loss": 3.8542924290137863, + "tokens_seen": 578945024 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004164493480441324, + "loss": 2.954, + "theoretical_loss": 3.8542466195270118, + "tokens_seen": 579010560 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004164393179538616, + "loss": 2.847, + "theoretical_loss": 3.8542008166765576, + "tokens_seen": 579076096 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004164292878635908, + "loss": 3.0589, + "theoretical_loss": 3.8541550204607127, + "tokens_seen": 579141632 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041641925777331995, + "loss": 2.8131, + "theoretical_loss": 3.8541092308777647, + "tokens_seen": 579207168 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004164092276830492, + "loss": 2.9362, + "theoretical_loss": 3.854063447926003, + "tokens_seen": 579272704 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004163991975927783, + "loss": 2.9235, + "theoretical_loss": 3.8540176716037164, + "tokens_seen": 579338240 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041638916750250755, + "loss": 2.8574, + "theoretical_loss": 3.853971901909196, + "tokens_seen": 579403776 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041637913741223673, + "loss": 2.827, + "theoretical_loss": 3.8539261388407327, + "tokens_seen": 579469312 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004163691073219659, + "loss": 2.7208, + "theoretical_loss": 3.8538803823966172, + "tokens_seen": 579534848 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004163590772316951, + "loss": 2.8829, + "theoretical_loss": 3.853834632575143, + "tokens_seen": 579600384 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004163490471414243, + "loss": 2.7364, + "theoretical_loss": 3.8537888893746013, + "tokens_seen": 579665920 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041633901705115346, + "loss": 2.6801, + "theoretical_loss": 3.853743152793287, + "tokens_seen": 579731456 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004163289869608827, + "loss": 2.6165, + "theoretical_loss": 3.8536974228294927, + "tokens_seen": 579796992 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004163189568706118, + "loss": 2.8427, + "theoretical_loss": 3.8536516994815146, + "tokens_seen": 579862528 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041630892678034105, + "loss": 2.951, + "theoretical_loss": 3.8536059827476477, + "tokens_seen": 579928064 + }, + { + "epoch": 0.18, + "objective/train/docs_used": 337878, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.160188913345337, + "objective/train/theoretical_loss": 3.8535602726261864, + "objective/train/tokens_used": 600453600, + "theoretical_loss": 3.8535602726261864, + "tokens_seen": 579993600 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004162988966900702, + "loss": 2.7548, + "theoretical_loss": 3.8535602726261864, + "tokens_seen": 579993600 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004162888665997994, + "loss": 2.6838, + "theoretical_loss": 3.853514569115429, + "tokens_seen": 580059136 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004162788365095286, + "loss": 2.9028, + "theoretical_loss": 3.8534688722136723, + "tokens_seen": 580124672 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004162688064192578, + "loss": 2.9418, + "theoretical_loss": 3.853423181919214, + "tokens_seen": 580190208 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041625877632898696, + "loss": 2.927, + "theoretical_loss": 3.8533774982303526, + "tokens_seen": 580255744 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004162487462387162, + "loss": 2.9875, + "theoretical_loss": 3.853331821145387, + "tokens_seen": 580321280 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004162387161484453, + "loss": 3.0165, + "theoretical_loss": 3.8532861506626173, + "tokens_seen": 580386816 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041622868605817456, + "loss": 3.0456, + "theoretical_loss": 3.853240486780344, + "tokens_seen": 580452352 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004162186559679037, + "loss": 2.7192, + "theoretical_loss": 3.8531948294968674, + "tokens_seen": 580517888 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004162086258776329, + "loss": 3.0205, + "theoretical_loss": 3.85314917881049, + "tokens_seen": 580583424 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004161985957873621, + "loss": 2.9209, + "theoretical_loss": 3.853103534719513, + "tokens_seen": 580648960 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004161885656970913, + "loss": 2.7941, + "theoretical_loss": 3.8530578972222402, + "tokens_seen": 580714496 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041617853560682046, + "loss": 3.1082, + "theoretical_loss": 3.8530122663169752, + "tokens_seen": 580780032 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041616850551654964, + "loss": 2.6093, + "theoretical_loss": 3.8529666420020217, + "tokens_seen": 580845568 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004161584754262788, + "loss": 2.8722, + "theoretical_loss": 3.852921024275684, + "tokens_seen": 580911104 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041614844533600806, + "loss": 2.9128, + "theoretical_loss": 3.8528754131362684, + "tokens_seen": 580976640 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004161384152457372, + "loss": 2.6595, + "theoretical_loss": 3.8528298085820802, + "tokens_seen": 581042176 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004161283851554664, + "loss": 2.7688, + "theoretical_loss": 3.8527842106114267, + "tokens_seen": 581107712 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041611835506519555, + "loss": 2.9458, + "theoretical_loss": 3.852738619222615, + "tokens_seen": 581173248 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004161083249749248, + "loss": 2.9801, + "theoretical_loss": 3.8526930344139525, + "tokens_seen": 581238784 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041609829488465397, + "loss": 2.8012, + "theoretical_loss": 3.8526474561837483, + "tokens_seen": 581304320 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041608826479438315, + "loss": 2.7945, + "theoretical_loss": 3.852601884530311, + "tokens_seen": 581369856 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041607823470411233, + "loss": 2.6186, + "theoretical_loss": 3.852556319451952, + "tokens_seen": 581435392 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041606820461384156, + "loss": 2.8717, + "theoretical_loss": 3.8525107609469793, + "tokens_seen": 581500928 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004160581745235707, + "loss": 2.7244, + "theoretical_loss": 3.852465209013705, + "tokens_seen": 581566464 + }, + { + "epoch": 0.18, + "objective/train/docs_used": 339259, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.0202083587646484, + "objective/train/theoretical_loss": 3.8524196636504415, + "objective/train/tokens_used": 602092000, + "theoretical_loss": 3.8524196636504415, + "tokens_seen": 581632000 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004160481444332999, + "loss": 3.1255, + "theoretical_loss": 3.8524196636504415, + "tokens_seen": 581632000 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041603811434302905, + "loss": 2.7626, + "theoretical_loss": 3.8523741248555, + "tokens_seen": 581697536 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004160280842527583, + "loss": 2.6162, + "theoretical_loss": 3.8523285926271935, + "tokens_seen": 581763072 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041601805416248747, + "loss": 3.1602, + "theoretical_loss": 3.852283066963836, + "tokens_seen": 581828608 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041600802407221665, + "loss": 2.6569, + "theoretical_loss": 3.8522375478637425, + "tokens_seen": 581894144 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041599799398194583, + "loss": 2.772, + "theoretical_loss": 3.852192035325226, + "tokens_seen": 581959680 + }, + { + "epoch": 0.18, + "learning_rate": 0.000415987963891675, + "loss": 2.6043, + "theoretical_loss": 3.8521465293466024, + "tokens_seen": 582025216 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004159779338014042, + "loss": 2.615, + "theoretical_loss": 3.852101029926188, + "tokens_seen": 582090752 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041596790371113343, + "loss": 2.8123, + "theoretical_loss": 3.8520555370622995, + "tokens_seen": 582156288 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041595787362086256, + "loss": 2.9079, + "theoretical_loss": 3.852010050753254, + "tokens_seen": 582221824 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004159478435305918, + "loss": 2.8539, + "theoretical_loss": 3.85196457099737, + "tokens_seen": 582287360 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004159378134403209, + "loss": 2.7228, + "theoretical_loss": 3.8519190977929645, + "tokens_seen": 582352896 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041592778335005015, + "loss": 2.8309, + "theoretical_loss": 3.851873631138358, + "tokens_seen": 582418432 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041591775325977934, + "loss": 2.8187, + "theoretical_loss": 3.85182817103187, + "tokens_seen": 582483968 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004159077231695085, + "loss": 2.9608, + "theoretical_loss": 3.8517827174718198, + "tokens_seen": 582549504 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041589769307923775, + "loss": 2.7939, + "theoretical_loss": 3.85173727045653, + "tokens_seen": 582615040 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041588766298896693, + "loss": 3.0776, + "theoretical_loss": 3.851691829984321, + "tokens_seen": 582680576 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004158776328986961, + "loss": 2.7862, + "theoretical_loss": 3.851646396053516, + "tokens_seen": 582746112 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004158676028084253, + "loss": 2.949, + "theoretical_loss": 3.851600968662437, + "tokens_seen": 582811648 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004158575727181545, + "loss": 2.9059, + "theoretical_loss": 3.851555547809408, + "tokens_seen": 582877184 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041584754262788366, + "loss": 3.1767, + "theoretical_loss": 3.851510133492752, + "tokens_seen": 582942720 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004158375125376129, + "loss": 2.8496, + "theoretical_loss": 3.8514647257107955, + "tokens_seen": 583008256 + }, + { + "epoch": 0.18, + "learning_rate": 0.000415827482447342, + "loss": 2.8678, + "theoretical_loss": 3.8514193244618617, + "tokens_seen": 583073792 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041581745235707126, + "loss": 2.9264, + "theoretical_loss": 3.8513739297442786, + "tokens_seen": 583139328 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004158074222668004, + "loss": 2.7656, + "theoretical_loss": 3.851328541556372, + "tokens_seen": 583204864 + }, + { + "epoch": 0.18, + "objective/train/docs_used": 340083, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.9477314949035645, + "objective/train/theoretical_loss": 3.851283159896468, + "objective/train/tokens_used": 603730400, + "theoretical_loss": 3.851283159896468, + "tokens_seen": 583270400 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004157973921765296, + "loss": 2.9254, + "theoretical_loss": 3.851283159896468, + "tokens_seen": 583270400 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004157873620862588, + "loss": 2.6633, + "theoretical_loss": 3.851237784762896, + "tokens_seen": 583335936 + }, + { + "epoch": 0.18, + "learning_rate": 0.000415777331995988, + "loss": 3.0265, + "theoretical_loss": 3.8511924161539826, + "tokens_seen": 583401472 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041576730190571716, + "loss": 2.748, + "theoretical_loss": 3.8511470540680586, + "tokens_seen": 583467008 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004157572718154464, + "loss": 3.0886, + "theoretical_loss": 3.8511016985034523, + "tokens_seen": 583532544 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004157472417251755, + "loss": 2.9146, + "theoretical_loss": 3.851056349458495, + "tokens_seen": 583598080 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041573721163490476, + "loss": 2.859, + "theoretical_loss": 3.8510110069315173, + "tokens_seen": 583663616 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004157271815446339, + "loss": 3.0352, + "theoretical_loss": 3.85096567092085, + "tokens_seen": 583729152 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004157171514543631, + "loss": 2.7757, + "theoretical_loss": 3.8509203414248256, + "tokens_seen": 583794688 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004157071213640923, + "loss": 2.912, + "theoretical_loss": 3.8508750184417764, + "tokens_seen": 583860224 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004156970912738215, + "loss": 2.6656, + "theoretical_loss": 3.850829701970037, + "tokens_seen": 583925760 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041568706118355066, + "loss": 2.7405, + "theoretical_loss": 3.85078439200794, + "tokens_seen": 583991296 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041567703109327984, + "loss": 2.572, + "theoretical_loss": 3.85073908855382, + "tokens_seen": 584056832 + }, + { + "epoch": 0.18, + "learning_rate": 0.000415667001003009, + "loss": 2.8036, + "theoretical_loss": 3.8506937916060133, + "tokens_seen": 584122368 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041565697091273826, + "loss": 2.7464, + "theoretical_loss": 3.8506485011628544, + "tokens_seen": 584187904 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004156469408224674, + "loss": 2.9262, + "theoretical_loss": 3.8506032172226803, + "tokens_seen": 584253440 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004156369107321966, + "loss": 2.6257, + "theoretical_loss": 3.850557939783828, + "tokens_seen": 584318976 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041562688064192575, + "loss": 2.8626, + "theoretical_loss": 3.850512668844635, + "tokens_seen": 584384512 + }, + { + "epoch": 0.18, + "learning_rate": 0.000415616850551655, + "loss": 2.7185, + "theoretical_loss": 3.8504674044034393, + "tokens_seen": 584450048 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041560682046138417, + "loss": 2.8801, + "theoretical_loss": 3.8504221464585795, + "tokens_seen": 584515584 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041559679037111335, + "loss": 2.8695, + "theoretical_loss": 3.850376895008396, + "tokens_seen": 584581120 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041558676028084253, + "loss": 2.8313, + "theoretical_loss": 3.8503316500512286, + "tokens_seen": 584646656 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041557673019057176, + "loss": 2.7877, + "theoretical_loss": 3.850286411585417, + "tokens_seen": 584712192 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004155667001003009, + "loss": 2.6076, + "theoretical_loss": 3.8502411796093035, + "tokens_seen": 584777728 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004155566700100301, + "loss": 2.8387, + "theoretical_loss": 3.8501959541212294, + "tokens_seen": 584843264 + }, + { + "epoch": 0.18, + "objective/train/docs_used": 341398, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.9583544731140137, + "objective/train/theoretical_loss": 3.850150735119537, + "objective/train/tokens_used": 605368800, + "theoretical_loss": 3.850150735119537, + "tokens_seen": 584908800 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041554663991975925, + "loss": 2.8463, + "theoretical_loss": 3.850150735119537, + "tokens_seen": 584908800 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004155366098294885, + "loss": 2.6436, + "theoretical_loss": 3.8501055226025707, + "tokens_seen": 584974336 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041552657973921767, + "loss": 2.6795, + "theoretical_loss": 3.8500603165686726, + "tokens_seen": 585039872 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041551654964894685, + "loss": 2.6834, + "theoretical_loss": 3.850015117016188, + "tokens_seen": 585105408 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041550651955867603, + "loss": 2.8015, + "theoretical_loss": 3.849969923943461, + "tokens_seen": 585170944 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004154964894684052, + "loss": 2.9964, + "theoretical_loss": 3.8499247373488377, + "tokens_seen": 585236480 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004154864593781344, + "loss": 2.7886, + "theoretical_loss": 3.8498795572306643, + "tokens_seen": 585302016 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041547642928786363, + "loss": 2.7726, + "theoretical_loss": 3.8498343835872872, + "tokens_seen": 585367552 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041546639919759276, + "loss": 2.8285, + "theoretical_loss": 3.8497892164170544, + "tokens_seen": 585433088 + }, + { + "epoch": 0.18, + "learning_rate": 0.000415456369107322, + "loss": 2.6364, + "theoretical_loss": 3.8497440557183134, + "tokens_seen": 585498624 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004154463390170511, + "loss": 2.9062, + "theoretical_loss": 3.849698901489412, + "tokens_seen": 585564160 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041543630892678035, + "loss": 2.6897, + "theoretical_loss": 3.849653753728701, + "tokens_seen": 585629696 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041542627883650954, + "loss": 2.7622, + "theoretical_loss": 3.849608612434529, + "tokens_seen": 585695232 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004154162487462387, + "loss": 2.9492, + "theoretical_loss": 3.8495634776052463, + "tokens_seen": 585760768 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004154062186559679, + "loss": 2.5872, + "theoretical_loss": 3.849518349239205, + "tokens_seen": 585826304 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041539618856569713, + "loss": 2.7244, + "theoretical_loss": 3.849473227334755, + "tokens_seen": 585891840 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041538615847542626, + "loss": 2.7746, + "theoretical_loss": 3.8494281118902505, + "tokens_seen": 585957376 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004153761283851555, + "loss": 2.8827, + "theoretical_loss": 3.849383002904043, + "tokens_seen": 586022912 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004153660982948846, + "loss": 2.6988, + "theoretical_loss": 3.8493379003744854, + "tokens_seen": 586088448 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041535606820461386, + "loss": 2.553, + "theoretical_loss": 3.8492928042999335, + "tokens_seen": 586153984 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041534603811434304, + "loss": 2.8868, + "theoretical_loss": 3.8492477146787403, + "tokens_seen": 586219520 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004153360080240722, + "loss": 2.7777, + "theoretical_loss": 3.849202631509262, + "tokens_seen": 586285056 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004153259779338014, + "loss": 2.8072, + "theoretical_loss": 3.849157554789853, + "tokens_seen": 586350592 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004153159478435306, + "loss": 2.7695, + "theoretical_loss": 3.8491124845188716, + "tokens_seen": 586416128 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041530591775325976, + "loss": 2.9147, + "theoretical_loss": 3.8490674206946736, + "tokens_seen": 586481664 + }, + { + "epoch": 0.18, + "objective/train/docs_used": 342071, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.2586634159088135, + "objective/train/theoretical_loss": 3.8490223633156173, + "objective/train/tokens_used": 607007200, + "theoretical_loss": 3.8490223633156173, + "tokens_seen": 586547200 + }, + { + "epoch": 0.18, + "learning_rate": 0.000415295887662989, + "loss": 3.1486, + "theoretical_loss": 3.8490223633156173, + "tokens_seen": 586547200 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004152858575727181, + "loss": 2.8847, + "theoretical_loss": 3.84897731238006, + "tokens_seen": 586612736 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041527582748244736, + "loss": 3.0224, + "theoretical_loss": 3.8489322678863624, + "tokens_seen": 586678272 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004152657973921765, + "loss": 2.5659, + "theoretical_loss": 3.8488872298328816, + "tokens_seen": 586743808 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004152557673019057, + "loss": 3.0194, + "theoretical_loss": 3.848842198217979, + "tokens_seen": 586809344 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004152457372116349, + "loss": 2.9042, + "theoretical_loss": 3.8487971730400146, + "tokens_seen": 586874880 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004152357071213641, + "loss": 2.8366, + "theoretical_loss": 3.84875215429735, + "tokens_seen": 586940416 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041522567703109327, + "loss": 2.7804, + "theoretical_loss": 3.8487071419883474, + "tokens_seen": 587005952 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004152156469408225, + "loss": 2.7865, + "theoretical_loss": 3.8486621361113684, + "tokens_seen": 587071488 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041520561685055163, + "loss": 2.7943, + "theoretical_loss": 3.848617136664777, + "tokens_seen": 587137024 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041519558676028086, + "loss": 2.8635, + "theoretical_loss": 3.848572143646936, + "tokens_seen": 587202560 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041518555667001, + "loss": 2.5945, + "theoretical_loss": 3.8485271570562096, + "tokens_seen": 587268096 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004151755265797392, + "loss": 2.9027, + "theoretical_loss": 3.8484821768909634, + "tokens_seen": 587333632 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004151654964894684, + "loss": 2.7172, + "theoretical_loss": 3.848437203149562, + "tokens_seen": 587399168 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004151554663991976, + "loss": 2.8817, + "theoretical_loss": 3.8483922358303717, + "tokens_seen": 587464704 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004151454363089268, + "loss": 2.6657, + "theoretical_loss": 3.8483472749317595, + "tokens_seen": 587530240 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041513540621865595, + "loss": 2.817, + "theoretical_loss": 3.8483023204520928, + "tokens_seen": 587595776 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004151253761283852, + "loss": 2.7708, + "theoretical_loss": 3.848257372389738, + "tokens_seen": 587661312 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041511534603811437, + "loss": 2.8142, + "theoretical_loss": 3.848212430743065, + "tokens_seen": 587726848 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041510531594784355, + "loss": 2.8943, + "theoretical_loss": 3.8481674955104426, + "tokens_seen": 587792384 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041509528585757273, + "loss": 2.6836, + "theoretical_loss": 3.848122566690239, + "tokens_seen": 587857920 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041508525576730196, + "loss": 2.9687, + "theoretical_loss": 3.8480776442808264, + "tokens_seen": 587923456 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004150752256770311, + "loss": 2.7452, + "theoretical_loss": 3.8480327282805744, + "tokens_seen": 587988992 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004150651955867603, + "loss": 2.8002, + "theoretical_loss": 3.8479878186878547, + "tokens_seen": 588054528 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041505516549648945, + "loss": 2.8321, + "theoretical_loss": 3.847942915501039, + "tokens_seen": 588120064 + }, + { + "epoch": 0.18, + "objective/train/docs_used": 342650, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.625291347503662, + "objective/train/theoretical_loss": 3.8478980187185003, + "objective/train/tokens_used": 608645600, + "theoretical_loss": 3.8478980187185003, + "tokens_seen": 588185600 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004150451354062187, + "loss": 2.8885, + "theoretical_loss": 3.8478980187185003, + "tokens_seen": 588185600 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041503510531594787, + "loss": 2.708, + "theoretical_loss": 3.8478531283386124, + "tokens_seen": 588251136 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041502507522567705, + "loss": 2.7076, + "theoretical_loss": 3.847808244359747, + "tokens_seen": 588316672 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041501504513540623, + "loss": 2.8783, + "theoretical_loss": 3.847763366780281, + "tokens_seen": 588382208 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004150050150451354, + "loss": 2.857, + "theoretical_loss": 3.8477184955985875, + "tokens_seen": 588447744 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004149949849548646, + "loss": 2.9214, + "theoretical_loss": 3.8476736308130426, + "tokens_seen": 588513280 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041498495486459383, + "loss": 2.86, + "theoretical_loss": 3.8476287724220226, + "tokens_seen": 588578816 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041497492477432296, + "loss": 2.5591, + "theoretical_loss": 3.8475839204239044, + "tokens_seen": 588644352 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004149648946840522, + "loss": 2.8309, + "theoretical_loss": 3.847539074817065, + "tokens_seen": 588709888 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004149548645937813, + "loss": 2.8306, + "theoretical_loss": 3.8474942355998825, + "tokens_seen": 588775424 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041494483450351055, + "loss": 2.983, + "theoretical_loss": 3.847449402770735, + "tokens_seen": 588840960 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041493480441323974, + "loss": 2.714, + "theoretical_loss": 3.8474045763280023, + "tokens_seen": 588906496 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004149247743229689, + "loss": 3.0201, + "theoretical_loss": 3.847359756270064, + "tokens_seen": 588972032 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004149147442326981, + "loss": 3.0067, + "theoretical_loss": 3.8473149425953, + "tokens_seen": 589037568 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041490471414242733, + "loss": 2.8129, + "theoretical_loss": 3.8472701353020913, + "tokens_seen": 589103104 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041489468405215646, + "loss": 2.7287, + "theoretical_loss": 3.8472253343888196, + "tokens_seen": 589168640 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004148846539618857, + "loss": 2.7721, + "theoretical_loss": 3.8471805398538663, + "tokens_seen": 589234176 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004148746238716148, + "loss": 2.6583, + "theoretical_loss": 3.8471357516956153, + "tokens_seen": 589299712 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041486459378134406, + "loss": 2.8003, + "theoretical_loss": 3.8470909699124487, + "tokens_seen": 589365248 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041485456369107324, + "loss": 2.7586, + "theoretical_loss": 3.8470461945027505, + "tokens_seen": 589430784 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004148445336008024, + "loss": 2.5329, + "theoretical_loss": 3.8470014254649056, + "tokens_seen": 589496320 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004148345035105316, + "loss": 2.8453, + "theoretical_loss": 3.8469566627972993, + "tokens_seen": 589561856 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004148244734202608, + "loss": 2.9672, + "theoretical_loss": 3.846911906498316, + "tokens_seen": 589627392 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041481444332998996, + "loss": 2.8493, + "theoretical_loss": 3.8468671565663426, + "tokens_seen": 589692928 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004148044132397192, + "loss": 2.7299, + "theoretical_loss": 3.8468224129997663, + "tokens_seen": 589758464 + }, + { + "epoch": 0.18, + "objective/train/docs_used": 343935, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6992390155792236, + "objective/train/theoretical_loss": 3.846777675796974, + "objective/train/tokens_used": 610284000, + "theoretical_loss": 3.846777675796974, + "tokens_seen": 589824000 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004147943831494483, + "loss": 2.9173, + "theoretical_loss": 3.846777675796974, + "tokens_seen": 589824000 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041478435305917756, + "loss": 2.799, + "theoretical_loss": 3.8467329449563534, + "tokens_seen": 589889536 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004147743229689067, + "loss": 2.8464, + "theoretical_loss": 3.8466882204762936, + "tokens_seen": 589955072 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004147642928786359, + "loss": 2.9946, + "theoretical_loss": 3.846643502355183, + "tokens_seen": 590020608 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004147542627883651, + "loss": 2.7192, + "theoretical_loss": 3.846598790591412, + "tokens_seen": 590086144 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004147442326980943, + "loss": 2.6515, + "theoretical_loss": 3.846554085183371, + "tokens_seen": 590151680 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041473420260782347, + "loss": 2.5479, + "theoretical_loss": 3.8465093861294504, + "tokens_seen": 590217216 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004147241725175527, + "loss": 2.7835, + "theoretical_loss": 3.8464646934280413, + "tokens_seen": 590282752 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041471414242728183, + "loss": 2.8182, + "theoretical_loss": 3.846420007077537, + "tokens_seen": 590348288 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041470411233701106, + "loss": 3.0853, + "theoretical_loss": 3.8463753270763297, + "tokens_seen": 590413824 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004146940822467402, + "loss": 2.9576, + "theoretical_loss": 3.846330653422812, + "tokens_seen": 590479360 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004146840521564694, + "loss": 2.6865, + "theoretical_loss": 3.8462859861153778, + "tokens_seen": 590544896 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004146740220661986, + "loss": 2.9227, + "theoretical_loss": 3.8462413251524215, + "tokens_seen": 590610432 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004146639919759278, + "loss": 2.8162, + "theoretical_loss": 3.846196670532339, + "tokens_seen": 590675968 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041465396188565697, + "loss": 2.7688, + "theoretical_loss": 3.8461520222535253, + "tokens_seen": 590741504 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041464393179538615, + "loss": 2.7389, + "theoretical_loss": 3.8461073803143764, + "tokens_seen": 590807040 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041463390170511533, + "loss": 2.8588, + "theoretical_loss": 3.846062744713289, + "tokens_seen": 590872576 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041462387161484457, + "loss": 2.8708, + "theoretical_loss": 3.8460181154486612, + "tokens_seen": 590938112 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004146138415245737, + "loss": 2.7913, + "theoretical_loss": 3.8459734925188895, + "tokens_seen": 591003648 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041460381143430293, + "loss": 2.8559, + "theoretical_loss": 3.8459288759223735, + "tokens_seen": 591069184 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004145937813440321, + "loss": 2.709, + "theoretical_loss": 3.8458842656575123, + "tokens_seen": 591134720 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004145837512537613, + "loss": 2.5506, + "theoretical_loss": 3.845839661722705, + "tokens_seen": 591200256 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004145737211634905, + "loss": 2.9009, + "theoretical_loss": 3.8457950641163516, + "tokens_seen": 591265792 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041456369107321965, + "loss": 3.1079, + "theoretical_loss": 3.845750472836854, + "tokens_seen": 591331328 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041455366098294884, + "loss": 2.7298, + "theoretical_loss": 3.845705887882612, + "tokens_seen": 591396864 + }, + { + "epoch": 0.18, + "objective/train/docs_used": 344744, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.8378853797912598, + "objective/train/theoretical_loss": 3.845661309252029, + "objective/train/tokens_used": 611922400, + "theoretical_loss": 3.845661309252029, + "tokens_seen": 591462400 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041454363089267807, + "loss": 2.8377, + "theoretical_loss": 3.845661309252029, + "tokens_seen": 591462400 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004145336008024072, + "loss": 2.8997, + "theoretical_loss": 3.8456167369435073, + "tokens_seen": 591527936 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041452357071213643, + "loss": 3.0789, + "theoretical_loss": 3.8455721709554496, + "tokens_seen": 591593472 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041451354062186556, + "loss": 2.8042, + "theoretical_loss": 3.84552761128626, + "tokens_seen": 591659008 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004145035105315948, + "loss": 2.9616, + "theoretical_loss": 3.8454830579343424, + "tokens_seen": 591724544 + }, + { + "epoch": 0.18, + "learning_rate": 0.000414493480441324, + "loss": 2.6906, + "theoretical_loss": 3.845438510898102, + "tokens_seen": 591790080 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041448345035105316, + "loss": 2.9381, + "theoretical_loss": 3.8453939701759436, + "tokens_seen": 591855616 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041447342026078234, + "loss": 2.7809, + "theoretical_loss": 3.8453494357662747, + "tokens_seen": 591921152 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004144633901705115, + "loss": 2.6349, + "theoretical_loss": 3.8453049076675008, + "tokens_seen": 591986688 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004144533600802407, + "loss": 2.7431, + "theoretical_loss": 3.8452603858780288, + "tokens_seen": 592052224 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041444332998996994, + "loss": 3.0355, + "theoretical_loss": 3.8452158703962676, + "tokens_seen": 592117760 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041443329989969906, + "loss": 2.7863, + "theoretical_loss": 3.8451713612206246, + "tokens_seen": 592183296 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004144232698094283, + "loss": 2.9746, + "theoretical_loss": 3.8451268583495084, + "tokens_seen": 592248832 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041441323971915753, + "loss": 2.8717, + "theoretical_loss": 3.84508236178133, + "tokens_seen": 592314368 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041440320962888666, + "loss": 2.8954, + "theoretical_loss": 3.8450378715144984, + "tokens_seen": 592379904 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004143931795386159, + "loss": 2.7278, + "theoretical_loss": 3.844993387547425, + "tokens_seen": 592445440 + }, + { + "epoch": 0.18, + "learning_rate": 0.000414383149448345, + "loss": 2.9246, + "theoretical_loss": 3.8449489098785197, + "tokens_seen": 592510976 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041437311935807426, + "loss": 2.8113, + "theoretical_loss": 3.8449044385061955, + "tokens_seen": 592576512 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041436308926780344, + "loss": 2.7338, + "theoretical_loss": 3.8448599734288647, + "tokens_seen": 592642048 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004143530591775326, + "loss": 2.5302, + "theoretical_loss": 3.8448155146449396, + "tokens_seen": 592707584 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004143430290872618, + "loss": 2.7848, + "theoretical_loss": 3.844771062152834, + "tokens_seen": 592773120 + }, + { + "epoch": 0.18, + "learning_rate": 0.000414332998996991, + "loss": 2.6598, + "theoretical_loss": 3.8447266159509628, + "tokens_seen": 592838656 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041432296890672016, + "loss": 2.7648, + "theoretical_loss": 3.8446821760377397, + "tokens_seen": 592904192 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004143129388164494, + "loss": 2.9814, + "theoretical_loss": 3.84463774241158, + "tokens_seen": 592969728 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004143029087261785, + "loss": 2.8138, + "theoretical_loss": 3.8445933150708997, + "tokens_seen": 593035264 + }, + { + "epoch": 0.18, + "objective/train/docs_used": 345976, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7285759449005127, + "objective/train/theoretical_loss": 3.844548894014116, + "objective/train/tokens_used": 613560800, + "theoretical_loss": 3.844548894014116, + "tokens_seen": 593100800 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041429287863590776, + "loss": 2.962, + "theoretical_loss": 3.844548894014116, + "tokens_seen": 593100800 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004142828485456369, + "loss": 2.7874, + "theoretical_loss": 3.844504479239645, + "tokens_seen": 593166336 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004142728184553661, + "loss": 2.9049, + "theoretical_loss": 3.844460070745905, + "tokens_seen": 593231872 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004142627883650953, + "loss": 2.7124, + "theoretical_loss": 3.8444156685313127, + "tokens_seen": 593297408 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004142527582748245, + "loss": 3.0542, + "theoretical_loss": 3.8443712725942882, + "tokens_seen": 593362944 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041424272818455367, + "loss": 2.8045, + "theoretical_loss": 3.8443268829332498, + "tokens_seen": 593428480 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004142326980942829, + "loss": 2.802, + "theoretical_loss": 3.844282499546619, + "tokens_seen": 593494016 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041422266800401203, + "loss": 2.7282, + "theoretical_loss": 3.844238122432814, + "tokens_seen": 593559552 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041421263791374126, + "loss": 2.8837, + "theoretical_loss": 3.8441937515902573, + "tokens_seen": 593625088 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004142026078234704, + "loss": 2.705, + "theoretical_loss": 3.84414938701737, + "tokens_seen": 593690624 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004141925777331996, + "loss": 2.8237, + "theoretical_loss": 3.8441050287125744, + "tokens_seen": 593756160 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004141825476429288, + "loss": 2.8771, + "theoretical_loss": 3.8440606766742933, + "tokens_seen": 593821696 + }, + { + "epoch": 0.18, + "learning_rate": 0.000414172517552658, + "loss": 2.7833, + "theoretical_loss": 3.844016330900949, + "tokens_seen": 593887232 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041416248746238717, + "loss": 2.8017, + "theoretical_loss": 3.8439719913909673, + "tokens_seen": 593952768 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041415245737211635, + "loss": 2.8719, + "theoretical_loss": 3.843927658142771, + "tokens_seen": 594018304 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041414242728184553, + "loss": 3.0091, + "theoretical_loss": 3.8438833311547853, + "tokens_seen": 594083840 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041413239719157477, + "loss": 2.7167, + "theoretical_loss": 3.843839010425436, + "tokens_seen": 594149376 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004141223671013039, + "loss": 2.9683, + "theoretical_loss": 3.8437946959531497, + "tokens_seen": 594214912 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041411233701103313, + "loss": 2.7069, + "theoretical_loss": 3.8437503877363524, + "tokens_seen": 594280448 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004141023069207623, + "loss": 2.7144, + "theoretical_loss": 3.8437060857734715, + "tokens_seen": 594345984 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004140922768304915, + "loss": 2.7049, + "theoretical_loss": 3.843661790062935, + "tokens_seen": 594411520 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004140822467402207, + "loss": 2.677, + "theoretical_loss": 3.8436175006031714, + "tokens_seen": 594477056 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041407221664994985, + "loss": 2.8057, + "theoretical_loss": 3.84357321739261, + "tokens_seen": 594542592 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041406218655967904, + "loss": 2.6801, + "theoretical_loss": 3.843528940429679, + "tokens_seen": 594608128 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041405215646940827, + "loss": 2.9674, + "theoretical_loss": 3.8434846697128098, + "tokens_seen": 594673664 + }, + { + "epoch": 0.18, + "objective/train/docs_used": 346603, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.512133836746216, + "objective/train/theoretical_loss": 3.843440405240433, + "objective/train/tokens_used": 615199200, + "theoretical_loss": 3.843440405240433, + "tokens_seen": 594739200 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004140421263791374, + "loss": 2.6842, + "theoretical_loss": 3.843440405240433, + "tokens_seen": 594739200 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041403209628886663, + "loss": 2.9031, + "theoretical_loss": 3.843396147010979, + "tokens_seen": 594804736 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041402206619859576, + "loss": 2.8196, + "theoretical_loss": 3.843351895022881, + "tokens_seen": 594870272 + }, + { + "epoch": 0.18, + "learning_rate": 0.000414012036108325, + "loss": 2.8598, + "theoretical_loss": 3.8433076492745695, + "tokens_seen": 594935808 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004140020060180542, + "loss": 2.7186, + "theoretical_loss": 3.8432634097644787, + "tokens_seen": 595001344 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041399197592778336, + "loss": 2.8506, + "theoretical_loss": 3.8432191764910426, + "tokens_seen": 595066880 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041398194583751254, + "loss": 2.8202, + "theoretical_loss": 3.8431749494526937, + "tokens_seen": 595132416 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004139719157472417, + "loss": 2.7127, + "theoretical_loss": 3.8431307286478678, + "tokens_seen": 595197952 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004139618856569709, + "loss": 2.8783, + "theoretical_loss": 3.843086514075, + "tokens_seen": 595263488 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041395185556670014, + "loss": 3.0094, + "theoretical_loss": 3.843042305732526, + "tokens_seen": 595329024 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041394182547642926, + "loss": 2.8636, + "theoretical_loss": 3.8429981036188816, + "tokens_seen": 595394560 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004139317953861585, + "loss": 2.6675, + "theoretical_loss": 3.8429539077325043, + "tokens_seen": 595460096 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004139217652958877, + "loss": 2.5632, + "theoretical_loss": 3.8429097180718315, + "tokens_seen": 595525632 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041391173520561686, + "loss": 2.639, + "theoretical_loss": 3.842865534635301, + "tokens_seen": 595591168 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041390170511534604, + "loss": 2.9456, + "theoretical_loss": 3.842821357421352, + "tokens_seen": 595656704 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004138916750250752, + "loss": 2.911, + "theoretical_loss": 3.842777186428423, + "tokens_seen": 595722240 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004138816449348044, + "loss": 2.6698, + "theoretical_loss": 3.8427330216549533, + "tokens_seen": 595787776 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041387161484453364, + "loss": 2.8877, + "theoretical_loss": 3.8426888630993847, + "tokens_seen": 595853312 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041386158475426277, + "loss": 2.805, + "theoretical_loss": 3.8426447107601565, + "tokens_seen": 595918848 + }, + { + "epoch": 0.18, + "learning_rate": 0.000413851554663992, + "loss": 2.7708, + "theoretical_loss": 3.8426005646357106, + "tokens_seen": 595984384 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041384152457372113, + "loss": 3.0422, + "theoretical_loss": 3.8425564247244894, + "tokens_seen": 596049920 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041383149448345036, + "loss": 2.7029, + "theoretical_loss": 3.842512291024935, + "tokens_seen": 596115456 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041382146439317954, + "loss": 2.6315, + "theoretical_loss": 3.842468163535491, + "tokens_seen": 596180992 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004138114343029087, + "loss": 2.7515, + "theoretical_loss": 3.842424042254601, + "tokens_seen": 596246528 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004138014042126379, + "loss": 2.9989, + "theoretical_loss": 3.8423799271807084, + "tokens_seen": 596312064 + }, + { + "epoch": 0.18, + "objective/train/docs_used": 348035, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6905086040496826, + "objective/train/theoretical_loss": 3.8423358183122582, + "objective/train/tokens_used": 616837600, + "theoretical_loss": 3.8423358183122582, + "tokens_seen": 596377600 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004137913741223671, + "loss": 2.8737, + "theoretical_loss": 3.8423358183122582, + "tokens_seen": 596377600 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041378134403209627, + "loss": 2.6198, + "theoretical_loss": 3.8422917156476966, + "tokens_seen": 596443136 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004137713139418255, + "loss": 2.8571, + "theoretical_loss": 3.8422476191854686, + "tokens_seen": 596508672 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041376128385155463, + "loss": 2.6298, + "theoretical_loss": 3.8422035289240215, + "tokens_seen": 596574208 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041375125376128387, + "loss": 2.8305, + "theoretical_loss": 3.8421594448618013, + "tokens_seen": 596639744 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041374122367101305, + "loss": 2.6677, + "theoretical_loss": 3.8421153669972563, + "tokens_seen": 596705280 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041373119358074223, + "loss": 2.6179, + "theoretical_loss": 3.842071295328835, + "tokens_seen": 596770816 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004137211634904714, + "loss": 2.9755, + "theoretical_loss": 3.8420272298549847, + "tokens_seen": 596836352 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004137111334002006, + "loss": 2.7474, + "theoretical_loss": 3.8419831705741556, + "tokens_seen": 596901888 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041370110330992977, + "loss": 2.8207, + "theoretical_loss": 3.841939117484798, + "tokens_seen": 596967424 + }, + { + "epoch": 0.18, + "learning_rate": 0.000413691073219659, + "loss": 2.7239, + "theoretical_loss": 3.841895070585361, + "tokens_seen": 597032960 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041368104312938813, + "loss": 2.7312, + "theoretical_loss": 3.8418510298742965, + "tokens_seen": 597098496 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041367101303911737, + "loss": 2.6136, + "theoretical_loss": 3.8418069953500558, + "tokens_seen": 597164032 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041366098294884655, + "loss": 2.6799, + "theoretical_loss": 3.8417629670110904, + "tokens_seen": 597229568 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041365095285857573, + "loss": 2.7027, + "theoretical_loss": 3.841718944855854, + "tokens_seen": 597295104 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041364092276830497, + "loss": 2.8793, + "theoretical_loss": 3.8416749288827985, + "tokens_seen": 597360640 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004136308926780341, + "loss": 2.7775, + "theoretical_loss": 3.8416309190903783, + "tokens_seen": 597426176 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041362086258776333, + "loss": 2.8909, + "theoretical_loss": 3.841586915477048, + "tokens_seen": 597491712 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004136108324974925, + "loss": 2.5833, + "theoretical_loss": 3.8415429180412617, + "tokens_seen": 597557248 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004136008024072217, + "loss": 2.814, + "theoretical_loss": 3.841498926781475, + "tokens_seen": 597622784 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004135907723169509, + "loss": 2.5627, + "theoretical_loss": 3.8414549416961443, + "tokens_seen": 597688320 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041358074222668005, + "loss": 2.999, + "theoretical_loss": 3.8414109627837254, + "tokens_seen": 597753856 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041357071213640924, + "loss": 2.7961, + "theoretical_loss": 3.841366990042676, + "tokens_seen": 597819392 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041356068204613847, + "loss": 2.7713, + "theoretical_loss": 3.8413230234714533, + "tokens_seen": 597884928 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004135506519558676, + "loss": 2.8248, + "theoretical_loss": 3.8412790630685154, + "tokens_seen": 597950464 + }, + { + "epoch": 0.18, + "objective/train/docs_used": 348629, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 1.8499125242233276, + "objective/train/theoretical_loss": 3.841235108832321, + "objective/train/tokens_used": 618476000, + "theoretical_loss": 3.841235108832321, + "tokens_seen": 598016000 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041354062186559683, + "loss": 2.6562, + "theoretical_loss": 3.841235108832321, + "tokens_seen": 598016000 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041353059177532596, + "loss": 2.7209, + "theoretical_loss": 3.84119116076133, + "tokens_seen": 598081536 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004135205616850552, + "loss": 2.796, + "theoretical_loss": 3.841147218854002, + "tokens_seen": 598147072 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004135105315947844, + "loss": 2.7772, + "theoretical_loss": 3.8411032831087963, + "tokens_seen": 598212608 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041350050150451356, + "loss": 2.7787, + "theoretical_loss": 3.841059353524175, + "tokens_seen": 598278144 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041349047141424274, + "loss": 2.9904, + "theoretical_loss": 3.841015430098599, + "tokens_seen": 598343680 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004134804413239719, + "loss": 2.7609, + "theoretical_loss": 3.8409715128305306, + "tokens_seen": 598409216 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004134704112337011, + "loss": 2.8453, + "theoretical_loss": 3.8409276017184326, + "tokens_seen": 598474752 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041346038114343034, + "loss": 2.7746, + "theoretical_loss": 3.840883696760768, + "tokens_seen": 598540288 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041345035105315946, + "loss": 2.7747, + "theoretical_loss": 3.840839797956, + "tokens_seen": 598605824 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004134403209628887, + "loss": 2.6503, + "theoretical_loss": 3.8407959053025937, + "tokens_seen": 598671360 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004134302908726179, + "loss": 2.8292, + "theoretical_loss": 3.8407520187990127, + "tokens_seen": 598736896 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041342026078234706, + "loss": 2.9153, + "theoretical_loss": 3.8407081384437234, + "tokens_seen": 598802432 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041341023069207624, + "loss": 2.9563, + "theoretical_loss": 3.8406642642351914, + "tokens_seen": 598867968 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004134002006018054, + "loss": 2.9276, + "theoretical_loss": 3.840620396171883, + "tokens_seen": 598933504 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004133901705115346, + "loss": 2.8121, + "theoretical_loss": 3.840576534252265, + "tokens_seen": 598999040 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041338014042126384, + "loss": 2.8858, + "theoretical_loss": 3.8405326784748057, + "tokens_seen": 599064576 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041337011033099297, + "loss": 2.7058, + "theoretical_loss": 3.840488828837973, + "tokens_seen": 599130112 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004133600802407222, + "loss": 2.726, + "theoretical_loss": 3.840444985340235, + "tokens_seen": 599195648 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041335005015045133, + "loss": 3.0285, + "theoretical_loss": 3.840401147980061, + "tokens_seen": 599261184 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041334002006018056, + "loss": 2.6381, + "theoretical_loss": 3.840357316755921, + "tokens_seen": 599326720 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041332998996990975, + "loss": 2.6953, + "theoretical_loss": 3.8403134916662847, + "tokens_seen": 599392256 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004133199598796389, + "loss": 2.9465, + "theoretical_loss": 3.840269672709624, + "tokens_seen": 599457792 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004133099297893681, + "loss": 2.8254, + "theoretical_loss": 3.8402258598844097, + "tokens_seen": 599523328 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004132998996990973, + "loss": 2.7709, + "theoretical_loss": 3.8401820531891135, + "tokens_seen": 599588864 + }, + { + "epoch": 0.18, + "objective/train/docs_used": 349996, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.159327268600464, + "objective/train/theoretical_loss": 3.840138252622208, + "objective/train/tokens_used": 620114400, + "theoretical_loss": 3.840138252622208, + "tokens_seen": 599654400 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041328986960882647, + "loss": 2.9845, + "theoretical_loss": 3.840138252622208, + "tokens_seen": 599654400 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004132798395185557, + "loss": 2.8462, + "theoretical_loss": 3.8400944581821674, + "tokens_seen": 599719936 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041326980942828483, + "loss": 2.8386, + "theoretical_loss": 3.8400506698674635, + "tokens_seen": 599785472 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041325977933801407, + "loss": 2.7926, + "theoretical_loss": 3.840006887676571, + "tokens_seen": 599851008 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041324974924774325, + "loss": 2.8419, + "theoretical_loss": 3.839963111607965, + "tokens_seen": 599916544 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041323971915747243, + "loss": 3.0014, + "theoretical_loss": 3.8399193416601207, + "tokens_seen": 599982080 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004132296890672016, + "loss": 2.7312, + "theoretical_loss": 3.8398755778315135, + "tokens_seen": 600047616 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004132196589769308, + "loss": 2.9378, + "theoretical_loss": 3.8398318201206196, + "tokens_seen": 600113152 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041320962888665997, + "loss": 2.5748, + "theoretical_loss": 3.8397880685259165, + "tokens_seen": 600178688 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004131995987963892, + "loss": 2.8941, + "theoretical_loss": 3.839744323045881, + "tokens_seen": 600244224 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041318956870611833, + "loss": 2.8358, + "theoretical_loss": 3.839700583678991, + "tokens_seen": 600309760 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041317953861584757, + "loss": 2.8855, + "theoretical_loss": 3.839656850423726, + "tokens_seen": 600375296 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004131695085255767, + "loss": 2.8487, + "theoretical_loss": 3.8396131232785637, + "tokens_seen": 600440832 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041315947843530593, + "loss": 2.9066, + "theoretical_loss": 3.8395694022419846, + "tokens_seen": 600506368 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004131494483450351, + "loss": 2.7708, + "theoretical_loss": 3.8395256873124683, + "tokens_seen": 600571904 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004131394182547643, + "loss": 2.867, + "theoretical_loss": 3.8394819784884957, + "tokens_seen": 600637440 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004131293881644935, + "loss": 2.8779, + "theoretical_loss": 3.8394382757685483, + "tokens_seen": 600702976 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004131193580742227, + "loss": 2.6322, + "theoretical_loss": 3.839394579151107, + "tokens_seen": 600768512 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041310932798395184, + "loss": 2.6564, + "theoretical_loss": 3.8393508886346552, + "tokens_seen": 600834048 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004130992978936811, + "loss": 2.828, + "theoretical_loss": 3.8393072042176755, + "tokens_seen": 600899584 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004130892678034102, + "loss": 2.7843, + "theoretical_loss": 3.83926352589865, + "tokens_seen": 600965120 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041307923771313944, + "loss": 2.7769, + "theoretical_loss": 3.839219853676064, + "tokens_seen": 601030656 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004130692076228686, + "loss": 2.6974, + "theoretical_loss": 3.839176187548402, + "tokens_seen": 601096192 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004130591775325978, + "loss": 3.2038, + "theoretical_loss": 3.839132527514148, + "tokens_seen": 601161728 + }, + { + "epoch": 0.18, + "learning_rate": 0.000413049147442327, + "loss": 2.5859, + "theoretical_loss": 3.8390888735717885, + "tokens_seen": 601227264 + }, + { + "epoch": 0.18, + "objective/train/docs_used": 350686, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.9660727977752686, + "objective/train/theoretical_loss": 3.8390452257198096, + "objective/train/tokens_used": 621752800, + "theoretical_loss": 3.8390452257198096, + "tokens_seen": 601292800 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041303911735205616, + "loss": 3.0267, + "theoretical_loss": 3.8390452257198096, + "tokens_seen": 601292800 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041302908726178534, + "loss": 2.6662, + "theoretical_loss": 3.839001583956697, + "tokens_seen": 601358336 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004130190571715146, + "loss": 2.8844, + "theoretical_loss": 3.838957948280939, + "tokens_seen": 601423872 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004130090270812437, + "loss": 2.7282, + "theoretical_loss": 3.8389143186910224, + "tokens_seen": 601489408 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041299899699097294, + "loss": 2.708, + "theoretical_loss": 3.8388706951854354, + "tokens_seen": 601554944 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041298896690070207, + "loss": 2.6505, + "theoretical_loss": 3.838827077762668, + "tokens_seen": 601620480 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004129789368104313, + "loss": 2.8487, + "theoretical_loss": 3.8387834664212086, + "tokens_seen": 601686016 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004129689067201605, + "loss": 2.4947, + "theoretical_loss": 3.8387398611595476, + "tokens_seen": 601751552 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041295887662988966, + "loss": 2.4742, + "theoretical_loss": 3.8386962619761746, + "tokens_seen": 601817088 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041294884653961884, + "loss": 2.8887, + "theoretical_loss": 3.838652668869581, + "tokens_seen": 601882624 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004129388164493481, + "loss": 2.8718, + "theoretical_loss": 3.8386090818382588, + "tokens_seen": 601948160 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004129287863590772, + "loss": 2.8474, + "theoretical_loss": 3.838565500880699, + "tokens_seen": 602013696 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041291875626880644, + "loss": 2.7326, + "theoretical_loss": 3.838521925995395, + "tokens_seen": 602079232 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004129087261785356, + "loss": 2.9752, + "theoretical_loss": 3.83847835718084, + "tokens_seen": 602144768 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004128986960882648, + "loss": 2.775, + "theoretical_loss": 3.838434794435527, + "tokens_seen": 602210304 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041288866599799404, + "loss": 2.8605, + "theoretical_loss": 3.8383912377579508, + "tokens_seen": 602275840 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041287863590772317, + "loss": 2.7048, + "theoretical_loss": 3.838347687146605, + "tokens_seen": 602341376 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004128686058174524, + "loss": 2.9368, + "theoretical_loss": 3.8383041425999864, + "tokens_seen": 602406912 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041285857572718153, + "loss": 2.9198, + "theoretical_loss": 3.83826060411659, + "tokens_seen": 602472448 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041284854563691076, + "loss": 2.874, + "theoretical_loss": 3.8382170716949124, + "tokens_seen": 602537984 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041283851554663995, + "loss": 3.0974, + "theoretical_loss": 3.83817354533345, + "tokens_seen": 602603520 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004128284854563691, + "loss": 2.7217, + "theoretical_loss": 3.8381300250307007, + "tokens_seen": 602669056 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004128184553660983, + "loss": 3.0917, + "theoretical_loss": 3.8380865107851623, + "tokens_seen": 602734592 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004128084252758275, + "loss": 2.8882, + "theoretical_loss": 3.8380430025953327, + "tokens_seen": 602800128 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041279839518555667, + "loss": 2.864, + "theoretical_loss": 3.837999500459712, + "tokens_seen": 602865664 + }, + { + "epoch": 0.18, + "objective/train/docs_used": 352047, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.20009183883667, + "objective/train/theoretical_loss": 3.837956004376799, + "objective/train/tokens_used": 623391200, + "theoretical_loss": 3.837956004376799, + "tokens_seen": 602931200 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004127883650952859, + "loss": 2.805, + "theoretical_loss": 3.837956004376799, + "tokens_seen": 602931200 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041277833500501503, + "loss": 2.5211, + "theoretical_loss": 3.837912514345094, + "tokens_seen": 602996736 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041276830491474427, + "loss": 2.9665, + "theoretical_loss": 3.8378690303630973, + "tokens_seen": 603062272 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041275827482447345, + "loss": 2.7414, + "theoretical_loss": 3.837825552429311, + "tokens_seen": 603127808 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041274824473420263, + "loss": 2.603, + "theoretical_loss": 3.837782080542236, + "tokens_seen": 603193344 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004127382146439318, + "loss": 2.8037, + "theoretical_loss": 3.8377386147003745, + "tokens_seen": 603258880 + }, + { + "epoch": 0.18, + "learning_rate": 0.000412728184553661, + "loss": 2.7224, + "theoretical_loss": 3.837695154902229, + "tokens_seen": 603324416 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041271815446339017, + "loss": 3.1136, + "theoretical_loss": 3.837651701146304, + "tokens_seen": 603389952 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004127081243731194, + "loss": 2.8866, + "theoretical_loss": 3.8376082534311022, + "tokens_seen": 603455488 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041269809428284854, + "loss": 2.8948, + "theoretical_loss": 3.8375648117551284, + "tokens_seen": 603521024 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041268806419257777, + "loss": 2.9363, + "theoretical_loss": 3.8375213761168867, + "tokens_seen": 603586560 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004126780341023069, + "loss": 3.028, + "theoretical_loss": 3.837477946514884, + "tokens_seen": 603652096 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041266800401203613, + "loss": 2.6492, + "theoretical_loss": 3.8374345229476248, + "tokens_seen": 603717632 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004126579739217653, + "loss": 2.8142, + "theoretical_loss": 3.8373911054136167, + "tokens_seen": 603783168 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004126479438314945, + "loss": 2.6301, + "theoretical_loss": 3.837347693911366, + "tokens_seen": 603848704 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004126379137412237, + "loss": 2.7895, + "theoretical_loss": 3.8373042884393813, + "tokens_seen": 603914240 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004126278836509529, + "loss": 2.9782, + "theoretical_loss": 3.837260888996169, + "tokens_seen": 603979776 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041261785356068204, + "loss": 2.8014, + "theoretical_loss": 3.837217495580239, + "tokens_seen": 604045312 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004126078234704113, + "loss": 2.8003, + "theoretical_loss": 3.8371741081900996, + "tokens_seen": 604110848 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004125977933801404, + "loss": 2.6928, + "theoretical_loss": 3.8371307268242614, + "tokens_seen": 604176384 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041258776328986964, + "loss": 2.7404, + "theoretical_loss": 3.837087351481234, + "tokens_seen": 604241920 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004125777331995988, + "loss": 2.8118, + "theoretical_loss": 3.8370439821595284, + "tokens_seen": 604307456 + }, + { + "epoch": 0.18, + "learning_rate": 0.000412567703109328, + "loss": 2.9591, + "theoretical_loss": 3.8370006188576555, + "tokens_seen": 604372992 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004125576730190572, + "loss": 2.7142, + "theoretical_loss": 3.8369572615741268, + "tokens_seen": 604438528 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041254764292878636, + "loss": 2.6995, + "theoretical_loss": 3.836913910307456, + "tokens_seen": 604504064 + }, + { + "epoch": 0.18, + "objective/train/docs_used": 352465, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.994818687438965, + "objective/train/theoretical_loss": 3.836870565056154, + "objective/train/tokens_used": 625029600, + "theoretical_loss": 3.836870565056154, + "tokens_seen": 604569600 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041253761283851554, + "loss": 2.7664, + "theoretical_loss": 3.836870565056154, + "tokens_seen": 604569600 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004125275827482448, + "loss": 2.7563, + "theoretical_loss": 3.8368272258187366, + "tokens_seen": 604635136 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004125175526579739, + "loss": 2.8653, + "theoretical_loss": 3.836783892593715, + "tokens_seen": 604700672 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041250752256770314, + "loss": 2.7511, + "theoretical_loss": 3.836740565379606, + "tokens_seen": 604766208 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041249749247743227, + "loss": 2.8707, + "theoretical_loss": 3.836697244174923, + "tokens_seen": 604831744 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004124874623871615, + "loss": 2.9738, + "theoretical_loss": 3.836653928978182, + "tokens_seen": 604897280 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004124774322968907, + "loss": 3.0348, + "theoretical_loss": 3.8366106197878995, + "tokens_seen": 604962816 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041246740220661986, + "loss": 2.7457, + "theoretical_loss": 3.8365673166025913, + "tokens_seen": 605028352 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041245737211634904, + "loss": 2.75, + "theoretical_loss": 3.8365240194207746, + "tokens_seen": 605093888 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004124473420260783, + "loss": 3.1115, + "theoretical_loss": 3.8364807282409674, + "tokens_seen": 605159424 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004124373119358074, + "loss": 2.8959, + "theoretical_loss": 3.836437443061688, + "tokens_seen": 605224960 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041242728184553664, + "loss": 2.7658, + "theoretical_loss": 3.8363941638814536, + "tokens_seen": 605290496 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041241725175526577, + "loss": 2.865, + "theoretical_loss": 3.8363508906987853, + "tokens_seen": 605356032 + }, + { + "epoch": 0.18, + "learning_rate": 0.000412407221664995, + "loss": 3.0766, + "theoretical_loss": 3.8363076235122016, + "tokens_seen": 605421568 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004123971915747242, + "loss": 2.7333, + "theoretical_loss": 3.8362643623202226, + "tokens_seen": 605487104 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041238716148445337, + "loss": 2.5842, + "theoretical_loss": 3.83622110712137, + "tokens_seen": 605552640 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041237713139418255, + "loss": 2.9508, + "theoretical_loss": 3.8361778579141648, + "tokens_seen": 605618176 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041236710130391173, + "loss": 2.6649, + "theoretical_loss": 3.8361346146971282, + "tokens_seen": 605683712 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004123570712136409, + "loss": 2.8695, + "theoretical_loss": 3.836091377468783, + "tokens_seen": 605749248 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041234704112337015, + "loss": 2.8449, + "theoretical_loss": 3.836048146227652, + "tokens_seen": 605814784 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041233701103309927, + "loss": 2.8439, + "theoretical_loss": 3.8360049209722584, + "tokens_seen": 605880320 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004123269809428285, + "loss": 2.8129, + "theoretical_loss": 3.8359617017011267, + "tokens_seen": 605945856 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041231695085255763, + "loss": 3.0075, + "theoretical_loss": 3.835918488412781, + "tokens_seen": 606011392 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041230692076228687, + "loss": 2.8721, + "theoretical_loss": 3.8358752811057455, + "tokens_seen": 606076928 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041229689067201605, + "loss": 2.7023, + "theoretical_loss": 3.8358320797785463, + "tokens_seen": 606142464 + }, + { + "epoch": 0.18, + "objective/train/docs_used": 353634, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.4573044776916504, + "objective/train/theoretical_loss": 3.8357888844297094, + "objective/train/tokens_used": 626668000, + "theoretical_loss": 3.8357888844297094, + "tokens_seen": 606208000 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041228686058174523, + "loss": 2.8458, + "theoretical_loss": 3.8357888844297094, + "tokens_seen": 606208000 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004122768304914744, + "loss": 2.8399, + "theoretical_loss": 3.8357456950577618, + "tokens_seen": 606273536 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041226680040120365, + "loss": 2.9228, + "theoretical_loss": 3.8357025116612293, + "tokens_seen": 606339072 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004122567703109328, + "loss": 2.6493, + "theoretical_loss": 3.8356593342386405, + "tokens_seen": 606404608 + }, + { + "epoch": 0.18, + "learning_rate": 0.000412246740220662, + "loss": 2.9215, + "theoretical_loss": 3.8356161627885235, + "tokens_seen": 606470144 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041223671013039114, + "loss": 2.8858, + "theoretical_loss": 3.835572997309406, + "tokens_seen": 606535680 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004122266800401204, + "loss": 2.9587, + "theoretical_loss": 3.835529837799818, + "tokens_seen": 606601216 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041221664994984955, + "loss": 2.902, + "theoretical_loss": 3.8354866842582895, + "tokens_seen": 606666752 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041220661985957874, + "loss": 2.7688, + "theoretical_loss": 3.835443536683349, + "tokens_seen": 606732288 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004121965897693079, + "loss": 2.7861, + "theoretical_loss": 3.835400395073529, + "tokens_seen": 606797824 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004121865596790371, + "loss": 2.9206, + "theoretical_loss": 3.8353572594273597, + "tokens_seen": 606863360 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004121765295887663, + "loss": 2.7887, + "theoretical_loss": 3.835314129743373, + "tokens_seen": 606928896 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004121664994984955, + "loss": 2.9723, + "theoretical_loss": 3.835271006020101, + "tokens_seen": 606994432 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004121564694082247, + "loss": 3.0446, + "theoretical_loss": 3.8352278882560764, + "tokens_seen": 607059968 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004121464393179539, + "loss": 3.0022, + "theoretical_loss": 3.8351847764498332, + "tokens_seen": 607125504 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004121364092276831, + "loss": 2.8775, + "theoretical_loss": 3.8351416705999055, + "tokens_seen": 607191040 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041212637913741224, + "loss": 3.0275, + "theoretical_loss": 3.835098570704826, + "tokens_seen": 607256576 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004121163490471415, + "loss": 2.9523, + "theoretical_loss": 3.8350554767631304, + "tokens_seen": 607322112 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004121063189568706, + "loss": 3.0455, + "theoretical_loss": 3.8350123887733543, + "tokens_seen": 607387648 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041209628886659984, + "loss": 2.907, + "theoretical_loss": 3.8349693067340334, + "tokens_seen": 607453184 + }, + { + "epoch": 0.18, + "learning_rate": 0.000412086258776329, + "loss": 2.7992, + "theoretical_loss": 3.8349262306437044, + "tokens_seen": 607518720 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004120762286860582, + "loss": 2.737, + "theoretical_loss": 3.8348831605009033, + "tokens_seen": 607584256 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004120661985957874, + "loss": 2.7433, + "theoretical_loss": 3.8348400963041684, + "tokens_seen": 607649792 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041205616850551656, + "loss": 2.7425, + "theoretical_loss": 3.8347970380520375, + "tokens_seen": 607715328 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041204613841524574, + "loss": 2.656, + "theoretical_loss": 3.834753985743049, + "tokens_seen": 607780864 + }, + { + "epoch": 0.18, + "objective/train/docs_used": 354263, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.0086724758148193, + "objective/train/theoretical_loss": 3.834710939375742, + "objective/train/tokens_used": 628306400, + "theoretical_loss": 3.834710939375742, + "tokens_seen": 607846400 + }, + { + "epoch": 0.18, + "learning_rate": 0.000412036108324975, + "loss": 3.1918, + "theoretical_loss": 3.834710939375742, + "tokens_seen": 607846400 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004120260782347041, + "loss": 2.9192, + "theoretical_loss": 3.8346678989486556, + "tokens_seen": 607911936 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041201604814443334, + "loss": 3.1419, + "theoretical_loss": 3.83462486446033, + "tokens_seen": 607977472 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041200601805416247, + "loss": 2.9224, + "theoretical_loss": 3.8345818359093062, + "tokens_seen": 608043008 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004119959879638917, + "loss": 3.0886, + "theoretical_loss": 3.8345388132941243, + "tokens_seen": 608108544 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004119859578736209, + "loss": 2.6788, + "theoretical_loss": 3.834495796613327, + "tokens_seen": 608174080 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041197592778335006, + "loss": 2.8325, + "theoretical_loss": 3.834452785865455, + "tokens_seen": 608239616 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041196589769307924, + "loss": 2.7408, + "theoretical_loss": 3.834409781049053, + "tokens_seen": 608305152 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004119558676028085, + "loss": 2.8954, + "theoretical_loss": 3.834366782162662, + "tokens_seen": 608370688 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004119458375125376, + "loss": 2.9108, + "theoretical_loss": 3.8343237892048263, + "tokens_seen": 608436224 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041193580742226684, + "loss": 2.866, + "theoretical_loss": 3.834280802174091, + "tokens_seen": 608501760 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041192577733199597, + "loss": 2.8815, + "theoretical_loss": 3.8342378210689994, + "tokens_seen": 608567296 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004119157472417252, + "loss": 3.0546, + "theoretical_loss": 3.834194845888097, + "tokens_seen": 608632832 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004119057171514544, + "loss": 2.7444, + "theoretical_loss": 3.83415187662993, + "tokens_seen": 608698368 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041189568706118357, + "loss": 2.9072, + "theoretical_loss": 3.8341089132930444, + "tokens_seen": 608763904 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041188565697091275, + "loss": 2.917, + "theoretical_loss": 3.834065955875987, + "tokens_seen": 608829440 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041187562688064193, + "loss": 2.9404, + "theoretical_loss": 3.8340230043773045, + "tokens_seen": 608894976 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004118655967903711, + "loss": 3.1211, + "theoretical_loss": 3.833980058795545, + "tokens_seen": 608960512 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041185556670010035, + "loss": 2.7573, + "theoretical_loss": 3.8339371191292573, + "tokens_seen": 609026048 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041184553660982947, + "loss": 3.0671, + "theoretical_loss": 3.8338941853769892, + "tokens_seen": 609091584 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004118355065195587, + "loss": 2.795, + "theoretical_loss": 3.8338512575372903, + "tokens_seen": 609157120 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041182547642928783, + "loss": 2.7125, + "theoretical_loss": 3.8338083356087105, + "tokens_seen": 609222656 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041181544633901707, + "loss": 3.0413, + "theoretical_loss": 3.8337654195898, + "tokens_seen": 609288192 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041180541624874625, + "loss": 2.8342, + "theoretical_loss": 3.8337225094791103, + "tokens_seen": 609353728 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041179538615847543, + "loss": 2.9156, + "theoretical_loss": 3.8336796052751914, + "tokens_seen": 609419264 + }, + { + "epoch": 0.18, + "objective/train/docs_used": 355721, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.934474229812622, + "objective/train/theoretical_loss": 3.8336367069765958, + "objective/train/tokens_used": 629944800, + "theoretical_loss": 3.8336367069765958, + "tokens_seen": 609484800 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004117853560682046, + "loss": 2.9516, + "theoretical_loss": 3.8336367069765958, + "tokens_seen": 609484800 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041177532597793385, + "loss": 2.6997, + "theoretical_loss": 3.833593814581876, + "tokens_seen": 609550336 + }, + { + "epoch": 0.18, + "learning_rate": 0.000411765295887663, + "loss": 2.7812, + "theoretical_loss": 3.8335509280895854, + "tokens_seen": 609615872 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004117552657973922, + "loss": 2.8464, + "theoretical_loss": 3.833508047498276, + "tokens_seen": 609681408 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041174523570712134, + "loss": 2.9426, + "theoretical_loss": 3.833465172806503, + "tokens_seen": 609746944 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004117352056168506, + "loss": 2.8335, + "theoretical_loss": 3.833422304012819, + "tokens_seen": 609812480 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041172517552657975, + "loss": 2.9878, + "theoretical_loss": 3.833379441115781, + "tokens_seen": 609878016 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041171514543630894, + "loss": 3.0314, + "theoretical_loss": 3.833336584113943, + "tokens_seen": 609943552 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004117051153460381, + "loss": 2.8506, + "theoretical_loss": 3.8332937330058616, + "tokens_seen": 610009088 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004116950852557673, + "loss": 2.9576, + "theoretical_loss": 3.8332508877900935, + "tokens_seen": 610074624 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004116850551654965, + "loss": 2.8403, + "theoretical_loss": 3.8332080484651945, + "tokens_seen": 610140160 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004116750250752257, + "loss": 2.8367, + "theoretical_loss": 3.8331652150297226, + "tokens_seen": 610205696 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041166499498495484, + "loss": 2.8702, + "theoretical_loss": 3.833122387482236, + "tokens_seen": 610271232 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004116549648946841, + "loss": 2.8323, + "theoretical_loss": 3.8330795658212926, + "tokens_seen": 610336768 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004116449348044132, + "loss": 2.8428, + "theoretical_loss": 3.833036750045452, + "tokens_seen": 610402304 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041163490471414244, + "loss": 3.0215, + "theoretical_loss": 3.832993940153273, + "tokens_seen": 610467840 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004116248746238716, + "loss": 2.9256, + "theoretical_loss": 3.8329511361433166, + "tokens_seen": 610533376 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004116148445336008, + "loss": 2.9917, + "theoretical_loss": 3.832908338014142, + "tokens_seen": 610598912 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041160481444333, + "loss": 2.7236, + "theoretical_loss": 3.832865545764311, + "tokens_seen": 610664448 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004115947843530592, + "loss": 2.6993, + "theoretical_loss": 3.8328227593923856, + "tokens_seen": 610729984 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041158475426278834, + "loss": 3.0371, + "theoretical_loss": 3.832779978896926, + "tokens_seen": 610795520 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004115747241725176, + "loss": 3.2142, + "theoretical_loss": 3.832737204276497, + "tokens_seen": 610861056 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004115646940822467, + "loss": 2.9671, + "theoretical_loss": 3.8326944355296595, + "tokens_seen": 610926592 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041155466399197594, + "loss": 2.9351, + "theoretical_loss": 3.832651672654978, + "tokens_seen": 610992128 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004115446339017051, + "loss": 2.5954, + "theoretical_loss": 3.832608915651017, + "tokens_seen": 611057664 + }, + { + "epoch": 0.19, + "objective/train/docs_used": 356348, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.924924850463867, + "objective/train/theoretical_loss": 3.83256616451634, + "objective/train/tokens_used": 631583200, + "theoretical_loss": 3.83256616451634, + "tokens_seen": 611123200 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004115346038114343, + "loss": 2.8526, + "theoretical_loss": 3.83256616451634, + "tokens_seen": 611123200 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004115245737211635, + "loss": 2.9963, + "theoretical_loss": 3.8325234192495126, + "tokens_seen": 611188736 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041151454363089267, + "loss": 2.7067, + "theoretical_loss": 3.832480679849101, + "tokens_seen": 611254272 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041150451354062185, + "loss": 2.851, + "theoretical_loss": 3.8324379463136697, + "tokens_seen": 611319808 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004114944834503511, + "loss": 2.7202, + "theoretical_loss": 3.8323952186417864, + "tokens_seen": 611385344 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004114844533600802, + "loss": 2.77, + "theoretical_loss": 3.832352496832018, + "tokens_seen": 611450880 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041147442326980944, + "loss": 2.8569, + "theoretical_loss": 3.832309780882932, + "tokens_seen": 611516416 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041146439317953857, + "loss": 2.7762, + "theoretical_loss": 3.8322670707930957, + "tokens_seen": 611581952 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004114543630892678, + "loss": 2.8274, + "theoretical_loss": 3.832224366561079, + "tokens_seen": 611647488 + }, + { + "epoch": 0.19, + "learning_rate": 0.000411444332998997, + "loss": 2.8144, + "theoretical_loss": 3.83218166818545, + "tokens_seen": 611713024 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041143430290872617, + "loss": 2.7575, + "theoretical_loss": 3.8321389756647783, + "tokens_seen": 611778560 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041142427281845535, + "loss": 2.7933, + "theoretical_loss": 3.8320962889976347, + "tokens_seen": 611844096 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004114142427281846, + "loss": 2.9317, + "theoretical_loss": 3.832053608182589, + "tokens_seen": 611909632 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041140421263791377, + "loss": 2.886, + "theoretical_loss": 3.8320109332182124, + "tokens_seen": 611975168 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041139418254764295, + "loss": 2.9534, + "theoretical_loss": 3.831968264103077, + "tokens_seen": 612040704 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041138415245737213, + "loss": 2.6557, + "theoretical_loss": 3.831925600835754, + "tokens_seen": 612106240 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004113741223671013, + "loss": 2.8374, + "theoretical_loss": 3.831882943414817, + "tokens_seen": 612171776 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041136409227683055, + "loss": 2.9132, + "theoretical_loss": 3.8318402918388386, + "tokens_seen": 612237312 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041135406218655967, + "loss": 2.855, + "theoretical_loss": 3.8317976461063923, + "tokens_seen": 612302848 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004113440320962889, + "loss": 2.9453, + "theoretical_loss": 3.831755006216052, + "tokens_seen": 612368384 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041133400200601803, + "loss": 2.9818, + "theoretical_loss": 3.8317123721663924, + "tokens_seen": 612433920 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041132397191574727, + "loss": 2.8159, + "theoretical_loss": 3.831669743955989, + "tokens_seen": 612499456 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041131394182547645, + "loss": 2.5259, + "theoretical_loss": 3.8316271215834172, + "tokens_seen": 612564992 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041130391173520563, + "loss": 3.0515, + "theoretical_loss": 3.8315845050472532, + "tokens_seen": 612630528 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004112938816449348, + "loss": 2.9738, + "theoretical_loss": 3.831541894346073, + "tokens_seen": 612696064 + }, + { + "epoch": 0.19, + "objective/train/docs_used": 356825, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.509188652038574, + "objective/train/theoretical_loss": 3.8314992894784536, + "objective/train/tokens_used": 633221600, + "theoretical_loss": 3.8314992894784536, + "tokens_seen": 612761600 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041128385155466405, + "loss": 2.8263, + "theoretical_loss": 3.8314992894784536, + "tokens_seen": 612761600 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004112738214643932, + "loss": 3.1219, + "theoretical_loss": 3.831456690442974, + "tokens_seen": 612827136 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004112637913741224, + "loss": 2.7939, + "theoretical_loss": 3.8314140972382105, + "tokens_seen": 612892672 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041125376128385154, + "loss": 3.0353, + "theoretical_loss": 3.831371509862743, + "tokens_seen": 612958208 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004112437311935808, + "loss": 2.5364, + "theoretical_loss": 3.8313289283151497, + "tokens_seen": 613023744 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041123370110330995, + "loss": 2.8265, + "theoretical_loss": 3.831286352594011, + "tokens_seen": 613089280 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041122367101303914, + "loss": 2.8228, + "theoretical_loss": 3.831243782697906, + "tokens_seen": 613154816 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004112136409227683, + "loss": 2.9346, + "theoretical_loss": 3.8312012186254156, + "tokens_seen": 613220352 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004112036108324975, + "loss": 2.9922, + "theoretical_loss": 3.831158660375122, + "tokens_seen": 613285888 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004111935807422267, + "loss": 2.9715, + "theoretical_loss": 3.831116107945605, + "tokens_seen": 613351424 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004111835506519559, + "loss": 2.8138, + "theoretical_loss": 3.8310735613354483, + "tokens_seen": 613416960 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041117352056168504, + "loss": 3.1155, + "theoretical_loss": 3.831031020543233, + "tokens_seen": 613482496 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004111634904714143, + "loss": 2.8944, + "theoretical_loss": 3.830988485567543, + "tokens_seen": 613548032 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004111534603811434, + "loss": 2.7865, + "theoretical_loss": 3.8309459564069615, + "tokens_seen": 613613568 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041114343029087264, + "loss": 2.6926, + "theoretical_loss": 3.830903433060073, + "tokens_seen": 613679104 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004111334002006018, + "loss": 2.8865, + "theoretical_loss": 3.8308609155254616, + "tokens_seen": 613744640 + }, + { + "epoch": 0.19, + "learning_rate": 0.000411123370110331, + "loss": 2.9062, + "theoretical_loss": 3.830818403801713, + "tokens_seen": 613810176 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004111133400200602, + "loss": 3.1555, + "theoretical_loss": 3.8307758978874116, + "tokens_seen": 613875712 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004111033099297894, + "loss": 2.7844, + "theoretical_loss": 3.830733397781144, + "tokens_seen": 613941248 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041109327983951854, + "loss": 3.0277, + "theoretical_loss": 3.830690903481497, + "tokens_seen": 614006784 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004110832497492478, + "loss": 2.9793, + "theoretical_loss": 3.830648414987057, + "tokens_seen": 614072320 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004110732196589769, + "loss": 2.805, + "theoretical_loss": 3.8306059322964128, + "tokens_seen": 614137856 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041106318956870614, + "loss": 2.8655, + "theoretical_loss": 3.830563455408151, + "tokens_seen": 614203392 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004110531594784353, + "loss": 2.9058, + "theoretical_loss": 3.8305209843208603, + "tokens_seen": 614268928 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004110431293881645, + "loss": 2.7876, + "theoretical_loss": 3.8304785190331305, + "tokens_seen": 614334464 + }, + { + "epoch": 0.19, + "objective/train/docs_used": 357491, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.4113407135009766, + "objective/train/theoretical_loss": 3.830436059543551, + "objective/train/tokens_used": 634860000, + "theoretical_loss": 3.830436059543551, + "tokens_seen": 614400000 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004110330992978937, + "loss": 2.6902, + "theoretical_loss": 3.830436059543551, + "tokens_seen": 614400000 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041102306920762287, + "loss": 2.7715, + "theoretical_loss": 3.830393605850711, + "tokens_seen": 614465536 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041101303911735205, + "loss": 2.8697, + "theoretical_loss": 3.8303511579532015, + "tokens_seen": 614531072 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004110030090270813, + "loss": 2.841, + "theoretical_loss": 3.8303087158496134, + "tokens_seen": 614596608 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004109929789368104, + "loss": 2.7685, + "theoretical_loss": 3.8302662795385385, + "tokens_seen": 614662144 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041098294884653965, + "loss": 2.7425, + "theoretical_loss": 3.8302238490185676, + "tokens_seen": 614727680 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041097291875626877, + "loss": 2.6689, + "theoretical_loss": 3.830181424288295, + "tokens_seen": 614793216 + }, + { + "epoch": 0.19, + "learning_rate": 0.000410962888665998, + "loss": 2.8161, + "theoretical_loss": 3.830139005346312, + "tokens_seen": 614858752 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004109528585757272, + "loss": 2.8926, + "theoretical_loss": 3.830096592191212, + "tokens_seen": 614924288 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041094282848545637, + "loss": 2.901, + "theoretical_loss": 3.8300541848215905, + "tokens_seen": 614989824 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041093279839518555, + "loss": 2.7902, + "theoretical_loss": 3.8300117832360403, + "tokens_seen": 615055360 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004109227683049148, + "loss": 2.919, + "theoretical_loss": 3.8299693874331573, + "tokens_seen": 615120896 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004109127382146439, + "loss": 2.9496, + "theoretical_loss": 3.829926997411537, + "tokens_seen": 615186432 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041090270812437315, + "loss": 3.0739, + "theoretical_loss": 3.8298846131697744, + "tokens_seen": 615251968 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004108926780341023, + "loss": 2.9957, + "theoretical_loss": 3.829842234706466, + "tokens_seen": 615317504 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004108826479438315, + "loss": 2.7602, + "theoretical_loss": 3.8297998620202094, + "tokens_seen": 615383040 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004108726178535607, + "loss": 2.7763, + "theoretical_loss": 3.829757495109602, + "tokens_seen": 615448576 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041086258776328987, + "loss": 2.809, + "theoretical_loss": 3.8297151339732407, + "tokens_seen": 615514112 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041085255767301905, + "loss": 2.6524, + "theoretical_loss": 3.8296727786097247, + "tokens_seen": 615579648 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041084252758274823, + "loss": 2.6245, + "theoretical_loss": 3.8296304290176524, + "tokens_seen": 615645184 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004108324974924774, + "loss": 2.8024, + "theoretical_loss": 3.829588085195623, + "tokens_seen": 615710720 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041082246740220665, + "loss": 2.677, + "theoretical_loss": 3.8295457471422365, + "tokens_seen": 615776256 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004108124373119358, + "loss": 2.8236, + "theoretical_loss": 3.8295034148560934, + "tokens_seen": 615841792 + }, + { + "epoch": 0.19, + "learning_rate": 0.000410802407221665, + "loss": 2.7574, + "theoretical_loss": 3.8294610883357945, + "tokens_seen": 615907328 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004107923771313942, + "loss": 2.5986, + "theoretical_loss": 3.8294187675799405, + "tokens_seen": 615972864 + }, + { + "epoch": 0.19, + "objective/train/docs_used": 358741, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.562229633331299, + "objective/train/theoretical_loss": 3.829376452587134, + "objective/train/tokens_used": 636498400, + "theoretical_loss": 3.829376452587134, + "tokens_seen": 616038400 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004107823470411234, + "loss": 2.749, + "theoretical_loss": 3.829376452587134, + "tokens_seen": 616038400 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041077231695085256, + "loss": 2.7358, + "theoretical_loss": 3.8293341433559767, + "tokens_seen": 616103936 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041076228686058174, + "loss": 2.8665, + "theoretical_loss": 3.8292918398850713, + "tokens_seen": 616169472 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004107522567703109, + "loss": 3.008, + "theoretical_loss": 3.829249542173021, + "tokens_seen": 616235008 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041074222668004015, + "loss": 2.9681, + "theoretical_loss": 3.82920725021843, + "tokens_seen": 616300544 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004107321965897693, + "loss": 3.0393, + "theoretical_loss": 3.8291649640199017, + "tokens_seen": 616366080 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004107221664994985, + "loss": 2.6672, + "theoretical_loss": 3.829122683576042, + "tokens_seen": 616431616 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041071213640922764, + "loss": 2.8898, + "theoretical_loss": 3.8290804088854546, + "tokens_seen": 616497152 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004107021063189569, + "loss": 2.9634, + "theoretical_loss": 3.8290381399467464, + "tokens_seen": 616562688 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041069207622868606, + "loss": 2.8602, + "theoretical_loss": 3.8289958767585226, + "tokens_seen": 616628224 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041068204613841524, + "loss": 2.8383, + "theoretical_loss": 3.8289536193193907, + "tokens_seen": 616693760 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004106720160481444, + "loss": 2.7516, + "theoretical_loss": 3.828911367627957, + "tokens_seen": 616759296 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004106619859578736, + "loss": 2.5165, + "theoretical_loss": 3.8288691216828297, + "tokens_seen": 616824832 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041065195586760284, + "loss": 3.1224, + "theoretical_loss": 3.828826881482617, + "tokens_seen": 616890368 + }, + { + "epoch": 0.19, + "learning_rate": 0.000410641925777332, + "loss": 2.5438, + "theoretical_loss": 3.8287846470259264, + "tokens_seen": 616955904 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004106318956870612, + "loss": 2.8022, + "theoretical_loss": 3.828742418311368, + "tokens_seen": 617021440 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004106218655967904, + "loss": 2.8263, + "theoretical_loss": 3.828700195337551, + "tokens_seen": 617086976 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004106118355065196, + "loss": 2.8831, + "theoretical_loss": 3.8286579781030863, + "tokens_seen": 617152512 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041060180541624874, + "loss": 2.8523, + "theoretical_loss": 3.8286157666065828, + "tokens_seen": 617218048 + }, + { + "epoch": 0.19, + "learning_rate": 0.000410591775325978, + "loss": 2.8952, + "theoretical_loss": 3.828573560846652, + "tokens_seen": 617283584 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004105817452357071, + "loss": 2.8318, + "theoretical_loss": 3.8285313608219056, + "tokens_seen": 617349120 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041057171514543634, + "loss": 2.8296, + "theoretical_loss": 3.8284891665309564, + "tokens_seen": 617414656 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004105616850551655, + "loss": 2.6925, + "theoretical_loss": 3.8284469779724155, + "tokens_seen": 617480192 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004105516549648947, + "loss": 2.6413, + "theoretical_loss": 3.8284047951448965, + "tokens_seen": 617545728 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004105416248746239, + "loss": 2.9738, + "theoretical_loss": 3.828362618047013, + "tokens_seen": 617611264 + }, + { + "epoch": 0.19, + "objective/train/docs_used": 359404, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6755175590515137, + "objective/train/theoretical_loss": 3.828320446677378, + "objective/train/tokens_used": 638136800, + "theoretical_loss": 3.828320446677378, + "tokens_seen": 617676800 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041053159478435307, + "loss": 2.7973, + "theoretical_loss": 3.828320446677378, + "tokens_seen": 617676800 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041052156469408225, + "loss": 2.8779, + "theoretical_loss": 3.8282782810346063, + "tokens_seen": 617742336 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004105115346038115, + "loss": 2.6723, + "theoretical_loss": 3.8282361211173135, + "tokens_seen": 617807872 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004105015045135406, + "loss": 2.8776, + "theoretical_loss": 3.8281939669241143, + "tokens_seen": 617873408 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041049147442326985, + "loss": 2.9594, + "theoretical_loss": 3.8281518184536245, + "tokens_seen": 617938944 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041048144433299897, + "loss": 3.041, + "theoretical_loss": 3.82810967570446, + "tokens_seen": 618004480 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004104714142427282, + "loss": 2.7117, + "theoretical_loss": 3.8280675386752385, + "tokens_seen": 618070016 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004104613841524574, + "loss": 2.6599, + "theoretical_loss": 3.828025407364577, + "tokens_seen": 618135552 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041045135406218657, + "loss": 2.865, + "theoretical_loss": 3.827983281771093, + "tokens_seen": 618201088 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041044132397191575, + "loss": 2.775, + "theoretical_loss": 3.8279411618934045, + "tokens_seen": 618266624 + }, + { + "epoch": 0.19, + "learning_rate": 0.000410431293881645, + "loss": 2.8597, + "theoretical_loss": 3.8278990477301305, + "tokens_seen": 618332160 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004104212637913741, + "loss": 2.8651, + "theoretical_loss": 3.82785693927989, + "tokens_seen": 618397696 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041041123370110335, + "loss": 2.8946, + "theoretical_loss": 3.827814836541303, + "tokens_seen": 618463232 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004104012036108325, + "loss": 3.0205, + "theoretical_loss": 3.82777273951299, + "tokens_seen": 618528768 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004103911735205617, + "loss": 2.6944, + "theoretical_loss": 3.8277306481935707, + "tokens_seen": 618594304 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004103811434302909, + "loss": 2.6417, + "theoretical_loss": 3.8276885625816663, + "tokens_seen": 618659840 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041037111334002007, + "loss": 2.7231, + "theoretical_loss": 3.827646482675899, + "tokens_seen": 618725376 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041036108324974925, + "loss": 2.8407, + "theoretical_loss": 3.8276044084748904, + "tokens_seen": 618790912 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041035105315947844, + "loss": 2.7093, + "theoretical_loss": 3.8275623399772636, + "tokens_seen": 618856448 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004103410230692076, + "loss": 2.8912, + "theoretical_loss": 3.827520277181641, + "tokens_seen": 618921984 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041033099297893685, + "loss": 2.931, + "theoretical_loss": 3.8274782200866464, + "tokens_seen": 618987520 + }, + { + "epoch": 0.19, + "learning_rate": 0.000410320962888666, + "loss": 2.665, + "theoretical_loss": 3.8274361686909035, + "tokens_seen": 619053056 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004103109327983952, + "loss": 2.6823, + "theoretical_loss": 3.827394122993037, + "tokens_seen": 619118592 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004103009027081244, + "loss": 2.8462, + "theoretical_loss": 3.827352082991672, + "tokens_seen": 619184128 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004102908726178536, + "loss": 2.964, + "theoretical_loss": 3.8273100486854332, + "tokens_seen": 619249664 + }, + { + "epoch": 0.19, + "objective/train/docs_used": 360794, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.980665445327759, + "objective/train/theoretical_loss": 3.827268020072948, + "objective/train/tokens_used": 639775200, + "theoretical_loss": 3.827268020072948, + "tokens_seen": 619315200 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041028084252758276, + "loss": 2.9076, + "theoretical_loss": 3.827268020072948, + "tokens_seen": 619315200 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041027081243731194, + "loss": 2.9329, + "theoretical_loss": 3.8272259971528406, + "tokens_seen": 619380736 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004102607823470411, + "loss": 2.8169, + "theoretical_loss": 3.8271839799237397, + "tokens_seen": 619446272 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041025075225677035, + "loss": 2.7872, + "theoretical_loss": 3.8271419683842716, + "tokens_seen": 619511808 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004102407221664995, + "loss": 2.7503, + "theoretical_loss": 3.8270999625330644, + "tokens_seen": 619577344 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004102306920762287, + "loss": 2.8183, + "theoretical_loss": 3.8270579623687464, + "tokens_seen": 619642880 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041022066198595784, + "loss": 2.9217, + "theoretical_loss": 3.8270159678899462, + "tokens_seen": 619708416 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004102106318956871, + "loss": 2.8363, + "theoretical_loss": 3.826973979095293, + "tokens_seen": 619773952 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041020060180541626, + "loss": 2.8259, + "theoretical_loss": 3.8269319959834167, + "tokens_seen": 619839488 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041019057171514544, + "loss": 2.9202, + "theoretical_loss": 3.8268900185529473, + "tokens_seen": 619905024 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004101805416248746, + "loss": 3.0615, + "theoretical_loss": 3.8268480468025157, + "tokens_seen": 619970560 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004101705115346038, + "loss": 2.9333, + "theoretical_loss": 3.826806080730753, + "tokens_seen": 620036096 + }, + { + "epoch": 0.19, + "learning_rate": 0.000410160481444333, + "loss": 2.7219, + "theoretical_loss": 3.82676412033629, + "tokens_seen": 620101632 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004101504513540622, + "loss": 2.8042, + "theoretical_loss": 3.8267221656177606, + "tokens_seen": 620167168 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041014042126379135, + "loss": 2.7574, + "theoretical_loss": 3.826680216573795, + "tokens_seen": 620232704 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004101303911735206, + "loss": 3.0536, + "theoretical_loss": 3.8266382732030277, + "tokens_seen": 620298240 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041012036108324976, + "loss": 2.8431, + "theoretical_loss": 3.8265963355040915, + "tokens_seen": 620363776 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041011033099297894, + "loss": 2.9546, + "theoretical_loss": 3.8265544034756216, + "tokens_seen": 620429312 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004101003009027081, + "loss": 2.7652, + "theoretical_loss": 3.8265124771162506, + "tokens_seen": 620494848 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004100902708124373, + "loss": 2.7911, + "theoretical_loss": 3.8264705564246153, + "tokens_seen": 620560384 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004100802407221665, + "loss": 2.7774, + "theoretical_loss": 3.8264286413993496, + "tokens_seen": 620625920 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004100702106318957, + "loss": 2.6415, + "theoretical_loss": 3.82638673203909, + "tokens_seen": 620691456 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041006018054162485, + "loss": 2.9728, + "theoretical_loss": 3.8263448283424726, + "tokens_seen": 620756992 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004100501504513541, + "loss": 2.7714, + "theoretical_loss": 3.8263029303081346, + "tokens_seen": 620822528 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004100401203610832, + "loss": 2.951, + "theoretical_loss": 3.826261037934713, + "tokens_seen": 620888064 + }, + { + "epoch": 0.19, + "objective/train/docs_used": 361341, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.8609559535980225, + "objective/train/theoretical_loss": 3.8262191512208448, + "objective/train/tokens_used": 641413600, + "theoretical_loss": 3.8262191512208448, + "tokens_seen": 620953600 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041003009027081245, + "loss": 3.0561, + "theoretical_loss": 3.8262191512208448, + "tokens_seen": 620953600 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041002006018054163, + "loss": 2.8569, + "theoretical_loss": 3.8261772701651697, + "tokens_seen": 621019136 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004100100300902708, + "loss": 2.9027, + "theoretical_loss": 3.8261353947663252, + "tokens_seen": 621084672 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041, + "loss": 3.0839, + "theoretical_loss": 3.8260935250229515, + "tokens_seen": 621150208 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040998996990972917, + "loss": 2.7312, + "theoretical_loss": 3.826051660933687, + "tokens_seen": 621215744 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040997993981945835, + "loss": 2.8798, + "theoretical_loss": 3.8260098024971727, + "tokens_seen": 621281280 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004099699097291876, + "loss": 2.9276, + "theoretical_loss": 3.825967949712049, + "tokens_seen": 621346816 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004099598796389167, + "loss": 3.1031, + "theoretical_loss": 3.8259261025769566, + "tokens_seen": 621412352 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040994984954864595, + "loss": 2.8031, + "theoretical_loss": 3.8258842610905375, + "tokens_seen": 621477888 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040993981945837513, + "loss": 2.7427, + "theoretical_loss": 3.825842425251434, + "tokens_seen": 621543424 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004099297893681043, + "loss": 2.8366, + "theoretical_loss": 3.8258005950582867, + "tokens_seen": 621608960 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004099197592778335, + "loss": 3.0436, + "theoretical_loss": 3.8257587705097404, + "tokens_seen": 621674496 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004099097291875627, + "loss": 2.9227, + "theoretical_loss": 3.825716951604438, + "tokens_seen": 621740032 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004098996990972919, + "loss": 2.7835, + "theoretical_loss": 3.8256751383410235, + "tokens_seen": 621805568 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004098896690070211, + "loss": 2.9456, + "theoretical_loss": 3.82563333071814, + "tokens_seen": 621871104 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004098796389167503, + "loss": 2.7466, + "theoretical_loss": 3.825591528734434, + "tokens_seen": 621936640 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040986960882647945, + "loss": 2.8019, + "theoretical_loss": 3.82554973238855, + "tokens_seen": 622002176 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040985957873620864, + "loss": 2.8368, + "theoretical_loss": 3.8255079416791338, + "tokens_seen": 622067712 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004098495486459378, + "loss": 2.4922, + "theoretical_loss": 3.825466156604831, + "tokens_seen": 622133248 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040983951855566705, + "loss": 2.6561, + "theoretical_loss": 3.8254243771642895, + "tokens_seen": 622198784 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004098294884653962, + "loss": 2.9194, + "theoretical_loss": 3.8253826033561555, + "tokens_seen": 622264320 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004098194583751254, + "loss": 2.9483, + "theoretical_loss": 3.8253408351790767, + "tokens_seen": 622329856 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004098094282848546, + "loss": 2.7821, + "theoretical_loss": 3.8252990726317018, + "tokens_seen": 622395392 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004097993981945838, + "loss": 2.9814, + "theoretical_loss": 3.825257315712678, + "tokens_seen": 622460928 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040978936810431296, + "loss": 3.0404, + "theoretical_loss": 3.825215564420656, + "tokens_seen": 622526464 + }, + { + "epoch": 0.19, + "objective/train/docs_used": 362042, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.844268321990967, + "objective/train/theoretical_loss": 3.8251738187542843, + "objective/train/tokens_used": 643052000, + "theoretical_loss": 3.8251738187542843, + "tokens_seen": 622592000 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040977933801404214, + "loss": 2.9836, + "theoretical_loss": 3.8251738187542843, + "tokens_seen": 622592000 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004097693079237713, + "loss": 2.9831, + "theoretical_loss": 3.8251320787122127, + "tokens_seen": 622657536 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040975927783350055, + "loss": 2.7079, + "theoretical_loss": 3.8250903442930917, + "tokens_seen": 622723072 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004097492477432297, + "loss": 2.961, + "theoretical_loss": 3.8250486154955725, + "tokens_seen": 622788608 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004097392176529589, + "loss": 2.9401, + "theoretical_loss": 3.8250068923183065, + "tokens_seen": 622854144 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040972918756268804, + "loss": 2.736, + "theoretical_loss": 3.824965174759945, + "tokens_seen": 622919680 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004097191574724173, + "loss": 3.0907, + "theoretical_loss": 3.8249234628191404, + "tokens_seen": 622985216 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040970912738214646, + "loss": 2.8815, + "theoretical_loss": 3.8248817564945465, + "tokens_seen": 623050752 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040969909729187564, + "loss": 3.0075, + "theoretical_loss": 3.8248400557848146, + "tokens_seen": 623116288 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004096890672016048, + "loss": 2.8003, + "theoretical_loss": 3.8247983606885994, + "tokens_seen": 623181824 + }, + { + "epoch": 0.19, + "learning_rate": 0.000409679037111334, + "loss": 2.833, + "theoretical_loss": 3.824756671204555, + "tokens_seen": 623247360 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004096690070210632, + "loss": 2.7482, + "theoretical_loss": 3.824714987331336, + "tokens_seen": 623312896 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004096589769307924, + "loss": 2.9531, + "theoretical_loss": 3.824673309067598, + "tokens_seen": 623378432 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040964894684052155, + "loss": 2.8552, + "theoretical_loss": 3.8246316364119948, + "tokens_seen": 623443968 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004096389167502508, + "loss": 2.9399, + "theoretical_loss": 3.824589969363184, + "tokens_seen": 623509504 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040962888665997996, + "loss": 2.9377, + "theoretical_loss": 3.8245483079198213, + "tokens_seen": 623575040 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040961885656970914, + "loss": 3.1203, + "theoretical_loss": 3.824506652080564, + "tokens_seen": 623640576 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004096088264794383, + "loss": 2.7411, + "theoretical_loss": 3.824465001844069, + "tokens_seen": 623706112 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004095987963891675, + "loss": 2.8339, + "theoretical_loss": 3.8244233572089943, + "tokens_seen": 623771648 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004095887662988967, + "loss": 2.8607, + "theoretical_loss": 3.824381718173999, + "tokens_seen": 623837184 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004095787362086259, + "loss": 3.0674, + "theoretical_loss": 3.8243400847377407, + "tokens_seen": 623902720 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040956870611835505, + "loss": 2.8358, + "theoretical_loss": 3.824298456898879, + "tokens_seen": 623968256 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004095586760280843, + "loss": 2.687, + "theoretical_loss": 3.8242568346560737, + "tokens_seen": 624033792 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004095486459378134, + "loss": 2.9343, + "theoretical_loss": 3.8242152180079847, + "tokens_seen": 624099328 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040953861584754265, + "loss": 2.9421, + "theoretical_loss": 3.8241736069532726, + "tokens_seen": 624164864 + }, + { + "epoch": 0.19, + "objective/train/docs_used": 363016, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.831054210662842, + "objective/train/theoretical_loss": 3.8241320014905993, + "objective/train/tokens_used": 644690400, + "theoretical_loss": 3.8241320014905993, + "tokens_seen": 624230400 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040952858575727183, + "loss": 2.6529, + "theoretical_loss": 3.8241320014905993, + "tokens_seen": 624230400 + }, + { + "epoch": 0.19, + "learning_rate": 0.000409518555667001, + "loss": 2.838, + "theoretical_loss": 3.824090401618625, + "tokens_seen": 624295936 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004095085255767302, + "loss": 2.8764, + "theoretical_loss": 3.8240488073360126, + "tokens_seen": 624361472 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040949849548645937, + "loss": 3.1084, + "theoretical_loss": 3.8240072186414245, + "tokens_seen": 624427008 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040948846539618855, + "loss": 3.0688, + "theoretical_loss": 3.8239656355335234, + "tokens_seen": 624492544 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004094784353059178, + "loss": 2.7764, + "theoretical_loss": 3.8239240580109723, + "tokens_seen": 624558080 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004094684052156469, + "loss": 2.8157, + "theoretical_loss": 3.8238824860724354, + "tokens_seen": 624623616 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040945837512537615, + "loss": 2.7175, + "theoretical_loss": 3.8238409197165772, + "tokens_seen": 624689152 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040944834503510533, + "loss": 2.9351, + "theoretical_loss": 3.823799358942062, + "tokens_seen": 624754688 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004094383149448345, + "loss": 2.8764, + "theoretical_loss": 3.8237578037475553, + "tokens_seen": 624820224 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004094282848545637, + "loss": 2.9202, + "theoretical_loss": 3.823716254131723, + "tokens_seen": 624885760 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004094182547642929, + "loss": 3.0774, + "theoretical_loss": 3.823674710093231, + "tokens_seen": 624951296 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040940822467402206, + "loss": 3.1001, + "theoretical_loss": 3.8236331716307452, + "tokens_seen": 625016832 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004093981945837513, + "loss": 2.6105, + "theoretical_loss": 3.8235916387429336, + "tokens_seen": 625082368 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004093881644934804, + "loss": 2.6535, + "theoretical_loss": 3.823550111428463, + "tokens_seen": 625147904 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040937813440320965, + "loss": 3.004, + "theoretical_loss": 3.8235085896860017, + "tokens_seen": 625213440 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004093681043129388, + "loss": 3.0837, + "theoretical_loss": 3.8234670735142187, + "tokens_seen": 625278976 + }, + { + "epoch": 0.19, + "learning_rate": 0.000409358074222668, + "loss": 2.8413, + "theoretical_loss": 3.8234255629117815, + "tokens_seen": 625344512 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004093480441323972, + "loss": 2.9418, + "theoretical_loss": 3.8233840578773606, + "tokens_seen": 625410048 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004093380140421264, + "loss": 2.7378, + "theoretical_loss": 3.8233425584096254, + "tokens_seen": 625475584 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040932798395185556, + "loss": 2.749, + "theoretical_loss": 3.8233010645072456, + "tokens_seen": 625541120 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004093179538615848, + "loss": 2.7658, + "theoretical_loss": 3.8232595761688932, + "tokens_seen": 625606656 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004093079237713139, + "loss": 2.7039, + "theoretical_loss": 3.8232180933932383, + "tokens_seen": 625672192 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040929789368104316, + "loss": 2.9206, + "theoretical_loss": 3.8231766161789524, + "tokens_seen": 625737728 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004092878635907723, + "loss": 2.7924, + "theoretical_loss": 3.8231351445247084, + "tokens_seen": 625803264 + }, + { + "epoch": 0.19, + "objective/train/docs_used": 363672, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.5218942165374756, + "objective/train/theoretical_loss": 3.8230936784291787, + "objective/train/tokens_used": 646328800, + "theoretical_loss": 3.8230936784291787, + "tokens_seen": 625868800 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004092778335005015, + "loss": 2.6696, + "theoretical_loss": 3.8230936784291787, + "tokens_seen": 625868800 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004092678034102307, + "loss": 2.7691, + "theoretical_loss": 3.8230522178910356, + "tokens_seen": 625934336 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004092577733199599, + "loss": 2.8193, + "theoretical_loss": 3.823010762908953, + "tokens_seen": 625999872 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040924774322968906, + "loss": 2.7196, + "theoretical_loss": 3.8229693134816043, + "tokens_seen": 626065408 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040923771313941824, + "loss": 2.8345, + "theoretical_loss": 3.822927869607664, + "tokens_seen": 626130944 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004092276830491474, + "loss": 2.7771, + "theoretical_loss": 3.8228864312858084, + "tokens_seen": 626196480 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040921765295887666, + "loss": 2.617, + "theoretical_loss": 3.822844998514711, + "tokens_seen": 626262016 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004092076228686058, + "loss": 2.8032, + "theoretical_loss": 3.822803571293048, + "tokens_seen": 626327552 + }, + { + "epoch": 0.19, + "learning_rate": 0.000409197592778335, + "loss": 3.0503, + "theoretical_loss": 3.8227621496194946, + "tokens_seen": 626393088 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040918756268806415, + "loss": 2.3918, + "theoretical_loss": 3.8227207334927296, + "tokens_seen": 626458624 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004091775325977934, + "loss": 2.663, + "theoretical_loss": 3.822679322911429, + "tokens_seen": 626524160 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040916750250752257, + "loss": 2.7565, + "theoretical_loss": 3.8226379178742693, + "tokens_seen": 626589696 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040915747241725175, + "loss": 2.6396, + "theoretical_loss": 3.82259651837993, + "tokens_seen": 626655232 + }, + { + "epoch": 0.19, + "learning_rate": 0.000409147442326981, + "loss": 2.8963, + "theoretical_loss": 3.822555124427089, + "tokens_seen": 626720768 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040913741223671016, + "loss": 2.7341, + "theoretical_loss": 3.8225137360144252, + "tokens_seen": 626786304 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040912738214643934, + "loss": 2.921, + "theoretical_loss": 3.8224723531406175, + "tokens_seen": 626851840 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004091173520561685, + "loss": 2.7428, + "theoretical_loss": 3.8224309758043464, + "tokens_seen": 626917376 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004091073219658977, + "loss": 2.8422, + "theoretical_loss": 3.8223896040042913, + "tokens_seen": 626982912 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004090972918756269, + "loss": 2.6143, + "theoretical_loss": 3.822348237739134, + "tokens_seen": 627048448 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004090872617853561, + "loss": 2.7723, + "theoretical_loss": 3.8223068770075552, + "tokens_seen": 627113984 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040907723169508525, + "loss": 2.6895, + "theoretical_loss": 3.822265521808236, + "tokens_seen": 627179520 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004090672016048145, + "loss": 2.5747, + "theoretical_loss": 3.822224172139859, + "tokens_seen": 627245056 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004090571715145436, + "loss": 2.5523, + "theoretical_loss": 3.8221828280011065, + "tokens_seen": 627310592 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040904714142427285, + "loss": 2.817, + "theoretical_loss": 3.8221414893906616, + "tokens_seen": 627376128 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040903711133400203, + "loss": 2.8352, + "theoretical_loss": 3.822100156307208, + "tokens_seen": 627441664 + }, + { + "epoch": 0.19, + "objective/train/docs_used": 364826, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.905837297439575, + "objective/train/theoretical_loss": 3.8220588287494284, + "objective/train/tokens_used": 647967200, + "theoretical_loss": 3.8220588287494284, + "tokens_seen": 627507200 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004090270812437312, + "loss": 2.828, + "theoretical_loss": 3.8220588287494284, + "tokens_seen": 627507200 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004090170511534604, + "loss": 3.0622, + "theoretical_loss": 3.8220175067160085, + "tokens_seen": 627572736 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040900702106318957, + "loss": 2.9543, + "theoretical_loss": 3.821976190205632, + "tokens_seen": 627638272 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040899699097291875, + "loss": 2.851, + "theoretical_loss": 3.821934879216985, + "tokens_seen": 627703808 + }, + { + "epoch": 0.19, + "learning_rate": 0.000408986960882648, + "loss": 2.8641, + "theoretical_loss": 3.8218935737487527, + "tokens_seen": 627769344 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004089769307923771, + "loss": 2.7105, + "theoretical_loss": 3.821852273799621, + "tokens_seen": 627834880 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040896690070210635, + "loss": 2.9023, + "theoretical_loss": 3.8218109793682773, + "tokens_seen": 627900416 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040895687061183553, + "loss": 2.7792, + "theoretical_loss": 3.821769690453407, + "tokens_seen": 627965952 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004089468405215647, + "loss": 2.6777, + "theoretical_loss": 3.8217284070536994, + "tokens_seen": 628031488 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004089368104312939, + "loss": 2.8971, + "theoretical_loss": 3.8216871291678416, + "tokens_seen": 628097024 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004089267803410231, + "loss": 2.7979, + "theoretical_loss": 3.8216458567945217, + "tokens_seen": 628162560 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040891675025075226, + "loss": 2.6513, + "theoretical_loss": 3.821604589932429, + "tokens_seen": 628228096 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004089067201604815, + "loss": 2.921, + "theoretical_loss": 3.821563328580253, + "tokens_seen": 628293632 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004088966900702106, + "loss": 2.8154, + "theoretical_loss": 3.8215220727366823, + "tokens_seen": 628359168 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040888665997993985, + "loss": 3.0392, + "theoretical_loss": 3.8214808224004075, + "tokens_seen": 628424704 + }, + { + "epoch": 0.19, + "learning_rate": 0.000408876629889669, + "loss": 3.0263, + "theoretical_loss": 3.82143957757012, + "tokens_seen": 628490240 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004088665997993982, + "loss": 2.7974, + "theoretical_loss": 3.82139833824451, + "tokens_seen": 628555776 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004088565697091274, + "loss": 2.8977, + "theoretical_loss": 3.8213571044222694, + "tokens_seen": 628621312 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004088465396188566, + "loss": 2.5061, + "theoretical_loss": 3.82131587610209, + "tokens_seen": 628686848 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040883650952858576, + "loss": 3.035, + "theoretical_loss": 3.8212746532826642, + "tokens_seen": 628752384 + }, + { + "epoch": 0.19, + "learning_rate": 0.000408826479438315, + "loss": 2.8943, + "theoretical_loss": 3.821233435962685, + "tokens_seen": 628817920 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004088164493480441, + "loss": 2.6748, + "theoretical_loss": 3.8211922241408454, + "tokens_seen": 628883456 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040880641925777336, + "loss": 2.8617, + "theoretical_loss": 3.821151017815839, + "tokens_seen": 628948992 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004087963891675025, + "loss": 3.1589, + "theoretical_loss": 3.8211098169863607, + "tokens_seen": 629014528 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004087863590772317, + "loss": 2.8188, + "theoretical_loss": 3.8210686216511043, + "tokens_seen": 629080064 + }, + { + "epoch": 0.19, + "objective/train/docs_used": 365375, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.0051121711730957, + "objective/train/theoretical_loss": 3.8210274318087656, + "objective/train/tokens_used": 649605600, + "theoretical_loss": 3.8210274318087656, + "tokens_seen": 629145600 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004087763289869609, + "loss": 2.9033, + "theoretical_loss": 3.8210274318087656, + "tokens_seen": 629145600 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004087662988966901, + "loss": 2.7331, + "theoretical_loss": 3.8209862474580394, + "tokens_seen": 629211136 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040875626880641926, + "loss": 2.962, + "theoretical_loss": 3.8209450685976227, + "tokens_seen": 629276672 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040874623871614844, + "loss": 2.5482, + "theoretical_loss": 3.820903895226211, + "tokens_seen": 629342208 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004087362086258776, + "loss": 2.8068, + "theoretical_loss": 3.820862727342501, + "tokens_seen": 629407744 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040872617853560686, + "loss": 2.7659, + "theoretical_loss": 3.8208215649451907, + "tokens_seen": 629473280 + }, + { + "epoch": 0.19, + "learning_rate": 0.000408716148445336, + "loss": 2.8289, + "theoretical_loss": 3.8207804080329772, + "tokens_seen": 629538816 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004087061183550652, + "loss": 2.8285, + "theoretical_loss": 3.8207392566045595, + "tokens_seen": 629604352 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040869608826479435, + "loss": 2.7572, + "theoretical_loss": 3.8206981106586357, + "tokens_seen": 629669888 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004086860581745236, + "loss": 3.0152, + "theoretical_loss": 3.820656970193905, + "tokens_seen": 629735424 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040867602808425277, + "loss": 3.1041, + "theoretical_loss": 3.8206158352090664, + "tokens_seen": 629800960 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040866599799398195, + "loss": 2.9477, + "theoretical_loss": 3.820574705702821, + "tokens_seen": 629866496 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040865596790371113, + "loss": 2.9567, + "theoretical_loss": 3.8205335816738684, + "tokens_seen": 629932032 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040864593781344036, + "loss": 2.8485, + "theoretical_loss": 3.8204924631209094, + "tokens_seen": 629997568 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004086359077231695, + "loss": 2.488, + "theoretical_loss": 3.820451350042646, + "tokens_seen": 630063104 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004086258776328987, + "loss": 2.843, + "theoretical_loss": 3.820410242437779, + "tokens_seen": 630128640 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040861584754262785, + "loss": 2.8967, + "theoretical_loss": 3.8203691403050106, + "tokens_seen": 630194176 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004086058174523571, + "loss": 3.094, + "theoretical_loss": 3.8203280436430447, + "tokens_seen": 630259712 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040859578736208627, + "loss": 2.9386, + "theoretical_loss": 3.8202869524505827, + "tokens_seen": 630325248 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040858575727181545, + "loss": 3.0392, + "theoretical_loss": 3.8202458667263297, + "tokens_seen": 630390784 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040857572718154463, + "loss": 2.8927, + "theoretical_loss": 3.8202047864689885, + "tokens_seen": 630456320 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004085656970912738, + "loss": 2.8877, + "theoretical_loss": 3.8201637116772638, + "tokens_seen": 630521856 + }, + { + "epoch": 0.19, + "learning_rate": 0.000408555667001003, + "loss": 2.9527, + "theoretical_loss": 3.820122642349861, + "tokens_seen": 630587392 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040854563691073223, + "loss": 2.9528, + "theoretical_loss": 3.8200815784854845, + "tokens_seen": 630652928 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040853560682046136, + "loss": 2.9753, + "theoretical_loss": 3.8200405200828405, + "tokens_seen": 630718464 + }, + { + "epoch": 0.19, + "objective/train/docs_used": 366612, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.0794413089752197, + "objective/train/theoretical_loss": 3.8199994671406348, + "objective/train/tokens_used": 651244000, + "theoretical_loss": 3.8199994671406348, + "tokens_seen": 630784000 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004085255767301906, + "loss": 2.7944, + "theoretical_loss": 3.8199994671406348, + "tokens_seen": 630784000 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004085155466399197, + "loss": 2.8903, + "theoretical_loss": 3.819958419657574, + "tokens_seen": 630849536 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040850551654964895, + "loss": 2.839, + "theoretical_loss": 3.8199173776323665, + "tokens_seen": 630915072 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040849548645937814, + "loss": 3.082, + "theoretical_loss": 3.819876341063718, + "tokens_seen": 630980608 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004084854563691073, + "loss": 2.8966, + "theoretical_loss": 3.8198353099503373, + "tokens_seen": 631046144 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004084754262788365, + "loss": 2.8914, + "theoretical_loss": 3.8197942842909325, + "tokens_seen": 631111680 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040846539618856573, + "loss": 2.9242, + "theoretical_loss": 3.8197532640842127, + "tokens_seen": 631177216 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040845536609829486, + "loss": 2.9136, + "theoretical_loss": 3.8197122493288864, + "tokens_seen": 631242752 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004084453360080241, + "loss": 2.7747, + "theoretical_loss": 3.819671240023664, + "tokens_seen": 631308288 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004084353059177532, + "loss": 3.0855, + "theoretical_loss": 3.8196302361672556, + "tokens_seen": 631373824 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040842527582748246, + "loss": 3.0077, + "theoretical_loss": 3.8195892377583713, + "tokens_seen": 631439360 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040841524573721164, + "loss": 2.7421, + "theoretical_loss": 3.8195482447957225, + "tokens_seen": 631504896 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004084052156469408, + "loss": 2.7754, + "theoretical_loss": 3.819507257278021, + "tokens_seen": 631570432 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040839518555667005, + "loss": 2.6845, + "theoretical_loss": 3.8194662752039776, + "tokens_seen": 631635968 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004083851554663992, + "loss": 2.7467, + "theoretical_loss": 3.8194252985723054, + "tokens_seen": 631701504 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004083751253761284, + "loss": 2.8189, + "theoretical_loss": 3.8193843273817167, + "tokens_seen": 631767040 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004083650952858576, + "loss": 3.0241, + "theoretical_loss": 3.819343361630925, + "tokens_seen": 631832576 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004083550651955868, + "loss": 2.7896, + "theoretical_loss": 3.8193024013186445, + "tokens_seen": 631898112 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040834503510531596, + "loss": 2.7943, + "theoretical_loss": 3.819261446443588, + "tokens_seen": 631963648 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004083350050150452, + "loss": 2.9116, + "theoretical_loss": 3.819220497004471, + "tokens_seen": 632029184 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004083249749247743, + "loss": 2.8442, + "theoretical_loss": 3.8191795530000077, + "tokens_seen": 632094720 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040831494483450356, + "loss": 2.9567, + "theoretical_loss": 3.8191386144289137, + "tokens_seen": 632160256 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004083049147442327, + "loss": 2.7852, + "theoretical_loss": 3.8190976812899056, + "tokens_seen": 632225792 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004082948846539619, + "loss": 2.7442, + "theoretical_loss": 3.819056753581699, + "tokens_seen": 632291328 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004082848545636911, + "loss": 3.0589, + "theoretical_loss": 3.8190158313030103, + "tokens_seen": 632356864 + }, + { + "epoch": 0.19, + "objective/train/docs_used": 367323, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.1488752365112305, + "objective/train/theoretical_loss": 3.818974914452557, + "objective/train/tokens_used": 652882400, + "theoretical_loss": 3.818974914452557, + "tokens_seen": 632422400 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004082748244734203, + "loss": 2.9677, + "theoretical_loss": 3.818974914452557, + "tokens_seen": 632422400 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040826479438314946, + "loss": 3.005, + "theoretical_loss": 3.8189340030290566, + "tokens_seen": 632487936 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040825476429287864, + "loss": 2.6826, + "theoretical_loss": 3.8188930970312267, + "tokens_seen": 632553472 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004082447342026078, + "loss": 2.9195, + "theoretical_loss": 3.8188521964577866, + "tokens_seen": 632619008 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040823470411233706, + "loss": 2.9644, + "theoretical_loss": 3.8188113013074547, + "tokens_seen": 632684544 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004082246740220662, + "loss": 2.8754, + "theoretical_loss": 3.8187704115789503, + "tokens_seen": 632750080 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004082146439317954, + "loss": 2.7909, + "theoretical_loss": 3.8187295272709934, + "tokens_seen": 632815616 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040820461384152455, + "loss": 2.5904, + "theoretical_loss": 3.8186886483823033, + "tokens_seen": 632881152 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004081945837512538, + "loss": 2.6039, + "theoretical_loss": 3.818647774911601, + "tokens_seen": 632946688 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040818455366098297, + "loss": 2.8456, + "theoretical_loss": 3.8186069068576085, + "tokens_seen": 633012224 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040817452357071215, + "loss": 2.7362, + "theoretical_loss": 3.818566044219046, + "tokens_seen": 633077760 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040816449348044133, + "loss": 2.9695, + "theoretical_loss": 3.818525186994636, + "tokens_seen": 633143296 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040815446339017056, + "loss": 2.7908, + "theoretical_loss": 3.8184843351831006, + "tokens_seen": 633208832 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004081444332998997, + "loss": 2.7368, + "theoretical_loss": 3.8184434887831626, + "tokens_seen": 633274368 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004081344032096289, + "loss": 2.9128, + "theoretical_loss": 3.8184026477935458, + "tokens_seen": 633339904 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040812437311935805, + "loss": 2.5912, + "theoretical_loss": 3.8183618122129737, + "tokens_seen": 633405440 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004081143430290873, + "loss": 2.8696, + "theoretical_loss": 3.818320982040169, + "tokens_seen": 633470976 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040810431293881647, + "loss": 3.0169, + "theoretical_loss": 3.8182801572738585, + "tokens_seen": 633536512 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040809428284854565, + "loss": 2.8764, + "theoretical_loss": 3.8182393379127655, + "tokens_seen": 633602048 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040808425275827483, + "loss": 2.6863, + "theoretical_loss": 3.8181985239556155, + "tokens_seen": 633667584 + }, + { + "epoch": 0.19, + "learning_rate": 0.000408074222668004, + "loss": 2.7174, + "theoretical_loss": 3.8181577154011346, + "tokens_seen": 633733120 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004080641925777332, + "loss": 3.0057, + "theoretical_loss": 3.8181169122480494, + "tokens_seen": 633798656 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040805416248746243, + "loss": 2.8978, + "theoretical_loss": 3.818076114495086, + "tokens_seen": 633864192 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040804413239719156, + "loss": 2.8676, + "theoretical_loss": 3.8180353221409717, + "tokens_seen": 633929728 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004080341023069208, + "loss": 2.925, + "theoretical_loss": 3.817994535184434, + "tokens_seen": 633995264 + }, + { + "epoch": 0.19, + "objective/train/docs_used": 367883, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.0935094356536865, + "objective/train/theoretical_loss": 3.8179537536242005, + "objective/train/tokens_used": 654520800, + "theoretical_loss": 3.8179537536242005, + "tokens_seen": 634060800 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004080240722166499, + "loss": 3.0077, + "theoretical_loss": 3.8179537536242005, + "tokens_seen": 634060800 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040801404212637915, + "loss": 2.6861, + "theoretical_loss": 3.8179129774590006, + "tokens_seen": 634126336 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040800401203610834, + "loss": 2.9862, + "theoretical_loss": 3.8178722066875626, + "tokens_seen": 634191872 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004079939819458375, + "loss": 3.0614, + "theoretical_loss": 3.8178314413086154, + "tokens_seen": 634257408 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004079839518555667, + "loss": 2.5927, + "theoretical_loss": 3.817790681320889, + "tokens_seen": 634322944 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040797392176529593, + "loss": 2.8403, + "theoretical_loss": 3.817749926723114, + "tokens_seen": 634388480 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040796389167502506, + "loss": 2.726, + "theoretical_loss": 3.8177091775140197, + "tokens_seen": 634454016 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004079538615847543, + "loss": 2.89, + "theoretical_loss": 3.8176684336923383, + "tokens_seen": 634519552 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004079438314944834, + "loss": 3.0981, + "theoretical_loss": 3.8176276952568005, + "tokens_seen": 634585088 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040793380140421266, + "loss": 2.8533, + "theoretical_loss": 3.8175869622061382, + "tokens_seen": 634650624 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040792377131394184, + "loss": 2.8895, + "theoretical_loss": 3.8175462345390843, + "tokens_seen": 634716160 + }, + { + "epoch": 0.19, + "learning_rate": 0.000407913741223671, + "loss": 2.8332, + "theoretical_loss": 3.817505512254371, + "tokens_seen": 634781696 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004079037111334002, + "loss": 3.0111, + "theoretical_loss": 3.8174647953507312, + "tokens_seen": 634847232 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004078936810431294, + "loss": 2.9046, + "theoretical_loss": 3.817424083826899, + "tokens_seen": 634912768 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040788365095285856, + "loss": 2.6449, + "theoretical_loss": 3.817383377681608, + "tokens_seen": 634978304 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004078736208625878, + "loss": 2.8857, + "theoretical_loss": 3.8173426769135927, + "tokens_seen": 635043840 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004078635907723169, + "loss": 2.8815, + "theoretical_loss": 3.8173019815215885, + "tokens_seen": 635109376 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040785356068204616, + "loss": 2.9979, + "theoretical_loss": 3.8172612915043294, + "tokens_seen": 635174912 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004078435305917753, + "loss": 3.0679, + "theoretical_loss": 3.8172206068605523, + "tokens_seen": 635240448 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004078335005015045, + "loss": 2.6561, + "theoretical_loss": 3.8171799275889926, + "tokens_seen": 635305984 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004078234704112337, + "loss": 2.7411, + "theoretical_loss": 3.8171392536883877, + "tokens_seen": 635371520 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004078134403209629, + "loss": 2.9754, + "theoretical_loss": 3.8170985851574732, + "tokens_seen": 635437056 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040780341023069207, + "loss": 2.9596, + "theoretical_loss": 3.8170579219949876, + "tokens_seen": 635502592 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004077933801404213, + "loss": 2.9114, + "theoretical_loss": 3.8170172641996682, + "tokens_seen": 635568128 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040778335005015043, + "loss": 2.9204, + "theoretical_loss": 3.816976611770254, + "tokens_seen": 635633664 + }, + { + "epoch": 0.19, + "objective/train/docs_used": 369288, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.2459752559661865, + "objective/train/theoretical_loss": 3.8169359647054835, + "objective/train/tokens_used": 656159200, + "theoretical_loss": 3.8169359647054835, + "tokens_seen": 635699200 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040777331995987966, + "loss": 2.7766, + "theoretical_loss": 3.8169359647054835, + "tokens_seen": 635699200 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004077632898696088, + "loss": 2.8547, + "theoretical_loss": 3.8168953230040947, + "tokens_seen": 635764736 + }, + { + "epoch": 0.19, + "learning_rate": 0.000407753259779338, + "loss": 2.8964, + "theoretical_loss": 3.8168546866648283, + "tokens_seen": 635830272 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004077432296890672, + "loss": 2.7998, + "theoretical_loss": 3.816814055686424, + "tokens_seen": 635895808 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004077331995987964, + "loss": 2.8474, + "theoretical_loss": 3.816773430067622, + "tokens_seen": 635961344 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040772316950852557, + "loss": 2.8972, + "theoretical_loss": 3.816732809807163, + "tokens_seen": 636026880 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040771313941825475, + "loss": 2.8695, + "theoretical_loss": 3.816692194903789, + "tokens_seen": 636092416 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040770310932798393, + "loss": 2.8568, + "theoretical_loss": 3.816651585356241, + "tokens_seen": 636157952 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040769307923771317, + "loss": 2.8861, + "theoretical_loss": 3.816610981163261, + "tokens_seen": 636223488 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004076830491474423, + "loss": 2.8353, + "theoretical_loss": 3.816570382323592, + "tokens_seen": 636289024 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040767301905717153, + "loss": 2.9671, + "theoretical_loss": 3.8165297888359766, + "tokens_seen": 636354560 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004076629889669007, + "loss": 2.9302, + "theoretical_loss": 3.8164892006991584, + "tokens_seen": 636420096 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004076529588766299, + "loss": 2.8664, + "theoretical_loss": 3.816448617911881, + "tokens_seen": 636485632 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004076429287863591, + "loss": 2.6786, + "theoretical_loss": 3.816408040472889, + "tokens_seen": 636551168 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040763289869608825, + "loss": 2.8933, + "theoretical_loss": 3.816367468380926, + "tokens_seen": 636616704 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004076228686058175, + "loss": 2.8954, + "theoretical_loss": 3.8163269016347385, + "tokens_seen": 636682240 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040761283851554667, + "loss": 3.0846, + "theoretical_loss": 3.8162863402330713, + "tokens_seen": 636747776 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040760280842527585, + "loss": 2.6784, + "theoretical_loss": 3.81624578417467, + "tokens_seen": 636813312 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040759277833500503, + "loss": 2.8148, + "theoretical_loss": 3.816205233458281, + "tokens_seen": 636878848 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004075827482447342, + "loss": 2.6165, + "theoretical_loss": 3.816164688082652, + "tokens_seen": 636944384 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004075727181544634, + "loss": 2.7548, + "theoretical_loss": 3.816124148046529, + "tokens_seen": 637009920 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040756268806419263, + "loss": 2.7754, + "theoretical_loss": 3.81608361334866, + "tokens_seen": 637075456 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040755265797392176, + "loss": 2.6995, + "theoretical_loss": 3.8160430839877937, + "tokens_seen": 637140992 + }, + { + "epoch": 0.19, + "learning_rate": 0.000407542627883651, + "loss": 2.9601, + "theoretical_loss": 3.8160025599626777, + "tokens_seen": 637206528 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004075325977933801, + "loss": 2.8804, + "theoretical_loss": 3.815962041272061, + "tokens_seen": 637272064 + }, + { + "epoch": 0.19, + "objective/train/docs_used": 369983, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.851693630218506, + "objective/train/theoretical_loss": 3.815921527914693, + "objective/train/tokens_used": 657797600, + "theoretical_loss": 3.815921527914693, + "tokens_seen": 637337600 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040752256770310935, + "loss": 2.9615, + "theoretical_loss": 3.815921527914693, + "tokens_seen": 637337600 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040751253761283854, + "loss": 2.7468, + "theoretical_loss": 3.815881019889324, + "tokens_seen": 637403136 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004075025075225677, + "loss": 3.1054, + "theoretical_loss": 3.8158405171947027, + "tokens_seen": 637468672 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004074924774322969, + "loss": 3.0256, + "theoretical_loss": 3.8158000198295814, + "tokens_seen": 637534208 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040748244734202613, + "loss": 2.7194, + "theoretical_loss": 3.81575952779271, + "tokens_seen": 637599744 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040747241725175526, + "loss": 2.9201, + "theoretical_loss": 3.81571904108284, + "tokens_seen": 637665280 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004074623871614845, + "loss": 2.6508, + "theoretical_loss": 3.815678559698723, + "tokens_seen": 637730816 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004074523570712136, + "loss": 2.6811, + "theoretical_loss": 3.8156380836391124, + "tokens_seen": 637796352 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040744232698094286, + "loss": 3.0323, + "theoretical_loss": 3.8155976129027596, + "tokens_seen": 637861888 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040743229689067204, + "loss": 3.0314, + "theoretical_loss": 3.815557147488418, + "tokens_seen": 637927424 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004074222668004012, + "loss": 2.9351, + "theoretical_loss": 3.8155166873948416, + "tokens_seen": 637992960 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004074122367101304, + "loss": 2.7764, + "theoretical_loss": 3.815476232620784, + "tokens_seen": 638058496 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004074022066198596, + "loss": 2.6456, + "theoretical_loss": 3.8154357831649994, + "tokens_seen": 638124032 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040739217652958876, + "loss": 2.984, + "theoretical_loss": 3.8153953390262427, + "tokens_seen": 638189568 + }, + { + "epoch": 0.19, + "learning_rate": 0.000407382146439318, + "loss": 2.9874, + "theoretical_loss": 3.815354900203269, + "tokens_seen": 638255104 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004073721163490471, + "loss": 3.0343, + "theoretical_loss": 3.8153144666948338, + "tokens_seen": 638320640 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040736208625877636, + "loss": 2.824, + "theoretical_loss": 3.8152740384996937, + "tokens_seen": 638386176 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004073520561685055, + "loss": 3.0618, + "theoretical_loss": 3.815233615616605, + "tokens_seen": 638451712 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004073420260782347, + "loss": 2.9456, + "theoretical_loss": 3.815193198044324, + "tokens_seen": 638517248 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004073319959879639, + "loss": 2.7513, + "theoretical_loss": 3.8151527857816077, + "tokens_seen": 638582784 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004073219658976931, + "loss": 2.8352, + "theoretical_loss": 3.815112378827215, + "tokens_seen": 638648320 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040731193580742227, + "loss": 2.8251, + "theoretical_loss": 3.815071977179903, + "tokens_seen": 638713856 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004073019057171515, + "loss": 2.9968, + "theoretical_loss": 3.815031580838431, + "tokens_seen": 638779392 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040729187562688063, + "loss": 2.7154, + "theoretical_loss": 3.814991189801557, + "tokens_seen": 638844928 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040728184553660986, + "loss": 2.7474, + "theoretical_loss": 3.8149508040680415, + "tokens_seen": 638910464 + }, + { + "epoch": 0.19, + "objective/train/docs_used": 370882, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.5294876098632812, + "objective/train/theoretical_loss": 3.8149104236366433, + "objective/train/tokens_used": 659436000, + "theoretical_loss": 3.8149104236366433, + "tokens_seen": 638976000 + }, + { + "epoch": 0.19, + "learning_rate": 0.000407271815446339, + "loss": 2.7506, + "theoretical_loss": 3.8149104236366433, + "tokens_seen": 638976000 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004072617853560682, + "loss": 3.0415, + "theoretical_loss": 3.8148700485061235, + "tokens_seen": 639041536 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004072517552657974, + "loss": 2.7941, + "theoretical_loss": 3.814829678675242, + "tokens_seen": 639107072 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004072417251755266, + "loss": 2.8105, + "theoretical_loss": 3.81478931414276, + "tokens_seen": 639172608 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040723169508525577, + "loss": 2.9813, + "theoretical_loss": 3.814748954907439, + "tokens_seen": 639238144 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040722166499498495, + "loss": 2.8723, + "theoretical_loss": 3.814708600968041, + "tokens_seen": 639303680 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040721163490471413, + "loss": 3.1523, + "theoretical_loss": 3.8146682523233286, + "tokens_seen": 639369216 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040720160481444337, + "loss": 2.9348, + "theoretical_loss": 3.8146279089720636, + "tokens_seen": 639434752 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004071915747241725, + "loss": 2.9485, + "theoretical_loss": 3.8145875709130093, + "tokens_seen": 639500288 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040718154463390173, + "loss": 2.6863, + "theoretical_loss": 3.8145472381449297, + "tokens_seen": 639565824 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004071715145436309, + "loss": 3.1441, + "theoretical_loss": 3.814506910666589, + "tokens_seen": 639631360 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004071614844533601, + "loss": 2.8225, + "theoretical_loss": 3.814466588476751, + "tokens_seen": 639696896 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040715145436308927, + "loss": 3.036, + "theoretical_loss": 3.81442627157418, + "tokens_seen": 639762432 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040714142427281845, + "loss": 2.8594, + "theoretical_loss": 3.814385959957643, + "tokens_seen": 639827968 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040713139418254763, + "loss": 2.8475, + "theoretical_loss": 3.8143456536259035, + "tokens_seen": 639893504 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040712136409227687, + "loss": 2.9374, + "theoretical_loss": 3.814305352577729, + "tokens_seen": 639959040 + }, + { + "epoch": 0.19, + "learning_rate": 0.000407111334002006, + "loss": 3.0184, + "theoretical_loss": 3.8142650568118848, + "tokens_seen": 640024576 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040710130391173523, + "loss": 2.9936, + "theoretical_loss": 3.8142247663271385, + "tokens_seen": 640090112 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040709127382146436, + "loss": 3.006, + "theoretical_loss": 3.814184481122257, + "tokens_seen": 640155648 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004070812437311936, + "loss": 2.9384, + "theoretical_loss": 3.8141442011960085, + "tokens_seen": 640221184 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004070712136409228, + "loss": 2.69, + "theoretical_loss": 3.814103926547161, + "tokens_seen": 640286720 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040706118355065196, + "loss": 2.8453, + "theoretical_loss": 3.8140636571744824, + "tokens_seen": 640352256 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040705115346038114, + "loss": 2.7686, + "theoretical_loss": 3.8140233930767424, + "tokens_seen": 640417792 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004070411233701103, + "loss": 2.8797, + "theoretical_loss": 3.8139831342527093, + "tokens_seen": 640483328 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004070310932798395, + "loss": 2.7827, + "theoretical_loss": 3.8139428807011537, + "tokens_seen": 640548864 + }, + { + "epoch": 0.19, + "objective/train/docs_used": 371543, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.3982415199279785, + "objective/train/theoretical_loss": 3.8139026324208456, + "objective/train/tokens_used": 661074400, + "theoretical_loss": 3.8139026324208456, + "tokens_seen": 640614400 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040702106318956874, + "loss": 2.9498, + "theoretical_loss": 3.8139026324208456, + "tokens_seen": 640614400 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040701103309929786, + "loss": 2.8623, + "theoretical_loss": 3.8138623894105557, + "tokens_seen": 640679936 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004070010030090271, + "loss": 2.9424, + "theoretical_loss": 3.8138221516690547, + "tokens_seen": 640745472 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004069909729187563, + "loss": 3.0576, + "theoretical_loss": 3.813781919195114, + "tokens_seen": 640811008 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040698094282848546, + "loss": 2.9864, + "theoretical_loss": 3.813741691987506, + "tokens_seen": 640876544 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040697091273821464, + "loss": 2.8182, + "theoretical_loss": 3.813701470045001, + "tokens_seen": 640942080 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004069608826479438, + "loss": 3.0685, + "theoretical_loss": 3.8136612533663747, + "tokens_seen": 641007616 + }, + { + "epoch": 0.19, + "learning_rate": 0.000406950852557673, + "loss": 2.973, + "theoretical_loss": 3.8136210419503977, + "tokens_seen": 641073152 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040694082246740224, + "loss": 3.2138, + "theoretical_loss": 3.813580835795844, + "tokens_seen": 641138688 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040693079237713137, + "loss": 2.922, + "theoretical_loss": 3.8135406349014884, + "tokens_seen": 641204224 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004069207622868606, + "loss": 2.8798, + "theoretical_loss": 3.8135004392661047, + "tokens_seen": 641269760 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040691073219658973, + "loss": 2.8194, + "theoretical_loss": 3.8134602488884664, + "tokens_seen": 641335296 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040690070210631896, + "loss": 2.6574, + "theoretical_loss": 3.8134200637673503, + "tokens_seen": 641400832 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004068906720160482, + "loss": 2.9714, + "theoretical_loss": 3.813379883901531, + "tokens_seen": 641466368 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004068806419257773, + "loss": 2.8454, + "theoretical_loss": 3.8133397092897843, + "tokens_seen": 641531904 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040687061183550656, + "loss": 2.9101, + "theoretical_loss": 3.8132995399308873, + "tokens_seen": 641597440 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004068605817452357, + "loss": 2.9595, + "theoretical_loss": 3.813259375823616, + "tokens_seen": 641662976 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004068505516549649, + "loss": 2.8636, + "theoretical_loss": 3.8132192169667483, + "tokens_seen": 641728512 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004068405215646941, + "loss": 2.8914, + "theoretical_loss": 3.813179063359061, + "tokens_seen": 641794048 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004068304914744233, + "loss": 2.7662, + "theoretical_loss": 3.813138914999332, + "tokens_seen": 641859584 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040682046138415247, + "loss": 2.9072, + "theoretical_loss": 3.8130987718863407, + "tokens_seen": 641925120 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004068104312938817, + "loss": 2.8274, + "theoretical_loss": 3.813058634018865, + "tokens_seen": 641990656 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040680040120361083, + "loss": 2.7849, + "theoretical_loss": 3.8130185013956837, + "tokens_seen": 642056192 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040679037111334006, + "loss": 3.0593, + "theoretical_loss": 3.812978374015578, + "tokens_seen": 642121728 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004067803410230692, + "loss": 2.9887, + "theoretical_loss": 3.8129382518773265, + "tokens_seen": 642187264 + }, + { + "epoch": 0.19, + "objective/train/docs_used": 373013, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.1482465267181396, + "objective/train/theoretical_loss": 3.8128981349797098, + "objective/train/tokens_used": 662712800, + "theoretical_loss": 3.8128981349797098, + "tokens_seen": 642252800 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004067703109327984, + "loss": 3.1431, + "theoretical_loss": 3.8128981349797098, + "tokens_seen": 642252800 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004067602808425276, + "loss": 2.8903, + "theoretical_loss": 3.8128580233215095, + "tokens_seen": 642318336 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004067502507522568, + "loss": 2.9454, + "theoretical_loss": 3.8128179169015057, + "tokens_seen": 642383872 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040674022066198597, + "loss": 3.1583, + "theoretical_loss": 3.8127778157184817, + "tokens_seen": 642449408 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040673019057171515, + "loss": 2.5443, + "theoretical_loss": 3.8127377197712176, + "tokens_seen": 642514944 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040672016048144433, + "loss": 2.8597, + "theoretical_loss": 3.8126976290584973, + "tokens_seen": 642580480 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040671013039117357, + "loss": 2.929, + "theoretical_loss": 3.812657543579103, + "tokens_seen": 642646016 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004067001003009027, + "loss": 2.7431, + "theoretical_loss": 3.812617463331818, + "tokens_seen": 642711552 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040669007021063193, + "loss": 3.1672, + "theoretical_loss": 3.812577388315426, + "tokens_seen": 642777088 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004066800401203611, + "loss": 2.7501, + "theoretical_loss": 3.8125373185287117, + "tokens_seen": 642842624 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004066700100300903, + "loss": 2.9092, + "theoretical_loss": 3.812497253970459, + "tokens_seen": 642908160 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040665997993981947, + "loss": 2.6322, + "theoretical_loss": 3.8124571946394528, + "tokens_seen": 642973696 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040664994984954865, + "loss": 2.9578, + "theoretical_loss": 3.812417140534478, + "tokens_seen": 643039232 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040663991975927783, + "loss": 3.0529, + "theoretical_loss": 3.8123770916543216, + "tokens_seen": 643104768 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040662988966900707, + "loss": 2.7786, + "theoretical_loss": 3.8123370479977687, + "tokens_seen": 643170304 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004066198595787362, + "loss": 2.7889, + "theoretical_loss": 3.812297009563606, + "tokens_seen": 643235840 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040660982948846543, + "loss": 2.9252, + "theoretical_loss": 3.8122569763506204, + "tokens_seen": 643301376 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040659979939819456, + "loss": 2.8931, + "theoretical_loss": 3.8122169483576, + "tokens_seen": 643366912 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004065897693079238, + "loss": 3.0525, + "theoretical_loss": 3.812176925583331, + "tokens_seen": 643432448 + }, + { + "epoch": 0.19, + "learning_rate": 0.000406579739217653, + "loss": 2.9719, + "theoretical_loss": 3.812136908026603, + "tokens_seen": 643497984 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040656970912738216, + "loss": 2.6867, + "theoretical_loss": 3.8120968956862034, + "tokens_seen": 643563520 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040655967903711134, + "loss": 2.8752, + "theoretical_loss": 3.8120568885609223, + "tokens_seen": 643629056 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004065496489468405, + "loss": 2.9349, + "theoretical_loss": 3.8120168866495483, + "tokens_seen": 643694592 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004065396188565697, + "loss": 3.0, + "theoretical_loss": 3.8119768899508713, + "tokens_seen": 643760128 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040652958876629894, + "loss": 2.826, + "theoretical_loss": 3.8119368984636814, + "tokens_seen": 643825664 + }, + { + "epoch": 0.2, + "objective/train/docs_used": 373533, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.1652634143829346, + "objective/train/theoretical_loss": 3.8118969121867696, + "objective/train/tokens_used": 664351200, + "theoretical_loss": 3.8118969121867696, + "tokens_seen": 643891200 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040651955867602806, + "loss": 3.0012, + "theoretical_loss": 3.8118969121867696, + "tokens_seen": 643891200 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004065095285857573, + "loss": 2.8095, + "theoretical_loss": 3.8118569311189265, + "tokens_seen": 643956736 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004064994984954865, + "loss": 3.0759, + "theoretical_loss": 3.8118169552589434, + "tokens_seen": 644022272 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040648946840521566, + "loss": 3.0136, + "theoretical_loss": 3.811776984605612, + "tokens_seen": 644087808 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040647943831494484, + "loss": 2.9811, + "theoretical_loss": 3.811737019157725, + "tokens_seen": 644153344 + }, + { + "epoch": 0.2, + "learning_rate": 0.000406469408224674, + "loss": 3.0463, + "theoretical_loss": 3.811697058914075, + "tokens_seen": 644218880 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004064593781344032, + "loss": 2.9082, + "theoretical_loss": 3.8116571038734546, + "tokens_seen": 644284416 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040644934804413244, + "loss": 2.846, + "theoretical_loss": 3.811617154034657, + "tokens_seen": 644349952 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040643931795386157, + "loss": 2.7425, + "theoretical_loss": 3.811577209396477, + "tokens_seen": 644415488 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004064292878635908, + "loss": 2.8223, + "theoretical_loss": 3.8115372699577073, + "tokens_seen": 644481024 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040641925777331993, + "loss": 2.872, + "theoretical_loss": 3.8114973357171436, + "tokens_seen": 644546560 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040640922768304916, + "loss": 2.8424, + "theoretical_loss": 3.8114574066735805, + "tokens_seen": 644612096 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040639919759277834, + "loss": 3.068, + "theoretical_loss": 3.8114174828258136, + "tokens_seen": 644677632 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004063891675025075, + "loss": 3.0501, + "theoretical_loss": 3.8113775641726386, + "tokens_seen": 644743168 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004063791374122367, + "loss": 2.7714, + "theoretical_loss": 3.8113376507128516, + "tokens_seen": 644808704 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004063691073219659, + "loss": 2.6974, + "theoretical_loss": 3.81129774244525, + "tokens_seen": 644874240 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040635907723169507, + "loss": 2.7727, + "theoretical_loss": 3.811257839368629, + "tokens_seen": 644939776 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004063490471414243, + "loss": 2.6825, + "theoretical_loss": 3.8112179414817877, + "tokens_seen": 645005312 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040633901705115343, + "loss": 2.868, + "theoretical_loss": 3.811178048783524, + "tokens_seen": 645070848 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040632898696088267, + "loss": 2.8024, + "theoretical_loss": 3.8111381612726345, + "tokens_seen": 645136384 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040631895687061185, + "loss": 2.8647, + "theoretical_loss": 3.8110982789479193, + "tokens_seen": 645201920 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040630892678034103, + "loss": 2.6108, + "theoretical_loss": 3.811058401808177, + "tokens_seen": 645267456 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004062988966900702, + "loss": 3.045, + "theoretical_loss": 3.8110185298522063, + "tokens_seen": 645332992 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004062888665997994, + "loss": 3.1537, + "theoretical_loss": 3.8109786630788083, + "tokens_seen": 645398528 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040627883650952857, + "loss": 2.8788, + "theoretical_loss": 3.8109388014867824, + "tokens_seen": 645464064 + }, + { + "epoch": 0.2, + "objective/train/docs_used": 374553, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.371049642562866, + "objective/train/theoretical_loss": 3.8108989450749293, + "objective/train/tokens_used": 665989600, + "theoretical_loss": 3.8108989450749293, + "tokens_seen": 645529600 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004062688064192578, + "loss": 2.9515, + "theoretical_loss": 3.8108989450749293, + "tokens_seen": 645529600 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040625877632898693, + "loss": 2.927, + "theoretical_loss": 3.81085909384205, + "tokens_seen": 645595136 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040624874623871617, + "loss": 2.9951, + "theoretical_loss": 3.810819247786946, + "tokens_seen": 645660672 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004062387161484453, + "loss": 2.893, + "theoretical_loss": 3.810779406908419, + "tokens_seen": 645726208 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040622868605817453, + "loss": 3.078, + "theoretical_loss": 3.8107395712052714, + "tokens_seen": 645791744 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004062186559679037, + "loss": 3.012, + "theoretical_loss": 3.810699740676306, + "tokens_seen": 645857280 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004062086258776329, + "loss": 2.8733, + "theoretical_loss": 3.8106599153203256, + "tokens_seen": 645922816 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004061985957873621, + "loss": 3.2076, + "theoretical_loss": 3.810620095136133, + "tokens_seen": 645988352 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040618856569709126, + "loss": 3.0625, + "theoretical_loss": 3.810580280122533, + "tokens_seen": 646053888 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040617853560682044, + "loss": 2.9701, + "theoretical_loss": 3.8105404702783288, + "tokens_seen": 646119424 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040616850551654967, + "loss": 2.7734, + "theoretical_loss": 3.8105006656023264, + "tokens_seen": 646184960 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004061584754262788, + "loss": 2.9347, + "theoretical_loss": 3.810460866093329, + "tokens_seen": 646250496 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040614844533600804, + "loss": 2.8789, + "theoretical_loss": 3.8104210717501434, + "tokens_seen": 646316032 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040613841524573727, + "loss": 2.9045, + "theoretical_loss": 3.8103812825715746, + "tokens_seen": 646381568 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004061283851554664, + "loss": 3.0062, + "theoretical_loss": 3.8103414985564292, + "tokens_seen": 646447104 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040611835506519563, + "loss": 3.077, + "theoretical_loss": 3.8103017197035145, + "tokens_seen": 646512640 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040610832497492476, + "loss": 2.7043, + "theoretical_loss": 3.810261946011636, + "tokens_seen": 646578176 + }, + { + "epoch": 0.2, + "learning_rate": 0.000406098294884654, + "loss": 2.8232, + "theoretical_loss": 3.8102221774796012, + "tokens_seen": 646643712 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004060882647943832, + "loss": 2.9394, + "theoretical_loss": 3.810182414106219, + "tokens_seen": 646709248 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040607823470411236, + "loss": 2.7536, + "theoretical_loss": 3.810142655890297, + "tokens_seen": 646774784 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040606820461384154, + "loss": 2.8398, + "theoretical_loss": 3.8101029028306437, + "tokens_seen": 646840320 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004060581745235707, + "loss": 3.0846, + "theoretical_loss": 3.810063154926068, + "tokens_seen": 646905856 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004060481444332999, + "loss": 3.0297, + "theoretical_loss": 3.8100234121753798, + "tokens_seen": 646971392 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040603811434302914, + "loss": 2.9815, + "theoretical_loss": 3.8099836745773885, + "tokens_seen": 647036928 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040602808425275826, + "loss": 2.9782, + "theoretical_loss": 3.8099439421309036, + "tokens_seen": 647102464 + }, + { + "epoch": 0.2, + "objective/train/docs_used": 375146, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.913668632507324, + "objective/train/theoretical_loss": 3.8099042148347366, + "objective/train/tokens_used": 667628000, + "theoretical_loss": 3.8099042148347366, + "tokens_seen": 647168000 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004060180541624875, + "loss": 2.7377, + "theoretical_loss": 3.8099042148347366, + "tokens_seen": 647168000 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004060080240722167, + "loss": 2.86, + "theoretical_loss": 3.809864492687698, + "tokens_seen": 647233536 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040599799398194586, + "loss": 2.9777, + "theoretical_loss": 3.8098247756885995, + "tokens_seen": 647299072 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040598796389167504, + "loss": 2.8704, + "theoretical_loss": 3.8097850638362516, + "tokens_seen": 647364608 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004059779338014042, + "loss": 3.0376, + "theoretical_loss": 3.8097453571294686, + "tokens_seen": 647430144 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004059679037111334, + "loss": 2.9493, + "theoretical_loss": 3.8097056555670608, + "tokens_seen": 647495680 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040595787362086264, + "loss": 2.9258, + "theoretical_loss": 3.809665959147843, + "tokens_seen": 647561216 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040594784353059177, + "loss": 2.9036, + "theoretical_loss": 3.8096262678706267, + "tokens_seen": 647626752 + }, + { + "epoch": 0.2, + "learning_rate": 0.000405937813440321, + "loss": 3.0442, + "theoretical_loss": 3.8095865817342274, + "tokens_seen": 647692288 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040592778335005013, + "loss": 2.8566, + "theoretical_loss": 3.809546900737457, + "tokens_seen": 647757824 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040591775325977936, + "loss": 3.1634, + "theoretical_loss": 3.809507224879132, + "tokens_seen": 647823360 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040590772316950854, + "loss": 2.9598, + "theoretical_loss": 3.809467554158067, + "tokens_seen": 647888896 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004058976930792377, + "loss": 3.0157, + "theoretical_loss": 3.809427888573077, + "tokens_seen": 647954432 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004058876629889669, + "loss": 3.0597, + "theoretical_loss": 3.8093882281229767, + "tokens_seen": 648019968 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004058776328986961, + "loss": 3.0888, + "theoretical_loss": 3.8093485728065835, + "tokens_seen": 648085504 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040586760280842527, + "loss": 2.7799, + "theoretical_loss": 3.809308922622713, + "tokens_seen": 648151040 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004058575727181545, + "loss": 3.0024, + "theoretical_loss": 3.8092692775701824, + "tokens_seen": 648216576 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040584754262788363, + "loss": 3.0655, + "theoretical_loss": 3.8092296376478094, + "tokens_seen": 648282112 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040583751253761287, + "loss": 3.1614, + "theoretical_loss": 3.8091900028544106, + "tokens_seen": 648347648 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040582748244734205, + "loss": 2.7886, + "theoretical_loss": 3.809150373188805, + "tokens_seen": 648413184 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040581745235707123, + "loss": 3.016, + "theoretical_loss": 3.80911074864981, + "tokens_seen": 648478720 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004058074222668004, + "loss": 2.9548, + "theoretical_loss": 3.8090711292362456, + "tokens_seen": 648544256 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004057973921765296, + "loss": 2.8439, + "theoretical_loss": 3.80903151494693, + "tokens_seen": 648609792 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040578736208625877, + "loss": 3.0615, + "theoretical_loss": 3.808991905780683, + "tokens_seen": 648675328 + }, + { + "epoch": 0.2, + "learning_rate": 0.000405777331995988, + "loss": 2.933, + "theoretical_loss": 3.8089523017363254, + "tokens_seen": 648740864 + }, + { + "epoch": 0.2, + "objective/train/docs_used": 375860, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.4426538944244385, + "objective/train/theoretical_loss": 3.8089127028126764, + "objective/train/tokens_used": 669266400, + "theoretical_loss": 3.8089127028126764, + "tokens_seen": 648806400 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040576730190571713, + "loss": 2.8438, + "theoretical_loss": 3.8089127028126764, + "tokens_seen": 648806400 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040575727181544637, + "loss": 2.7353, + "theoretical_loss": 3.8088731090085575, + "tokens_seen": 648871936 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004057472417251755, + "loss": 3.3057, + "theoretical_loss": 3.8088335203227897, + "tokens_seen": 648937472 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040573721163490473, + "loss": 2.923, + "theoretical_loss": 3.8087939367541943, + "tokens_seen": 649003008 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004057271815446339, + "loss": 2.972, + "theoretical_loss": 3.8087543583015933, + "tokens_seen": 649068544 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004057171514543631, + "loss": 2.7037, + "theoretical_loss": 3.8087147849638097, + "tokens_seen": 649134080 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004057071213640923, + "loss": 2.9425, + "theoretical_loss": 3.808675216739665, + "tokens_seen": 649199616 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040569709127382146, + "loss": 3.2247, + "theoretical_loss": 3.808635653627983, + "tokens_seen": 649265152 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040568706118355064, + "loss": 3.0832, + "theoretical_loss": 3.8085960956275873, + "tokens_seen": 649330688 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004056770310932799, + "loss": 2.9939, + "theoretical_loss": 3.8085565427373016, + "tokens_seen": 649396224 + }, + { + "epoch": 0.2, + "learning_rate": 0.000405667001003009, + "loss": 2.7575, + "theoretical_loss": 3.8085169949559505, + "tokens_seen": 649461760 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040565697091273824, + "loss": 3.0567, + "theoretical_loss": 3.808477452282358, + "tokens_seen": 649527296 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004056469408224674, + "loss": 2.682, + "theoretical_loss": 3.8084379147153493, + "tokens_seen": 649592832 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004056369107321966, + "loss": 3.0957, + "theoretical_loss": 3.808398382253751, + "tokens_seen": 649658368 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004056268806419258, + "loss": 2.8599, + "theoretical_loss": 3.8083588548963867, + "tokens_seen": 649723904 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040561685055165496, + "loss": 2.8648, + "theoretical_loss": 3.8083193326420846, + "tokens_seen": 649789440 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040560682046138414, + "loss": 2.9996, + "theoretical_loss": 3.808279815489671, + "tokens_seen": 649854976 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004055967903711134, + "loss": 2.986, + "theoretical_loss": 3.8082403034379713, + "tokens_seen": 649920512 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004055867602808425, + "loss": 2.9914, + "theoretical_loss": 3.808200796485815, + "tokens_seen": 649986048 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040557673019057174, + "loss": 3.0049, + "theoretical_loss": 3.8081612946320287, + "tokens_seen": 650051584 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040556670010030087, + "loss": 2.9477, + "theoretical_loss": 3.8081217978754407, + "tokens_seen": 650117120 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004055566700100301, + "loss": 2.9083, + "theoretical_loss": 3.80808230621488, + "tokens_seen": 650182656 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004055466399197593, + "loss": 2.9676, + "theoretical_loss": 3.8080428196491747, + "tokens_seen": 650248192 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040553660982948846, + "loss": 2.9914, + "theoretical_loss": 3.808003338177154, + "tokens_seen": 650313728 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040552657973921764, + "loss": 2.7487, + "theoretical_loss": 3.8079638617976492, + "tokens_seen": 650379264 + }, + { + "epoch": 0.2, + "objective/train/docs_used": 377374, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.0465145111083984, + "objective/train/theoretical_loss": 3.8079243905094886, + "objective/train/tokens_used": 670904800, + "theoretical_loss": 3.8079243905094886, + "tokens_seen": 650444800 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004055165496489469, + "loss": 2.9739, + "theoretical_loss": 3.8079243905094886, + "tokens_seen": 650444800 + }, + { + "epoch": 0.2, + "learning_rate": 0.000405506519558676, + "loss": 3.0502, + "theoretical_loss": 3.8078849243115043, + "tokens_seen": 650510336 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040549648946840524, + "loss": 3.146, + "theoretical_loss": 3.8078454632025256, + "tokens_seen": 650575872 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040548645937813437, + "loss": 2.8227, + "theoretical_loss": 3.8078060071813846, + "tokens_seen": 650641408 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004054764292878636, + "loss": 2.9306, + "theoretical_loss": 3.8077665562469134, + "tokens_seen": 650706944 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004054663991975928, + "loss": 3.0301, + "theoretical_loss": 3.807727110397942, + "tokens_seen": 650772480 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040545636910732197, + "loss": 3.0228, + "theoretical_loss": 3.8076876696333057, + "tokens_seen": 650838016 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040544633901705115, + "loss": 2.9481, + "theoretical_loss": 3.807648233951835, + "tokens_seen": 650903552 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040543630892678033, + "loss": 2.7749, + "theoretical_loss": 3.8076088033523643, + "tokens_seen": 650969088 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004054262788365095, + "loss": 2.8163, + "theoretical_loss": 3.807569377833726, + "tokens_seen": 651034624 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040541624874623874, + "loss": 2.7201, + "theoretical_loss": 3.8075299573947556, + "tokens_seen": 651100160 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040540621865596787, + "loss": 3.0412, + "theoretical_loss": 3.8074905420342864, + "tokens_seen": 651165696 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004053961885656971, + "loss": 2.8397, + "theoretical_loss": 3.8074511317511535, + "tokens_seen": 651231232 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004053861584754263, + "loss": 3.1449, + "theoretical_loss": 3.807411726544191, + "tokens_seen": 651296768 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040537612838515547, + "loss": 2.8009, + "theoretical_loss": 3.8073723264122363, + "tokens_seen": 651362304 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004053660982948847, + "loss": 2.9038, + "theoretical_loss": 3.8073329313541233, + "tokens_seen": 651427840 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040535606820461383, + "loss": 2.8607, + "theoretical_loss": 3.80729354136869, + "tokens_seen": 651493376 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040534603811434307, + "loss": 3.0246, + "theoretical_loss": 3.807254156454772, + "tokens_seen": 651558912 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040533600802407225, + "loss": 2.9812, + "theoretical_loss": 3.807214776611206, + "tokens_seen": 651624448 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040532597793380143, + "loss": 2.7247, + "theoretical_loss": 3.8071754018368305, + "tokens_seen": 651689984 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004053159478435306, + "loss": 2.9776, + "theoretical_loss": 3.807136032130483, + "tokens_seen": 651755520 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004053059177532598, + "loss": 2.9831, + "theoretical_loss": 3.8070966674910007, + "tokens_seen": 651821056 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040529588766298897, + "loss": 2.7516, + "theoretical_loss": 3.8070573079172227, + "tokens_seen": 651886592 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004052858575727182, + "loss": 2.8783, + "theoretical_loss": 3.807017953407989, + "tokens_seen": 651952128 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040527582748244733, + "loss": 3.0798, + "theoretical_loss": 3.806978603962137, + "tokens_seen": 652017664 + }, + { + "epoch": 0.2, + "objective/train/docs_used": 377873, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.429914951324463, + "objective/train/theoretical_loss": 3.8069392595785083, + "objective/train/tokens_used": 672543200, + "theoretical_loss": 3.8069392595785083, + "tokens_seen": 652083200 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040526579739217657, + "loss": 2.6429, + "theoretical_loss": 3.8069392595785083, + "tokens_seen": 652083200 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004052557673019057, + "loss": 2.9397, + "theoretical_loss": 3.806899920255942, + "tokens_seen": 652148736 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040524573721163493, + "loss": 3.0315, + "theoretical_loss": 3.8068605859932783, + "tokens_seen": 652214272 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004052357071213641, + "loss": 3.0197, + "theoretical_loss": 3.806821256789359, + "tokens_seen": 652279808 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004052256770310933, + "loss": 2.8544, + "theoretical_loss": 3.806781932643024, + "tokens_seen": 652345344 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004052156469408225, + "loss": 2.7467, + "theoretical_loss": 3.8067426135531157, + "tokens_seen": 652410880 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040520561685055166, + "loss": 3.1048, + "theoretical_loss": 3.8067032995184764, + "tokens_seen": 652476416 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040519558676028084, + "loss": 2.7708, + "theoretical_loss": 3.8066639905379485, + "tokens_seen": 652541952 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004051855566700101, + "loss": 2.9568, + "theoretical_loss": 3.806624686610374, + "tokens_seen": 652607488 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004051755265797392, + "loss": 2.9305, + "theoretical_loss": 3.8065853877345965, + "tokens_seen": 652673024 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040516549648946844, + "loss": 3.0848, + "theoretical_loss": 3.8065460939094597, + "tokens_seen": 652738560 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004051554663991976, + "loss": 3.0073, + "theoretical_loss": 3.8065068051338073, + "tokens_seen": 652804096 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004051454363089268, + "loss": 2.8468, + "theoretical_loss": 3.806467521406484, + "tokens_seen": 652869632 + }, + { + "epoch": 0.2, + "learning_rate": 0.000405135406218656, + "loss": 2.7456, + "theoretical_loss": 3.806428242726333, + "tokens_seen": 652935168 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040512537612838516, + "loss": 2.8552, + "theoretical_loss": 3.8063889690922013, + "tokens_seen": 653000704 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040511534603811434, + "loss": 2.9067, + "theoretical_loss": 3.806349700502933, + "tokens_seen": 653066240 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004051053159478436, + "loss": 2.8893, + "theoretical_loss": 3.806310436957374, + "tokens_seen": 653131776 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004050952858575727, + "loss": 2.7838, + "theoretical_loss": 3.8062711784543715, + "tokens_seen": 653197312 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040508525576730194, + "loss": 2.6882, + "theoretical_loss": 3.8062319249927707, + "tokens_seen": 653262848 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040507522567703107, + "loss": 3.025, + "theoretical_loss": 3.8061926765714196, + "tokens_seen": 653328384 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004050651955867603, + "loss": 2.9965, + "theoretical_loss": 3.806153433189165, + "tokens_seen": 653393920 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004050551654964895, + "loss": 2.7691, + "theoretical_loss": 3.806114194844855, + "tokens_seen": 653459456 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040504513540621866, + "loss": 3.0947, + "theoretical_loss": 3.8060749615373375, + "tokens_seen": 653524992 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040503510531594784, + "loss": 2.6537, + "theoretical_loss": 3.8060357332654604, + "tokens_seen": 653590528 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004050250752256771, + "loss": 3.1121, + "theoretical_loss": 3.8059965100280735, + "tokens_seen": 653656064 + }, + { + "epoch": 0.2, + "objective/train/docs_used": 379015, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.1280202865600586, + "objective/train/theoretical_loss": 3.8059572918240248, + "objective/train/tokens_used": 674181600, + "theoretical_loss": 3.8059572918240248, + "tokens_seen": 653721600 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004050150451354062, + "loss": 2.8693, + "theoretical_loss": 3.8059572918240248, + "tokens_seen": 653721600 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040500501504513544, + "loss": 2.9088, + "theoretical_loss": 3.8059180786521654, + "tokens_seen": 653787136 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040499498495486457, + "loss": 2.9063, + "theoretical_loss": 3.805878870511344, + "tokens_seen": 653852672 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004049849548645938, + "loss": 2.9036, + "theoretical_loss": 3.805839667400412, + "tokens_seen": 653918208 + }, + { + "epoch": 0.2, + "learning_rate": 0.000404974924774323, + "loss": 3.0185, + "theoretical_loss": 3.8058004693182195, + "tokens_seen": 653983744 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040496489468405217, + "loss": 2.7107, + "theoretical_loss": 3.8057612762636173, + "tokens_seen": 654049280 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040495486459378135, + "loss": 2.7581, + "theoretical_loss": 3.8057220882354574, + "tokens_seen": 654114816 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040494483450351053, + "loss": 2.7486, + "theoretical_loss": 3.8056829052325916, + "tokens_seen": 654180352 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004049348044132397, + "loss": 2.5744, + "theoretical_loss": 3.805643727253872, + "tokens_seen": 654245888 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040492477432296894, + "loss": 3.1129, + "theoretical_loss": 3.8056045542981516, + "tokens_seen": 654311424 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040491474423269807, + "loss": 3.1035, + "theoretical_loss": 3.805565386364283, + "tokens_seen": 654376960 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004049047141424273, + "loss": 3.0437, + "theoretical_loss": 3.8055262234511193, + "tokens_seen": 654442496 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040489468405215643, + "loss": 2.784, + "theoretical_loss": 3.8054870655575153, + "tokens_seen": 654508032 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040488465396188567, + "loss": 2.8953, + "theoretical_loss": 3.805447912682324, + "tokens_seen": 654573568 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040487462387161485, + "loss": 2.9645, + "theoretical_loss": 3.8054087648244, + "tokens_seen": 654639104 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040486459378134403, + "loss": 2.5963, + "theoretical_loss": 3.8053696219825994, + "tokens_seen": 654704640 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004048545636910732, + "loss": 3.1099, + "theoretical_loss": 3.805330484155776, + "tokens_seen": 654770176 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040484453360080245, + "loss": 2.6921, + "theoretical_loss": 3.8052913513427864, + "tokens_seen": 654835712 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004048345035105316, + "loss": 2.9852, + "theoretical_loss": 3.8052522235424853, + "tokens_seen": 654901248 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004048244734202608, + "loss": 3.1504, + "theoretical_loss": 3.805213100753731, + "tokens_seen": 654966784 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040481444332998994, + "loss": 2.8946, + "theoretical_loss": 3.8051739829753792, + "tokens_seen": 655032320 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040480441323971917, + "loss": 2.8164, + "theoretical_loss": 3.8051348702062864, + "tokens_seen": 655097856 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040479438314944835, + "loss": 2.9156, + "theoretical_loss": 3.805095762445311, + "tokens_seen": 655163392 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040478435305917753, + "loss": 2.8516, + "theoretical_loss": 3.805056659691311, + "tokens_seen": 655228928 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004047743229689067, + "loss": 2.9378, + "theoretical_loss": 3.805017561943144, + "tokens_seen": 655294464 + }, + { + "debugging/Self-BLEU-5": 0.39752547377166153, + "debugging/distinct-1-grams": 0.789457132193548, + "debugging/distinct-2-grams": 0.9593000578965885, + "debugging/entropy-1-grams": 5.703465459916224, + "debugging/entropy-2-grams": 6.500444773010346, + "debugging/length": 460.7, + "debugging/num_segments": 10, + "debugging/score": 0.00455369169278577, + "debugging/score_std": 0.00653759201806481, + "epoch": 0.2, + "objective/train/docs_used": 379813, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6106984615325928, + "objective/train/theoretical_loss": 3.804978469199669, + "objective/train/tokens_used": 675820000, + "theoretical_loss": 3.804978469199669, + "tokens_seen": 655360000 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004047642928786359, + "loss": 2.8037, + "theoretical_loss": 3.804978469199669, + "tokens_seen": 655360000 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004047542627883651, + "loss": 3.0539, + "theoretical_loss": 3.804939381459745, + "tokens_seen": 655425536 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004047442326980943, + "loss": 2.7599, + "theoretical_loss": 3.8049002987222313, + "tokens_seen": 655491072 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040473420260782344, + "loss": 2.8744, + "theoretical_loss": 3.8048612209859876, + "tokens_seen": 655556608 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004047241725175527, + "loss": 2.7876, + "theoretical_loss": 3.8048221482498743, + "tokens_seen": 655622144 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004047141424272818, + "loss": 3.0235, + "theoretical_loss": 3.8047830805127516, + "tokens_seen": 655687680 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040470411233701104, + "loss": 2.9379, + "theoretical_loss": 3.8047440177734804, + "tokens_seen": 655753216 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004046940822467402, + "loss": 2.7527, + "theoretical_loss": 3.8047049600309224, + "tokens_seen": 655818752 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004046840521564694, + "loss": 2.9035, + "theoretical_loss": 3.804665907283938, + "tokens_seen": 655884288 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004046740220661986, + "loss": 2.7228, + "theoretical_loss": 3.804626859531391, + "tokens_seen": 655949824 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004046639919759278, + "loss": 2.697, + "theoretical_loss": 3.804587816772142, + "tokens_seen": 656015360 + }, + { + "epoch": 0.2, + "learning_rate": 0.000404653961885657, + "loss": 2.9764, + "theoretical_loss": 3.8045487790050547, + "tokens_seen": 656080896 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004046439317953862, + "loss": 2.8793, + "theoretical_loss": 3.8045097462289923, + "tokens_seen": 656146432 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040463390170511536, + "loss": 2.9416, + "theoretical_loss": 3.804470718442818, + "tokens_seen": 656211968 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040462387161484454, + "loss": 2.8612, + "theoretical_loss": 3.8044316956453956, + "tokens_seen": 656277504 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004046138415245738, + "loss": 2.6803, + "theoretical_loss": 3.8043926778355894, + "tokens_seen": 656343040 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004046038114343029, + "loss": 2.7724, + "theoretical_loss": 3.804353665012264, + "tokens_seen": 656408576 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040459378134403214, + "loss": 3.1158, + "theoretical_loss": 3.8043146571742845, + "tokens_seen": 656474112 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040458375125376127, + "loss": 2.9497, + "theoretical_loss": 3.8042756543205156, + "tokens_seen": 656539648 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004045737211634905, + "loss": 3.0363, + "theoretical_loss": 3.8042366564498242, + "tokens_seen": 656605184 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004045636910732197, + "loss": 2.6614, + "theoretical_loss": 3.8041976635610752, + "tokens_seen": 656670720 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040455366098294886, + "loss": 3.0031, + "theoretical_loss": 3.8041586756531363, + "tokens_seen": 656736256 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040454363089267804, + "loss": 2.8858, + "theoretical_loss": 3.804119692724873, + "tokens_seen": 656801792 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004045336008024073, + "loss": 2.879, + "theoretical_loss": 3.804080714775153, + "tokens_seen": 656867328 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004045235707121364, + "loss": 2.8286, + "theoretical_loss": 3.8040417418028443, + "tokens_seen": 656932864 + }, + { + "epoch": 0.2, + "objective/train/docs_used": 381022, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.140230178833008, + "objective/train/theoretical_loss": 3.8040027738068143, + "objective/train/tokens_used": 677458400, + "theoretical_loss": 3.8040027738068143, + "tokens_seen": 656998400 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040451354062186564, + "loss": 2.7224, + "theoretical_loss": 3.8040027738068143, + "tokens_seen": 656998400 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040450351053159477, + "loss": 2.9484, + "theoretical_loss": 3.8039638107859317, + "tokens_seen": 657063936 + }, + { + "epoch": 0.2, + "learning_rate": 0.000404493480441324, + "loss": 3.0023, + "theoretical_loss": 3.803924852739065, + "tokens_seen": 657129472 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004044834503510532, + "loss": 2.9376, + "theoretical_loss": 3.803885899665083, + "tokens_seen": 657195008 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040447342026078237, + "loss": 2.8174, + "theoretical_loss": 3.8038469515628557, + "tokens_seen": 657260544 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040446339017051155, + "loss": 2.8703, + "theoretical_loss": 3.803808008431252, + "tokens_seen": 657326080 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040445336008024073, + "loss": 2.944, + "theoretical_loss": 3.8037690702691433, + "tokens_seen": 657391616 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004044433299899699, + "loss": 3.0208, + "theoretical_loss": 3.8037301370753993, + "tokens_seen": 657457152 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040443329989969915, + "loss": 2.9923, + "theoretical_loss": 3.803691208848891, + "tokens_seen": 657522688 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040442326980942827, + "loss": 2.6483, + "theoretical_loss": 3.8036522855884893, + "tokens_seen": 657588224 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004044132397191575, + "loss": 2.9095, + "theoretical_loss": 3.803613367293067, + "tokens_seen": 657653760 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040440320962888663, + "loss": 2.8386, + "theoretical_loss": 3.803574453961495, + "tokens_seen": 657719296 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040439317953861587, + "loss": 2.7497, + "theoretical_loss": 3.803535545592646, + "tokens_seen": 657784832 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040438314944834505, + "loss": 2.6138, + "theoretical_loss": 3.803496642185393, + "tokens_seen": 657850368 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040437311935807423, + "loss": 2.995, + "theoretical_loss": 3.8034577437386083, + "tokens_seen": 657915904 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004043630892678034, + "loss": 3.0055, + "theoretical_loss": 3.8034188502511665, + "tokens_seen": 657981440 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040435305917753265, + "loss": 2.8259, + "theoretical_loss": 3.803379961721941, + "tokens_seen": 658046976 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004043430290872618, + "loss": 2.915, + "theoretical_loss": 3.8033410781498054, + "tokens_seen": 658112512 + }, + { + "epoch": 0.2, + "learning_rate": 0.000404332998996991, + "loss": 2.728, + "theoretical_loss": 3.803302199533635, + "tokens_seen": 658178048 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040432296890672014, + "loss": 2.7651, + "theoretical_loss": 3.8032633258723045, + "tokens_seen": 658243584 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040431293881644937, + "loss": 3.0034, + "theoretical_loss": 3.8032244571646894, + "tokens_seen": 658309120 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040430290872617855, + "loss": 2.7287, + "theoretical_loss": 3.8031855934096654, + "tokens_seen": 658374656 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040429287863590773, + "loss": 3.0284, + "theoretical_loss": 3.8031467346061083, + "tokens_seen": 658440192 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004042828485456369, + "loss": 2.9762, + "theoretical_loss": 3.8031078807528944, + "tokens_seen": 658505728 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004042728184553661, + "loss": 2.7698, + "theoretical_loss": 3.803069031848901, + "tokens_seen": 658571264 + }, + { + "epoch": 0.2, + "objective/train/docs_used": 381585, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.8466696739196777, + "objective/train/theoretical_loss": 3.803030187893005, + "objective/train/tokens_used": 679096800, + "theoretical_loss": 3.803030187893005, + "tokens_seen": 658636800 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004042627883650953, + "loss": 2.7268, + "theoretical_loss": 3.803030187893005, + "tokens_seen": 658636800 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004042527582748245, + "loss": 2.6081, + "theoretical_loss": 3.802991348884084, + "tokens_seen": 658702336 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040424272818455364, + "loss": 2.8475, + "theoretical_loss": 3.8029525148210146, + "tokens_seen": 658767872 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004042326980942829, + "loss": 2.9978, + "theoretical_loss": 3.802913685702677, + "tokens_seen": 658833408 + }, + { + "epoch": 0.2, + "learning_rate": 0.000404222668004012, + "loss": 2.8434, + "theoretical_loss": 3.8028748615279495, + "tokens_seen": 658898944 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040421263791374124, + "loss": 2.9989, + "theoretical_loss": 3.80283604229571, + "tokens_seen": 658964480 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004042026078234704, + "loss": 2.8352, + "theoretical_loss": 3.802797228004839, + "tokens_seen": 659030016 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004041925777331996, + "loss": 3.1865, + "theoretical_loss": 3.802758418654215, + "tokens_seen": 659095552 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004041825476429288, + "loss": 3.0958, + "theoretical_loss": 3.8027196142427195, + "tokens_seen": 659161088 + }, + { + "epoch": 0.2, + "learning_rate": 0.000404172517552658, + "loss": 2.8457, + "theoretical_loss": 3.802680814769232, + "tokens_seen": 659226624 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040416248746238714, + "loss": 3.02, + "theoretical_loss": 3.802642020232633, + "tokens_seen": 659292160 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004041524573721164, + "loss": 2.8611, + "theoretical_loss": 3.8026032306318047, + "tokens_seen": 659357696 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004041424272818455, + "loss": 2.8806, + "theoretical_loss": 3.8025644459656283, + "tokens_seen": 659423232 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040413239719157474, + "loss": 3.0605, + "theoretical_loss": 3.802525666232985, + "tokens_seen": 659488768 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004041223671013039, + "loss": 2.8626, + "theoretical_loss": 3.802486891432758, + "tokens_seen": 659554304 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004041123370110331, + "loss": 3.386, + "theoretical_loss": 3.802448121563829, + "tokens_seen": 659619840 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004041023069207623, + "loss": 2.8694, + "theoretical_loss": 3.802409356625082, + "tokens_seen": 659685376 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040409227683049147, + "loss": 3.0799, + "theoretical_loss": 3.8023705966153996, + "tokens_seen": 659750912 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040408224674022065, + "loss": 3.0324, + "theoretical_loss": 3.802331841533666, + "tokens_seen": 659816448 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004040722166499499, + "loss": 2.7588, + "theoretical_loss": 3.8022930913787647, + "tokens_seen": 659881984 + }, + { + "epoch": 0.2, + "learning_rate": 0.000404062186559679, + "loss": 3.0444, + "theoretical_loss": 3.8022543461495806, + "tokens_seen": 659947520 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040405215646940824, + "loss": 3.2894, + "theoretical_loss": 3.8022156058449985, + "tokens_seen": 660013056 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040404212637913737, + "loss": 2.7804, + "theoretical_loss": 3.802176870463904, + "tokens_seen": 660078592 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004040320962888666, + "loss": 2.7903, + "theoretical_loss": 3.802138140005182, + "tokens_seen": 660144128 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004040220661985958, + "loss": 3.0366, + "theoretical_loss": 3.8020994144677185, + "tokens_seen": 660209664 + }, + { + "epoch": 0.2, + "objective/train/docs_used": 382875, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.026890993118286, + "objective/train/theoretical_loss": 3.8020606938504, + "objective/train/tokens_used": 680735200, + "theoretical_loss": 3.8020606938504, + "tokens_seen": 660275200 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040401203610832497, + "loss": 2.8705, + "theoretical_loss": 3.8020606938504, + "tokens_seen": 660275200 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040400200601805415, + "loss": 2.7666, + "theoretical_loss": 3.8020219781521125, + "tokens_seen": 660340736 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004039919759277834, + "loss": 2.8945, + "theoretical_loss": 3.8019832673717437, + "tokens_seen": 660406272 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004039819458375125, + "loss": 3.1173, + "theoretical_loss": 3.801944561508181, + "tokens_seen": 660471808 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040397191574724175, + "loss": 2.9012, + "theoretical_loss": 3.801905860560311, + "tokens_seen": 660537344 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004039618856569709, + "loss": 2.7743, + "theoretical_loss": 3.8018671645270237, + "tokens_seen": 660602880 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004039518555667001, + "loss": 2.8718, + "theoretical_loss": 3.8018284734072054, + "tokens_seen": 660668416 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004039418254764293, + "loss": 2.7487, + "theoretical_loss": 3.8017897871997466, + "tokens_seen": 660733952 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040393179538615847, + "loss": 2.947, + "theoretical_loss": 3.801751105903536, + "tokens_seen": 660799488 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040392176529588765, + "loss": 2.7729, + "theoretical_loss": 3.8017124295174622, + "tokens_seen": 660865024 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040391173520561683, + "loss": 2.659, + "theoretical_loss": 3.8016737580404163, + "tokens_seen": 660930560 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040390170511534607, + "loss": 2.885, + "theoretical_loss": 3.8016350914712884, + "tokens_seen": 660996096 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040389167502507525, + "loss": 3.058, + "theoretical_loss": 3.801596429808968, + "tokens_seen": 661061632 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040388164493480443, + "loss": 2.7235, + "theoretical_loss": 3.8015577730523473, + "tokens_seen": 661127168 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004038716148445336, + "loss": 2.3908, + "theoretical_loss": 3.801519121200317, + "tokens_seen": 661192704 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040386158475426285, + "loss": 2.6493, + "theoretical_loss": 3.801480474251769, + "tokens_seen": 661258240 + }, + { + "epoch": 0.2, + "learning_rate": 0.000403851554663992, + "loss": 2.7979, + "theoretical_loss": 3.801441832205595, + "tokens_seen": 661323776 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004038415245737212, + "loss": 2.9626, + "theoretical_loss": 3.801403195060688, + "tokens_seen": 661389312 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040383149448345034, + "loss": 2.9596, + "theoretical_loss": 3.801364562815941, + "tokens_seen": 661454848 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040382146439317957, + "loss": 2.7746, + "theoretical_loss": 3.8013259354702456, + "tokens_seen": 661520384 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040381143430290875, + "loss": 2.8874, + "theoretical_loss": 3.801287313022497, + "tokens_seen": 661585920 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040380140421263794, + "loss": 2.8737, + "theoretical_loss": 3.801248695471588, + "tokens_seen": 661651456 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004037913741223671, + "loss": 2.9437, + "theoretical_loss": 3.8012100828164135, + "tokens_seen": 661716992 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004037813440320963, + "loss": 2.8516, + "theoretical_loss": 3.801171475055867, + "tokens_seen": 661782528 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004037713139418255, + "loss": 2.9149, + "theoretical_loss": 3.8011328721888447, + "tokens_seen": 661848064 + }, + { + "epoch": 0.2, + "objective/train/docs_used": 383560, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.94755482673645, + "objective/train/theoretical_loss": 3.8010942742142415, + "objective/train/tokens_used": 682373600, + "theoretical_loss": 3.8010942742142415, + "tokens_seen": 661913600 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004037612838515547, + "loss": 2.8702, + "theoretical_loss": 3.8010942742142415, + "tokens_seen": 661913600 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040375125376128384, + "loss": 2.4864, + "theoretical_loss": 3.8010556811309524, + "tokens_seen": 661979136 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004037412236710131, + "loss": 3.0884, + "theoretical_loss": 3.801017092937874, + "tokens_seen": 662044672 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004037311935807422, + "loss": 2.9156, + "theoretical_loss": 3.800978509633903, + "tokens_seen": 662110208 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040372116349047144, + "loss": 2.3746, + "theoretical_loss": 3.8009399312179353, + "tokens_seen": 662175744 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004037111334002006, + "loss": 2.8447, + "theoretical_loss": 3.800901357688868, + "tokens_seen": 662241280 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004037011033099298, + "loss": 2.895, + "theoretical_loss": 3.800862789045599, + "tokens_seen": 662306816 + }, + { + "epoch": 0.2, + "learning_rate": 0.000403691073219659, + "loss": 2.7383, + "theoretical_loss": 3.800824225287026, + "tokens_seen": 662372352 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004036810431293882, + "loss": 2.8769, + "theoretical_loss": 3.8007856664120476, + "tokens_seen": 662437888 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040367101303911734, + "loss": 2.9749, + "theoretical_loss": 3.800747112419561, + "tokens_seen": 662503424 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004036609829488466, + "loss": 2.8727, + "theoretical_loss": 3.8007085633084663, + "tokens_seen": 662568960 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004036509528585757, + "loss": 2.9192, + "theoretical_loss": 3.8006700190776623, + "tokens_seen": 662634496 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040364092276830494, + "loss": 2.9784, + "theoretical_loss": 3.8006314797260488, + "tokens_seen": 662700032 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004036308926780341, + "loss": 2.8827, + "theoretical_loss": 3.800592945252525, + "tokens_seen": 662765568 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004036208625877633, + "loss": 2.8283, + "theoretical_loss": 3.8005544156559923, + "tokens_seen": 662831104 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004036108324974925, + "loss": 2.4836, + "theoretical_loss": 3.80051589093535, + "tokens_seen": 662896640 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040360080240722167, + "loss": 2.9974, + "theoretical_loss": 3.8004773710895003, + "tokens_seen": 662962176 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040359077231695085, + "loss": 2.8594, + "theoretical_loss": 3.800438856117344, + "tokens_seen": 663027712 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004035807422266801, + "loss": 2.8728, + "theoretical_loss": 3.800400346017783, + "tokens_seen": 663093248 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004035707121364092, + "loss": 2.9307, + "theoretical_loss": 3.8003618407897193, + "tokens_seen": 663158784 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040356068204613844, + "loss": 2.7888, + "theoretical_loss": 3.800323340432055, + "tokens_seen": 663224320 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040355065195586757, + "loss": 2.9874, + "theoretical_loss": 3.8002848449436932, + "tokens_seen": 663289856 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004035406218655968, + "loss": 2.7677, + "theoretical_loss": 3.800246354323538, + "tokens_seen": 663355392 + }, + { + "epoch": 0.2, + "learning_rate": 0.000403530591775326, + "loss": 2.7384, + "theoretical_loss": 3.8002078685704914, + "tokens_seen": 663420928 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040352056168505517, + "loss": 2.8846, + "theoretical_loss": 3.8001693876834572, + "tokens_seen": 663486464 + }, + { + "epoch": 0.2, + "objective/train/docs_used": 384103, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.909592628479004, + "objective/train/theoretical_loss": 3.8001309116613413, + "objective/train/tokens_used": 684012000, + "theoretical_loss": 3.8001309116613413, + "tokens_seen": 663552000 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040351053159478435, + "loss": 2.8122, + "theoretical_loss": 3.8001309116613413, + "tokens_seen": 663552000 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004035005015045136, + "loss": 2.9387, + "theoretical_loss": 3.8000924405030467, + "tokens_seen": 663617536 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004034904714142427, + "loss": 2.9296, + "theoretical_loss": 3.8000539742074784, + "tokens_seen": 663683072 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040348044132397195, + "loss": 2.6277, + "theoretical_loss": 3.8000155127735424, + "tokens_seen": 663748608 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004034704112337011, + "loss": 2.8678, + "theoretical_loss": 3.7999770562001443, + "tokens_seen": 663814144 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004034603811434303, + "loss": 2.9365, + "theoretical_loss": 3.799938604486189, + "tokens_seen": 663879680 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004034503510531595, + "loss": 2.9277, + "theoretical_loss": 3.799900157630584, + "tokens_seen": 663945216 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040344032096288867, + "loss": 2.8401, + "theoretical_loss": 3.799861715632236, + "tokens_seen": 664010752 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040343029087261785, + "loss": 2.9198, + "theoretical_loss": 3.7998232784900514, + "tokens_seen": 664076288 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040342026078234703, + "loss": 2.7077, + "theoretical_loss": 3.7997848462029378, + "tokens_seen": 664141824 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004034102306920762, + "loss": 2.6015, + "theoretical_loss": 3.7997464187698027, + "tokens_seen": 664207360 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040340020060180545, + "loss": 2.9637, + "theoretical_loss": 3.799707996189555, + "tokens_seen": 664272896 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004033901705115346, + "loss": 2.8308, + "theoretical_loss": 3.7996695784611023, + "tokens_seen": 664338432 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004033801404212638, + "loss": 2.5064, + "theoretical_loss": 3.799631165583354, + "tokens_seen": 664403968 + }, + { + "epoch": 0.2, + "learning_rate": 0.000403370110330993, + "loss": 2.7838, + "theoretical_loss": 3.7995927575552186, + "tokens_seen": 664469504 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004033600802407222, + "loss": 2.7744, + "theoretical_loss": 3.7995543543756063, + "tokens_seen": 664535040 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040335005015045136, + "loss": 2.7371, + "theoretical_loss": 3.799515956043427, + "tokens_seen": 664600576 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040334002006018054, + "loss": 2.5989, + "theoretical_loss": 3.7994775625575903, + "tokens_seen": 664666112 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004033299899699097, + "loss": 2.7306, + "theoretical_loss": 3.799439173917007, + "tokens_seen": 664731648 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040331995987963895, + "loss": 2.7472, + "theoretical_loss": 3.799400790120588, + "tokens_seen": 664797184 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004033099297893681, + "loss": 2.7923, + "theoretical_loss": 3.799362411167245, + "tokens_seen": 664862720 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004032998996990973, + "loss": 2.7568, + "theoretical_loss": 3.7993240370558894, + "tokens_seen": 664928256 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040328986960882644, + "loss": 2.9197, + "theoretical_loss": 3.7992856677854334, + "tokens_seen": 664993792 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004032798395185557, + "loss": 3.1946, + "theoretical_loss": 3.7992473033547887, + "tokens_seen": 665059328 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040326980942828486, + "loss": 3.0812, + "theoretical_loss": 3.799208943762868, + "tokens_seen": 665124864 + }, + { + "epoch": 0.2, + "objective/train/docs_used": 385247, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.238375663757324, + "objective/train/theoretical_loss": 3.799170589008585, + "objective/train/tokens_used": 685650400, + "theoretical_loss": 3.799170589008585, + "tokens_seen": 665190400 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040325977933801404, + "loss": 2.8898, + "theoretical_loss": 3.799170589008585, + "tokens_seen": 665190400 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004032497492477432, + "loss": 2.9089, + "theoretical_loss": 3.799132239090852, + "tokens_seen": 665255936 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004032397191574724, + "loss": 2.7847, + "theoretical_loss": 3.7990938940085845, + "tokens_seen": 665321472 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004032296890672016, + "loss": 2.919, + "theoretical_loss": 3.799055553760695, + "tokens_seen": 665387008 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004032196589769308, + "loss": 2.8573, + "theoretical_loss": 3.7990172183460995, + "tokens_seen": 665452544 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040320962888665995, + "loss": 2.7393, + "theoretical_loss": 3.7989788877637105, + "tokens_seen": 665518080 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004031995987963892, + "loss": 2.7331, + "theoretical_loss": 3.7989405620124455, + "tokens_seen": 665583616 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040318956870611836, + "loss": 2.8972, + "theoretical_loss": 3.7989022410912185, + "tokens_seen": 665649152 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040317953861584754, + "loss": 2.7189, + "theoretical_loss": 3.7988639249989458, + "tokens_seen": 665714688 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004031695085255767, + "loss": 2.8076, + "theoretical_loss": 3.7988256137345435, + "tokens_seen": 665780224 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004031594784353059, + "loss": 2.8447, + "theoretical_loss": 3.7987873072969283, + "tokens_seen": 665845760 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040314944834503514, + "loss": 3.0372, + "theoretical_loss": 3.798749005685017, + "tokens_seen": 665911296 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004031394182547643, + "loss": 2.7098, + "theoretical_loss": 3.798710708897727, + "tokens_seen": 665976832 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004031293881644935, + "loss": 2.8833, + "theoretical_loss": 3.7986724169339756, + "tokens_seen": 666042368 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004031193580742227, + "loss": 2.8547, + "theoretical_loss": 3.798634129792681, + "tokens_seen": 666107904 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040310932798395187, + "loss": 2.9972, + "theoretical_loss": 3.798595847472761, + "tokens_seen": 666173440 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040309929789368105, + "loss": 2.7587, + "theoretical_loss": 3.798557569973135, + "tokens_seen": 666238976 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004030892678034103, + "loss": 2.8401, + "theoretical_loss": 3.7985192972927218, + "tokens_seen": 666304512 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004030792377131394, + "loss": 2.8261, + "theoretical_loss": 3.79848102943044, + "tokens_seen": 666370048 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040306920762286864, + "loss": 2.6952, + "theoretical_loss": 3.79844276638521, + "tokens_seen": 666435584 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040305917753259777, + "loss": 3.0011, + "theoretical_loss": 3.7984045081559517, + "tokens_seen": 666501120 + }, + { + "epoch": 0.2, + "learning_rate": 0.000403049147442327, + "loss": 2.6255, + "theoretical_loss": 3.7983662547415853, + "tokens_seen": 666566656 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004030391173520562, + "loss": 2.8306, + "theoretical_loss": 3.798328006141032, + "tokens_seen": 666632192 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040302908726178537, + "loss": 3.0386, + "theoretical_loss": 3.798289762353212, + "tokens_seen": 666697728 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040301905717151455, + "loss": 2.8491, + "theoretical_loss": 3.798251523377048, + "tokens_seen": 666763264 + }, + { + "epoch": 0.2, + "objective/train/docs_used": 385855, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6259422302246094, + "objective/train/theoretical_loss": 3.79821328921146, + "objective/train/tokens_used": 687288800, + "theoretical_loss": 3.79821328921146, + "tokens_seen": 666828800 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004030090270812438, + "loss": 2.6368, + "theoretical_loss": 3.79821328921146, + "tokens_seen": 666828800 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004029989969909729, + "loss": 2.6636, + "theoretical_loss": 3.798175059855372, + "tokens_seen": 666894336 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040298896690070215, + "loss": 2.9531, + "theoretical_loss": 3.7981368353077056, + "tokens_seen": 666959872 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004029789368104313, + "loss": 2.8866, + "theoretical_loss": 3.7980986155673833, + "tokens_seen": 667025408 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004029689067201605, + "loss": 2.7862, + "theoretical_loss": 3.7980604006333287, + "tokens_seen": 667090944 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004029588766298897, + "loss": 2.8463, + "theoretical_loss": 3.7980221905044655, + "tokens_seen": 667156480 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040294884653961887, + "loss": 2.7244, + "theoretical_loss": 3.797983985179717, + "tokens_seen": 667222016 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040293881644934805, + "loss": 2.8068, + "theoretical_loss": 3.797945784658008, + "tokens_seen": 667287552 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040292878635907723, + "loss": 2.8942, + "theoretical_loss": 3.797907588938262, + "tokens_seen": 667353088 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004029187562688064, + "loss": 2.6061, + "theoretical_loss": 3.7978693980194054, + "tokens_seen": 667418624 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040290872617853565, + "loss": 2.8004, + "theoretical_loss": 3.797831211900362, + "tokens_seen": 667484160 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004028986960882648, + "loss": 3.0971, + "theoretical_loss": 3.7977930305800585, + "tokens_seen": 667549696 + }, + { + "epoch": 0.2, + "learning_rate": 0.000402888665997994, + "loss": 2.8088, + "theoretical_loss": 3.7977548540574206, + "tokens_seen": 667615232 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004028786359077232, + "loss": 3.0338, + "theoretical_loss": 3.7977166823313735, + "tokens_seen": 667680768 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004028686058174524, + "loss": 2.5565, + "theoretical_loss": 3.7976785154008454, + "tokens_seen": 667746304 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040285857572718156, + "loss": 2.8942, + "theoretical_loss": 3.797640353264762, + "tokens_seen": 667811840 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040284854563691074, + "loss": 2.7999, + "theoretical_loss": 3.797602195922052, + "tokens_seen": 667877376 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004028385155466399, + "loss": 2.8825, + "theoretical_loss": 3.7975640433716413, + "tokens_seen": 667942912 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040282848545636915, + "loss": 2.9716, + "theoretical_loss": 3.797525895612459, + "tokens_seen": 668008448 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004028184553660983, + "loss": 2.8631, + "theoretical_loss": 3.797487752643434, + "tokens_seen": 668073984 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004028084252758275, + "loss": 2.9298, + "theoretical_loss": 3.7974496144634937, + "tokens_seen": 668139520 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040279839518555664, + "loss": 2.7151, + "theoretical_loss": 3.797411481071568, + "tokens_seen": 668205056 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004027883650952859, + "loss": 3.0341, + "theoretical_loss": 3.797373352466586, + "tokens_seen": 668270592 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040277833500501506, + "loss": 2.8544, + "theoretical_loss": 3.7973352286474773, + "tokens_seen": 668336128 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040276830491474424, + "loss": 2.9154, + "theoretical_loss": 3.7972971096131722, + "tokens_seen": 668401664 + }, + { + "epoch": 0.2, + "objective/train/docs_used": 387212, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6597354412078857, + "objective/train/theoretical_loss": 3.7972589953626006, + "objective/train/tokens_used": 688927200, + "theoretical_loss": 3.7972589953626006, + "tokens_seen": 668467200 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004027582748244734, + "loss": 2.8495, + "theoretical_loss": 3.7972589953626006, + "tokens_seen": 668467200 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004027482447342026, + "loss": 2.6392, + "theoretical_loss": 3.797220885894694, + "tokens_seen": 668532736 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004027382146439318, + "loss": 2.477, + "theoretical_loss": 3.7971827812083836, + "tokens_seen": 668598272 + }, + { + "epoch": 0.2, + "learning_rate": 0.000402728184553661, + "loss": 2.7564, + "theoretical_loss": 3.7971446813026, + "tokens_seen": 668663808 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040271815446339015, + "loss": 2.8408, + "theoretical_loss": 3.7971065861762754, + "tokens_seen": 668729344 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004027081243731194, + "loss": 2.8359, + "theoretical_loss": 3.7970684958283423, + "tokens_seen": 668794880 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040269809428284856, + "loss": 2.9813, + "theoretical_loss": 3.797030410257733, + "tokens_seen": 668860416 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040268806419257774, + "loss": 2.6872, + "theoretical_loss": 3.7969923294633796, + "tokens_seen": 668925952 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004026780341023069, + "loss": 2.8946, + "theoretical_loss": 3.796954253444216, + "tokens_seen": 668991488 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004026680040120361, + "loss": 2.901, + "theoretical_loss": 3.7969161821991757, + "tokens_seen": 669057024 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004026579739217653, + "loss": 2.7655, + "theoretical_loss": 3.7968781157271922, + "tokens_seen": 669122560 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004026479438314945, + "loss": 2.8933, + "theoretical_loss": 3.7968400540272, + "tokens_seen": 669188096 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040263791374122365, + "loss": 2.6813, + "theoretical_loss": 3.796801997098134, + "tokens_seen": 669253632 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004026278836509529, + "loss": 2.8661, + "theoretical_loss": 3.7967639449389283, + "tokens_seen": 669319168 + }, + { + "epoch": 0.2, + "learning_rate": 0.000402617853560682, + "loss": 2.9676, + "theoretical_loss": 3.7967258975485185, + "tokens_seen": 669384704 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040260782347041125, + "loss": 2.8605, + "theoretical_loss": 3.79668785492584, + "tokens_seen": 669450240 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040259779338014043, + "loss": 2.9737, + "theoretical_loss": 3.796649817069828, + "tokens_seen": 669515776 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004025877632898696, + "loss": 2.815, + "theoretical_loss": 3.7966117839794205, + "tokens_seen": 669581312 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004025777331995988, + "loss": 2.863, + "theoretical_loss": 3.796573755653553, + "tokens_seen": 669646848 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040256770310932797, + "loss": 2.6706, + "theoretical_loss": 3.7965357320911624, + "tokens_seen": 669712384 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040255767301905715, + "loss": 2.7118, + "theoretical_loss": 3.7964977132911866, + "tokens_seen": 669777920 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004025476429287864, + "loss": 2.9375, + "theoretical_loss": 3.796459699252562, + "tokens_seen": 669843456 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004025376128385155, + "loss": 2.7698, + "theoretical_loss": 3.7964216899742276, + "tokens_seen": 669908992 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040252758274824475, + "loss": 2.6378, + "theoretical_loss": 3.7963836854551216, + "tokens_seen": 669974528 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040251755265797393, + "loss": 2.7482, + "theoretical_loss": 3.796345685694182, + "tokens_seen": 670040064 + }, + { + "epoch": 0.2, + "objective/train/docs_used": 387914, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.9556262493133545, + "objective/train/theoretical_loss": 3.796307690690348, + "objective/train/tokens_used": 690565600, + "theoretical_loss": 3.796307690690348, + "tokens_seen": 670105600 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004025075225677031, + "loss": 2.8599, + "theoretical_loss": 3.796307690690348, + "tokens_seen": 670105600 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004024974924774323, + "loss": 2.9695, + "theoretical_loss": 3.7962697004425596, + "tokens_seen": 670171136 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004024874623871615, + "loss": 2.9686, + "theoretical_loss": 3.796231714949756, + "tokens_seen": 670236672 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040247743229689066, + "loss": 2.9333, + "theoretical_loss": 3.7961937342108767, + "tokens_seen": 670302208 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004024674022066199, + "loss": 3.0246, + "theoretical_loss": 3.796155758224863, + "tokens_seen": 670367744 + }, + { + "epoch": 0.2, + "learning_rate": 0.000402457372116349, + "loss": 2.9733, + "theoretical_loss": 3.796117786990655, + "tokens_seen": 670433280 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040244734202607825, + "loss": 2.7806, + "theoretical_loss": 3.7960798205071935, + "tokens_seen": 670498816 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004024373119358074, + "loss": 2.9088, + "theoretical_loss": 3.79604185877342, + "tokens_seen": 670564352 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004024272818455366, + "loss": 2.8015, + "theoretical_loss": 3.7960039017882767, + "tokens_seen": 670629888 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004024172517552658, + "loss": 2.6111, + "theoretical_loss": 3.7959659495507045, + "tokens_seen": 670695424 + }, + { + "epoch": 0.2, + "learning_rate": 0.000402407221664995, + "loss": 2.7582, + "theoretical_loss": 3.7959280020596475, + "tokens_seen": 670760960 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004023971915747242, + "loss": 2.6639, + "theoretical_loss": 3.795890059314047, + "tokens_seen": 670826496 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004023871614844534, + "loss": 2.777, + "theoretical_loss": 3.795852121312846, + "tokens_seen": 670892032 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004023771313941826, + "loss": 2.7667, + "theoretical_loss": 3.7958141880549885, + "tokens_seen": 670957568 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040236710130391176, + "loss": 2.8056, + "theoretical_loss": 3.7957762595394184, + "tokens_seen": 671023104 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040235707121364094, + "loss": 2.7246, + "theoretical_loss": 3.79573833576508, + "tokens_seen": 671088640 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004023470411233701, + "loss": 2.773, + "theoretical_loss": 3.795700416730916, + "tokens_seen": 671154176 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040233701103309935, + "loss": 2.855, + "theoretical_loss": 3.7956625024358726, + "tokens_seen": 671219712 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004023269809428285, + "loss": 2.8859, + "theoretical_loss": 3.795624592878895, + "tokens_seen": 671285248 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004023169508525577, + "loss": 2.68, + "theoretical_loss": 3.795586688058928, + "tokens_seen": 671350784 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040230692076228684, + "loss": 2.7463, + "theoretical_loss": 3.7955487879749175, + "tokens_seen": 671416320 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004022968906720161, + "loss": 2.8671, + "theoretical_loss": 3.795510892625809, + "tokens_seen": 671481856 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040228686058174526, + "loss": 2.8666, + "theoretical_loss": 3.7954730020105503, + "tokens_seen": 671547392 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040227683049147444, + "loss": 2.8142, + "theoretical_loss": 3.795435116128087, + "tokens_seen": 671612928 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004022668004012036, + "loss": 2.6166, + "theoretical_loss": 3.795397234977367, + "tokens_seen": 671678464 + }, + { + "epoch": 0.2, + "objective/train/docs_used": 389368, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6852924823760986, + "objective/train/theoretical_loss": 3.795359358557337, + "objective/train/tokens_used": 692204000, + "theoretical_loss": 3.795359358557337, + "tokens_seen": 671744000 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004022567703109328, + "loss": 2.899, + "theoretical_loss": 3.795359358557337, + "tokens_seen": 671744000 + }, + { + "epoch": 0.2, + "learning_rate": 0.000402246740220662, + "loss": 2.9861, + "theoretical_loss": 3.7953214868669454, + "tokens_seen": 671809536 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004022367101303912, + "loss": 2.9788, + "theoretical_loss": 3.79528361990514, + "tokens_seen": 671875072 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040222668004012035, + "loss": 2.7628, + "theoretical_loss": 3.7952457576708687, + "tokens_seen": 671940608 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004022166499498496, + "loss": 2.9801, + "theoretical_loss": 3.7952079001630814, + "tokens_seen": 672006144 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040220661985957876, + "loss": 2.8589, + "theoretical_loss": 3.7951700473807266, + "tokens_seen": 672071680 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040219658976930794, + "loss": 2.9261, + "theoretical_loss": 3.795132199322754, + "tokens_seen": 672137216 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004021865596790371, + "loss": 2.7907, + "theoretical_loss": 3.795094355988113, + "tokens_seen": 672202752 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004021765295887663, + "loss": 2.6389, + "theoretical_loss": 3.795056517375754, + "tokens_seen": 672268288 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004021664994984955, + "loss": 2.9392, + "theoretical_loss": 3.795018683484628, + "tokens_seen": 672333824 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004021564694082247, + "loss": 2.6858, + "theoretical_loss": 3.7949808543136845, + "tokens_seen": 672399360 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040214643931795385, + "loss": 3.003, + "theoretical_loss": 3.7949430298618756, + "tokens_seen": 672464896 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004021364092276831, + "loss": 2.9455, + "theoretical_loss": 3.794905210128152, + "tokens_seen": 672530432 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004021263791374122, + "loss": 3.1507, + "theoretical_loss": 3.794867395111467, + "tokens_seen": 672595968 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040211634904714145, + "loss": 3.196, + "theoretical_loss": 3.7948295848107714, + "tokens_seen": 672661504 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040210631895687063, + "loss": 2.9244, + "theoretical_loss": 3.794791779225018, + "tokens_seen": 672727040 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004020962888665998, + "loss": 2.7954, + "theoretical_loss": 3.7947539783531594, + "tokens_seen": 672792576 + }, + { + "epoch": 0.2, + "learning_rate": 0.000402086258776329, + "loss": 2.7656, + "theoretical_loss": 3.7947161821941497, + "tokens_seen": 672858112 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040207622868605817, + "loss": 2.7689, + "theoretical_loss": 3.794678390746941, + "tokens_seen": 672923648 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040206619859578735, + "loss": 2.9119, + "theoretical_loss": 3.794640604010488, + "tokens_seen": 672989184 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004020561685055166, + "loss": 2.7778, + "theoretical_loss": 3.7946028219837453, + "tokens_seen": 673054720 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004020461384152457, + "loss": 2.6834, + "theoretical_loss": 3.7945650446656662, + "tokens_seen": 673120256 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040203610832497495, + "loss": 2.8575, + "theoretical_loss": 3.794527272055206, + "tokens_seen": 673185792 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040202607823470413, + "loss": 2.9762, + "theoretical_loss": 3.79448950415132, + "tokens_seen": 673251328 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004020160481444333, + "loss": 3.0, + "theoretical_loss": 3.7944517409529634, + "tokens_seen": 673316864 + }, + { + "epoch": 0.2, + "objective/train/docs_used": 390091, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.931178331375122, + "objective/train/theoretical_loss": 3.7944139824590923, + "objective/train/tokens_used": 693842400, + "theoretical_loss": 3.7944139824590923, + "tokens_seen": 673382400 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004020060180541625, + "loss": 2.664, + "theoretical_loss": 3.7944139824590923, + "tokens_seen": 673382400 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004019959879638917, + "loss": 2.86, + "theoretical_loss": 3.794376228668663, + "tokens_seen": 673447936 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040198595787362086, + "loss": 2.7169, + "theoretical_loss": 3.794338479580632, + "tokens_seen": 673513472 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004019759277833501, + "loss": 2.8822, + "theoretical_loss": 3.7943007351939553, + "tokens_seen": 673579008 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004019658976930792, + "loss": 2.8794, + "theoretical_loss": 3.794262995507591, + "tokens_seen": 673644544 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040195586760280845, + "loss": 2.8051, + "theoretical_loss": 3.794225260520496, + "tokens_seen": 673710080 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004019458375125376, + "loss": 2.8605, + "theoretical_loss": 3.794187530231628, + "tokens_seen": 673775616 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004019358074222668, + "loss": 2.7264, + "theoretical_loss": 3.7941498046399458, + "tokens_seen": 673841152 + }, + { + "epoch": 0.2, + "learning_rate": 0.000401925777331996, + "loss": 2.9611, + "theoretical_loss": 3.794112083744407, + "tokens_seen": 673906688 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004019157472417252, + "loss": 2.6618, + "theoretical_loss": 3.7940743675439714, + "tokens_seen": 673972224 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040190571715145436, + "loss": 2.6166, + "theoretical_loss": 3.7940366560375978, + "tokens_seen": 674037760 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004018956870611836, + "loss": 2.815, + "theoretical_loss": 3.793998949224245, + "tokens_seen": 674103296 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004018856569709127, + "loss": 3.0881, + "theoretical_loss": 3.793961247102874, + "tokens_seen": 674168832 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040187562688064196, + "loss": 2.9754, + "theoretical_loss": 3.793923549672444, + "tokens_seen": 674234368 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004018655967903711, + "loss": 2.9012, + "theoretical_loss": 3.793885856931916, + "tokens_seen": 674299904 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004018555667001003, + "loss": 3.0228, + "theoretical_loss": 3.7938481688802503, + "tokens_seen": 674365440 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004018455366098295, + "loss": 2.6691, + "theoretical_loss": 3.793810485516409, + "tokens_seen": 674430976 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004018355065195587, + "loss": 2.7718, + "theoretical_loss": 3.793772806839352, + "tokens_seen": 674496512 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040182547642928786, + "loss": 2.8209, + "theoretical_loss": 3.793735132848042, + "tokens_seen": 674562048 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040181544633901704, + "loss": 2.9048, + "theoretical_loss": 3.7936974635414416, + "tokens_seen": 674627584 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004018054162487462, + "loss": 2.7399, + "theoretical_loss": 3.7936597989185126, + "tokens_seen": 674693120 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040179538615847546, + "loss": 2.8621, + "theoretical_loss": 3.793622138978218, + "tokens_seen": 674758656 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004017853560682046, + "loss": 2.711, + "theoretical_loss": 3.7935844837195205, + "tokens_seen": 674824192 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004017753259779338, + "loss": 3.0176, + "theoretical_loss": 3.7935468331413844, + "tokens_seen": 674889728 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040176529588766295, + "loss": 2.7994, + "theoretical_loss": 3.7935091872427726, + "tokens_seen": 674955264 + }, + { + "epoch": 0.2, + "objective/train/docs_used": 391518, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.5851986408233643, + "objective/train/theoretical_loss": 3.79347154602265, + "objective/train/tokens_used": 695480800, + "theoretical_loss": 3.79347154602265, + "tokens_seen": 675020800 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004017552657973922, + "loss": 2.8448, + "theoretical_loss": 3.79347154602265, + "tokens_seen": 675020800 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040174523570712137, + "loss": 2.8627, + "theoretical_loss": 3.7934339094799805, + "tokens_seen": 675086336 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040173520561685055, + "loss": 2.8723, + "theoretical_loss": 3.7933962776137284, + "tokens_seen": 675151872 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040172517552657973, + "loss": 2.8036, + "theoretical_loss": 3.79335865042286, + "tokens_seen": 675217408 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040171514543630896, + "loss": 2.7785, + "theoretical_loss": 3.79332102790634, + "tokens_seen": 675282944 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004017051153460381, + "loss": 2.8239, + "theoretical_loss": 3.7932834100631343, + "tokens_seen": 675348480 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004016950852557673, + "loss": 2.6686, + "theoretical_loss": 3.793245796892209, + "tokens_seen": 675414016 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040168505516549645, + "loss": 2.7119, + "theoretical_loss": 3.7932081883925304, + "tokens_seen": 675479552 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004016750250752257, + "loss": 2.6828, + "theoretical_loss": 3.7931705845630654, + "tokens_seen": 675545088 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040166499498495487, + "loss": 2.6504, + "theoretical_loss": 3.7931329854027807, + "tokens_seen": 675610624 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040165496489468405, + "loss": 2.9984, + "theoretical_loss": 3.7930953909106444, + "tokens_seen": 675676160 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004016449348044133, + "loss": 3.0036, + "theoretical_loss": 3.7930578010856233, + "tokens_seen": 675741696 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004016349047141424, + "loss": 2.9051, + "theoretical_loss": 3.793020215926686, + "tokens_seen": 675807232 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040162487462387165, + "loss": 2.5337, + "theoretical_loss": 3.7929826354328013, + "tokens_seen": 675872768 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040161484453360083, + "loss": 2.713, + "theoretical_loss": 3.7929450596029373, + "tokens_seen": 675938304 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040160481444333, + "loss": 2.843, + "theoretical_loss": 3.792907488436063, + "tokens_seen": 676003840 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004015947843530592, + "loss": 2.7535, + "theoretical_loss": 3.7928699219311484, + "tokens_seen": 676069376 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040158475426278837, + "loss": 2.6306, + "theoretical_loss": 3.7928323600871625, + "tokens_seen": 676134912 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040157472417251755, + "loss": 2.716, + "theoretical_loss": 3.7927948029030754, + "tokens_seen": 676200448 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004015646940822468, + "loss": 2.6361, + "theoretical_loss": 3.792757250377858, + "tokens_seen": 676265984 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004015546639919759, + "loss": 2.7268, + "theoretical_loss": 3.792719702510481, + "tokens_seen": 676331520 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040154463390170515, + "loss": 2.691, + "theoretical_loss": 3.7926821592999147, + "tokens_seen": 676397056 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040153460381143433, + "loss": 2.9191, + "theoretical_loss": 3.79264462074513, + "tokens_seen": 676462592 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004015245737211635, + "loss": 2.7752, + "theoretical_loss": 3.7926070868451003, + "tokens_seen": 676528128 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004015145436308927, + "loss": 2.6885, + "theoretical_loss": 3.7925695575987963, + "tokens_seen": 676593664 + }, + { + "epoch": 0.21, + "objective/train/docs_used": 392138, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.301459312438965, + "objective/train/theoretical_loss": 3.792532033005191, + "objective/train/tokens_used": 697119200, + "theoretical_loss": 3.792532033005191, + "tokens_seen": 676659200 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004015045135406219, + "loss": 2.6319, + "theoretical_loss": 3.792532033005191, + "tokens_seen": 676659200 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040149448345035106, + "loss": 2.7037, + "theoretical_loss": 3.7924945130632564, + "tokens_seen": 676724736 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004014844533600803, + "loss": 2.8042, + "theoretical_loss": 3.7924569977719655, + "tokens_seen": 676790272 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004014744232698094, + "loss": 3.0209, + "theoretical_loss": 3.792419487130292, + "tokens_seen": 676855808 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040146439317953865, + "loss": 2.923, + "theoretical_loss": 3.792381981137209, + "tokens_seen": 676921344 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004014543630892678, + "loss": 2.8851, + "theoretical_loss": 3.7923444797916916, + "tokens_seen": 676986880 + }, + { + "epoch": 0.21, + "learning_rate": 0.000401444332998997, + "loss": 2.9247, + "theoretical_loss": 3.7923069830927125, + "tokens_seen": 677052416 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004014343029087262, + "loss": 3.0124, + "theoretical_loss": 3.7922694910392476, + "tokens_seen": 677117952 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004014242728184554, + "loss": 3.1205, + "theoretical_loss": 3.792232003630271, + "tokens_seen": 677183488 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040141424272818456, + "loss": 2.8678, + "theoretical_loss": 3.7921945208647583, + "tokens_seen": 677249024 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004014042126379138, + "loss": 2.7988, + "theoretical_loss": 3.792157042741685, + "tokens_seen": 677314560 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004013941825476429, + "loss": 3.0703, + "theoretical_loss": 3.792119569260027, + "tokens_seen": 677380096 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040138415245737216, + "loss": 2.6792, + "theoretical_loss": 3.79208210041876, + "tokens_seen": 677445632 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004013741223671013, + "loss": 3.0643, + "theoretical_loss": 3.792044636216862, + "tokens_seen": 677511168 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004013640922768305, + "loss": 2.89, + "theoretical_loss": 3.7920071766533088, + "tokens_seen": 677576704 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004013540621865597, + "loss": 2.8468, + "theoretical_loss": 3.791969721727077, + "tokens_seen": 677642240 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004013440320962889, + "loss": 2.9783, + "theoretical_loss": 3.7919322714371457, + "tokens_seen": 677707776 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040133400200601806, + "loss": 3.0253, + "theoretical_loss": 3.7918948257824914, + "tokens_seen": 677773312 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040132397191574724, + "loss": 2.7954, + "theoretical_loss": 3.7918573847620927, + "tokens_seen": 677838848 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004013139418254764, + "loss": 2.8825, + "theoretical_loss": 3.791819948374928, + "tokens_seen": 677904384 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040130391173520566, + "loss": 2.8253, + "theoretical_loss": 3.791782516619977, + "tokens_seen": 677969920 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004012938816449348, + "loss": 2.5945, + "theoretical_loss": 3.7917450894962177, + "tokens_seen": 678035456 + }, + { + "epoch": 0.21, + "learning_rate": 0.000401283851554664, + "loss": 3.0434, + "theoretical_loss": 3.7917076670026297, + "tokens_seen": 678100992 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040127382146439315, + "loss": 2.7672, + "theoretical_loss": 3.7916702491381935, + "tokens_seen": 678166528 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004012637913741224, + "loss": 3.021, + "theoretical_loss": 3.791632835901889, + "tokens_seen": 678232064 + }, + { + "epoch": 0.21, + "objective/train/docs_used": 393619, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.3123815059661865, + "objective/train/theoretical_loss": 3.7915954272926955, + "objective/train/tokens_used": 698757600, + "theoretical_loss": 3.7915954272926955, + "tokens_seen": 678297600 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040125376128385157, + "loss": 2.9455, + "theoretical_loss": 3.7915954272926955, + "tokens_seen": 678297600 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040124373119358075, + "loss": 2.8319, + "theoretical_loss": 3.7915580233095953, + "tokens_seen": 678363136 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040123370110330993, + "loss": 2.5695, + "theoretical_loss": 3.791520623951569, + "tokens_seen": 678428672 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040122367101303916, + "loss": 2.6046, + "theoretical_loss": 3.7914832292175973, + "tokens_seen": 678494208 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004012136409227683, + "loss": 2.7709, + "theoretical_loss": 3.7914458391066628, + "tokens_seen": 678559744 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004012036108324975, + "loss": 2.8296, + "theoretical_loss": 3.7914084536177475, + "tokens_seen": 678625280 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040119358074222665, + "loss": 3.0844, + "theoretical_loss": 3.7913710727498335, + "tokens_seen": 678690816 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004011835506519559, + "loss": 2.8567, + "theoretical_loss": 3.791333696501903, + "tokens_seen": 678756352 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040117352056168507, + "loss": 2.716, + "theoretical_loss": 3.7912963248729397, + "tokens_seen": 678821888 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040116349047141425, + "loss": 2.7171, + "theoretical_loss": 3.7912589578619267, + "tokens_seen": 678887424 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040115346038114343, + "loss": 2.9645, + "theoretical_loss": 3.7912215954678476, + "tokens_seen": 678952960 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004011434302908726, + "loss": 2.8967, + "theoretical_loss": 3.791184237689687, + "tokens_seen": 679018496 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004011334002006018, + "loss": 2.7813, + "theoretical_loss": 3.791146884526428, + "tokens_seen": 679084032 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040112337011033103, + "loss": 2.9556, + "theoretical_loss": 3.791109535977056, + "tokens_seen": 679149568 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040111334002006016, + "loss": 2.8205, + "theoretical_loss": 3.7910721920405566, + "tokens_seen": 679215104 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004011033099297894, + "loss": 2.8423, + "theoretical_loss": 3.7910348527159137, + "tokens_seen": 679280640 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004010932798395185, + "loss": 2.7842, + "theoretical_loss": 3.790997518002113, + "tokens_seen": 679346176 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040108324974924775, + "loss": 2.7179, + "theoretical_loss": 3.790960187898142, + "tokens_seen": 679411712 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040107321965897693, + "loss": 2.8114, + "theoretical_loss": 3.7909228624029856, + "tokens_seen": 679477248 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004010631895687061, + "loss": 3.0359, + "theoretical_loss": 3.79088554151563, + "tokens_seen": 679542784 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004010531594784353, + "loss": 2.8679, + "theoretical_loss": 3.790848225235063, + "tokens_seen": 679608320 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040104312938816453, + "loss": 2.7087, + "theoretical_loss": 3.7908109135602714, + "tokens_seen": 679673856 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040103309929789366, + "loss": 2.6258, + "theoretical_loss": 3.790773606490243, + "tokens_seen": 679739392 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004010230692076229, + "loss": 2.7007, + "theoretical_loss": 3.7907363040239654, + "tokens_seen": 679804928 + }, + { + "epoch": 0.21, + "learning_rate": 0.000401013039117352, + "loss": 2.8675, + "theoretical_loss": 3.790699006160427, + "tokens_seen": 679870464 + }, + { + "epoch": 0.21, + "objective/train/docs_used": 393966, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.9207396507263184, + "objective/train/theoretical_loss": 3.7906617128986153, + "objective/train/tokens_used": 700396000, + "theoretical_loss": 3.7906617128986153, + "tokens_seen": 679936000 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040100300902708126, + "loss": 2.907, + "theoretical_loss": 3.7906617128986153, + "tokens_seen": 679936000 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040099297893681044, + "loss": 2.758, + "theoretical_loss": 3.79062442423752, + "tokens_seen": 680001536 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004009829488465396, + "loss": 2.6981, + "theoretical_loss": 3.7905871401761306, + "tokens_seen": 680067072 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004009729187562688, + "loss": 2.8391, + "theoretical_loss": 3.790549860713435, + "tokens_seen": 680132608 + }, + { + "epoch": 0.21, + "learning_rate": 0.000400962888665998, + "loss": 2.7831, + "theoretical_loss": 3.7905125858484245, + "tokens_seen": 680198144 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040095285857572716, + "loss": 2.8503, + "theoretical_loss": 3.7904753155800885, + "tokens_seen": 680263680 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004009428284854564, + "loss": 2.8111, + "theoretical_loss": 3.7904380499074177, + "tokens_seen": 680329216 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004009327983951855, + "loss": 2.505, + "theoretical_loss": 3.7904007888294027, + "tokens_seen": 680394752 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040092276830491476, + "loss": 3.0351, + "theoretical_loss": 3.790363532345034, + "tokens_seen": 680460288 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004009127382146439, + "loss": 2.9324, + "theoretical_loss": 3.790326280453303, + "tokens_seen": 680525824 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004009027081243731, + "loss": 3.1533, + "theoretical_loss": 3.790289033153202, + "tokens_seen": 680591360 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040089267803410236, + "loss": 2.7432, + "theoretical_loss": 3.7902517904437225, + "tokens_seen": 680656896 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004008826479438315, + "loss": 2.8723, + "theoretical_loss": 3.7902145523238575, + "tokens_seen": 680722432 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004008726178535607, + "loss": 2.5839, + "theoretical_loss": 3.790177318792599, + "tokens_seen": 680787968 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004008625877632899, + "loss": 2.8556, + "theoretical_loss": 3.79014008984894, + "tokens_seen": 680853504 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004008525576730191, + "loss": 2.8399, + "theoretical_loss": 3.790102865491874, + "tokens_seen": 680919040 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040084252758274826, + "loss": 2.6629, + "theoretical_loss": 3.7900656457203934, + "tokens_seen": 680984576 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040083249749247744, + "loss": 2.9075, + "theoretical_loss": 3.7900284305334937, + "tokens_seen": 681050112 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004008224674022066, + "loss": 2.797, + "theoretical_loss": 3.7899912199301684, + "tokens_seen": 681115648 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040081243731193586, + "loss": 2.8778, + "theoretical_loss": 3.789954013909412, + "tokens_seen": 681181184 + }, + { + "epoch": 0.21, + "learning_rate": 0.000400802407221665, + "loss": 2.6526, + "theoretical_loss": 3.78991681247022, + "tokens_seen": 681246720 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004007923771313942, + "loss": 2.6062, + "theoretical_loss": 3.7898796156115866, + "tokens_seen": 681312256 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040078234704112335, + "loss": 2.6582, + "theoretical_loss": 3.7898424233325074, + "tokens_seen": 681377792 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004007723169508526, + "loss": 3.0625, + "theoretical_loss": 3.7898052356319787, + "tokens_seen": 681443328 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040076228686058177, + "loss": 2.9488, + "theoretical_loss": 3.7897680525089967, + "tokens_seen": 681508864 + }, + { + "epoch": 0.21, + "objective/train/docs_used": 395160, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.8984808921813965, + "objective/train/theoretical_loss": 3.789730873962557, + "objective/train/tokens_used": 702034400, + "theoretical_loss": 3.789730873962557, + "tokens_seen": 681574400 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040075225677031095, + "loss": 2.9792, + "theoretical_loss": 3.789730873962557, + "tokens_seen": 681574400 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040074222668004013, + "loss": 3.074, + "theoretical_loss": 3.789693699991657, + "tokens_seen": 681639936 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040073219658976936, + "loss": 3.0624, + "theoretical_loss": 3.7896565305952934, + "tokens_seen": 681705472 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004007221664994985, + "loss": 2.9794, + "theoretical_loss": 3.789619365772464, + "tokens_seen": 681771008 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004007121364092277, + "loss": 3.0109, + "theoretical_loss": 3.7895822055221657, + "tokens_seen": 681836544 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040070210631895685, + "loss": 2.7651, + "theoretical_loss": 3.7895450498433982, + "tokens_seen": 681902080 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004006920762286861, + "loss": 2.9595, + "theoretical_loss": 3.789507898735158, + "tokens_seen": 681967616 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040068204613841527, + "loss": 2.8614, + "theoretical_loss": 3.7894707521964444, + "tokens_seen": 682033152 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040067201604814445, + "loss": 2.9915, + "theoretical_loss": 3.789433610226256, + "tokens_seen": 682098688 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040066198595787363, + "loss": 2.7755, + "theoretical_loss": 3.789396472823593, + "tokens_seen": 682164224 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004006519558676028, + "loss": 2.7368, + "theoretical_loss": 3.7893593399874548, + "tokens_seen": 682229760 + }, + { + "epoch": 0.21, + "learning_rate": 0.000400641925777332, + "loss": 3.0648, + "theoretical_loss": 3.78932221171684, + "tokens_seen": 682295296 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040063189568706123, + "loss": 3.2184, + "theoretical_loss": 3.7892850880107503, + "tokens_seen": 682360832 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040062186559679036, + "loss": 2.9323, + "theoretical_loss": 3.789247968868186, + "tokens_seen": 682426368 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004006118355065196, + "loss": 2.7098, + "theoretical_loss": 3.7892108542881475, + "tokens_seen": 682491904 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004006018054162487, + "loss": 2.7786, + "theoretical_loss": 3.7891737442696356, + "tokens_seen": 682557440 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040059177532597795, + "loss": 3.0218, + "theoretical_loss": 3.7891366388116525, + "tokens_seen": 682622976 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040058174523570713, + "loss": 3.0288, + "theoretical_loss": 3.7890995379131995, + "tokens_seen": 682688512 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004005717151454363, + "loss": 2.7651, + "theoretical_loss": 3.7890624415732796, + "tokens_seen": 682754048 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004005616850551655, + "loss": 2.7824, + "theoretical_loss": 3.789025349790894, + "tokens_seen": 682819584 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040055165496489473, + "loss": 2.65, + "theoretical_loss": 3.7889882625650464, + "tokens_seen": 682885120 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040054162487462386, + "loss": 2.7582, + "theoretical_loss": 3.788951179894739, + "tokens_seen": 682950656 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004005315947843531, + "loss": 2.7901, + "theoretical_loss": 3.788914101778976, + "tokens_seen": 683016192 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004005215646940822, + "loss": 3.2101, + "theoretical_loss": 3.7888770282167608, + "tokens_seen": 683081728 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040051153460381146, + "loss": 3.0466, + "theoretical_loss": 3.788839959207097, + "tokens_seen": 683147264 + }, + { + "epoch": 0.21, + "objective/train/docs_used": 395993, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.792558193206787, + "objective/train/theoretical_loss": 3.788802894748989, + "objective/train/tokens_used": 703672800, + "theoretical_loss": 3.788802894748989, + "tokens_seen": 683212800 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040050150451354064, + "loss": 2.6948, + "theoretical_loss": 3.788802894748989, + "tokens_seen": 683212800 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004004914744232698, + "loss": 2.683, + "theoretical_loss": 3.788765834841442, + "tokens_seen": 683278336 + }, + { + "epoch": 0.21, + "learning_rate": 0.000400481444332999, + "loss": 2.7205, + "theoretical_loss": 3.78872877948346, + "tokens_seen": 683343872 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004004714142427282, + "loss": 3.1439, + "theoretical_loss": 3.788691728674049, + "tokens_seen": 683409408 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040046138415245736, + "loss": 2.7111, + "theoretical_loss": 3.788654682412215, + "tokens_seen": 683474944 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004004513540621866, + "loss": 2.9913, + "theoretical_loss": 3.7886176406969625, + "tokens_seen": 683540480 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004004413239719157, + "loss": 2.6972, + "theoretical_loss": 3.788580603527298, + "tokens_seen": 683606016 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040043129388164496, + "loss": 2.9007, + "theoretical_loss": 3.7885435709022284, + "tokens_seen": 683671552 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004004212637913741, + "loss": 2.9944, + "theoretical_loss": 3.788506542820761, + "tokens_seen": 683737088 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004004112337011033, + "loss": 3.0228, + "theoretical_loss": 3.788469519281902, + "tokens_seen": 683802624 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004004012036108325, + "loss": 3.0442, + "theoretical_loss": 3.7884325002846593, + "tokens_seen": 683868160 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004003911735205617, + "loss": 3.0193, + "theoretical_loss": 3.7883954858280404, + "tokens_seen": 683933696 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040038114343029087, + "loss": 2.7901, + "theoretical_loss": 3.788358475911054, + "tokens_seen": 683999232 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004003711133400201, + "loss": 2.8192, + "theoretical_loss": 3.788321470532707, + "tokens_seen": 684064768 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040036108324974923, + "loss": 2.8012, + "theoretical_loss": 3.7882844696920097, + "tokens_seen": 684130304 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040035105315947846, + "loss": 2.874, + "theoretical_loss": 3.78824747338797, + "tokens_seen": 684195840 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004003410230692076, + "loss": 2.8143, + "theoretical_loss": 3.788210481619598, + "tokens_seen": 684261376 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004003309929789368, + "loss": 2.6845, + "theoretical_loss": 3.7881734943859025, + "tokens_seen": 684326912 + }, + { + "epoch": 0.21, + "learning_rate": 0.000400320962888666, + "loss": 2.9729, + "theoretical_loss": 3.788136511685894, + "tokens_seen": 684392448 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004003109327983952, + "loss": 2.7388, + "theoretical_loss": 3.788099533518582, + "tokens_seen": 684457984 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040030090270812437, + "loss": 2.7836, + "theoretical_loss": 3.7880625598829782, + "tokens_seen": 684523520 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040029087261785355, + "loss": 2.573, + "theoretical_loss": 3.7880255907780924, + "tokens_seen": 684589056 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040028084252758273, + "loss": 2.5967, + "theoretical_loss": 3.787988626202937, + "tokens_seen": 684654592 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040027081243731197, + "loss": 2.9593, + "theoretical_loss": 3.7879516661565216, + "tokens_seen": 684720128 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004002607823470411, + "loss": 2.7774, + "theoretical_loss": 3.7879147106378595, + "tokens_seen": 684785664 + }, + { + "epoch": 0.21, + "objective/train/docs_used": 397274, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.975048542022705, + "objective/train/theoretical_loss": 3.787877759645963, + "objective/train/tokens_used": 705311200, + "theoretical_loss": 3.787877759645963, + "tokens_seen": 684851200 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040025075225677033, + "loss": 3.0113, + "theoretical_loss": 3.787877759645963, + "tokens_seen": 684851200 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040024072216649946, + "loss": 2.888, + "theoretical_loss": 3.7878408131798427, + "tokens_seen": 684916736 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004002306920762287, + "loss": 2.7738, + "theoretical_loss": 3.787803871238513, + "tokens_seen": 684982272 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040022066198595787, + "loss": 2.5958, + "theoretical_loss": 3.7877669338209863, + "tokens_seen": 685047808 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040021063189568705, + "loss": 2.801, + "theoretical_loss": 3.787730000926276, + "tokens_seen": 685113344 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040020060180541623, + "loss": 3.0578, + "theoretical_loss": 3.787693072553396, + "tokens_seen": 685178880 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040019057171514547, + "loss": 3.0213, + "theoretical_loss": 3.78765614870136, + "tokens_seen": 685244416 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004001805416248746, + "loss": 2.8021, + "theoretical_loss": 3.7876192293691817, + "tokens_seen": 685309952 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040017051153460383, + "loss": 2.9207, + "theoretical_loss": 3.787582314555877, + "tokens_seen": 685375488 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040016048144433296, + "loss": 2.81, + "theoretical_loss": 3.7875454042604595, + "tokens_seen": 685441024 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004001504513540622, + "loss": 2.7681, + "theoretical_loss": 3.787508498481945, + "tokens_seen": 685506560 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040014042126379143, + "loss": 2.4868, + "theoretical_loss": 3.7874715972193487, + "tokens_seen": 685572096 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040013039117352056, + "loss": 2.6161, + "theoretical_loss": 3.7874347004716866, + "tokens_seen": 685637632 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004001203610832498, + "loss": 2.8467, + "theoretical_loss": 3.787397808237975, + "tokens_seen": 685703168 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004001103309929789, + "loss": 2.7188, + "theoretical_loss": 3.78736092051723, + "tokens_seen": 685768704 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040010030090270815, + "loss": 2.807, + "theoretical_loss": 3.7873240373084687, + "tokens_seen": 685834240 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040009027081243733, + "loss": 2.6013, + "theoretical_loss": 3.7872871586107077, + "tokens_seen": 685899776 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004000802407221665, + "loss": 2.8663, + "theoretical_loss": 3.7872502844229645, + "tokens_seen": 685965312 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004000702106318957, + "loss": 3.1228, + "theoretical_loss": 3.787213414744257, + "tokens_seen": 686030848 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040006018054162493, + "loss": 2.951, + "theoretical_loss": 3.787176549573603, + "tokens_seen": 686096384 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040005015045135406, + "loss": 2.9166, + "theoretical_loss": 3.7871396889100204, + "tokens_seen": 686161920 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004000401203610833, + "loss": 2.6314, + "theoretical_loss": 3.7871028327525287, + "tokens_seen": 686227456 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004000300902708124, + "loss": 2.6956, + "theoretical_loss": 3.7870659811001457, + "tokens_seen": 686292992 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040002006018054166, + "loss": 2.7525, + "theoretical_loss": 3.787029133951891, + "tokens_seen": 686358528 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040001003009027084, + "loss": 3.0182, + "theoretical_loss": 3.7869922913067846, + "tokens_seen": 686424064 + }, + { + "epoch": 0.21, + "objective/train/docs_used": 397954, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.8832666873931885, + "objective/train/theoretical_loss": 3.786955453163846, + "objective/train/tokens_used": 706949600, + "theoretical_loss": 3.786955453163846, + "tokens_seen": 686489600 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004, + "loss": 2.8264, + "theoretical_loss": 3.786955453163846, + "tokens_seen": 686489600 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003999899699097292, + "loss": 2.5334, + "theoretical_loss": 3.7869186195220945, + "tokens_seen": 686555136 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003999799398194584, + "loss": 2.8664, + "theoretical_loss": 3.786881790380552, + "tokens_seen": 686620672 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039996990972918756, + "loss": 2.8542, + "theoretical_loss": 3.7868449657382377, + "tokens_seen": 686686208 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003999598796389168, + "loss": 3.0112, + "theoretical_loss": 3.786808145594174, + "tokens_seen": 686751744 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003999498495486459, + "loss": 2.9331, + "theoretical_loss": 3.7867713299473813, + "tokens_seen": 686817280 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039993981945837516, + "loss": 2.798, + "theoretical_loss": 3.786734518796882, + "tokens_seen": 686882816 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003999297893681043, + "loss": 2.5032, + "theoretical_loss": 3.786697712141697, + "tokens_seen": 686948352 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003999197592778335, + "loss": 2.9318, + "theoretical_loss": 3.7866609099808493, + "tokens_seen": 687013888 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003999097291875627, + "loss": 2.8875, + "theoretical_loss": 3.7866241123133615, + "tokens_seen": 687079424 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003998996990972919, + "loss": 2.7277, + "theoretical_loss": 3.786587319138256, + "tokens_seen": 687144960 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039988966900702107, + "loss": 2.7086, + "theoretical_loss": 3.786550530454557, + "tokens_seen": 687210496 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003998796389167503, + "loss": 2.5312, + "theoretical_loss": 3.786513746261287, + "tokens_seen": 687276032 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039986960882647943, + "loss": 2.7448, + "theoretical_loss": 3.78647696655747, + "tokens_seen": 687341568 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039985957873620866, + "loss": 2.731, + "theoretical_loss": 3.7864401913421295, + "tokens_seen": 687407104 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003998495486459378, + "loss": 2.9249, + "theoretical_loss": 3.7864034206142914, + "tokens_seen": 687472640 + }, + { + "epoch": 0.21, + "learning_rate": 0.000399839518555667, + "loss": 3.0563, + "theoretical_loss": 3.786366654372979, + "tokens_seen": 687538176 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003998294884653962, + "loss": 2.8605, + "theoretical_loss": 3.7863298926172186, + "tokens_seen": 687603712 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003998194583751254, + "loss": 2.8705, + "theoretical_loss": 3.786293135346034, + "tokens_seen": 687669248 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039980942828485457, + "loss": 3.0669, + "theoretical_loss": 3.786256382558452, + "tokens_seen": 687734784 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039979939819458375, + "loss": 2.9938, + "theoretical_loss": 3.7862196342534977, + "tokens_seen": 687800320 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039978936810431293, + "loss": 2.7179, + "theoretical_loss": 3.786182890430198, + "tokens_seen": 687865856 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039977933801404217, + "loss": 2.8097, + "theoretical_loss": 3.7861461510875793, + "tokens_seen": 687931392 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003997693079237713, + "loss": 2.6891, + "theoretical_loss": 3.786109416224668, + "tokens_seen": 687996928 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039975927783350053, + "loss": 2.7367, + "theoretical_loss": 3.786072685840492, + "tokens_seen": 688062464 + }, + { + "epoch": 0.21, + "objective/train/docs_used": 398552, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.9904754161834717, + "objective/train/theoretical_loss": 3.7860359599340776, + "objective/train/tokens_used": 708588000, + "theoretical_loss": 3.7860359599340776, + "tokens_seen": 688128000 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039974924774322966, + "loss": 2.8214, + "theoretical_loss": 3.7860359599340776, + "tokens_seen": 688128000 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003997392176529589, + "loss": 2.551, + "theoretical_loss": 3.7859992385044534, + "tokens_seen": 688193536 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039972918756268807, + "loss": 2.664, + "theoretical_loss": 3.7859625215506476, + "tokens_seen": 688259072 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039971915747241725, + "loss": 2.614, + "theoretical_loss": 3.7859258090716876, + "tokens_seen": 688324608 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039970912738214643, + "loss": 2.6916, + "theoretical_loss": 3.785889101066603, + "tokens_seen": 688390144 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039969909729187567, + "loss": 2.9462, + "theoretical_loss": 3.785852397534423, + "tokens_seen": 688455680 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003996890672016048, + "loss": 2.9353, + "theoretical_loss": 3.7858156984741758, + "tokens_seen": 688521216 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039967903711133403, + "loss": 2.679, + "theoretical_loss": 3.7857790038848913, + "tokens_seen": 688586752 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039966900702106316, + "loss": 2.6621, + "theoretical_loss": 3.7857423137655997, + "tokens_seen": 688652288 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003996589769307924, + "loss": 2.9201, + "theoretical_loss": 3.785705628115331, + "tokens_seen": 688717824 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003996489468405216, + "loss": 3.0937, + "theoretical_loss": 3.7856689469331153, + "tokens_seen": 688783360 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039963891675025076, + "loss": 2.6647, + "theoretical_loss": 3.785632270217984, + "tokens_seen": 688848896 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039962888665997994, + "loss": 2.7568, + "theoretical_loss": 3.7855955979689684, + "tokens_seen": 688914432 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003996188565697091, + "loss": 2.6805, + "theoretical_loss": 3.7855589301850987, + "tokens_seen": 688979968 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003996088264794383, + "loss": 2.8601, + "theoretical_loss": 3.7855222668654074, + "tokens_seen": 689045504 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039959879638916754, + "loss": 2.9986, + "theoretical_loss": 3.7854856080089268, + "tokens_seen": 689111040 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039958876629889666, + "loss": 2.8372, + "theoretical_loss": 3.7854489536146883, + "tokens_seen": 689176576 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003995787362086259, + "loss": 2.9725, + "theoretical_loss": 3.785412303681725, + "tokens_seen": 689242112 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003995687061183551, + "loss": 2.8084, + "theoretical_loss": 3.7853756582090705, + "tokens_seen": 689307648 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039955867602808426, + "loss": 2.7216, + "theoretical_loss": 3.7853390171957564, + "tokens_seen": 689373184 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039954864593781344, + "loss": 2.8981, + "theoretical_loss": 3.7853023806408173, + "tokens_seen": 689438720 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003995386158475426, + "loss": 2.9901, + "theoretical_loss": 3.785265748543287, + "tokens_seen": 689504256 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003995285857572718, + "loss": 2.9998, + "theoretical_loss": 3.7852291209021987, + "tokens_seen": 689569792 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039951855566700104, + "loss": 2.7837, + "theoretical_loss": 3.785192497716588, + "tokens_seen": 689635328 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039950852557673017, + "loss": 2.8644, + "theoretical_loss": 3.7851558789854884, + "tokens_seen": 689700864 + }, + { + "epoch": 0.21, + "objective/train/docs_used": 399765, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.939901828765869, + "objective/train/theoretical_loss": 3.785119264707936, + "objective/train/tokens_used": 710226400, + "theoretical_loss": 3.785119264707936, + "tokens_seen": 689766400 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003994984954864594, + "loss": 2.985, + "theoretical_loss": 3.785119264707936, + "tokens_seen": 689766400 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039948846539618853, + "loss": 2.8666, + "theoretical_loss": 3.7850826548829652, + "tokens_seen": 689831936 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039947843530591776, + "loss": 2.6676, + "theoretical_loss": 3.7850460495096123, + "tokens_seen": 689897472 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039946840521564694, + "loss": 2.8564, + "theoretical_loss": 3.785009448586913, + "tokens_seen": 689963008 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003994583751253761, + "loss": 2.9739, + "theoretical_loss": 3.784972852113903, + "tokens_seen": 690028544 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003994483450351053, + "loss": 2.8601, + "theoretical_loss": 3.78493626008962, + "tokens_seen": 690094080 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003994383149448345, + "loss": 2.6314, + "theoretical_loss": 3.7848996725130992, + "tokens_seen": 690159616 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039942828485456367, + "loss": 3.0499, + "theoretical_loss": 3.7848630893833786, + "tokens_seen": 690225152 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003994182547642929, + "loss": 2.8139, + "theoretical_loss": 3.784826510699496, + "tokens_seen": 690290688 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039940822467402203, + "loss": 2.8819, + "theoretical_loss": 3.7847899364604887, + "tokens_seen": 690356224 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039939819458375127, + "loss": 2.8537, + "theoretical_loss": 3.7847533666653943, + "tokens_seen": 690421760 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003993881644934805, + "loss": 2.8592, + "theoretical_loss": 3.7847168013132513, + "tokens_seen": 690487296 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039937813440320963, + "loss": 2.8396, + "theoretical_loss": 3.7846802404030986, + "tokens_seen": 690552832 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039936810431293886, + "loss": 2.874, + "theoretical_loss": 3.7846436839339743, + "tokens_seen": 690618368 + }, + { + "epoch": 0.21, + "learning_rate": 0.000399358074222668, + "loss": 2.6484, + "theoretical_loss": 3.7846071319049193, + "tokens_seen": 690683904 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003993480441323972, + "loss": 2.8152, + "theoretical_loss": 3.7845705843149715, + "tokens_seen": 690749440 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003993380140421264, + "loss": 2.8489, + "theoretical_loss": 3.7845340411631714, + "tokens_seen": 690814976 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003993279839518556, + "loss": 2.6405, + "theoretical_loss": 3.7844975024485583, + "tokens_seen": 690880512 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039931795386158477, + "loss": 2.79, + "theoretical_loss": 3.7844609681701735, + "tokens_seen": 690946048 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039930792377131395, + "loss": 2.8363, + "theoretical_loss": 3.7844244383270578, + "tokens_seen": 691011584 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039929789368104313, + "loss": 2.7311, + "theoretical_loss": 3.7843879129182514, + "tokens_seen": 691077120 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039928786359077237, + "loss": 2.6962, + "theoretical_loss": 3.7843513919427965, + "tokens_seen": 691142656 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003992778335005015, + "loss": 2.7641, + "theoretical_loss": 3.7843148753997333, + "tokens_seen": 691208192 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039926780341023073, + "loss": 2.7931, + "theoretical_loss": 3.784278363288105, + "tokens_seen": 691273728 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039925777331995986, + "loss": 2.8702, + "theoretical_loss": 3.7842418556069535, + "tokens_seen": 691339264 + }, + { + "epoch": 0.21, + "objective/train/docs_used": 400468, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.811363935470581, + "objective/train/theoretical_loss": 3.784205352355321, + "objective/train/tokens_used": 711864800, + "theoretical_loss": 3.784205352355321, + "tokens_seen": 691404800 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003992477432296891, + "loss": 2.5283, + "theoretical_loss": 3.784205352355321, + "tokens_seen": 691404800 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039923771313941827, + "loss": 2.7441, + "theoretical_loss": 3.7841688535322504, + "tokens_seen": 691470336 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039922768304914745, + "loss": 2.9421, + "theoretical_loss": 3.7841323591367844, + "tokens_seen": 691535872 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039921765295887663, + "loss": 2.8069, + "theoretical_loss": 3.7840958691679667, + "tokens_seen": 691601408 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039920762286860587, + "loss": 2.8315, + "theoretical_loss": 3.7840593836248413, + "tokens_seen": 691666944 + }, + { + "epoch": 0.21, + "learning_rate": 0.000399197592778335, + "loss": 2.6754, + "theoretical_loss": 3.784022902506452, + "tokens_seen": 691732480 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039918756268806423, + "loss": 2.5659, + "theoretical_loss": 3.7839864258118427, + "tokens_seen": 691798016 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039917753259779336, + "loss": 2.7707, + "theoretical_loss": 3.7839499535400583, + "tokens_seen": 691863552 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003991675025075226, + "loss": 2.7279, + "theoretical_loss": 3.7839134856901437, + "tokens_seen": 691929088 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003991574724172518, + "loss": 3.0561, + "theoretical_loss": 3.7838770222611435, + "tokens_seen": 691994624 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039914744232698096, + "loss": 2.9778, + "theoretical_loss": 3.7838405632521037, + "tokens_seen": 692060160 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039913741223671014, + "loss": 2.5886, + "theoretical_loss": 3.7838041086620704, + "tokens_seen": 692125696 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003991273821464393, + "loss": 2.8461, + "theoretical_loss": 3.7837676584900883, + "tokens_seen": 692191232 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003991173520561685, + "loss": 2.7891, + "theoretical_loss": 3.7837312127352054, + "tokens_seen": 692256768 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039910732196589774, + "loss": 2.9199, + "theoretical_loss": 3.7836947713964673, + "tokens_seen": 692322304 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039909729187562686, + "loss": 2.7669, + "theoretical_loss": 3.783658334472921, + "tokens_seen": 692387840 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003990872617853561, + "loss": 2.8963, + "theoretical_loss": 3.783621901963614, + "tokens_seen": 692453376 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003990772316950853, + "loss": 2.8769, + "theoretical_loss": 3.7835854738675936, + "tokens_seen": 692518912 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039906720160481446, + "loss": 2.8156, + "theoretical_loss": 3.7835490501839075, + "tokens_seen": 692584448 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039905717151454364, + "loss": 2.9119, + "theoretical_loss": 3.7835126309116047, + "tokens_seen": 692649984 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003990471414242728, + "loss": 3.0404, + "theoretical_loss": 3.7834762160497326, + "tokens_seen": 692715520 + }, + { + "epoch": 0.21, + "learning_rate": 0.000399037111334002, + "loss": 2.9171, + "theoretical_loss": 3.7834398055973404, + "tokens_seen": 692781056 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039902708124373124, + "loss": 2.805, + "theoretical_loss": 3.7834033995534764, + "tokens_seen": 692846592 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039901705115346037, + "loss": 2.9773, + "theoretical_loss": 3.7833669979171916, + "tokens_seen": 692912128 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003990070210631896, + "loss": 2.5461, + "theoretical_loss": 3.7833306006875334, + "tokens_seen": 692977664 + }, + { + "epoch": 0.21, + "objective/train/docs_used": 401917, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.8451545238494873, + "objective/train/theoretical_loss": 3.7832942078635527, + "objective/train/tokens_used": 713503200, + "theoretical_loss": 3.7832942078635527, + "tokens_seen": 693043200 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039899699097291873, + "loss": 2.823, + "theoretical_loss": 3.7832942078635527, + "tokens_seen": 693043200 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039898696088264796, + "loss": 2.7804, + "theoretical_loss": 3.7832578194443007, + "tokens_seen": 693108736 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039897693079237714, + "loss": 2.8775, + "theoretical_loss": 3.7832214354288265, + "tokens_seen": 693174272 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003989669007021063, + "loss": 2.5998, + "theoretical_loss": 3.783185055816181, + "tokens_seen": 693239808 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003989568706118355, + "loss": 2.6169, + "theoretical_loss": 3.7831486806054158, + "tokens_seen": 693305344 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003989468405215647, + "loss": 2.732, + "theoretical_loss": 3.783112309795582, + "tokens_seen": 693370880 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039893681043129387, + "loss": 2.9819, + "theoretical_loss": 3.783075943385731, + "tokens_seen": 693436416 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003989267803410231, + "loss": 2.8156, + "theoretical_loss": 3.783039581374915, + "tokens_seen": 693501952 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039891675025075223, + "loss": 2.9239, + "theoretical_loss": 3.7830032237621865, + "tokens_seen": 693567488 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039890672016048147, + "loss": 2.883, + "theoretical_loss": 3.7829668705465975, + "tokens_seen": 693633024 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039889669007021065, + "loss": 2.8467, + "theoretical_loss": 3.7829305217272013, + "tokens_seen": 693698560 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039888665997993983, + "loss": 2.882, + "theoretical_loss": 3.7828941773030507, + "tokens_seen": 693764096 + }, + { + "epoch": 0.21, + "learning_rate": 0.000398876629889669, + "loss": 3.0187, + "theoretical_loss": 3.782857837273199, + "tokens_seen": 693829632 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003988665997993982, + "loss": 2.7486, + "theoretical_loss": 3.782821501636701, + "tokens_seen": 693895168 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039885656970912737, + "loss": 2.9178, + "theoretical_loss": 3.782785170392609, + "tokens_seen": 693960704 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003988465396188566, + "loss": 2.804, + "theoretical_loss": 3.782748843539978, + "tokens_seen": 694026240 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039883650952858573, + "loss": 2.8876, + "theoretical_loss": 3.7827125210778636, + "tokens_seen": 694091776 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039882647943831497, + "loss": 2.7313, + "theoretical_loss": 3.7826762030053196, + "tokens_seen": 694157312 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003988164493480441, + "loss": 3.0362, + "theoretical_loss": 3.7826398893214006, + "tokens_seen": 694222848 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039880641925777333, + "loss": 2.737, + "theoretical_loss": 3.782603580025164, + "tokens_seen": 694288384 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003987963891675025, + "loss": 2.9627, + "theoretical_loss": 3.782567275115663, + "tokens_seen": 694353920 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003987863590772317, + "loss": 2.8431, + "theoretical_loss": 3.782530974591956, + "tokens_seen": 694419456 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003987763289869609, + "loss": 2.5091, + "theoretical_loss": 3.782494678453098, + "tokens_seen": 694484992 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039876629889669006, + "loss": 3.0296, + "theoretical_loss": 3.7824583866981465, + "tokens_seen": 694550528 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039875626880641924, + "loss": 2.7676, + "theoretical_loss": 3.7824220993261575, + "tokens_seen": 694616064 + }, + { + "epoch": 0.21, + "objective/train/docs_used": 402587, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.668381690979004, + "objective/train/theoretical_loss": 3.782385816336189, + "objective/train/tokens_used": 715141600, + "theoretical_loss": 3.782385816336189, + "tokens_seen": 694681600 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039874623871614847, + "loss": 2.75, + "theoretical_loss": 3.782385816336189, + "tokens_seen": 694681600 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003987362086258776, + "loss": 2.8727, + "theoretical_loss": 3.7823495377272978, + "tokens_seen": 694747136 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039872617853560683, + "loss": 2.6111, + "theoretical_loss": 3.7823132634985424, + "tokens_seen": 694812672 + }, + { + "epoch": 0.21, + "learning_rate": 0.000398716148445336, + "loss": 2.5202, + "theoretical_loss": 3.7822769936489804, + "tokens_seen": 694878208 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003987061183550652, + "loss": 2.8302, + "theoretical_loss": 3.78224072817767, + "tokens_seen": 694943744 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003986960882647944, + "loss": 2.9181, + "theoretical_loss": 3.7822044670836705, + "tokens_seen": 695009280 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039868605817452356, + "loss": 2.6447, + "theoretical_loss": 3.782168210366041, + "tokens_seen": 695074816 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039867602808425274, + "loss": 2.6707, + "theoretical_loss": 3.78213195802384, + "tokens_seen": 695140352 + }, + { + "epoch": 0.21, + "learning_rate": 0.000398665997993982, + "loss": 2.8557, + "theoretical_loss": 3.7820957100561277, + "tokens_seen": 695205888 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003986559679037111, + "loss": 2.6936, + "theoretical_loss": 3.782059466461963, + "tokens_seen": 695271424 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039864593781344034, + "loss": 3.0645, + "theoretical_loss": 3.782023227240407, + "tokens_seen": 695336960 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003986359077231695, + "loss": 2.6558, + "theoretical_loss": 3.78198699239052, + "tokens_seen": 695402496 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003986258776328987, + "loss": 2.8949, + "theoretical_loss": 3.781950761911362, + "tokens_seen": 695468032 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039861584754262794, + "loss": 2.6127, + "theoretical_loss": 3.7819145358019948, + "tokens_seen": 695533568 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039860581745235706, + "loss": 2.5769, + "theoretical_loss": 3.7818783140614793, + "tokens_seen": 695599104 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003985957873620863, + "loss": 2.5067, + "theoretical_loss": 3.7818420966888775, + "tokens_seen": 695664640 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003985857572718155, + "loss": 2.7049, + "theoretical_loss": 3.7818058836832504, + "tokens_seen": 695730176 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039857572718154466, + "loss": 2.6489, + "theoretical_loss": 3.7817696750436607, + "tokens_seen": 695795712 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039856569709127384, + "loss": 2.8479, + "theoretical_loss": 3.781733470769171, + "tokens_seen": 695861248 + }, + { + "epoch": 0.21, + "learning_rate": 0.000398555667001003, + "loss": 2.8889, + "theoretical_loss": 3.7816972708588446, + "tokens_seen": 695926784 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003985456369107322, + "loss": 2.844, + "theoretical_loss": 3.7816610753117432, + "tokens_seen": 695992320 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039853560682046144, + "loss": 2.8438, + "theoretical_loss": 3.7816248841269307, + "tokens_seen": 696057856 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039852557673019057, + "loss": 2.8653, + "theoretical_loss": 3.7815886973034707, + "tokens_seen": 696123392 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003985155466399198, + "loss": 2.9573, + "theoretical_loss": 3.781552514840427, + "tokens_seen": 696188928 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039850551654964893, + "loss": 2.6189, + "theoretical_loss": 3.781516336736864, + "tokens_seen": 696254464 + }, + { + "epoch": 0.21, + "objective/train/docs_used": 403173, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6805615425109863, + "objective/train/theoretical_loss": 3.781480162991847, + "objective/train/tokens_used": 716780000, + "theoretical_loss": 3.781480162991847, + "tokens_seen": 696320000 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039849548645937816, + "loss": 2.7638, + "theoretical_loss": 3.781480162991847, + "tokens_seen": 696320000 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039848545636910734, + "loss": 2.7707, + "theoretical_loss": 3.7814439936044386, + "tokens_seen": 696385536 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003984754262788365, + "loss": 2.7766, + "theoretical_loss": 3.781407828573706, + "tokens_seen": 696451072 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003984653961885657, + "loss": 2.7234, + "theoretical_loss": 3.7813716678987133, + "tokens_seen": 696516608 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003984553660982949, + "loss": 2.6632, + "theoretical_loss": 3.781335511578526, + "tokens_seen": 696582144 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039844533600802407, + "loss": 3.1248, + "theoretical_loss": 3.781299359612211, + "tokens_seen": 696647680 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003984353059177533, + "loss": 2.8936, + "theoretical_loss": 3.781263211998834, + "tokens_seen": 696713216 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039842527582748243, + "loss": 2.7857, + "theoretical_loss": 3.7812270687374614, + "tokens_seen": 696778752 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039841524573721167, + "loss": 2.9456, + "theoretical_loss": 3.7811909298271607, + "tokens_seen": 696844288 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039840521564694085, + "loss": 2.7911, + "theoretical_loss": 3.7811547952669975, + "tokens_seen": 696909824 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039839518555667003, + "loss": 2.7668, + "theoretical_loss": 3.78111866505604, + "tokens_seen": 696975360 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003983851554663992, + "loss": 2.8271, + "theoretical_loss": 3.7810825391933562, + "tokens_seen": 697040896 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003983751253761284, + "loss": 2.8923, + "theoretical_loss": 3.781046417678014, + "tokens_seen": 697106432 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039836509528585757, + "loss": 2.8448, + "theoretical_loss": 3.781010300509081, + "tokens_seen": 697171968 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003983550651955868, + "loss": 2.7279, + "theoretical_loss": 3.7809741876856258, + "tokens_seen": 697237504 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039834503510531593, + "loss": 2.5467, + "theoretical_loss": 3.780938079206718, + "tokens_seen": 697303040 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039833500501504517, + "loss": 2.8478, + "theoretical_loss": 3.7809019750714254, + "tokens_seen": 697368576 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003983249749247743, + "loss": 2.7018, + "theoretical_loss": 3.7808658752788187, + "tokens_seen": 697434112 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039831494483450353, + "loss": 2.7938, + "theoretical_loss": 3.7808297798279664, + "tokens_seen": 697499648 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003983049147442327, + "loss": 2.7658, + "theoretical_loss": 3.7807936887179396, + "tokens_seen": 697565184 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003982948846539619, + "loss": 2.7483, + "theoretical_loss": 3.780757601947807, + "tokens_seen": 697630720 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003982848545636911, + "loss": 2.8211, + "theoretical_loss": 3.780721519516641, + "tokens_seen": 697696256 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039827482447342026, + "loss": 2.849, + "theoretical_loss": 3.7806854414235103, + "tokens_seen": 697761792 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039826479438314944, + "loss": 2.7717, + "theoretical_loss": 3.780649367667488, + "tokens_seen": 697827328 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039825476429287867, + "loss": 2.8079, + "theoretical_loss": 3.780613298247644, + "tokens_seen": 697892864 + }, + { + "epoch": 0.21, + "objective/train/docs_used": 404489, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.075455665588379, + "objective/train/theoretical_loss": 3.7805772331630516, + "objective/train/tokens_used": 718418400, + "theoretical_loss": 3.7805772331630516, + "tokens_seen": 697958400 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003982447342026078, + "loss": 2.7553, + "theoretical_loss": 3.7805772331630516, + "tokens_seen": 697958400 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039823470411233703, + "loss": 2.7964, + "theoretical_loss": 3.7805411724127804, + "tokens_seen": 698023936 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003982246740220662, + "loss": 2.7858, + "theoretical_loss": 3.780505115995904, + "tokens_seen": 698089472 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003982146439317954, + "loss": 2.7204, + "theoretical_loss": 3.7804690639114957, + "tokens_seen": 698155008 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003982046138415246, + "loss": 2.6261, + "theoretical_loss": 3.780433016158627, + "tokens_seen": 698220544 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039819458375125376, + "loss": 2.5031, + "theoretical_loss": 3.780396972736371, + "tokens_seen": 698286080 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039818455366098294, + "loss": 2.8176, + "theoretical_loss": 3.780360933643802, + "tokens_seen": 698351616 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003981745235707122, + "loss": 3.0303, + "theoretical_loss": 3.780324898879993, + "tokens_seen": 698417152 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003981644934804413, + "loss": 2.9263, + "theoretical_loss": 3.780288868444018, + "tokens_seen": 698482688 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039815446339017054, + "loss": 2.893, + "theoretical_loss": 3.7802528423349515, + "tokens_seen": 698548224 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039814443329989966, + "loss": 2.9738, + "theoretical_loss": 3.7802168205518676, + "tokens_seen": 698613760 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003981344032096289, + "loss": 2.7414, + "theoretical_loss": 3.7801808030938417, + "tokens_seen": 698679296 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003981243731193581, + "loss": 2.8169, + "theoretical_loss": 3.780144789959948, + "tokens_seen": 698744832 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039811434302908726, + "loss": 2.8726, + "theoretical_loss": 3.780108781149263, + "tokens_seen": 698810368 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039810431293881644, + "loss": 2.8116, + "theoretical_loss": 3.7800727766608615, + "tokens_seen": 698875904 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003980942828485457, + "loss": 2.7424, + "theoretical_loss": 3.7800367764938194, + "tokens_seen": 698941440 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003980842527582748, + "loss": 2.974, + "theoretical_loss": 3.7800007806472133, + "tokens_seen": 699006976 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039807422266800404, + "loss": 2.7141, + "theoretical_loss": 3.77996478912012, + "tokens_seen": 699072512 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039806419257773317, + "loss": 2.9876, + "theoretical_loss": 3.7799288019116153, + "tokens_seen": 699138048 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003980541624874624, + "loss": 2.8239, + "theoretical_loss": 3.7798928190207772, + "tokens_seen": 699203584 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003980441323971916, + "loss": 2.8128, + "theoretical_loss": 3.7798568404466826, + "tokens_seen": 699269120 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039803410230692077, + "loss": 2.912, + "theoretical_loss": 3.779820866188409, + "tokens_seen": 699334656 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039802407221664995, + "loss": 2.7765, + "theoretical_loss": 3.779784896245035, + "tokens_seen": 699400192 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039801404212637913, + "loss": 2.6396, + "theoretical_loss": 3.7797489306156384, + "tokens_seen": 699465728 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003980040120361083, + "loss": 2.7244, + "theoretical_loss": 3.7797129692992977, + "tokens_seen": 699531264 + }, + { + "epoch": 0.21, + "objective/train/docs_used": 405172, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.034540891647339, + "objective/train/theoretical_loss": 3.779677012295091, + "objective/train/tokens_used": 720056800, + "theoretical_loss": 3.779677012295091, + "tokens_seen": 699596800 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039799398194583754, + "loss": 2.8546, + "theoretical_loss": 3.779677012295091, + "tokens_seen": 699596800 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039798395185556667, + "loss": 2.7322, + "theoretical_loss": 3.7796410596020986, + "tokens_seen": 699662336 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003979739217652959, + "loss": 2.9482, + "theoretical_loss": 3.779605111219399, + "tokens_seen": 699727872 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039796389167502503, + "loss": 2.8466, + "theoretical_loss": 3.7795691671460725, + "tokens_seen": 699793408 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039795386158475427, + "loss": 2.68, + "theoretical_loss": 3.779533227381198, + "tokens_seen": 699858944 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039794383149448345, + "loss": 2.7141, + "theoretical_loss": 3.7794972919238563, + "tokens_seen": 699924480 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039793380140421263, + "loss": 2.5528, + "theoretical_loss": 3.779461360773128, + "tokens_seen": 699990016 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003979237713139418, + "loss": 2.7442, + "theoretical_loss": 3.7794254339280937, + "tokens_seen": 700055552 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039791374122367105, + "loss": 2.7758, + "theoretical_loss": 3.7793895113878344, + "tokens_seen": 700121088 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003979037111334002, + "loss": 2.8044, + "theoretical_loss": 3.7793535931514315, + "tokens_seen": 700186624 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003978936810431294, + "loss": 2.9932, + "theoretical_loss": 3.779317679217966, + "tokens_seen": 700252160 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003978836509528586, + "loss": 2.6386, + "theoretical_loss": 3.779281769586521, + "tokens_seen": 700317696 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039787362086258777, + "loss": 2.7325, + "theoretical_loss": 3.7792458642561777, + "tokens_seen": 700383232 + }, + { + "epoch": 0.21, + "learning_rate": 0.000397863590772317, + "loss": 2.704, + "theoretical_loss": 3.7792099632260188, + "tokens_seen": 700448768 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039785356068204613, + "loss": 2.717, + "theoretical_loss": 3.779174066495127, + "tokens_seen": 700514304 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039784353059177537, + "loss": 2.8388, + "theoretical_loss": 3.7791381740625853, + "tokens_seen": 700579840 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003978335005015045, + "loss": 2.9585, + "theoretical_loss": 3.779102285927477, + "tokens_seen": 700645376 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039782347041123373, + "loss": 2.8581, + "theoretical_loss": 3.779066402088886, + "tokens_seen": 700710912 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003978134403209629, + "loss": 2.8195, + "theoretical_loss": 3.779030522545896, + "tokens_seen": 700776448 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003978034102306921, + "loss": 2.6504, + "theoretical_loss": 3.7789946472975906, + "tokens_seen": 700841984 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003977933801404213, + "loss": 2.8794, + "theoretical_loss": 3.7789587763430545, + "tokens_seen": 700907520 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039778335005015046, + "loss": 2.6254, + "theoretical_loss": 3.7789229096813726, + "tokens_seen": 700973056 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039777331995987964, + "loss": 2.9458, + "theoretical_loss": 3.77888704731163, + "tokens_seen": 701038592 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039776328986960887, + "loss": 2.6863, + "theoretical_loss": 3.7788511892329115, + "tokens_seen": 701104128 + }, + { + "epoch": 0.21, + "learning_rate": 0.000397753259779338, + "loss": 2.6573, + "theoretical_loss": 3.7788153354443024, + "tokens_seen": 701169664 + }, + { + "epoch": 0.21, + "objective/train/docs_used": 406046, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.1960320472717285, + "objective/train/theoretical_loss": 3.7787794859448898, + "objective/train/tokens_used": 721695200, + "theoretical_loss": 3.7787794859448898, + "tokens_seen": 701235200 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039774322968906723, + "loss": 2.8094, + "theoretical_loss": 3.7787794859448898, + "tokens_seen": 701235200 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003977331995987964, + "loss": 2.9694, + "theoretical_loss": 3.7787436407337585, + "tokens_seen": 701300736 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003977231695085256, + "loss": 2.7062, + "theoretical_loss": 3.7787077998099954, + "tokens_seen": 701366272 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003977131394182548, + "loss": 2.6571, + "theoretical_loss": 3.778671963172687, + "tokens_seen": 701431808 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039770310932798396, + "loss": 2.6306, + "theoretical_loss": 3.77863613082092, + "tokens_seen": 701497344 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039769307923771314, + "loss": 2.7091, + "theoretical_loss": 3.778600302753783, + "tokens_seen": 701562880 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003976830491474424, + "loss": 3.0083, + "theoretical_loss": 3.7785644789703614, + "tokens_seen": 701628416 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003976730190571715, + "loss": 2.8071, + "theoretical_loss": 3.7785286594697443, + "tokens_seen": 701693952 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039766298896690074, + "loss": 2.757, + "theoretical_loss": 3.7784928442510193, + "tokens_seen": 701759488 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039765295887662987, + "loss": 2.728, + "theoretical_loss": 3.778457033313275, + "tokens_seen": 701825024 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003976429287863591, + "loss": 2.5176, + "theoretical_loss": 3.7784212266556008, + "tokens_seen": 701890560 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003976328986960883, + "loss": 2.7215, + "theoretical_loss": 3.778385424277084, + "tokens_seen": 701956096 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039762286860581746, + "loss": 2.9712, + "theoretical_loss": 3.778349626176814, + "tokens_seen": 702021632 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039761283851554664, + "loss": 2.926, + "theoretical_loss": 3.7783138323538807, + "tokens_seen": 702087168 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003976028084252759, + "loss": 2.705, + "theoretical_loss": 3.778278042807375, + "tokens_seen": 702152704 + }, + { + "epoch": 0.21, + "learning_rate": 0.000397592778335005, + "loss": 2.7145, + "theoretical_loss": 3.7782422575363848, + "tokens_seen": 702218240 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039758274824473424, + "loss": 2.7362, + "theoretical_loss": 3.778206476540001, + "tokens_seen": 702283776 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039757271815446337, + "loss": 3.0193, + "theoretical_loss": 3.7781706998173155, + "tokens_seen": 702349312 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003975626880641926, + "loss": 2.9259, + "theoretical_loss": 3.778134927367417, + "tokens_seen": 702414848 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003975526579739218, + "loss": 2.9853, + "theoretical_loss": 3.778099159189399, + "tokens_seen": 702480384 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039754262788365097, + "loss": 2.8845, + "theoretical_loss": 3.7780633952823512, + "tokens_seen": 702545920 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039753259779338015, + "loss": 2.8962, + "theoretical_loss": 3.7780276356453655, + "tokens_seen": 702611456 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039752256770310933, + "loss": 3.0626, + "theoretical_loss": 3.7779918802775345, + "tokens_seen": 702676992 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003975125376128385, + "loss": 2.6853, + "theoretical_loss": 3.7779561291779498, + "tokens_seen": 702742528 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039750250752256774, + "loss": 2.888, + "theoretical_loss": 3.777920382345704, + "tokens_seen": 702808064 + }, + { + "epoch": 0.21, + "objective/train/docs_used": 406800, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7845449447631836, + "objective/train/theoretical_loss": 3.7778846397798898, + "objective/train/tokens_used": 723333600, + "theoretical_loss": 3.7778846397798898, + "tokens_seen": 702873600 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039749247743229687, + "loss": 2.728, + "theoretical_loss": 3.7778846397798898, + "tokens_seen": 702873600 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003974824473420261, + "loss": 2.8862, + "theoretical_loss": 3.777848901479601, + "tokens_seen": 702939136 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039747241725175523, + "loss": 2.9406, + "theoretical_loss": 3.77781316744393, + "tokens_seen": 703004672 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039746238716148447, + "loss": 2.9106, + "theoretical_loss": 3.777777437671971, + "tokens_seen": 703070208 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039745235707121365, + "loss": 3.0021, + "theoretical_loss": 3.777741712162818, + "tokens_seen": 703135744 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039744232698094283, + "loss": 2.8516, + "theoretical_loss": 3.777705990915565, + "tokens_seen": 703201280 + }, + { + "epoch": 0.21, + "learning_rate": 0.000397432296890672, + "loss": 2.6443, + "theoretical_loss": 3.7776702739293055, + "tokens_seen": 703266816 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039742226680040125, + "loss": 2.6048, + "theoretical_loss": 3.777634561203136, + "tokens_seen": 703332352 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003974122367101304, + "loss": 2.6758, + "theoretical_loss": 3.7775988527361504, + "tokens_seen": 703397888 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003974022066198596, + "loss": 2.7794, + "theoretical_loss": 3.7775631485274435, + "tokens_seen": 703463424 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039739217652958874, + "loss": 2.5, + "theoretical_loss": 3.7775274485761123, + "tokens_seen": 703528960 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039738214643931797, + "loss": 2.9331, + "theoretical_loss": 3.7774917528812515, + "tokens_seen": 703594496 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039737211634904715, + "loss": 2.7329, + "theoretical_loss": 3.7774560614419577, + "tokens_seen": 703660032 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039736208625877633, + "loss": 2.8243, + "theoretical_loss": 3.777420374257327, + "tokens_seen": 703725568 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003973520561685055, + "loss": 2.6111, + "theoretical_loss": 3.7773846913264566, + "tokens_seen": 703791104 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003973420260782347, + "loss": 2.835, + "theoretical_loss": 3.777349012648443, + "tokens_seen": 703856640 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003973319959879639, + "loss": 2.7486, + "theoretical_loss": 3.777313338222383, + "tokens_seen": 703922176 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003973219658976931, + "loss": 2.9408, + "theoretical_loss": 3.7772776680473745, + "tokens_seen": 703987712 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039731193580742224, + "loss": 2.8539, + "theoretical_loss": 3.7772420021225157, + "tokens_seen": 704053248 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003973019057171515, + "loss": 3.0435, + "theoretical_loss": 3.777206340446904, + "tokens_seen": 704118784 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003972918756268806, + "loss": 2.9196, + "theoretical_loss": 3.7771706830196385, + "tokens_seen": 704184320 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039728184553660984, + "loss": 2.7243, + "theoretical_loss": 3.777135029839817, + "tokens_seen": 704249856 + }, + { + "epoch": 0.21, + "learning_rate": 0.000397271815446339, + "loss": 2.8336, + "theoretical_loss": 3.777099380906538, + "tokens_seen": 704315392 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003972617853560682, + "loss": 2.7218, + "theoretical_loss": 3.777063736218902, + "tokens_seen": 704380928 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003972517552657974, + "loss": 2.7917, + "theoretical_loss": 3.7770280957760076, + "tokens_seen": 704446464 + }, + { + "epoch": 0.21, + "objective/train/docs_used": 408057, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.068608283996582, + "objective/train/theoretical_loss": 3.7769924595769546, + "objective/train/tokens_used": 724972000, + "theoretical_loss": 3.7769924595769546, + "tokens_seen": 704512000 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003972417251755266, + "loss": 3.1235, + "theoretical_loss": 3.7769924595769546, + "tokens_seen": 704512000 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039723169508525574, + "loss": 2.8954, + "theoretical_loss": 3.776956827620843, + "tokens_seen": 704577536 + }, + { + "epoch": 0.21, + "learning_rate": 0.000397221664994985, + "loss": 2.7839, + "theoretical_loss": 3.7769211999067727, + "tokens_seen": 704643072 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003972116349047141, + "loss": 2.7807, + "theoretical_loss": 3.7768855764338447, + "tokens_seen": 704708608 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039720160481444334, + "loss": 2.9969, + "theoretical_loss": 3.776849957201159, + "tokens_seen": 704774144 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003971915747241725, + "loss": 2.6231, + "theoretical_loss": 3.776814342207818, + "tokens_seen": 704839680 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003971815446339017, + "loss": 2.7981, + "theoretical_loss": 3.7767787314529215, + "tokens_seen": 704905216 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003971715145436309, + "loss": 2.9135, + "theoretical_loss": 3.7767431249355727, + "tokens_seen": 704970752 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039716148445336007, + "loss": 2.7559, + "theoretical_loss": 3.7767075226548723, + "tokens_seen": 705036288 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039715145436308925, + "loss": 2.9006, + "theoretical_loss": 3.7766719246099227, + "tokens_seen": 705101824 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003971414242728185, + "loss": 2.8278, + "theoretical_loss": 3.7766363307998265, + "tokens_seen": 705167360 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039713139418254766, + "loss": 2.6968, + "theoretical_loss": 3.7766007412236866, + "tokens_seen": 705232896 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039712136409227684, + "loss": 2.7226, + "theoretical_loss": 3.7765651558806055, + "tokens_seen": 705298432 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003971113340020061, + "loss": 2.6984, + "theoretical_loss": 3.776529574769687, + "tokens_seen": 705363968 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003971013039117352, + "loss": 2.9837, + "theoretical_loss": 3.776493997890034, + "tokens_seen": 705429504 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039709127382146444, + "loss": 3.0822, + "theoretical_loss": 3.776458425240751, + "tokens_seen": 705495040 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039708124373119357, + "loss": 2.721, + "theoretical_loss": 3.776422856820941, + "tokens_seen": 705560576 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003970712136409228, + "loss": 2.9578, + "theoretical_loss": 3.7763872926297095, + "tokens_seen": 705626112 + }, + { + "epoch": 0.21, + "learning_rate": 0.000397061183550652, + "loss": 2.9142, + "theoretical_loss": 3.7763517326661606, + "tokens_seen": 705691648 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039705115346038117, + "loss": 2.8568, + "theoretical_loss": 3.7763161769293996, + "tokens_seen": 705757184 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039704112337011035, + "loss": 2.9059, + "theoretical_loss": 3.776280625418531, + "tokens_seen": 705822720 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039703109327983953, + "loss": 2.9186, + "theoretical_loss": 3.77624507813266, + "tokens_seen": 705888256 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003970210631895687, + "loss": 2.9294, + "theoretical_loss": 3.7762095350708935, + "tokens_seen": 705953792 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039701103309929794, + "loss": 2.9142, + "theoretical_loss": 3.7761739962323366, + "tokens_seen": 706019328 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039700100300902707, + "loss": 2.7371, + "theoretical_loss": 3.776138461616096, + "tokens_seen": 706084864 + }, + { + "epoch": 0.21, + "objective/train/docs_used": 408602, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.9465298652648926, + "objective/train/theoretical_loss": 3.7761029312212777, + "objective/train/tokens_used": 726610400, + "theoretical_loss": 3.7761029312212777, + "tokens_seen": 706150400 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003969909729187563, + "loss": 2.8961, + "theoretical_loss": 3.7761029312212777, + "tokens_seen": 706150400 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039698094282848543, + "loss": 2.9136, + "theoretical_loss": 3.776067405046989, + "tokens_seen": 706215936 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039697091273821467, + "loss": 2.7726, + "theoretical_loss": 3.7760318830923367, + "tokens_seen": 706281472 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039696088264794385, + "loss": 2.9769, + "theoretical_loss": 3.775996365356428, + "tokens_seen": 706347008 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039695085255767303, + "loss": 2.6792, + "theoretical_loss": 3.775960851838371, + "tokens_seen": 706412544 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003969408224674022, + "loss": 2.6868, + "theoretical_loss": 3.7759253425372736, + "tokens_seen": 706478080 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039693079237713145, + "loss": 2.9147, + "theoretical_loss": 3.775889837452243, + "tokens_seen": 706543616 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003969207622868606, + "loss": 3.2163, + "theoretical_loss": 3.7758543365823884, + "tokens_seen": 706609152 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003969107321965898, + "loss": 2.9538, + "theoretical_loss": 3.775818839926819, + "tokens_seen": 706674688 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039690070210631894, + "loss": 2.7733, + "theoretical_loss": 3.7757833474846425, + "tokens_seen": 706740224 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039689067201604817, + "loss": 3.0128, + "theoretical_loss": 3.775747859254969, + "tokens_seen": 706805760 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039688064192577735, + "loss": 2.9053, + "theoretical_loss": 3.7757123752369077, + "tokens_seen": 706871296 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039687061183550653, + "loss": 2.7967, + "theoretical_loss": 3.775676895429569, + "tokens_seen": 706936832 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003968605817452357, + "loss": 2.6981, + "theoretical_loss": 3.775641419832062, + "tokens_seen": 707002368 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003968505516549649, + "loss": 2.8559, + "theoretical_loss": 3.775605948443497, + "tokens_seen": 707067904 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003968405215646941, + "loss": 2.8295, + "theoretical_loss": 3.7755704812629856, + "tokens_seen": 707133440 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003968304914744233, + "loss": 2.5673, + "theoretical_loss": 3.775535018289638, + "tokens_seen": 707198976 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039682046138415244, + "loss": 2.8696, + "theoretical_loss": 3.7754995595225656, + "tokens_seen": 707264512 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003968104312938817, + "loss": 2.8667, + "theoretical_loss": 3.7754641049608795, + "tokens_seen": 707330048 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003968004012036108, + "loss": 3.0868, + "theoretical_loss": 3.775428654603691, + "tokens_seen": 707395584 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039679037111334004, + "loss": 2.7584, + "theoretical_loss": 3.7753932084501134, + "tokens_seen": 707461120 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003967803410230692, + "loss": 2.7852, + "theoretical_loss": 3.7753577664992575, + "tokens_seen": 707526656 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003967703109327984, + "loss": 2.8679, + "theoretical_loss": 3.775322328750237, + "tokens_seen": 707592192 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003967602808425276, + "loss": 2.9832, + "theoretical_loss": 3.7752868952021634, + "tokens_seen": 707657728 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003967502507522568, + "loss": 2.8163, + "theoretical_loss": 3.7752514658541503, + "tokens_seen": 707723264 + }, + { + "epoch": 0.21, + "objective/train/docs_used": 409727, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.47202730178833, + "objective/train/theoretical_loss": 3.7752160407053115, + "objective/train/tokens_used": 728248800, + "theoretical_loss": 3.7752160407053115, + "tokens_seen": 707788800 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039674022066198594, + "loss": 2.6648, + "theoretical_loss": 3.7752160407053115, + "tokens_seen": 707788800 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003967301905717152, + "loss": 2.8107, + "theoretical_loss": 3.7751806197547593, + "tokens_seen": 707854336 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003967201604814443, + "loss": 2.7098, + "theoretical_loss": 3.775145203001609, + "tokens_seen": 707919872 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039671013039117354, + "loss": 2.8968, + "theoretical_loss": 3.775109790444974, + "tokens_seen": 707985408 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003967001003009027, + "loss": 2.7904, + "theoretical_loss": 3.775074382083968, + "tokens_seen": 708050944 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003966900702106319, + "loss": 2.8493, + "theoretical_loss": 3.775038977917707, + "tokens_seen": 708116480 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003966800401203611, + "loss": 2.8811, + "theoretical_loss": 3.775003577945305, + "tokens_seen": 708182016 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039667001003009027, + "loss": 2.7193, + "theoretical_loss": 3.7749681821658774, + "tokens_seen": 708247552 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039665997993981945, + "loss": 2.7779, + "theoretical_loss": 3.7749327905785393, + "tokens_seen": 708313088 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003966499498495487, + "loss": 2.8851, + "theoretical_loss": 3.774897403182407, + "tokens_seen": 708378624 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003966399197592778, + "loss": 3.0948, + "theoretical_loss": 3.774862019976596, + "tokens_seen": 708444160 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039662988966900704, + "loss": 2.9039, + "theoretical_loss": 3.7748266409602236, + "tokens_seen": 708509696 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039661985957873617, + "loss": 2.5842, + "theoretical_loss": 3.7747912661324046, + "tokens_seen": 708575232 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003966098294884654, + "loss": 2.8454, + "theoretical_loss": 3.774755895492257, + "tokens_seen": 708640768 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003965997993981946, + "loss": 2.9818, + "theoretical_loss": 3.7747205290388974, + "tokens_seen": 708706304 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039658976930792377, + "loss": 2.9379, + "theoretical_loss": 3.7746851667714436, + "tokens_seen": 708771840 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039657973921765295, + "loss": 2.9218, + "theoretical_loss": 3.774649808689013, + "tokens_seen": 708837376 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003965697091273822, + "loss": 2.8962, + "theoretical_loss": 3.774614454790722, + "tokens_seen": 708902912 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003965596790371113, + "loss": 2.6493, + "theoretical_loss": 3.7745791050756914, + "tokens_seen": 708968448 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039654964894684055, + "loss": 3.022, + "theoretical_loss": 3.774543759543037, + "tokens_seen": 709033984 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003965396188565697, + "loss": 2.8301, + "theoretical_loss": 3.7745084181918793, + "tokens_seen": 709099520 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003965295887662989, + "loss": 2.7331, + "theoretical_loss": 3.7744730810213367, + "tokens_seen": 709165056 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003965195586760281, + "loss": 2.859, + "theoretical_loss": 3.774437748030528, + "tokens_seen": 709230592 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039650952858575727, + "loss": 2.977, + "theoretical_loss": 3.774402419218573, + "tokens_seen": 709296128 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039649949849548645, + "loss": 2.8577, + "theoretical_loss": 3.7743670945845915, + "tokens_seen": 709361664 + }, + { + "epoch": 0.21, + "objective/train/docs_used": 410190, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.9692718982696533, + "objective/train/theoretical_loss": 3.774331774127703, + "objective/train/tokens_used": 729887200, + "theoretical_loss": 3.774331774127703, + "tokens_seen": 709427200 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039648946840521563, + "loss": 2.9606, + "theoretical_loss": 3.774331774127703, + "tokens_seen": 709427200 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003964794383149448, + "loss": 3.0036, + "theoretical_loss": 3.7742964578470284, + "tokens_seen": 709492736 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039646940822467405, + "loss": 3.0042, + "theoretical_loss": 3.7742611457416877, + "tokens_seen": 709558272 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003964593781344032, + "loss": 2.8063, + "theoretical_loss": 3.7742258378108016, + "tokens_seen": 709623808 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003964493480441324, + "loss": 2.871, + "theoretical_loss": 3.774190534053492, + "tokens_seen": 709689344 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003964393179538616, + "loss": 3.0059, + "theoretical_loss": 3.7741552344688793, + "tokens_seen": 709754880 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003964292878635908, + "loss": 2.8091, + "theoretical_loss": 3.774119939056086, + "tokens_seen": 709820416 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039641925777331996, + "loss": 2.9103, + "theoretical_loss": 3.7740846478142327, + "tokens_seen": 709885952 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039640922768304914, + "loss": 2.7316, + "theoretical_loss": 3.774049360742443, + "tokens_seen": 709951488 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003963991975927783, + "loss": 2.9193, + "theoretical_loss": 3.774014077839838, + "tokens_seen": 710017024 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039638916750250755, + "loss": 3.0491, + "theoretical_loss": 3.7739787991055413, + "tokens_seen": 710082560 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039637913741223673, + "loss": 2.9005, + "theoretical_loss": 3.773943524538675, + "tokens_seen": 710148096 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003963691073219659, + "loss": 2.6386, + "theoretical_loss": 3.773908254138363, + "tokens_seen": 710213632 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003963590772316951, + "loss": 2.8656, + "theoretical_loss": 3.7738729879037285, + "tokens_seen": 710279168 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003963490471414243, + "loss": 2.8592, + "theoretical_loss": 3.7738377258338947, + "tokens_seen": 710344704 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003963390170511535, + "loss": 2.6913, + "theoretical_loss": 3.773802467927986, + "tokens_seen": 710410240 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039632898696088264, + "loss": 2.9048, + "theoretical_loss": 3.7737672141851273, + "tokens_seen": 710475776 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003963189568706119, + "loss": 2.7769, + "theoretical_loss": 3.7737319646044423, + "tokens_seen": 710541312 + }, + { + "epoch": 0.22, + "learning_rate": 0.000396308926780341, + "loss": 2.8757, + "theoretical_loss": 3.7736967191850557, + "tokens_seen": 710606848 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039629889669007024, + "loss": 2.6992, + "theoretical_loss": 3.773661477926093, + "tokens_seen": 710672384 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003962888665997994, + "loss": 3.052, + "theoretical_loss": 3.773626240826679, + "tokens_seen": 710737920 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003962788365095286, + "loss": 3.0545, + "theoretical_loss": 3.7735910078859396, + "tokens_seen": 710803456 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003962688064192578, + "loss": 2.6514, + "theoretical_loss": 3.7735557791030008, + "tokens_seen": 710868992 + }, + { + "epoch": 0.22, + "learning_rate": 0.000396258776328987, + "loss": 2.9845, + "theoretical_loss": 3.773520554476989, + "tokens_seen": 710934528 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039624874623871614, + "loss": 2.9835, + "theoretical_loss": 3.7734853340070296, + "tokens_seen": 711000064 + }, + { + "epoch": 0.22, + "objective/train/docs_used": 411414, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.157093048095703, + "objective/train/theoretical_loss": 3.7734501176922493, + "objective/train/tokens_used": 731525600, + "theoretical_loss": 3.7734501176922493, + "tokens_seen": 711065600 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003962387161484454, + "loss": 2.7747, + "theoretical_loss": 3.7734501176922493, + "tokens_seen": 711065600 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003962286860581745, + "loss": 2.6618, + "theoretical_loss": 3.7734149055317756, + "tokens_seen": 711131136 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039621865596790374, + "loss": 2.8653, + "theoretical_loss": 3.773379697524736, + "tokens_seen": 711196672 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003962086258776329, + "loss": 2.8468, + "theoretical_loss": 3.773344493670257, + "tokens_seen": 711262208 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003961985957873621, + "loss": 2.6692, + "theoretical_loss": 3.7733092939674666, + "tokens_seen": 711327744 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003961885656970913, + "loss": 2.886, + "theoretical_loss": 3.7732740984154924, + "tokens_seen": 711393280 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039617853560682047, + "loss": 3.101, + "theoretical_loss": 3.7732389070134635, + "tokens_seen": 711458816 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039616850551654965, + "loss": 2.9392, + "theoretical_loss": 3.7732037197605077, + "tokens_seen": 711524352 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003961584754262789, + "loss": 2.7167, + "theoretical_loss": 3.7731685366557537, + "tokens_seen": 711589888 + }, + { + "epoch": 0.22, + "learning_rate": 0.000396148445336008, + "loss": 2.697, + "theoretical_loss": 3.773133357698331, + "tokens_seen": 711655424 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039613841524573724, + "loss": 2.7666, + "theoretical_loss": 3.7730981828873684, + "tokens_seen": 711720960 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039612838515546637, + "loss": 2.8228, + "theoretical_loss": 3.7730630122219955, + "tokens_seen": 711786496 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003961183550651956, + "loss": 2.8, + "theoretical_loss": 3.773027845701342, + "tokens_seen": 711852032 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003961083249749248, + "loss": 2.6197, + "theoretical_loss": 3.7729926833245386, + "tokens_seen": 711917568 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039609829488465397, + "loss": 3.1873, + "theoretical_loss": 3.772957525090715, + "tokens_seen": 711983104 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039608826479438315, + "loss": 2.8796, + "theoretical_loss": 3.7729223709990016, + "tokens_seen": 712048640 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003960782347041124, + "loss": 2.9836, + "theoretical_loss": 3.7728872210485296, + "tokens_seen": 712114176 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003960682046138415, + "loss": 3.0497, + "theoretical_loss": 3.77285207523843, + "tokens_seen": 712179712 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039605817452357075, + "loss": 2.8392, + "theoretical_loss": 3.7728169335678343, + "tokens_seen": 712245248 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003960481444332999, + "loss": 2.9066, + "theoretical_loss": 3.772781796035874, + "tokens_seen": 712310784 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003960381143430291, + "loss": 2.8291, + "theoretical_loss": 3.772746662641681, + "tokens_seen": 712376320 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003960280842527583, + "loss": 2.9252, + "theoretical_loss": 3.7727115333843875, + "tokens_seen": 712441856 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039601805416248747, + "loss": 2.787, + "theoretical_loss": 3.772676408263126, + "tokens_seen": 712507392 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039600802407221665, + "loss": 2.7013, + "theoretical_loss": 3.772641287277029, + "tokens_seen": 712572928 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039599799398194583, + "loss": 2.7019, + "theoretical_loss": 3.772606170425229, + "tokens_seen": 712638464 + }, + { + "epoch": 0.22, + "objective/train/docs_used": 412148, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.140347480773926, + "objective/train/theoretical_loss": 3.77257105770686, + "objective/train/tokens_used": 733164000, + "theoretical_loss": 3.77257105770686, + "tokens_seen": 712704000 + }, + { + "epoch": 0.22, + "learning_rate": 0.000395987963891675, + "loss": 2.8638, + "theoretical_loss": 3.77257105770686, + "tokens_seen": 712704000 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039597793380140425, + "loss": 3.1233, + "theoretical_loss": 3.772535949121055, + "tokens_seen": 712769536 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003959679037111334, + "loss": 2.8731, + "theoretical_loss": 3.772500844666948, + "tokens_seen": 712835072 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003959578736208626, + "loss": 2.6943, + "theoretical_loss": 3.7724657443436724, + "tokens_seen": 712900608 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003959478435305918, + "loss": 2.8287, + "theoretical_loss": 3.7724306481503636, + "tokens_seen": 712966144 + }, + { + "epoch": 0.22, + "learning_rate": 0.000395937813440321, + "loss": 2.777, + "theoretical_loss": 3.7723955560861544, + "tokens_seen": 713031680 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039592778335005016, + "loss": 2.8369, + "theoretical_loss": 3.7723604681501812, + "tokens_seen": 713097216 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039591775325977934, + "loss": 2.918, + "theoretical_loss": 3.7723253843415785, + "tokens_seen": 713162752 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003959077231695085, + "loss": 2.7466, + "theoretical_loss": 3.7722903046594807, + "tokens_seen": 713228288 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039589769307923775, + "loss": 2.7408, + "theoretical_loss": 3.772255229103025, + "tokens_seen": 713293824 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003958876629889669, + "loss": 3.0104, + "theoretical_loss": 3.772220157671345, + "tokens_seen": 713359360 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003958776328986961, + "loss": 2.8837, + "theoretical_loss": 3.7721850903635787, + "tokens_seen": 713424896 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039586760280842524, + "loss": 2.6186, + "theoretical_loss": 3.7721500271788617, + "tokens_seen": 713490432 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003958575727181545, + "loss": 2.8276, + "theoretical_loss": 3.772114968116331, + "tokens_seen": 713555968 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039584754262788366, + "loss": 2.7722, + "theoretical_loss": 3.772079913175123, + "tokens_seen": 713621504 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039583751253761284, + "loss": 2.8812, + "theoretical_loss": 3.7720448623543743, + "tokens_seen": 713687040 + }, + { + "epoch": 0.22, + "learning_rate": 0.000395827482447342, + "loss": 2.9712, + "theoretical_loss": 3.7720098156532234, + "tokens_seen": 713752576 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003958174523570712, + "loss": 2.7475, + "theoretical_loss": 3.771974773070807, + "tokens_seen": 713818112 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003958074222668004, + "loss": 2.9953, + "theoretical_loss": 3.7719397346062635, + "tokens_seen": 713883648 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003957973921765296, + "loss": 2.7824, + "theoretical_loss": 3.771904700258731, + "tokens_seen": 713949184 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039578736208625875, + "loss": 2.8511, + "theoretical_loss": 3.771869670027348, + "tokens_seen": 714014720 + }, + { + "epoch": 0.22, + "learning_rate": 0.000395777331995988, + "loss": 2.8225, + "theoretical_loss": 3.7718346439112524, + "tokens_seen": 714080256 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039576730190571716, + "loss": 2.8774, + "theoretical_loss": 3.771799621909584, + "tokens_seen": 714145792 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039575727181544634, + "loss": 3.0086, + "theoretical_loss": 3.771764604021482, + "tokens_seen": 714211328 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003957472417251755, + "loss": 2.9911, + "theoretical_loss": 3.771729590246085, + "tokens_seen": 714276864 + }, + { + "epoch": 0.22, + "objective/train/docs_used": 413348, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.798063278198242, + "objective/train/theoretical_loss": 3.7716945805825337, + "objective/train/tokens_used": 734802400, + "theoretical_loss": 3.7716945805825337, + "tokens_seen": 714342400 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003957372116349047, + "loss": 2.7625, + "theoretical_loss": 3.7716945805825337, + "tokens_seen": 714342400 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003957271815446339, + "loss": 2.9822, + "theoretical_loss": 3.7716595750299673, + "tokens_seen": 714407936 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003957171514543631, + "loss": 3.0668, + "theoretical_loss": 3.771624573587526, + "tokens_seen": 714473472 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039570712136409225, + "loss": 3.0342, + "theoretical_loss": 3.771589576254351, + "tokens_seen": 714539008 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003956970912738215, + "loss": 3.138, + "theoretical_loss": 3.7715545830295825, + "tokens_seen": 714604544 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003956870611835506, + "loss": 3.0589, + "theoretical_loss": 3.7715195939123616, + "tokens_seen": 714670080 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039567703109327985, + "loss": 2.6719, + "theoretical_loss": 3.7714846089018295, + "tokens_seen": 714735616 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039566700100300903, + "loss": 2.8142, + "theoretical_loss": 3.7714496279971277, + "tokens_seen": 714801152 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003956569709127382, + "loss": 2.9613, + "theoretical_loss": 3.7714146511973983, + "tokens_seen": 714866688 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003956469408224674, + "loss": 2.6564, + "theoretical_loss": 3.7713796785017824, + "tokens_seen": 714932224 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039563691073219657, + "loss": 2.9044, + "theoretical_loss": 3.7713447099094233, + "tokens_seen": 714997760 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003956268806419258, + "loss": 2.959, + "theoretical_loss": 3.7713097454194635, + "tokens_seen": 715063296 + }, + { + "epoch": 0.22, + "learning_rate": 0.000395616850551655, + "loss": 2.8488, + "theoretical_loss": 3.7712747850310446, + "tokens_seen": 715128832 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039560682046138417, + "loss": 2.994, + "theoretical_loss": 3.7712398287433113, + "tokens_seen": 715194368 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039559679037111335, + "loss": 2.9303, + "theoretical_loss": 3.771204876555406, + "tokens_seen": 715259904 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003955867602808426, + "loss": 2.8726, + "theoretical_loss": 3.771169928466472, + "tokens_seen": 715325440 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003955767301905717, + "loss": 2.8746, + "theoretical_loss": 3.7711349844756543, + "tokens_seen": 715390976 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039556670010030095, + "loss": 2.9297, + "theoretical_loss": 3.771100044582096, + "tokens_seen": 715456512 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003955566700100301, + "loss": 2.9217, + "theoretical_loss": 3.771065108784941, + "tokens_seen": 715522048 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003955466399197593, + "loss": 2.9171, + "theoretical_loss": 3.771030177083335, + "tokens_seen": 715587584 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003955366098294885, + "loss": 2.8871, + "theoretical_loss": 3.7709952494764227, + "tokens_seen": 715653120 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039552657973921767, + "loss": 2.8705, + "theoretical_loss": 3.7709603259633484, + "tokens_seen": 715718656 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039551654964894685, + "loss": 2.8481, + "theoretical_loss": 3.7709254065432583, + "tokens_seen": 715784192 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039550651955867603, + "loss": 2.7286, + "theoretical_loss": 3.770890491215298, + "tokens_seen": 715849728 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003954964894684052, + "loss": 2.688, + "theoretical_loss": 3.7708555799786128, + "tokens_seen": 715915264 + }, + { + "epoch": 0.22, + "objective/train/docs_used": 413910, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.5926387310028076, + "objective/train/theoretical_loss": 3.7708206728323495, + "objective/train/tokens_used": 736440800, + "theoretical_loss": 3.7708206728323495, + "tokens_seen": 715980800 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039548645937813445, + "loss": 2.6346, + "theoretical_loss": 3.7708206728323495, + "tokens_seen": 715980800 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003954764292878636, + "loss": 2.8919, + "theoretical_loss": 3.770785769775654, + "tokens_seen": 716046336 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003954663991975928, + "loss": 2.9641, + "theoretical_loss": 3.7707508708076727, + "tokens_seen": 716111872 + }, + { + "epoch": 0.22, + "learning_rate": 0.000395456369107322, + "loss": 2.6675, + "theoretical_loss": 3.770715975927553, + "tokens_seen": 716177408 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003954463390170512, + "loss": 2.91, + "theoretical_loss": 3.7706810851344423, + "tokens_seen": 716242944 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039543630892678036, + "loss": 2.8812, + "theoretical_loss": 3.7706461984274875, + "tokens_seen": 716308480 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039542627883650954, + "loss": 2.9537, + "theoretical_loss": 3.7706113158058363, + "tokens_seen": 716374016 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003954162487462387, + "loss": 2.7365, + "theoretical_loss": 3.7705764372686374, + "tokens_seen": 716439552 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039540621865596795, + "loss": 3.026, + "theoretical_loss": 3.770541562815038, + "tokens_seen": 716505088 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003953961885656971, + "loss": 2.8576, + "theoretical_loss": 3.770506692444187, + "tokens_seen": 716570624 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003953861584754263, + "loss": 2.8465, + "theoretical_loss": 3.7704718261552332, + "tokens_seen": 716636160 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039537612838515544, + "loss": 2.9258, + "theoretical_loss": 3.770436963947325, + "tokens_seen": 716701696 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003953660982948847, + "loss": 2.7709, + "theoretical_loss": 3.7704021058196124, + "tokens_seen": 716767232 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039535606820461386, + "loss": 2.9063, + "theoretical_loss": 3.770367251771245, + "tokens_seen": 716832768 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039534603811434304, + "loss": 3.0919, + "theoretical_loss": 3.7703324018013706, + "tokens_seen": 716898304 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003953360080240722, + "loss": 2.8019, + "theoretical_loss": 3.7702975559091416, + "tokens_seen": 716963840 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003953259779338014, + "loss": 2.804, + "theoretical_loss": 3.7702627140937075, + "tokens_seen": 717029376 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003953159478435306, + "loss": 2.7027, + "theoretical_loss": 3.7702278763542174, + "tokens_seen": 717094912 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003953059177532598, + "loss": 2.5267, + "theoretical_loss": 3.770193042689824, + "tokens_seen": 717160448 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039529588766298895, + "loss": 2.7953, + "theoretical_loss": 3.770158213099677, + "tokens_seen": 717225984 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003952858575727182, + "loss": 2.7399, + "theoretical_loss": 3.7701233875829283, + "tokens_seen": 717291520 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039527582748244736, + "loss": 2.872, + "theoretical_loss": 3.7700885661387287, + "tokens_seen": 717357056 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039526579739217654, + "loss": 2.7063, + "theoretical_loss": 3.770053748766231, + "tokens_seen": 717422592 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003952557673019057, + "loss": 2.7113, + "theoretical_loss": 3.770018935464586, + "tokens_seen": 717488128 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003952457372116349, + "loss": 2.9319, + "theoretical_loss": 3.7699841262329477, + "tokens_seen": 717553664 + }, + { + "epoch": 0.22, + "objective/train/docs_used": 414484, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.5700790882110596, + "objective/train/theoretical_loss": 3.7699493210704667, + "objective/train/tokens_used": 738079200, + "theoretical_loss": 3.7699493210704667, + "tokens_seen": 717619200 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003952357071213641, + "loss": 2.8101, + "theoretical_loss": 3.7699493210704667, + "tokens_seen": 717619200 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003952256770310933, + "loss": 2.7085, + "theoretical_loss": 3.7699145199762967, + "tokens_seen": 717684736 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039521564694082245, + "loss": 2.86, + "theoretical_loss": 3.7698797229495904, + "tokens_seen": 717750272 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003952056168505517, + "loss": 3.0002, + "theoretical_loss": 3.7698449299895014, + "tokens_seen": 717815808 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003951955867602808, + "loss": 2.8867, + "theoretical_loss": 3.769810141095183, + "tokens_seen": 717881344 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039518555667001005, + "loss": 2.9841, + "theoretical_loss": 3.76977535626579, + "tokens_seen": 717946880 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039517552657973923, + "loss": 2.8474, + "theoretical_loss": 3.769740575500475, + "tokens_seen": 718012416 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003951654964894684, + "loss": 2.6073, + "theoretical_loss": 3.769705798798393, + "tokens_seen": 718077952 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003951554663991976, + "loss": 2.7891, + "theoretical_loss": 3.769671026158698, + "tokens_seen": 718143488 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039514543630892677, + "loss": 2.7263, + "theoretical_loss": 3.769636257580545, + "tokens_seen": 718209024 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039513540621865595, + "loss": 2.9137, + "theoretical_loss": 3.76960149306309, + "tokens_seen": 718274560 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003951253761283852, + "loss": 2.9736, + "theoretical_loss": 3.769566732605487, + "tokens_seen": 718340096 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003951153460381143, + "loss": 2.8924, + "theoretical_loss": 3.769531976206893, + "tokens_seen": 718405632 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039510531594784355, + "loss": 2.816, + "theoretical_loss": 3.7694972238664617, + "tokens_seen": 718471168 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039509528585757273, + "loss": 2.8018, + "theoretical_loss": 3.7694624755833512, + "tokens_seen": 718536704 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003950852557673019, + "loss": 2.813, + "theoretical_loss": 3.769427731356717, + "tokens_seen": 718602240 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003950752256770311, + "loss": 2.7627, + "theoretical_loss": 3.7693929911857156, + "tokens_seen": 718667776 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003950651955867603, + "loss": 2.8254, + "theoretical_loss": 3.7693582550695037, + "tokens_seen": 718733312 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039505516549648946, + "loss": 2.8885, + "theoretical_loss": 3.7693235230072384, + "tokens_seen": 718798848 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003950451354062187, + "loss": 2.907, + "theoretical_loss": 3.7692887949980776, + "tokens_seen": 718864384 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003950351053159478, + "loss": 2.7439, + "theoretical_loss": 3.7692540710411784, + "tokens_seen": 718929920 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039502507522567705, + "loss": 2.7825, + "theoretical_loss": 3.769219351135699, + "tokens_seen": 718995456 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003950150451354062, + "loss": 2.6845, + "theoretical_loss": 3.769184635280797, + "tokens_seen": 719060992 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003950050150451354, + "loss": 3.0903, + "theoretical_loss": 3.7691499234756303, + "tokens_seen": 719126528 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003949949849548646, + "loss": 3.027, + "theoretical_loss": 3.7691152157193586, + "tokens_seen": 719192064 + }, + { + "epoch": 0.22, + "objective/train/docs_used": 415542, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.169238567352295, + "objective/train/theoretical_loss": 3.76908051201114, + "objective/train/tokens_used": 739717600, + "theoretical_loss": 3.76908051201114, + "tokens_seen": 719257600 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003949849548645938, + "loss": 3.091, + "theoretical_loss": 3.76908051201114, + "tokens_seen": 719257600 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039497492477432296, + "loss": 2.7458, + "theoretical_loss": 3.769045812350134, + "tokens_seen": 719323136 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003949648946840522, + "loss": 2.885, + "theoretical_loss": 3.7690111167355, + "tokens_seen": 719388672 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003949548645937813, + "loss": 2.5503, + "theoretical_loss": 3.768976425166397, + "tokens_seen": 719454208 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039494483450351056, + "loss": 2.8851, + "theoretical_loss": 3.7689417376419856, + "tokens_seen": 719519744 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003949348044132397, + "loss": 2.8298, + "theoretical_loss": 3.768907054161425, + "tokens_seen": 719585280 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003949247743229689, + "loss": 2.712, + "theoretical_loss": 3.768872374723876, + "tokens_seen": 719650816 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003949147442326981, + "loss": 2.767, + "theoretical_loss": 3.7688376993284995, + "tokens_seen": 719716352 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003949047141424273, + "loss": 2.9895, + "theoretical_loss": 3.768803027974456, + "tokens_seen": 719781888 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003948946840521565, + "loss": 2.9242, + "theoretical_loss": 3.768768360660907, + "tokens_seen": 719847424 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039488465396188564, + "loss": 2.9074, + "theoretical_loss": 3.768733697387013, + "tokens_seen": 719912960 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003948746238716149, + "loss": 2.7716, + "theoretical_loss": 3.7686990381519365, + "tokens_seen": 719978496 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039486459378134406, + "loss": 2.9496, + "theoretical_loss": 3.7686643829548387, + "tokens_seen": 720044032 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039485456369107324, + "loss": 2.874, + "theoretical_loss": 3.768629731794882, + "tokens_seen": 720109568 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003948445336008024, + "loss": 3.0257, + "theoretical_loss": 3.7685950846712286, + "tokens_seen": 720175104 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003948345035105316, + "loss": 2.6757, + "theoretical_loss": 3.768560441583041, + "tokens_seen": 720240640 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003948244734202608, + "loss": 2.8433, + "theoretical_loss": 3.7685258025294823, + "tokens_seen": 720306176 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039481444332999, + "loss": 3.0277, + "theoretical_loss": 3.768491167509716, + "tokens_seen": 720371712 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039480441323971915, + "loss": 2.6604, + "theoretical_loss": 3.768456536522905, + "tokens_seen": 720437248 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003947943831494484, + "loss": 2.749, + "theoretical_loss": 3.768421909568212, + "tokens_seen": 720502784 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039478435305917756, + "loss": 2.861, + "theoretical_loss": 3.768387286644802, + "tokens_seen": 720568320 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039477432296890674, + "loss": 3.0266, + "theoretical_loss": 3.768352667751839, + "tokens_seen": 720633856 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003947642928786359, + "loss": 2.8026, + "theoretical_loss": 3.768318052888487, + "tokens_seen": 720699392 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003947542627883651, + "loss": 2.8675, + "theoretical_loss": 3.768283442053911, + "tokens_seen": 720764928 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003947442326980943, + "loss": 2.8569, + "theoretical_loss": 3.7682488352472756, + "tokens_seen": 720830464 + }, + { + "epoch": 0.22, + "objective/train/docs_used": 416165, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.952298641204834, + "objective/train/theoretical_loss": 3.7682142324677455, + "objective/train/tokens_used": 741356000, + "theoretical_loss": 3.7682142324677455, + "tokens_seen": 720896000 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003947342026078235, + "loss": 3.0552, + "theoretical_loss": 3.7682142324677455, + "tokens_seen": 720896000 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039472417251755265, + "loss": 2.6991, + "theoretical_loss": 3.7681796337144866, + "tokens_seen": 720961536 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003947141424272819, + "loss": 2.7814, + "theoretical_loss": 3.768145038986664, + "tokens_seen": 721027072 + }, + { + "epoch": 0.22, + "learning_rate": 0.000394704112337011, + "loss": 3.0046, + "theoretical_loss": 3.768110448283444, + "tokens_seen": 721092608 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039469408224674025, + "loss": 2.8124, + "theoretical_loss": 3.768075861603993, + "tokens_seen": 721158144 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039468405215646943, + "loss": 2.6781, + "theoretical_loss": 3.7680412789474764, + "tokens_seen": 721223680 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003946740220661986, + "loss": 2.8248, + "theoretical_loss": 3.7680067003130615, + "tokens_seen": 721289216 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003946639919759278, + "loss": 2.6626, + "theoretical_loss": 3.767972125699915, + "tokens_seen": 721354752 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039465396188565697, + "loss": 2.6927, + "theoretical_loss": 3.767937555107203, + "tokens_seen": 721420288 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039464393179538615, + "loss": 2.7999, + "theoretical_loss": 3.767902988534094, + "tokens_seen": 721485824 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003946339017051154, + "loss": 2.7896, + "theoretical_loss": 3.7678684259797555, + "tokens_seen": 721551360 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003946238716148445, + "loss": 2.9314, + "theoretical_loss": 3.767833867443355, + "tokens_seen": 721616896 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039461384152457375, + "loss": 2.7502, + "theoretical_loss": 3.767799312924061, + "tokens_seen": 721682432 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039460381143430293, + "loss": 3.0307, + "theoretical_loss": 3.7677647624210406, + "tokens_seen": 721747968 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003945937813440321, + "loss": 2.53, + "theoretical_loss": 3.7677302159334642, + "tokens_seen": 721813504 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003945837512537613, + "loss": 2.701, + "theoretical_loss": 3.7676956734604987, + "tokens_seen": 721879040 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003945737211634905, + "loss": 2.8273, + "theoretical_loss": 3.7676611350013145, + "tokens_seen": 721944576 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039456369107321966, + "loss": 2.861, + "theoretical_loss": 3.7676266005550803, + "tokens_seen": 722010112 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003945536609829489, + "loss": 2.8848, + "theoretical_loss": 3.7675920701209664, + "tokens_seen": 722075648 + }, + { + "epoch": 0.22, + "learning_rate": 0.000394543630892678, + "loss": 2.9749, + "theoretical_loss": 3.7675575436981417, + "tokens_seen": 722141184 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039453360080240725, + "loss": 2.8751, + "theoretical_loss": 3.767523021285777, + "tokens_seen": 722206720 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003945235707121364, + "loss": 2.8746, + "theoretical_loss": 3.7674885028830416, + "tokens_seen": 722272256 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003945135406218656, + "loss": 3.072, + "theoretical_loss": 3.767453988489107, + "tokens_seen": 722337792 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003945035105315948, + "loss": 2.8291, + "theoretical_loss": 3.767419478103144, + "tokens_seen": 722403328 + }, + { + "epoch": 0.22, + "learning_rate": 0.000394493480441324, + "loss": 2.8067, + "theoretical_loss": 3.7673849717243226, + "tokens_seen": 722468864 + }, + { + "epoch": 0.22, + "objective/train/docs_used": 417606, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.0366592407226562, + "objective/train/theoretical_loss": 3.7673504693518156, + "objective/train/tokens_used": 742994400, + "theoretical_loss": 3.7673504693518156, + "tokens_seen": 722534400 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039448345035105316, + "loss": 2.9699, + "theoretical_loss": 3.7673504693518156, + "tokens_seen": 722534400 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003944734202607824, + "loss": 2.832, + "theoretical_loss": 3.767315970984793, + "tokens_seen": 722599936 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003944633901705115, + "loss": 2.913, + "theoretical_loss": 3.7672814766224274, + "tokens_seen": 722665472 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039445336008024076, + "loss": 3.0233, + "theoretical_loss": 3.767246986263891, + "tokens_seen": 722731008 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003944433299899699, + "loss": 2.7665, + "theoretical_loss": 3.7672124999083554, + "tokens_seen": 722796544 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003944332998996991, + "loss": 2.632, + "theoretical_loss": 3.767178017554994, + "tokens_seen": 722862080 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003944232698094283, + "loss": 2.6896, + "theoretical_loss": 3.7671435392029786, + "tokens_seen": 722927616 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003944132397191575, + "loss": 2.7693, + "theoretical_loss": 3.767109064851483, + "tokens_seen": 722993152 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039440320962888666, + "loss": 2.7244, + "theoretical_loss": 3.7670745944996797, + "tokens_seen": 723058688 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039439317953861584, + "loss": 2.8937, + "theoretical_loss": 3.7670401281467427, + "tokens_seen": 723124224 + }, + { + "epoch": 0.22, + "learning_rate": 0.000394383149448345, + "loss": 2.9876, + "theoretical_loss": 3.767005665791846, + "tokens_seen": 723189760 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039437311935807426, + "loss": 2.8755, + "theoretical_loss": 3.766971207434163, + "tokens_seen": 723255296 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003943630892678034, + "loss": 2.9191, + "theoretical_loss": 3.766936753072869, + "tokens_seen": 723320832 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003943530591775326, + "loss": 2.7756, + "theoretical_loss": 3.7669023027071367, + "tokens_seen": 723386368 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039434302908726175, + "loss": 2.7826, + "theoretical_loss": 3.766867856336142, + "tokens_seen": 723451904 + }, + { + "epoch": 0.22, + "learning_rate": 0.000394332998996991, + "loss": 2.889, + "theoretical_loss": 3.76683341395906, + "tokens_seen": 723517440 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039432296890672017, + "loss": 2.7759, + "theoretical_loss": 3.7667989755750657, + "tokens_seen": 723582976 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039431293881644935, + "loss": 2.9682, + "theoretical_loss": 3.766764541183334, + "tokens_seen": 723648512 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039430290872617853, + "loss": 2.928, + "theoretical_loss": 3.7667301107830413, + "tokens_seen": 723714048 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039429287863590776, + "loss": 2.8197, + "theoretical_loss": 3.766695684373363, + "tokens_seen": 723779584 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003942828485456369, + "loss": 2.954, + "theoretical_loss": 3.766661261953476, + "tokens_seen": 723845120 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003942728184553661, + "loss": 2.9678, + "theoretical_loss": 3.7666268435225563, + "tokens_seen": 723910656 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039426278836509525, + "loss": 2.7975, + "theoretical_loss": 3.7665924290797808, + "tokens_seen": 723976192 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003942527582748245, + "loss": 2.71, + "theoretical_loss": 3.766558018624326, + "tokens_seen": 724041728 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039424272818455367, + "loss": 2.9967, + "theoretical_loss": 3.7665236121553693, + "tokens_seen": 724107264 + }, + { + "epoch": 0.22, + "objective/train/docs_used": 418215, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.9476613998413086, + "objective/train/theoretical_loss": 3.7664892096720886, + "objective/train/tokens_used": 744632800, + "theoretical_loss": 3.7664892096720886, + "tokens_seen": 724172800 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039423269809428285, + "loss": 2.8225, + "theoretical_loss": 3.7664892096720886, + "tokens_seen": 724172800 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039422266800401203, + "loss": 2.8369, + "theoretical_loss": 3.766454811173661, + "tokens_seen": 724238336 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003942126379137412, + "loss": 2.5846, + "theoretical_loss": 3.7664204166592645, + "tokens_seen": 724303872 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003942026078234704, + "loss": 2.8447, + "theoretical_loss": 3.7663860261280773, + "tokens_seen": 724369408 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039419257773319963, + "loss": 2.9258, + "theoretical_loss": 3.7663516395792778, + "tokens_seen": 724434944 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039418254764292876, + "loss": 2.8501, + "theoretical_loss": 3.7663172570120445, + "tokens_seen": 724500480 + }, + { + "epoch": 0.22, + "learning_rate": 0.000394172517552658, + "loss": 2.9418, + "theoretical_loss": 3.7662828784255566, + "tokens_seen": 724566016 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003941624874623871, + "loss": 2.8616, + "theoretical_loss": 3.766248503818993, + "tokens_seen": 724631552 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039415245737211635, + "loss": 2.9151, + "theoretical_loss": 3.766214133191533, + "tokens_seen": 724697088 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003941424272818456, + "loss": 2.75, + "theoretical_loss": 3.7661797665423564, + "tokens_seen": 724762624 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003941323971915747, + "loss": 2.8404, + "theoretical_loss": 3.766145403870643, + "tokens_seen": 724828160 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039412236710130395, + "loss": 2.7451, + "theoretical_loss": 3.766111045175573, + "tokens_seen": 724893696 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039411233701103313, + "loss": 2.8126, + "theoretical_loss": 3.7660766904563268, + "tokens_seen": 724959232 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003941023069207623, + "loss": 2.9938, + "theoretical_loss": 3.7660423397120844, + "tokens_seen": 725024768 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003940922768304915, + "loss": 2.7133, + "theoretical_loss": 3.7660079929420274, + "tokens_seen": 725090304 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003940822467402207, + "loss": 2.7613, + "theoretical_loss": 3.765973650145336, + "tokens_seen": 725155840 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039407221664994986, + "loss": 2.7846, + "theoretical_loss": 3.7659393113211923, + "tokens_seen": 725221376 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003940621865596791, + "loss": 2.9663, + "theoretical_loss": 3.7659049764687778, + "tokens_seen": 725286912 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003940521564694082, + "loss": 2.7069, + "theoretical_loss": 3.765870645587274, + "tokens_seen": 725352448 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039404212637913745, + "loss": 2.7764, + "theoretical_loss": 3.7658363186758628, + "tokens_seen": 725417984 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003940320962888666, + "loss": 2.8289, + "theoretical_loss": 3.765801995733727, + "tokens_seen": 725483520 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003940220661985958, + "loss": 3.1114, + "theoretical_loss": 3.7657676767600488, + "tokens_seen": 725549056 + }, + { + "epoch": 0.22, + "learning_rate": 0.000394012036108325, + "loss": 2.6524, + "theoretical_loss": 3.7657333617540107, + "tokens_seen": 725614592 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003940020060180542, + "loss": 3.0244, + "theoretical_loss": 3.7656990507147965, + "tokens_seen": 725680128 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039399197592778336, + "loss": 2.8608, + "theoretical_loss": 3.765664743641589, + "tokens_seen": 725745664 + }, + { + "epoch": 0.22, + "objective/train/docs_used": 419550, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.600438117980957, + "objective/train/theoretical_loss": 3.765630440533571, + "objective/train/tokens_used": 746271200, + "theoretical_loss": 3.765630440533571, + "tokens_seen": 725811200 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003939819458375126, + "loss": 2.4983, + "theoretical_loss": 3.765630440533571, + "tokens_seen": 725811200 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003939719157472417, + "loss": 2.9337, + "theoretical_loss": 3.7655961413899277, + "tokens_seen": 725876736 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039396188565697096, + "loss": 2.923, + "theoretical_loss": 3.765561846209842, + "tokens_seen": 725942272 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003939518555667001, + "loss": 2.5997, + "theoretical_loss": 3.7655275549924987, + "tokens_seen": 726007808 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003939418254764293, + "loss": 3.0696, + "theoretical_loss": 3.7654932677370816, + "tokens_seen": 726073344 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003939317953861585, + "loss": 2.8452, + "theoretical_loss": 3.765458984442776, + "tokens_seen": 726138880 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003939217652958877, + "loss": 2.7695, + "theoretical_loss": 3.7654247051087664, + "tokens_seen": 726204416 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039391173520561686, + "loss": 2.9168, + "theoretical_loss": 3.765390429734238, + "tokens_seen": 726269952 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039390170511534604, + "loss": 2.9919, + "theoretical_loss": 3.765356158318377, + "tokens_seen": 726335488 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003938916750250752, + "loss": 2.8605, + "theoretical_loss": 3.765321890860368, + "tokens_seen": 726401024 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039388164493480446, + "loss": 3.0699, + "theoretical_loss": 3.7652876273593976, + "tokens_seen": 726466560 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003938716148445336, + "loss": 3.0389, + "theoretical_loss": 3.765253367814652, + "tokens_seen": 726532096 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003938615847542628, + "loss": 3.0639, + "theoretical_loss": 3.765219112225317, + "tokens_seen": 726597632 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039385155466399195, + "loss": 2.8729, + "theoretical_loss": 3.7651848605905798, + "tokens_seen": 726663168 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003938415245737212, + "loss": 2.7502, + "theoretical_loss": 3.7651506129096273, + "tokens_seen": 726728704 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039383149448345037, + "loss": 2.8262, + "theoretical_loss": 3.765116369181646, + "tokens_seen": 726794240 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039382146439317955, + "loss": 2.8354, + "theoretical_loss": 3.7650821294058234, + "tokens_seen": 726859776 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039381143430290873, + "loss": 2.8642, + "theoretical_loss": 3.7650478935813476, + "tokens_seen": 726925312 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039380140421263796, + "loss": 2.7676, + "theoretical_loss": 3.7650136617074064, + "tokens_seen": 726990848 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003937913741223671, + "loss": 2.9191, + "theoretical_loss": 3.7649794337831874, + "tokens_seen": 727056384 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003937813440320963, + "loss": 3.0492, + "theoretical_loss": 3.7649452098078795, + "tokens_seen": 727121920 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039377131394182545, + "loss": 2.6498, + "theoretical_loss": 3.7649109897806703, + "tokens_seen": 727187456 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003937612838515547, + "loss": 2.8335, + "theoretical_loss": 3.7648767737007494, + "tokens_seen": 727252992 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039375125376128387, + "loss": 2.837, + "theoretical_loss": 3.7648425615673053, + "tokens_seen": 727318528 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039374122367101305, + "loss": 2.9133, + "theoretical_loss": 3.7648083533795282, + "tokens_seen": 727384064 + }, + { + "epoch": 0.22, + "objective/train/docs_used": 420047, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.0645720958709717, + "objective/train/theoretical_loss": 3.7647741491366067, + "objective/train/tokens_used": 747909600, + "theoretical_loss": 3.7647741491366067, + "tokens_seen": 727449600 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039373119358074223, + "loss": 2.8217, + "theoretical_loss": 3.7647741491366067, + "tokens_seen": 727449600 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003937211634904714, + "loss": 2.9632, + "theoretical_loss": 3.764739948837731, + "tokens_seen": 727515136 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003937111334002006, + "loss": 2.8915, + "theoretical_loss": 3.764705752482091, + "tokens_seen": 727580672 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039370110330992983, + "loss": 2.8291, + "theoretical_loss": 3.7646715600688765, + "tokens_seen": 727646208 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039369107321965896, + "loss": 2.6642, + "theoretical_loss": 3.7646373715972787, + "tokens_seen": 727711744 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003936810431293882, + "loss": 2.8879, + "theoretical_loss": 3.7646031870664878, + "tokens_seen": 727777280 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003936710130391173, + "loss": 2.9941, + "theoretical_loss": 3.764569006475695, + "tokens_seen": 727842816 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039366098294884655, + "loss": 2.7726, + "theoretical_loss": 3.7645348298240915, + "tokens_seen": 727908352 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039365095285857573, + "loss": 2.8907, + "theoretical_loss": 3.7645006571108683, + "tokens_seen": 727973888 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003936409227683049, + "loss": 3.0053, + "theoretical_loss": 3.7644664883352172, + "tokens_seen": 728039424 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003936308926780341, + "loss": 2.701, + "theoretical_loss": 3.764432323496331, + "tokens_seen": 728104960 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039362086258776333, + "loss": 2.6887, + "theoretical_loss": 3.764398162593401, + "tokens_seen": 728170496 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039361083249749246, + "loss": 3.0269, + "theoretical_loss": 3.764364005625619, + "tokens_seen": 728236032 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003936008024072217, + "loss": 2.9996, + "theoretical_loss": 3.764329852592179, + "tokens_seen": 728301568 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003935907723169508, + "loss": 2.7975, + "theoretical_loss": 3.764295703492273, + "tokens_seen": 728367104 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039358074222668006, + "loss": 2.8269, + "theoretical_loss": 3.764261558325094, + "tokens_seen": 728432640 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039357071213640924, + "loss": 2.7227, + "theoretical_loss": 3.764227417089836, + "tokens_seen": 728498176 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003935606820461384, + "loss": 2.7352, + "theoretical_loss": 3.764193279785692, + "tokens_seen": 728563712 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003935506519558676, + "loss": 2.6766, + "theoretical_loss": 3.7641591464118562, + "tokens_seen": 728629248 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003935406218655968, + "loss": 2.9174, + "theoretical_loss": 3.7641250169675224, + "tokens_seen": 728694784 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039353059177532596, + "loss": 2.6721, + "theoretical_loss": 3.7640908914518847, + "tokens_seen": 728760320 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003935205616850552, + "loss": 2.6247, + "theoretical_loss": 3.764056769864138, + "tokens_seen": 728825856 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003935105315947843, + "loss": 2.8767, + "theoretical_loss": 3.7640226522034768, + "tokens_seen": 728891392 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039350050150451356, + "loss": 2.9291, + "theoretical_loss": 3.7639885384690963, + "tokens_seen": 728956928 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003934904714142427, + "loss": 3.0908, + "theoretical_loss": 3.763954428660192, + "tokens_seen": 729022464 + }, + { + "epoch": 0.22, + "objective/train/docs_used": 421294, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.16310453414917, + "objective/train/theoretical_loss": 3.7639203227759586, + "objective/train/tokens_used": 749548000, + "theoretical_loss": 3.7639203227759586, + "tokens_seen": 729088000 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003934804413239719, + "loss": 2.7638, + "theoretical_loss": 3.7639203227759586, + "tokens_seen": 729088000 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003934704112337011, + "loss": 3.0142, + "theoretical_loss": 3.763886220815592, + "tokens_seen": 729153536 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003934603811434303, + "loss": 2.8515, + "theoretical_loss": 3.763852122778289, + "tokens_seen": 729219072 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039345035105315946, + "loss": 2.8388, + "theoretical_loss": 3.7638180286632443, + "tokens_seen": 729284608 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003934403209628887, + "loss": 2.8996, + "theoretical_loss": 3.763783938469656, + "tokens_seen": 729350144 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039343029087261783, + "loss": 2.6174, + "theoretical_loss": 3.763749852196719, + "tokens_seen": 729415680 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039342026078234706, + "loss": 2.8401, + "theoretical_loss": 3.763715769843632, + "tokens_seen": 729481216 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003934102306920762, + "loss": 3.0257, + "theoretical_loss": 3.763681691409591, + "tokens_seen": 729546752 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003934002006018054, + "loss": 2.945, + "theoretical_loss": 3.7636476168937936, + "tokens_seen": 729612288 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039339017051153466, + "loss": 2.757, + "theoretical_loss": 3.7636135462954368, + "tokens_seen": 729677824 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003933801404212638, + "loss": 2.9165, + "theoretical_loss": 3.76357947961372, + "tokens_seen": 729743360 + }, + { + "epoch": 0.22, + "learning_rate": 0.000393370110330993, + "loss": 2.864, + "theoretical_loss": 3.76354541684784, + "tokens_seen": 729808896 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039336008024072215, + "loss": 2.7285, + "theoretical_loss": 3.763511357996995, + "tokens_seen": 729874432 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003933500501504514, + "loss": 2.8437, + "theoretical_loss": 3.7634773030603847, + "tokens_seen": 729939968 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039334002006018057, + "loss": 2.8042, + "theoretical_loss": 3.7634432520372068, + "tokens_seen": 730005504 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039332998996990975, + "loss": 2.6502, + "theoretical_loss": 3.7634092049266608, + "tokens_seen": 730071040 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039331995987963893, + "loss": 3.1138, + "theoretical_loss": 3.763375161727946, + "tokens_seen": 730136576 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039330992978936816, + "loss": 2.4887, + "theoretical_loss": 3.7633411224402615, + "tokens_seen": 730202112 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003932998996990973, + "loss": 2.7633, + "theoretical_loss": 3.7633070870628074, + "tokens_seen": 730267648 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003932898696088265, + "loss": 2.6567, + "theoretical_loss": 3.7632730555947833, + "tokens_seen": 730333184 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039327983951855565, + "loss": 2.7214, + "theoretical_loss": 3.76323902803539, + "tokens_seen": 730398720 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003932698094282849, + "loss": 2.9217, + "theoretical_loss": 3.763205004383827, + "tokens_seen": 730464256 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039325977933801407, + "loss": 2.7799, + "theoretical_loss": 3.7631709846392964, + "tokens_seen": 730529792 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039324974924774325, + "loss": 2.9608, + "theoretical_loss": 3.763136968800998, + "tokens_seen": 730595328 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039323971915747243, + "loss": 2.8059, + "theoretical_loss": 3.7631029568681327, + "tokens_seen": 730660864 + }, + { + "epoch": 0.22, + "objective/train/docs_used": 422072, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.0749242305755615, + "objective/train/theoretical_loss": 3.7630689488399027, + "objective/train/tokens_used": 751186400, + "theoretical_loss": 3.7630689488399027, + "tokens_seen": 730726400 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003932296890672016, + "loss": 2.8552, + "theoretical_loss": 3.7630689488399027, + "tokens_seen": 730726400 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003932196589769308, + "loss": 2.9487, + "theoretical_loss": 3.763034944715509, + "tokens_seen": 730791936 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039320962888666003, + "loss": 2.7472, + "theoretical_loss": 3.763000944494154, + "tokens_seen": 730857472 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039319959879638916, + "loss": 2.6415, + "theoretical_loss": 3.7629669481750394, + "tokens_seen": 730923008 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003931895687061184, + "loss": 2.8201, + "theoretical_loss": 3.7629329557573676, + "tokens_seen": 730988544 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003931795386158475, + "loss": 2.7187, + "theoretical_loss": 3.762898967240341, + "tokens_seen": 731054080 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039316950852557675, + "loss": 2.9508, + "theoretical_loss": 3.762864982623163, + "tokens_seen": 731119616 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039315947843530593, + "loss": 2.5658, + "theoretical_loss": 3.762831001905036, + "tokens_seen": 731185152 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003931494483450351, + "loss": 2.6415, + "theoretical_loss": 3.7627970250851632, + "tokens_seen": 731250688 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003931394182547643, + "loss": 2.6982, + "theoretical_loss": 3.7627630521627484, + "tokens_seen": 731316224 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039312938816449353, + "loss": 2.9946, + "theoretical_loss": 3.7627290831369953, + "tokens_seen": 731381760 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039311935807422266, + "loss": 2.7562, + "theoretical_loss": 3.762695118007107, + "tokens_seen": 731447296 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003931093279839519, + "loss": 2.9938, + "theoretical_loss": 3.7626611567722894, + "tokens_seen": 731512832 + }, + { + "epoch": 0.22, + "learning_rate": 0.000393099297893681, + "loss": 3.0203, + "theoretical_loss": 3.7626271994317455, + "tokens_seen": 731578368 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039308926780341026, + "loss": 2.9158, + "theoretical_loss": 3.7625932459846805, + "tokens_seen": 731643904 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039307923771313944, + "loss": 2.9955, + "theoretical_loss": 3.7625592964302994, + "tokens_seen": 731709440 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003930692076228686, + "loss": 2.8138, + "theoretical_loss": 3.762525350767807, + "tokens_seen": 731774976 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003930591775325978, + "loss": 2.6556, + "theoretical_loss": 3.762491408996409, + "tokens_seen": 731840512 + }, + { + "epoch": 0.22, + "learning_rate": 0.000393049147442327, + "loss": 2.8742, + "theoretical_loss": 3.7624574711153107, + "tokens_seen": 731906048 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039303911735205616, + "loss": 3.0189, + "theoretical_loss": 3.7624235371237176, + "tokens_seen": 731971584 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003930290872617854, + "loss": 2.8103, + "theoretical_loss": 3.7623896070208365, + "tokens_seen": 732037120 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003930190571715145, + "loss": 2.8711, + "theoretical_loss": 3.7623556808058733, + "tokens_seen": 732102656 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039300902708124376, + "loss": 2.8375, + "theoretical_loss": 3.762321758478034, + "tokens_seen": 732168192 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003929989969909729, + "loss": 2.6643, + "theoretical_loss": 3.762287840036527, + "tokens_seen": 732233728 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003929889669007021, + "loss": 2.6375, + "theoretical_loss": 3.7622539254805574, + "tokens_seen": 732299264 + }, + { + "epoch": 0.22, + "objective/train/docs_used": 423458, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.801943302154541, + "objective/train/theoretical_loss": 3.7622200148093334, + "objective/train/tokens_used": 752824800, + "theoretical_loss": 3.7622200148093334, + "tokens_seen": 732364800 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003929789368104313, + "loss": 2.928, + "theoretical_loss": 3.7622200148093334, + "tokens_seen": 732364800 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003929689067201605, + "loss": 2.9963, + "theoretical_loss": 3.7621861080220627, + "tokens_seen": 732430336 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039295887662988967, + "loss": 2.8082, + "theoretical_loss": 3.7621522051179523, + "tokens_seen": 732495872 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003929488465396189, + "loss": 2.763, + "theoretical_loss": 3.7621183060962102, + "tokens_seen": 732561408 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039293881644934803, + "loss": 2.8364, + "theoretical_loss": 3.762084410956045, + "tokens_seen": 732626944 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039292878635907726, + "loss": 2.8703, + "theoretical_loss": 3.7620505196966647, + "tokens_seen": 732692480 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003929187562688064, + "loss": 2.6946, + "theoretical_loss": 3.762016632317278, + "tokens_seen": 732758016 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003929087261785356, + "loss": 2.9394, + "theoretical_loss": 3.761982748817094, + "tokens_seen": 732823552 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003928986960882648, + "loss": 2.619, + "theoretical_loss": 3.7619488691953213, + "tokens_seen": 732889088 + }, + { + "epoch": 0.22, + "learning_rate": 0.000392888665997994, + "loss": 2.8621, + "theoretical_loss": 3.7619149934511693, + "tokens_seen": 732954624 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039287863590772317, + "loss": 2.5965, + "theoretical_loss": 3.761881121583848, + "tokens_seen": 733020160 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039286860581745235, + "loss": 2.9244, + "theoretical_loss": 3.761847253592567, + "tokens_seen": 733085696 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039285857572718153, + "loss": 2.8055, + "theoretical_loss": 3.7618133894765364, + "tokens_seen": 733151232 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039284854563691077, + "loss": 2.8574, + "theoretical_loss": 3.761779529234966, + "tokens_seen": 733216768 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003928385155466399, + "loss": 2.812, + "theoretical_loss": 3.761745672867066, + "tokens_seen": 733282304 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039282848545636913, + "loss": 2.7803, + "theoretical_loss": 3.7617118203720485, + "tokens_seen": 733347840 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039281845536609825, + "loss": 2.7938, + "theoretical_loss": 3.7616779717491227, + "tokens_seen": 733413376 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003928084252758275, + "loss": 2.6736, + "theoretical_loss": 3.761644126997501, + "tokens_seen": 733478912 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039279839518555667, + "loss": 2.9487, + "theoretical_loss": 3.7616102861163947, + "tokens_seen": 733544448 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039278836509528585, + "loss": 2.7945, + "theoretical_loss": 3.7615764491050143, + "tokens_seen": 733609984 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039277833500501503, + "loss": 2.6866, + "theoretical_loss": 3.761542615962573, + "tokens_seen": 733675520 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039276830491474427, + "loss": 2.9171, + "theoretical_loss": 3.7615087866882826, + "tokens_seen": 733741056 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003927582748244734, + "loss": 2.7848, + "theoretical_loss": 3.7614749612813547, + "tokens_seen": 733806592 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039274824473420263, + "loss": 2.8896, + "theoretical_loss": 3.761441139741003, + "tokens_seen": 733872128 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039273821464393176, + "loss": 2.6323, + "theoretical_loss": 3.7614073220664395, + "tokens_seen": 733937664 + }, + { + "epoch": 0.22, + "objective/train/docs_used": 424116, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.8282527923583984, + "objective/train/theoretical_loss": 3.7613735082568764, + "objective/train/tokens_used": 754463200, + "theoretical_loss": 3.7613735082568764, + "tokens_seen": 734003200 + }, + { + "epoch": 0.22, + "learning_rate": 0.000392728184553661, + "loss": 2.8828, + "theoretical_loss": 3.7613735082568764, + "tokens_seen": 734003200 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003927181544633902, + "loss": 2.8582, + "theoretical_loss": 3.7613396983115286, + "tokens_seen": 734068736 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039270812437311936, + "loss": 2.934, + "theoretical_loss": 3.761305892229609, + "tokens_seen": 734134272 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039269809428284854, + "loss": 2.8317, + "theoretical_loss": 3.7612720900103307, + "tokens_seen": 734199808 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003926880641925777, + "loss": 2.8033, + "theoretical_loss": 3.7612382916529086, + "tokens_seen": 734265344 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003926780341023069, + "loss": 2.7635, + "theoretical_loss": 3.7612044971565557, + "tokens_seen": 734330880 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039266800401203613, + "loss": 2.8057, + "theoretical_loss": 3.7611707065204874, + "tokens_seen": 734396416 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039265797392176526, + "loss": 2.8957, + "theoretical_loss": 3.761136919743918, + "tokens_seen": 734461952 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003926479438314945, + "loss": 2.8024, + "theoretical_loss": 3.761103136826062, + "tokens_seen": 734527488 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039263791374122373, + "loss": 2.835, + "theoretical_loss": 3.761069357766135, + "tokens_seen": 734593024 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039262788365095286, + "loss": 2.6485, + "theoretical_loss": 3.761035582563352, + "tokens_seen": 734658560 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003926178535606821, + "loss": 2.9575, + "theoretical_loss": 3.7610018112169286, + "tokens_seen": 734724096 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003926078234704112, + "loss": 2.7646, + "theoretical_loss": 3.760968043726081, + "tokens_seen": 734789632 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039259779338014046, + "loss": 2.6534, + "theoretical_loss": 3.7609342800900247, + "tokens_seen": 734855168 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039258776328986964, + "loss": 2.947, + "theoretical_loss": 3.7609005203079757, + "tokens_seen": 734920704 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003925777331995988, + "loss": 2.6984, + "theoretical_loss": 3.760866764379151, + "tokens_seen": 734986240 + }, + { + "epoch": 0.22, + "learning_rate": 0.000392567703109328, + "loss": 2.9421, + "theoretical_loss": 3.7608330123027667, + "tokens_seen": 735051776 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003925576730190572, + "loss": 2.6986, + "theoretical_loss": 3.7607992640780408, + "tokens_seen": 735117312 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039254764292878636, + "loss": 2.7074, + "theoretical_loss": 3.760765519704189, + "tokens_seen": 735182848 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003925376128385156, + "loss": 2.787, + "theoretical_loss": 3.7607317791804302, + "tokens_seen": 735248384 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003925275827482447, + "loss": 2.7682, + "theoretical_loss": 3.7606980425059806, + "tokens_seen": 735313920 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039251755265797396, + "loss": 2.7863, + "theoretical_loss": 3.7606643096800587, + "tokens_seen": 735379456 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003925075225677031, + "loss": 2.8101, + "theoretical_loss": 3.760630580701883, + "tokens_seen": 735444992 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003924974924774323, + "loss": 2.7665, + "theoretical_loss": 3.760596855570671, + "tokens_seen": 735510528 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003924874623871615, + "loss": 2.8509, + "theoretical_loss": 3.7605631342856416, + "tokens_seen": 735576064 + }, + { + "epoch": 0.22, + "objective/train/docs_used": 424834, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7749907970428467, + "objective/train/theoretical_loss": 3.7605294168460133, + "objective/train/tokens_used": 756101600, + "theoretical_loss": 3.7605294168460133, + "tokens_seen": 735641600 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003924774322968907, + "loss": 2.9718, + "theoretical_loss": 3.7605294168460133, + "tokens_seen": 735641600 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039246740220661987, + "loss": 2.8459, + "theoretical_loss": 3.7604957032510056, + "tokens_seen": 735707136 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003924573721163491, + "loss": 2.6657, + "theoretical_loss": 3.760461993499837, + "tokens_seen": 735772672 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039244734202607823, + "loss": 3.1802, + "theoretical_loss": 3.7604282875917274, + "tokens_seen": 735838208 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039243731193580746, + "loss": 2.7035, + "theoretical_loss": 3.760394585525896, + "tokens_seen": 735903744 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003924272818455366, + "loss": 2.8095, + "theoretical_loss": 3.7603608873015633, + "tokens_seen": 735969280 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003924172517552658, + "loss": 2.9598, + "theoretical_loss": 3.7603271929179494, + "tokens_seen": 736034816 + }, + { + "epoch": 0.22, + "learning_rate": 0.000392407221664995, + "loss": 2.8553, + "theoretical_loss": 3.7602935023742736, + "tokens_seen": 736100352 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003923971915747242, + "loss": 2.5848, + "theoretical_loss": 3.760259815669758, + "tokens_seen": 736165888 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039238716148445337, + "loss": 2.7623, + "theoretical_loss": 3.7602261328036226, + "tokens_seen": 736231424 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039237713139418255, + "loss": 2.7988, + "theoretical_loss": 3.760192453775088, + "tokens_seen": 736296960 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039236710130391173, + "loss": 2.6708, + "theoretical_loss": 3.760158778583376, + "tokens_seen": 736362496 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039235707121364097, + "loss": 2.7437, + "theoretical_loss": 3.7601251072277084, + "tokens_seen": 736428032 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003923470411233701, + "loss": 2.8454, + "theoretical_loss": 3.760091439707306, + "tokens_seen": 736493568 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039233701103309933, + "loss": 2.829, + "theoretical_loss": 3.760057776021392, + "tokens_seen": 736559104 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039232698094282846, + "loss": 2.8942, + "theoretical_loss": 3.7600241161691876, + "tokens_seen": 736624640 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003923169508525577, + "loss": 2.8214, + "theoretical_loss": 3.759990460149915, + "tokens_seen": 736690176 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039230692076228687, + "loss": 2.6976, + "theoretical_loss": 3.759956807962797, + "tokens_seen": 736755712 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039229689067201605, + "loss": 2.9868, + "theoretical_loss": 3.759923159607057, + "tokens_seen": 736821248 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039228686058174523, + "loss": 2.6928, + "theoretical_loss": 3.759889515081918, + "tokens_seen": 736886784 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039227683049147447, + "loss": 2.7411, + "theoretical_loss": 3.759855874386602, + "tokens_seen": 736952320 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003922668004012036, + "loss": 2.8783, + "theoretical_loss": 3.7598222375203347, + "tokens_seen": 737017856 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039225677031093283, + "loss": 2.8159, + "theoretical_loss": 3.7597886044823383, + "tokens_seen": 737083392 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039224674022066196, + "loss": 2.9971, + "theoretical_loss": 3.7597549752718367, + "tokens_seen": 737148928 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003922367101303912, + "loss": 2.9457, + "theoretical_loss": 3.7597213498880553, + "tokens_seen": 737214464 + }, + { + "epoch": 0.22, + "objective/train/docs_used": 426204, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.457033157348633, + "objective/train/theoretical_loss": 3.759687728330217, + "objective/train/tokens_used": 757740000, + "theoretical_loss": 3.759687728330217, + "tokens_seen": 737280000 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003922266800401204, + "loss": 2.9082, + "theoretical_loss": 3.759687728330217, + "tokens_seen": 737280000 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039221664994984956, + "loss": 2.5964, + "theoretical_loss": 3.759654110597548, + "tokens_seen": 737345536 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039220661985957874, + "loss": 2.6955, + "theoretical_loss": 3.759620496689272, + "tokens_seen": 737411072 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003921965897693079, + "loss": 2.7703, + "theoretical_loss": 3.7595868866046143, + "tokens_seen": 737476608 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003921865596790371, + "loss": 2.8794, + "theoretical_loss": 3.759553280342801, + "tokens_seen": 737542144 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039217652958876633, + "loss": 2.6278, + "theoretical_loss": 3.7595196779030564, + "tokens_seen": 737607680 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039216649949849546, + "loss": 2.6098, + "theoretical_loss": 3.759486079284607, + "tokens_seen": 737673216 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003921564694082247, + "loss": 2.6276, + "theoretical_loss": 3.7594524844866797, + "tokens_seen": 737738752 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003921464393179539, + "loss": 2.8393, + "theoretical_loss": 3.7594188935084993, + "tokens_seen": 737804288 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039213640922768306, + "loss": 2.8491, + "theoretical_loss": 3.7593853063492926, + "tokens_seen": 737869824 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039212637913741224, + "loss": 2.7298, + "theoretical_loss": 3.7593517230082867, + "tokens_seen": 737935360 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003921163490471414, + "loss": 2.6339, + "theoretical_loss": 3.759318143484708, + "tokens_seen": 738000896 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003921063189568706, + "loss": 2.6485, + "theoretical_loss": 3.759284567777784, + "tokens_seen": 738066432 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039209628886659984, + "loss": 2.4511, + "theoretical_loss": 3.7592509958867426, + "tokens_seen": 738131968 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039208625877632896, + "loss": 3.043, + "theoretical_loss": 3.75921742781081, + "tokens_seen": 738197504 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003920762286860582, + "loss": 2.8424, + "theoretical_loss": 3.759183863549215, + "tokens_seen": 738263040 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003920661985957873, + "loss": 2.8083, + "theoretical_loss": 3.759150303101185, + "tokens_seen": 738328576 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039205616850551656, + "loss": 2.9238, + "theoretical_loss": 3.759116746465949, + "tokens_seen": 738394112 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039204613841524574, + "loss": 2.706, + "theoretical_loss": 3.7590831936427356, + "tokens_seen": 738459648 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003920361083249749, + "loss": 2.843, + "theoretical_loss": 3.759049644630773, + "tokens_seen": 738525184 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003920260782347041, + "loss": 2.7555, + "theoretical_loss": 3.7590160994292896, + "tokens_seen": 738590720 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003920160481444333, + "loss": 2.9035, + "theoretical_loss": 3.7589825580375154, + "tokens_seen": 738656256 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039200601805416247, + "loss": 2.9236, + "theoretical_loss": 3.758949020454679, + "tokens_seen": 738721792 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003919959879638917, + "loss": 2.8534, + "theoretical_loss": 3.758915486680011, + "tokens_seen": 738787328 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039198595787362083, + "loss": 2.8563, + "theoretical_loss": 3.7588819567127407, + "tokens_seen": 738852864 + }, + { + "epoch": 0.22, + "objective/train/docs_used": 426747, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6297125816345215, + "objective/train/theoretical_loss": 3.7588484305520984, + "objective/train/tokens_used": 759378400, + "theoretical_loss": 3.7588484305520984, + "tokens_seen": 738918400 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039197592778335007, + "loss": 2.8277, + "theoretical_loss": 3.7588484305520984, + "tokens_seen": 738918400 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039196589769307925, + "loss": 3.0829, + "theoretical_loss": 3.7588149081973143, + "tokens_seen": 738983936 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039195586760280843, + "loss": 2.6553, + "theoretical_loss": 3.7587813896476185, + "tokens_seen": 739049472 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003919458375125376, + "loss": 2.715, + "theoretical_loss": 3.758747874902242, + "tokens_seen": 739115008 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003919358074222668, + "loss": 2.6567, + "theoretical_loss": 3.758714363960416, + "tokens_seen": 739180544 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039192577733199597, + "loss": 2.5722, + "theoretical_loss": 3.758680856821371, + "tokens_seen": 739246080 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003919157472417252, + "loss": 2.9656, + "theoretical_loss": 3.7586473534843394, + "tokens_seen": 739311616 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039190571715145433, + "loss": 2.8007, + "theoretical_loss": 3.7586138539485514, + "tokens_seen": 739377152 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039189568706118357, + "loss": 2.8261, + "theoretical_loss": 3.7585803582132407, + "tokens_seen": 739442688 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039188565697091275, + "loss": 2.7802, + "theoretical_loss": 3.7585468662776376, + "tokens_seen": 739508224 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039187562688064193, + "loss": 2.8741, + "theoretical_loss": 3.758513378140975, + "tokens_seen": 739573760 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039186559679037117, + "loss": 2.7766, + "theoretical_loss": 3.7584798938024866, + "tokens_seen": 739639296 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003918555667001003, + "loss": 2.858, + "theoretical_loss": 3.7584464132614035, + "tokens_seen": 739704832 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039184553660982953, + "loss": 2.883, + "theoretical_loss": 3.758412936516959, + "tokens_seen": 739770368 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039183550651955866, + "loss": 2.9446, + "theoretical_loss": 3.758379463568387, + "tokens_seen": 739835904 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003918254764292879, + "loss": 3.0587, + "theoretical_loss": 3.7583459944149205, + "tokens_seen": 739901440 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039181544633901707, + "loss": 2.82, + "theoretical_loss": 3.7583125290557926, + "tokens_seen": 739966976 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039180541624874625, + "loss": 2.8348, + "theoretical_loss": 3.7582790674902378, + "tokens_seen": 740032512 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039179538615847543, + "loss": 2.8726, + "theoretical_loss": 3.7582456097174903, + "tokens_seen": 740098048 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039178535606820467, + "loss": 2.786, + "theoretical_loss": 3.758212155736784, + "tokens_seen": 740163584 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003917753259779338, + "loss": 2.7433, + "theoretical_loss": 3.758178705547353, + "tokens_seen": 740229120 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039176529588766303, + "loss": 2.851, + "theoretical_loss": 3.7581452591484323, + "tokens_seen": 740294656 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039175526579739216, + "loss": 2.819, + "theoretical_loss": 3.758111816539258, + "tokens_seen": 740360192 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003917452357071214, + "loss": 2.9126, + "theoretical_loss": 3.7580783777190643, + "tokens_seen": 740425728 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003917352056168506, + "loss": 2.8882, + "theoretical_loss": 3.7580449426870866, + "tokens_seen": 740491264 + }, + { + "epoch": 0.22, + "objective/train/docs_used": 427723, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7428524494171143, + "objective/train/theoretical_loss": 3.75801151144256, + "objective/train/tokens_used": 761016800, + "theoretical_loss": 3.75801151144256, + "tokens_seen": 740556800 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039172517552657976, + "loss": 2.9239, + "theoretical_loss": 3.75801151144256, + "tokens_seen": 740556800 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039171514543630894, + "loss": 2.7239, + "theoretical_loss": 3.757978083984722, + "tokens_seen": 740622336 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003917051153460381, + "loss": 3.0594, + "theoretical_loss": 3.7579446603128064, + "tokens_seen": 740687872 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003916950852557673, + "loss": 2.7663, + "theoretical_loss": 3.7579112404260515, + "tokens_seen": 740753408 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039168505516549653, + "loss": 2.9605, + "theoretical_loss": 3.7578778243236934, + "tokens_seen": 740818944 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039167502507522566, + "loss": 2.8039, + "theoretical_loss": 3.757844412004968, + "tokens_seen": 740884480 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003916649949849549, + "loss": 2.6737, + "theoretical_loss": 3.7578110034691123, + "tokens_seen": 740950016 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003916549648946841, + "loss": 2.5398, + "theoretical_loss": 3.7577775987153648, + "tokens_seen": 741015552 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039164493480441326, + "loss": 2.8808, + "theoretical_loss": 3.7577441977429613, + "tokens_seen": 741081088 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039163490471414244, + "loss": 2.8235, + "theoretical_loss": 3.7577108005511404, + "tokens_seen": 741146624 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003916248746238716, + "loss": 2.9614, + "theoretical_loss": 3.7576774071391403, + "tokens_seen": 741212160 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003916148445336008, + "loss": 2.8559, + "theoretical_loss": 3.7576440175061974, + "tokens_seen": 741277696 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039160481444333004, + "loss": 2.6585, + "theoretical_loss": 3.757610631651552, + "tokens_seen": 741343232 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039159478435305916, + "loss": 2.975, + "theoretical_loss": 3.757577249574441, + "tokens_seen": 741408768 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003915847542627884, + "loss": 2.8601, + "theoretical_loss": 3.7575438712741036, + "tokens_seen": 741474304 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003915747241725175, + "loss": 2.8656, + "theoretical_loss": 3.75751049674978, + "tokens_seen": 741539840 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039156469408224676, + "loss": 2.7535, + "theoretical_loss": 3.7574771260007074, + "tokens_seen": 741605376 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039155466399197594, + "loss": 2.9709, + "theoretical_loss": 3.7574437590261263, + "tokens_seen": 741670912 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003915446339017051, + "loss": 2.763, + "theoretical_loss": 3.757410395825276, + "tokens_seen": 741736448 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003915346038114343, + "loss": 2.7076, + "theoretical_loss": 3.757377036397396, + "tokens_seen": 741801984 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003915245737211635, + "loss": 2.8834, + "theoretical_loss": 3.7573436807417275, + "tokens_seen": 741867520 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039151454363089267, + "loss": 2.7392, + "theoretical_loss": 3.7573103288575096, + "tokens_seen": 741933056 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003915045135406219, + "loss": 2.7391, + "theoretical_loss": 3.757276980743983, + "tokens_seen": 741998592 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039149448345035103, + "loss": 2.6135, + "theoretical_loss": 3.757243636400389, + "tokens_seen": 742064128 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039148445336008027, + "loss": 2.7537, + "theoretical_loss": 3.757210295825968, + "tokens_seen": 742129664 + }, + { + "epoch": 0.22, + "objective/train/docs_used": 429091, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.591966390609741, + "objective/train/theoretical_loss": 3.7571769590199615, + "objective/train/tokens_used": 762655200, + "theoretical_loss": 3.7571769590199615, + "tokens_seen": 742195200 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039147442326980945, + "loss": 2.6163, + "theoretical_loss": 3.7571769590199615, + "tokens_seen": 742195200 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039146439317953863, + "loss": 2.8168, + "theoretical_loss": 3.7571436259816107, + "tokens_seen": 742260736 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003914543630892678, + "loss": 2.615, + "theoretical_loss": 3.757110296710157, + "tokens_seen": 742326272 + }, + { + "epoch": 0.22, + "learning_rate": 0.000391444332998997, + "loss": 2.8025, + "theoretical_loss": 3.7570769712048424, + "tokens_seen": 742391808 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039143430290872617, + "loss": 3.0868, + "theoretical_loss": 3.757043649464909, + "tokens_seen": 742457344 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003914242728184554, + "loss": 2.8381, + "theoretical_loss": 3.7570103314895986, + "tokens_seen": 742522880 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039141424272818453, + "loss": 2.5077, + "theoretical_loss": 3.756977017278154, + "tokens_seen": 742588416 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039140421263791377, + "loss": 3.0984, + "theoretical_loss": 3.7569437068298184, + "tokens_seen": 742653952 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003913941825476429, + "loss": 2.9864, + "theoretical_loss": 3.756910400143833, + "tokens_seen": 742719488 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039138415245737213, + "loss": 2.8554, + "theoretical_loss": 3.7568770972194434, + "tokens_seen": 742785024 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003913741223671013, + "loss": 3.0056, + "theoretical_loss": 3.7568437980558915, + "tokens_seen": 742850560 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003913640922768305, + "loss": 2.7288, + "theoretical_loss": 3.756810502652421, + "tokens_seen": 742916096 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003913540621865597, + "loss": 2.7873, + "theoretical_loss": 3.7567772110082753, + "tokens_seen": 742981632 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039134403209628886, + "loss": 3.0103, + "theoretical_loss": 3.756743923122699, + "tokens_seen": 743047168 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039133400200601804, + "loss": 2.8987, + "theoretical_loss": 3.756710638994936, + "tokens_seen": 743112704 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039132397191574727, + "loss": 2.929, + "theoretical_loss": 3.756677358624231, + "tokens_seen": 743178240 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003913139418254764, + "loss": 3.0462, + "theoretical_loss": 3.7566440820098284, + "tokens_seen": 743243776 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039130391173520563, + "loss": 2.7603, + "theoretical_loss": 3.756610809150973, + "tokens_seen": 743309312 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003912938816449348, + "loss": 3.0916, + "theoretical_loss": 3.7565775400469104, + "tokens_seen": 743374848 + }, + { + "epoch": 0.23, + "learning_rate": 0.000391283851554664, + "loss": 2.6973, + "theoretical_loss": 3.7565442746968856, + "tokens_seen": 743440384 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003912738214643932, + "loss": 2.5707, + "theoretical_loss": 3.7565110131001442, + "tokens_seen": 743505920 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039126379137412236, + "loss": 2.9152, + "theoretical_loss": 3.756477755255932, + "tokens_seen": 743571456 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039125376128385154, + "loss": 3.0235, + "theoretical_loss": 3.7564445011634944, + "tokens_seen": 743636992 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003912437311935808, + "loss": 2.9465, + "theoretical_loss": 3.7564112508220777, + "tokens_seen": 743702528 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003912337011033099, + "loss": 2.8228, + "theoretical_loss": 3.756378004230929, + "tokens_seen": 743768064 + }, + { + "epoch": 0.23, + "objective/train/docs_used": 429832, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.1936089992523193, + "objective/train/theoretical_loss": 3.756344761389295, + "objective/train/tokens_used": 764293600, + "theoretical_loss": 3.756344761389295, + "tokens_seen": 743833600 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039122367101303914, + "loss": 3.0362, + "theoretical_loss": 3.756344761389295, + "tokens_seen": 743833600 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039121364092276826, + "loss": 2.9838, + "theoretical_loss": 3.7563115222964214, + "tokens_seen": 743899136 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003912036108324975, + "loss": 2.7325, + "theoretical_loss": 3.7562782869515567, + "tokens_seen": 743964672 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003911935807422267, + "loss": 3.0409, + "theoretical_loss": 3.756245055353947, + "tokens_seen": 744030208 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039118355065195586, + "loss": 2.837, + "theoretical_loss": 3.75621182750284, + "tokens_seen": 744095744 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039117352056168504, + "loss": 2.7614, + "theoretical_loss": 3.7561786033974833, + "tokens_seen": 744161280 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003911634904714143, + "loss": 2.9199, + "theoretical_loss": 3.7561453830371256, + "tokens_seen": 744226816 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003911534603811434, + "loss": 2.747, + "theoretical_loss": 3.7561121664210146, + "tokens_seen": 744292352 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039114343029087264, + "loss": 2.8664, + "theoretical_loss": 3.756078953548398, + "tokens_seen": 744357888 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003911334002006018, + "loss": 2.9967, + "theoretical_loss": 3.756045744418526, + "tokens_seen": 744423424 + }, + { + "epoch": 0.23, + "learning_rate": 0.000391123370110331, + "loss": 2.9658, + "theoretical_loss": 3.7560125390306456, + "tokens_seen": 744488960 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039111334002006024, + "loss": 2.9419, + "theoretical_loss": 3.755979337384007, + "tokens_seen": 744554496 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039110330992978936, + "loss": 3.0731, + "theoretical_loss": 3.7559461394778584, + "tokens_seen": 744620032 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003910932798395186, + "loss": 2.7457, + "theoretical_loss": 3.755912945311451, + "tokens_seen": 744685568 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039108324974924773, + "loss": 2.8382, + "theoretical_loss": 3.755879754884032, + "tokens_seen": 744751104 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039107321965897696, + "loss": 2.7646, + "theoretical_loss": 3.755846568194853, + "tokens_seen": 744816640 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039106318956870614, + "loss": 2.8971, + "theoretical_loss": 3.755813385243164, + "tokens_seen": 744882176 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003910531594784353, + "loss": 3.0498, + "theoretical_loss": 3.7557802060282146, + "tokens_seen": 744947712 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003910431293881645, + "loss": 2.9768, + "theoretical_loss": 3.755747030549256, + "tokens_seen": 745013248 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003910330992978937, + "loss": 2.9746, + "theoretical_loss": 3.7557138588055388, + "tokens_seen": 745078784 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039102306920762287, + "loss": 2.6984, + "theoretical_loss": 3.7556806907963134, + "tokens_seen": 745144320 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003910130391173521, + "loss": 2.8482, + "theoretical_loss": 3.7556475265208316, + "tokens_seen": 745209856 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039100300902708123, + "loss": 2.7164, + "theoretical_loss": 3.7556143659783445, + "tokens_seen": 745275392 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039099297893681047, + "loss": 2.8489, + "theoretical_loss": 3.755581209168104, + "tokens_seen": 745340928 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039098294884653965, + "loss": 2.6978, + "theoretical_loss": 3.7555480560893617, + "tokens_seen": 745406464 + }, + { + "epoch": 0.23, + "objective/train/docs_used": 430489, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.059129238128662, + "objective/train/theoretical_loss": 3.7555149067413693, + "objective/train/tokens_used": 765932000, + "theoretical_loss": 3.7555149067413693, + "tokens_seen": 745472000 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039097291875626883, + "loss": 2.9399, + "theoretical_loss": 3.7555149067413693, + "tokens_seen": 745472000 + }, + { + "epoch": 0.23, + "learning_rate": 0.000390962888665998, + "loss": 2.6395, + "theoretical_loss": 3.755481761123379, + "tokens_seen": 745537536 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003909528585757272, + "loss": 3.0138, + "theoretical_loss": 3.7554486192346443, + "tokens_seen": 745603072 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039094282848545637, + "loss": 2.816, + "theoretical_loss": 3.7554154810744174, + "tokens_seen": 745668608 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003909327983951856, + "loss": 2.6564, + "theoretical_loss": 3.7553823466419507, + "tokens_seen": 745734144 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039092276830491473, + "loss": 2.8054, + "theoretical_loss": 3.755349215936498, + "tokens_seen": 745799680 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039091273821464397, + "loss": 2.6408, + "theoretical_loss": 3.755316088957312, + "tokens_seen": 745865216 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003909027081243731, + "loss": 3.0102, + "theoretical_loss": 3.755282965703646, + "tokens_seen": 745930752 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039089267803410233, + "loss": 2.8772, + "theoretical_loss": 3.755249846174755, + "tokens_seen": 745996288 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003908826479438315, + "loss": 2.8151, + "theoretical_loss": 3.7552167303698925, + "tokens_seen": 746061824 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003908726178535607, + "loss": 2.9385, + "theoretical_loss": 3.755183618288312, + "tokens_seen": 746127360 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003908625877632899, + "loss": 2.8539, + "theoretical_loss": 3.755150509929268, + "tokens_seen": 746192896 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039085255767301906, + "loss": 2.7237, + "theoretical_loss": 3.7551174052920158, + "tokens_seen": 746258432 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039084252758274824, + "loss": 2.9449, + "theoretical_loss": 3.75508430437581, + "tokens_seen": 746323968 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039083249749247747, + "loss": 2.8255, + "theoretical_loss": 3.755051207179905, + "tokens_seen": 746389504 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003908224674022066, + "loss": 3.0098, + "theoretical_loss": 3.7550181137035574, + "tokens_seen": 746455040 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039081243731193583, + "loss": 2.5888, + "theoretical_loss": 3.7549850239460216, + "tokens_seen": 746520576 + }, + { + "epoch": 0.23, + "learning_rate": 0.000390802407221665, + "loss": 2.7872, + "theoretical_loss": 3.7549519379065535, + "tokens_seen": 746586112 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003907923771313942, + "loss": 2.9184, + "theoretical_loss": 3.7549188555844095, + "tokens_seen": 746651648 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003907823470411234, + "loss": 2.8595, + "theoretical_loss": 3.7548857769788446, + "tokens_seen": 746717184 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039077231695085256, + "loss": 2.8949, + "theoretical_loss": 3.7548527020891163, + "tokens_seen": 746782720 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039076228686058174, + "loss": 2.8507, + "theoretical_loss": 3.754819630914481, + "tokens_seen": 746848256 + }, + { + "epoch": 0.23, + "learning_rate": 0.000390752256770311, + "loss": 2.9951, + "theoretical_loss": 3.7547865634541946, + "tokens_seen": 746913792 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003907422266800401, + "loss": 2.7173, + "theoretical_loss": 3.7547534997075145, + "tokens_seen": 746979328 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039073219658976934, + "loss": 2.6399, + "theoretical_loss": 3.754720439673698, + "tokens_seen": 747044864 + }, + { + "epoch": 0.23, + "objective/train/docs_used": 431396, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6308023929595947, + "objective/train/theoretical_loss": 3.754687383352003, + "objective/train/tokens_used": 767570400, + "theoretical_loss": 3.754687383352003, + "tokens_seen": 747110400 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039072216649949846, + "loss": 2.858, + "theoretical_loss": 3.754687383352003, + "tokens_seen": 747110400 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003907121364092277, + "loss": 2.9568, + "theoretical_loss": 3.754654330741687, + "tokens_seen": 747175936 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003907021063189569, + "loss": 2.8312, + "theoretical_loss": 3.754621281842007, + "tokens_seen": 747241472 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039069207622868606, + "loss": 2.9293, + "theoretical_loss": 3.7545882366522214, + "tokens_seen": 747307008 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039068204613841524, + "loss": 2.7484, + "theoretical_loss": 3.7545551951715885, + "tokens_seen": 747372544 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003906720160481445, + "loss": 2.6245, + "theoretical_loss": 3.754522157399367, + "tokens_seen": 747438080 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003906619859578736, + "loss": 2.8434, + "theoretical_loss": 3.7544891233348157, + "tokens_seen": 747503616 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039065195586760284, + "loss": 2.9989, + "theoretical_loss": 3.7544560929771924, + "tokens_seen": 747569152 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039064192577733197, + "loss": 2.7289, + "theoretical_loss": 3.7544230663257583, + "tokens_seen": 747634688 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003906318956870612, + "loss": 2.8148, + "theoretical_loss": 3.7543900433797703, + "tokens_seen": 747700224 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003906218655967904, + "loss": 2.8105, + "theoretical_loss": 3.75435702413849, + "tokens_seen": 747765760 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039061183550651957, + "loss": 3.1121, + "theoretical_loss": 3.7543240086011753, + "tokens_seen": 747831296 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039060180541624875, + "loss": 3.0859, + "theoretical_loss": 3.7542909967670877, + "tokens_seen": 747896832 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039059177532597793, + "loss": 2.9044, + "theoretical_loss": 3.754257988635487, + "tokens_seen": 747962368 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003905817452357071, + "loss": 2.8967, + "theoretical_loss": 3.754224984205633, + "tokens_seen": 748027904 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039057171514543634, + "loss": 2.897, + "theoretical_loss": 3.7541919834767867, + "tokens_seen": 748093440 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039056168505516547, + "loss": 2.8435, + "theoretical_loss": 3.754158986448209, + "tokens_seen": 748158976 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003905516549648947, + "loss": 3.0243, + "theoretical_loss": 3.7541259931191604, + "tokens_seen": 748224512 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039054162487462383, + "loss": 3.1334, + "theoretical_loss": 3.7540930034889035, + "tokens_seen": 748290048 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039053159478435307, + "loss": 2.7682, + "theoretical_loss": 3.7540600175566983, + "tokens_seen": 748355584 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039052156469408225, + "loss": 2.8338, + "theoretical_loss": 3.7540270353218066, + "tokens_seen": 748421120 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039051153460381143, + "loss": 2.8192, + "theoretical_loss": 3.7539940567834913, + "tokens_seen": 748486656 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003905015045135406, + "loss": 2.7454, + "theoretical_loss": 3.753961081941013, + "tokens_seen": 748552192 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039049147442326985, + "loss": 2.7857, + "theoretical_loss": 3.7539281107936358, + "tokens_seen": 748617728 + }, + { + "epoch": 0.23, + "learning_rate": 0.000390481444332999, + "loss": 2.8768, + "theoretical_loss": 3.7538951433406202, + "tokens_seen": 748683264 + }, + { + "epoch": 0.23, + "objective/train/docs_used": 432599, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.837986707687378, + "objective/train/theoretical_loss": 3.7538621795812306, + "objective/train/tokens_used": 769208800, + "theoretical_loss": 3.7538621795812306, + "tokens_seen": 748748800 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003904714142427282, + "loss": 2.8327, + "theoretical_loss": 3.7538621795812306, + "tokens_seen": 748748800 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039046138415245734, + "loss": 2.7929, + "theoretical_loss": 3.7538292195147287, + "tokens_seen": 748814336 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039045135406218657, + "loss": 2.7708, + "theoretical_loss": 3.753796263140379, + "tokens_seen": 748879872 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039044132397191575, + "loss": 2.9958, + "theoretical_loss": 3.753763310457444, + "tokens_seen": 748945408 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039043129388164493, + "loss": 2.9547, + "theoretical_loss": 3.7537303614651867, + "tokens_seen": 749010944 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003904212637913741, + "loss": 2.9264, + "theoretical_loss": 3.753697416162872, + "tokens_seen": 749076480 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003904112337011033, + "loss": 2.9024, + "theoretical_loss": 3.753664474549763, + "tokens_seen": 749142016 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003904012036108325, + "loss": 2.9812, + "theoretical_loss": 3.753631536625125, + "tokens_seen": 749207552 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003903911735205617, + "loss": 2.8139, + "theoretical_loss": 3.7535986023882213, + "tokens_seen": 749273088 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003903811434302909, + "loss": 2.9811, + "theoretical_loss": 3.753565671838317, + "tokens_seen": 749338624 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003903711133400201, + "loss": 2.7124, + "theoretical_loss": 3.753532744974677, + "tokens_seen": 749404160 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039036108324974926, + "loss": 2.9426, + "theoretical_loss": 3.753499821796566, + "tokens_seen": 749469696 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039035105315947844, + "loss": 2.6793, + "theoretical_loss": 3.7534669023032494, + "tokens_seen": 749535232 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039034102306920767, + "loss": 2.8192, + "theoretical_loss": 3.7534339864939934, + "tokens_seen": 749600768 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003903309929789368, + "loss": 3.0412, + "theoretical_loss": 3.7534010743680626, + "tokens_seen": 749666304 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039032096288866603, + "loss": 2.8082, + "theoretical_loss": 3.7533681659247233, + "tokens_seen": 749731840 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003903109327983952, + "loss": 3.0036, + "theoretical_loss": 3.7533352611632416, + "tokens_seen": 749797376 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003903009027081244, + "loss": 2.9253, + "theoretical_loss": 3.753302360082884, + "tokens_seen": 749862912 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003902908726178536, + "loss": 3.0689, + "theoretical_loss": 3.7532694626829164, + "tokens_seen": 749928448 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039028084252758276, + "loss": 2.7392, + "theoretical_loss": 3.753236568962606, + "tokens_seen": 749993984 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039027081243731194, + "loss": 2.9229, + "theoretical_loss": 3.7532036789212206, + "tokens_seen": 750059520 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003902607823470412, + "loss": 2.9861, + "theoretical_loss": 3.7531707925580253, + "tokens_seen": 750125056 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003902507522567703, + "loss": 2.8246, + "theoretical_loss": 3.7531379098722897, + "tokens_seen": 750190592 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039024072216649954, + "loss": 2.9943, + "theoretical_loss": 3.7531050308632796, + "tokens_seen": 750256128 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039023069207622866, + "loss": 2.6532, + "theoretical_loss": 3.7530721555302637, + "tokens_seen": 750321664 + }, + { + "epoch": 0.23, + "objective/train/docs_used": 433353, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.474212169647217, + "objective/train/theoretical_loss": 3.7530392838725097, + "objective/train/tokens_used": 770847200, + "theoretical_loss": 3.7530392838725097, + "tokens_seen": 750387200 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003902206619859579, + "loss": 2.7147, + "theoretical_loss": 3.7530392838725097, + "tokens_seen": 750387200 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003902106318956871, + "loss": 2.8331, + "theoretical_loss": 3.7530064158892857, + "tokens_seen": 750452736 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039020060180541626, + "loss": 2.8898, + "theoretical_loss": 3.7529735515798603, + "tokens_seen": 750518272 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039019057171514544, + "loss": 3.0382, + "theoretical_loss": 3.752940690943502, + "tokens_seen": 750583808 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003901805416248747, + "loss": 2.759, + "theoretical_loss": 3.7529078339794806, + "tokens_seen": 750649344 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003901705115346038, + "loss": 2.6074, + "theoretical_loss": 3.7528749806870634, + "tokens_seen": 750714880 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039016048144433304, + "loss": 2.8372, + "theoretical_loss": 3.7528421310655204, + "tokens_seen": 750780416 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039015045135406217, + "loss": 2.611, + "theoretical_loss": 3.7528092851141213, + "tokens_seen": 750845952 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003901404212637914, + "loss": 3.0834, + "theoretical_loss": 3.7527764428321357, + "tokens_seen": 750911488 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003901303911735206, + "loss": 2.9887, + "theoretical_loss": 3.752743604218834, + "tokens_seen": 750977024 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039012036108324977, + "loss": 2.6841, + "theoretical_loss": 3.7527107692734845, + "tokens_seen": 751042560 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039011033099297895, + "loss": 2.7793, + "theoretical_loss": 3.752677937995359, + "tokens_seen": 751108096 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039010030090270813, + "loss": 2.8275, + "theoretical_loss": 3.752645110383728, + "tokens_seen": 751173632 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003900902708124373, + "loss": 2.8719, + "theoretical_loss": 3.752612286437862, + "tokens_seen": 751239168 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039008024072216654, + "loss": 3.0531, + "theoretical_loss": 3.752579466157031, + "tokens_seen": 751304704 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039007021063189567, + "loss": 3.0352, + "theoretical_loss": 3.7525466495405073, + "tokens_seen": 751370240 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003900601805416249, + "loss": 2.9086, + "theoretical_loss": 3.7525138365875623, + "tokens_seen": 751435776 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039005015045135403, + "loss": 2.8744, + "theoretical_loss": 3.752481027297466, + "tokens_seen": 751501312 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039004012036108327, + "loss": 2.9484, + "theoretical_loss": 3.7524482216694923, + "tokens_seen": 751566848 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039003009027081245, + "loss": 2.7338, + "theoretical_loss": 3.7524154197029116, + "tokens_seen": 751632384 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039002006018054163, + "loss": 2.9289, + "theoretical_loss": 3.752382621396997, + "tokens_seen": 751697920 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003900100300902708, + "loss": 2.6153, + "theoretical_loss": 3.7523498267510202, + "tokens_seen": 751763456 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039000000000000005, + "loss": 2.9596, + "theoretical_loss": 3.7523170357642535, + "tokens_seen": 751828992 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003899899699097292, + "loss": 3.0263, + "theoretical_loss": 3.752284248435971, + "tokens_seen": 751894528 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003899799398194584, + "loss": 2.7721, + "theoretical_loss": 3.752251464765444, + "tokens_seen": 751960064 + }, + { + "epoch": 0.23, + "objective/train/docs_used": 434593, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.5289430618286133, + "objective/train/theoretical_loss": 3.7522186847519476, + "objective/train/tokens_used": 772485600, + "theoretical_loss": 3.7522186847519476, + "tokens_seen": 752025600 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038996990972918754, + "loss": 2.957, + "theoretical_loss": 3.7522186847519476, + "tokens_seen": 752025600 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038995987963891677, + "loss": 2.8376, + "theoretical_loss": 3.752185908394754, + "tokens_seen": 752091136 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038994984954864595, + "loss": 2.985, + "theoretical_loss": 3.7521531356931366, + "tokens_seen": 752156672 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038993981945837513, + "loss": 3.0036, + "theoretical_loss": 3.75212036664637, + "tokens_seen": 752222208 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003899297893681043, + "loss": 2.8278, + "theoretical_loss": 3.7520876012537276, + "tokens_seen": 752287744 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003899197592778335, + "loss": 2.8, + "theoretical_loss": 3.7520548395144844, + "tokens_seen": 752353280 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003899097291875627, + "loss": 3.0135, + "theoretical_loss": 3.7520220814279144, + "tokens_seen": 752418816 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003898996990972919, + "loss": 3.0658, + "theoretical_loss": 3.751989326993292, + "tokens_seen": 752484352 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038988966900702104, + "loss": 3.1185, + "theoretical_loss": 3.751956576209892, + "tokens_seen": 752549888 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003898796389167503, + "loss": 2.7999, + "theoretical_loss": 3.7519238290769907, + "tokens_seen": 752615424 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003898696088264794, + "loss": 2.867, + "theoretical_loss": 3.751891085593862, + "tokens_seen": 752680960 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038985957873620864, + "loss": 2.6724, + "theoretical_loss": 3.751858345759782, + "tokens_seen": 752746496 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003898495486459378, + "loss": 2.7922, + "theoretical_loss": 3.751825609574026, + "tokens_seen": 752812032 + }, + { + "epoch": 0.23, + "learning_rate": 0.000389839518555667, + "loss": 2.8414, + "theoretical_loss": 3.7517928770358706, + "tokens_seen": 752877568 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003898294884653962, + "loss": 2.7745, + "theoretical_loss": 3.7517601481445917, + "tokens_seen": 752943104 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003898194583751254, + "loss": 2.8785, + "theoretical_loss": 3.7517274228994646, + "tokens_seen": 753008640 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038980942828485454, + "loss": 3.0157, + "theoretical_loss": 3.7516947012997672, + "tokens_seen": 753074176 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003897993981945838, + "loss": 2.7983, + "theoretical_loss": 3.7516619833447757, + "tokens_seen": 753139712 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003897893681043129, + "loss": 2.9848, + "theoretical_loss": 3.7516292690337667, + "tokens_seen": 753205248 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038977933801404214, + "loss": 3.1573, + "theoretical_loss": 3.7515965583660176, + "tokens_seen": 753270784 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003897693079237713, + "loss": 2.8656, + "theoretical_loss": 3.751563851340806, + "tokens_seen": 753336320 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003897592778335005, + "loss": 2.9367, + "theoretical_loss": 3.7515311479574094, + "tokens_seen": 753401856 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003897492477432297, + "loss": 2.7383, + "theoretical_loss": 3.7514984482151053, + "tokens_seen": 753467392 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038973921765295886, + "loss": 2.8614, + "theoretical_loss": 3.7514657521131713, + "tokens_seen": 753532928 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038972918756268805, + "loss": 3.1543, + "theoretical_loss": 3.7514330596508865, + "tokens_seen": 753598464 + }, + { + "epoch": 0.23, + "objective/train/docs_used": 435245, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.9846413135528564, + "objective/train/theoretical_loss": 3.751400370827529, + "objective/train/tokens_used": 774124000, + "theoretical_loss": 3.751400370827529, + "tokens_seen": 753664000 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003897191574724173, + "loss": 2.9658, + "theoretical_loss": 3.751400370827529, + "tokens_seen": 753664000 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003897091273821464, + "loss": 2.8185, + "theoretical_loss": 3.7513676856423768, + "tokens_seen": 753729536 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038969909729187564, + "loss": 3.01, + "theoretical_loss": 3.751335004094709, + "tokens_seen": 753795072 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038968906720160477, + "loss": 2.9995, + "theoretical_loss": 3.751302326183805, + "tokens_seen": 753860608 + }, + { + "epoch": 0.23, + "learning_rate": 0.000389679037111334, + "loss": 2.7125, + "theoretical_loss": 3.7512696519089435, + "tokens_seen": 753926144 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003896690070210632, + "loss": 2.4205, + "theoretical_loss": 3.751236981269404, + "tokens_seen": 753991680 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038965897693079237, + "loss": 2.7507, + "theoretical_loss": 3.7512043142644664, + "tokens_seen": 754057216 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038964894684052155, + "loss": 2.6949, + "theoretical_loss": 3.7511716508934105, + "tokens_seen": 754122752 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003896389167502508, + "loss": 3.093, + "theoretical_loss": 3.7511389911555155, + "tokens_seen": 754188288 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038962888665997997, + "loss": 2.9774, + "theoretical_loss": 3.7511063350500624, + "tokens_seen": 754253824 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038961885656970915, + "loss": 3.0856, + "theoretical_loss": 3.7510736825763322, + "tokens_seen": 754319360 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038960882647943833, + "loss": 2.8426, + "theoretical_loss": 3.751041033733604, + "tokens_seen": 754384896 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003895987963891675, + "loss": 2.8553, + "theoretical_loss": 3.75100838852116, + "tokens_seen": 754450432 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038958876629889674, + "loss": 2.6753, + "theoretical_loss": 3.7509757469382805, + "tokens_seen": 754515968 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038957873620862587, + "loss": 2.6767, + "theoretical_loss": 3.750943108984247, + "tokens_seen": 754581504 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003895687061183551, + "loss": 2.8497, + "theoretical_loss": 3.7509104746583413, + "tokens_seen": 754647040 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038955867602808423, + "loss": 2.8663, + "theoretical_loss": 3.7508778439598442, + "tokens_seen": 754712576 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038954864593781347, + "loss": 2.7238, + "theoretical_loss": 3.750845216888038, + "tokens_seen": 754778112 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038953861584754265, + "loss": 2.768, + "theoretical_loss": 3.7508125934422054, + "tokens_seen": 754843648 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038952858575727183, + "loss": 2.6991, + "theoretical_loss": 3.7507799736216274, + "tokens_seen": 754909184 + }, + { + "epoch": 0.23, + "learning_rate": 0.000389518555667001, + "loss": 2.8243, + "theoretical_loss": 3.7507473574255874, + "tokens_seen": 754974720 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038950852557673025, + "loss": 2.7035, + "theoretical_loss": 3.7507147448533678, + "tokens_seen": 755040256 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003894984954864594, + "loss": 2.8476, + "theoretical_loss": 3.7506821359042517, + "tokens_seen": 755105792 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003894884653961886, + "loss": 2.886, + "theoretical_loss": 3.7506495305775225, + "tokens_seen": 755171328 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038947843530591774, + "loss": 2.8924, + "theoretical_loss": 3.7506169288724625, + "tokens_seen": 755236864 + }, + { + "epoch": 0.23, + "objective/train/docs_used": 436337, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.917560577392578, + "objective/train/theoretical_loss": 3.7505843307883557, + "objective/train/tokens_used": 775762400, + "theoretical_loss": 3.7505843307883557, + "tokens_seen": 755302400 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038946840521564697, + "loss": 2.7928, + "theoretical_loss": 3.7505843307883557, + "tokens_seen": 755302400 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038945837512537615, + "loss": 2.8857, + "theoretical_loss": 3.750551736324486, + "tokens_seen": 755367936 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038944834503510533, + "loss": 2.9049, + "theoretical_loss": 3.750519145480137, + "tokens_seen": 755433472 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003894383149448345, + "loss": 2.5214, + "theoretical_loss": 3.750486558254593, + "tokens_seen": 755499008 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003894282848545637, + "loss": 2.9032, + "theoretical_loss": 3.7504539746471384, + "tokens_seen": 755564544 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003894182547642929, + "loss": 2.9992, + "theoretical_loss": 3.7504213946570575, + "tokens_seen": 755630080 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003894082246740221, + "loss": 2.813, + "theoretical_loss": 3.7503888182836356, + "tokens_seen": 755695616 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038939819458375124, + "loss": 2.9544, + "theoretical_loss": 3.7503562455261568, + "tokens_seen": 755761152 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003893881644934805, + "loss": 2.8684, + "theoretical_loss": 3.750323676383906, + "tokens_seen": 755826688 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003893781344032096, + "loss": 2.9263, + "theoretical_loss": 3.7502911108561694, + "tokens_seen": 755892224 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038936810431293884, + "loss": 2.9308, + "theoretical_loss": 3.7502585489422327, + "tokens_seen": 755957760 + }, + { + "epoch": 0.23, + "learning_rate": 0.000389358074222668, + "loss": 2.7316, + "theoretical_loss": 3.750225990641381, + "tokens_seen": 756023296 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003893480441323972, + "loss": 2.7115, + "theoretical_loss": 3.7501934359529, + "tokens_seen": 756088832 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003893380140421264, + "loss": 2.9176, + "theoretical_loss": 3.7501608848760766, + "tokens_seen": 756154368 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003893279839518556, + "loss": 2.927, + "theoretical_loss": 3.750128337410197, + "tokens_seen": 756219904 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038931795386158474, + "loss": 3.1039, + "theoretical_loss": 3.7500957935545474, + "tokens_seen": 756285440 + }, + { + "epoch": 0.23, + "learning_rate": 0.000389307923771314, + "loss": 2.919, + "theoretical_loss": 3.750063253308414, + "tokens_seen": 756350976 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003892978936810431, + "loss": 3.0676, + "theoretical_loss": 3.750030716671086, + "tokens_seen": 756416512 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038928786359077234, + "loss": 2.6327, + "theoretical_loss": 3.7499981836418477, + "tokens_seen": 756482048 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003892778335005015, + "loss": 2.948, + "theoretical_loss": 3.7499656542199884, + "tokens_seen": 756547584 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003892678034102307, + "loss": 2.8615, + "theoretical_loss": 3.7499331284047948, + "tokens_seen": 756613120 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003892577733199599, + "loss": 2.9161, + "theoretical_loss": 3.749900606195555, + "tokens_seen": 756678656 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038924774322968906, + "loss": 3.0664, + "theoretical_loss": 3.749868087591557, + "tokens_seen": 756744192 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038923771313941825, + "loss": 2.8028, + "theoretical_loss": 3.7498355725920884, + "tokens_seen": 756809728 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003892276830491475, + "loss": 3.0503, + "theoretical_loss": 3.7498030611964386, + "tokens_seen": 756875264 + }, + { + "epoch": 0.23, + "objective/train/docs_used": 436904, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.38763427734375, + "objective/train/theoretical_loss": 3.749770553403895, + "objective/train/tokens_used": 777400800, + "theoretical_loss": 3.749770553403895, + "tokens_seen": 756940800 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003892176529588766, + "loss": 3.0464, + "theoretical_loss": 3.749770553403895, + "tokens_seen": 756940800 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038920762286860584, + "loss": 3.0071, + "theoretical_loss": 3.7497380492137475, + "tokens_seen": 757006336 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038919759277833497, + "loss": 2.8865, + "theoretical_loss": 3.7497055486252844, + "tokens_seen": 757071872 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003891875626880642, + "loss": 2.8315, + "theoretical_loss": 3.749673051637795, + "tokens_seen": 757137408 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003891775325977934, + "loss": 2.8508, + "theoretical_loss": 3.7496405582505687, + "tokens_seen": 757202944 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038916750250752257, + "loss": 2.9239, + "theoretical_loss": 3.7496080684628956, + "tokens_seen": 757268480 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038915747241725175, + "loss": 2.7625, + "theoretical_loss": 3.7495755822740646, + "tokens_seen": 757334016 + }, + { + "epoch": 0.23, + "learning_rate": 0.000389147442326981, + "loss": 2.6653, + "theoretical_loss": 3.749543099683366, + "tokens_seen": 757399552 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003891374122367101, + "loss": 2.8183, + "theoretical_loss": 3.7495106206900903, + "tokens_seen": 757465088 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038912738214643935, + "loss": 2.9024, + "theoretical_loss": 3.7494781452935273, + "tokens_seen": 757530624 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003891173520561685, + "loss": 2.7759, + "theoretical_loss": 3.749445673492968, + "tokens_seen": 757596160 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003891073219658977, + "loss": 2.5628, + "theoretical_loss": 3.7494132052877034, + "tokens_seen": 757661696 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003890972918756269, + "loss": 2.734, + "theoretical_loss": 3.749380740677024, + "tokens_seen": 757727232 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038908726178535607, + "loss": 2.9379, + "theoretical_loss": 3.7493482796602216, + "tokens_seen": 757792768 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038907723169508525, + "loss": 3.1088, + "theoretical_loss": 3.749315822236587, + "tokens_seen": 757858304 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038906720160481443, + "loss": 2.8032, + "theoretical_loss": 3.749283368405412, + "tokens_seen": 757923840 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003890571715145436, + "loss": 2.8034, + "theoretical_loss": 3.7492509181659885, + "tokens_seen": 757989376 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038904714142427285, + "loss": 2.8875, + "theoretical_loss": 3.749218471517608, + "tokens_seen": 758054912 + }, + { + "epoch": 0.23, + "learning_rate": 0.000389037111334002, + "loss": 2.9086, + "theoretical_loss": 3.749186028459564, + "tokens_seen": 758120448 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003890270812437312, + "loss": 2.9218, + "theoretical_loss": 3.749153588991147, + "tokens_seen": 758185984 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003890170511534604, + "loss": 2.8673, + "theoretical_loss": 3.7491211531116515, + "tokens_seen": 758251520 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003890070210631896, + "loss": 2.9495, + "theoretical_loss": 3.749088720820369, + "tokens_seen": 758317056 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038899699097291876, + "loss": 2.7937, + "theoretical_loss": 3.749056292116593, + "tokens_seen": 758382592 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038898696088264794, + "loss": 2.78, + "theoretical_loss": 3.7490238669996163, + "tokens_seen": 758448128 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003889769307923771, + "loss": 3.0258, + "theoretical_loss": 3.748991445468733, + "tokens_seen": 758513664 + }, + { + "epoch": 0.23, + "objective/train/docs_used": 438157, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.0058841705322266, + "objective/train/theoretical_loss": 3.7489590275232367, + "objective/train/tokens_used": 779039200, + "theoretical_loss": 3.7489590275232367, + "tokens_seen": 758579200 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038896690070210635, + "loss": 2.8619, + "theoretical_loss": 3.7489590275232367, + "tokens_seen": 758579200 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003889568706118355, + "loss": 2.8127, + "theoretical_loss": 3.7489266131624204, + "tokens_seen": 758644736 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003889468405215647, + "loss": 2.6365, + "theoretical_loss": 3.7488942023855785, + "tokens_seen": 758710272 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038893681043129384, + "loss": 2.8195, + "theoretical_loss": 3.7488617951920054, + "tokens_seen": 758775808 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003889267803410231, + "loss": 2.8967, + "theoretical_loss": 3.748829391580995, + "tokens_seen": 758841344 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038891675025075226, + "loss": 2.9357, + "theoretical_loss": 3.748796991551843, + "tokens_seen": 758906880 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038890672016048144, + "loss": 2.853, + "theoretical_loss": 3.7487645951038426, + "tokens_seen": 758972416 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003888966900702106, + "loss": 2.782, + "theoretical_loss": 3.7487322022362903, + "tokens_seen": 759037952 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003888866599799398, + "loss": 2.7737, + "theoretical_loss": 3.7486998129484803, + "tokens_seen": 759103488 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038887662988966904, + "loss": 2.8009, + "theoretical_loss": 3.7486674272397082, + "tokens_seen": 759169024 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003888665997993982, + "loss": 2.8684, + "theoretical_loss": 3.7486350451092703, + "tokens_seen": 759234560 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003888565697091274, + "loss": 2.7507, + "theoretical_loss": 3.7486026665564616, + "tokens_seen": 759300096 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003888465396188566, + "loss": 2.9314, + "theoretical_loss": 3.7485702915805783, + "tokens_seen": 759365632 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003888365095285858, + "loss": 2.7055, + "theoretical_loss": 3.7485379201809166, + "tokens_seen": 759431168 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038882647943831494, + "loss": 2.7818, + "theoretical_loss": 3.7485055523567734, + "tokens_seen": 759496704 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003888164493480442, + "loss": 2.9687, + "theoretical_loss": 3.748473188107444, + "tokens_seen": 759562240 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003888064192577733, + "loss": 2.8432, + "theoretical_loss": 3.7484408274322267, + "tokens_seen": 759627776 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038879638916750254, + "loss": 2.9142, + "theoretical_loss": 3.7484084703304177, + "tokens_seen": 759693312 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003887863590772317, + "loss": 2.8028, + "theoretical_loss": 3.7483761168013143, + "tokens_seen": 759758848 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003887763289869609, + "loss": 2.9619, + "theoretical_loss": 3.748343766844214, + "tokens_seen": 759824384 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003887662988966901, + "loss": 3.0116, + "theoretical_loss": 3.7483114204584145, + "tokens_seen": 759889920 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038875626880641927, + "loss": 2.8142, + "theoretical_loss": 3.748279077643213, + "tokens_seen": 759955456 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038874623871614845, + "loss": 2.9845, + "theoretical_loss": 3.748246738397908, + "tokens_seen": 760020992 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003887362086258777, + "loss": 2.7345, + "theoretical_loss": 3.748214402721798, + "tokens_seen": 760086528 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003887261785356068, + "loss": 3.0422, + "theoretical_loss": 3.7481820706141806, + "tokens_seen": 760152064 + }, + { + "epoch": 0.23, + "objective/train/docs_used": 438864, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.791293144226074, + "objective/train/theoretical_loss": 3.748149742074355, + "objective/train/tokens_used": 780677600, + "theoretical_loss": 3.748149742074355, + "tokens_seen": 760217600 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038871614844533604, + "loss": 3.1178, + "theoretical_loss": 3.748149742074355, + "tokens_seen": 760217600 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038870611835506517, + "loss": 2.6967, + "theoretical_loss": 3.7481174171016196, + "tokens_seen": 760283136 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003886960882647944, + "loss": 2.907, + "theoretical_loss": 3.7480850956952736, + "tokens_seen": 760348672 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003886860581745236, + "loss": 2.9835, + "theoretical_loss": 3.7480527778546167, + "tokens_seen": 760414208 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038867602808425277, + "loss": 2.925, + "theoretical_loss": 3.7480204635789462, + "tokens_seen": 760479744 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038866599799398195, + "loss": 2.9349, + "theoretical_loss": 3.7479881528675643, + "tokens_seen": 760545280 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003886559679037112, + "loss": 3.0262, + "theoretical_loss": 3.74795584571977, + "tokens_seen": 760610816 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003886459378134403, + "loss": 3.0668, + "theoretical_loss": 3.7479235421348616, + "tokens_seen": 760676352 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038863590772316955, + "loss": 2.846, + "theoretical_loss": 3.747891242112142, + "tokens_seen": 760741888 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003886258776328987, + "loss": 2.8821, + "theoretical_loss": 3.747858945650909, + "tokens_seen": 760807424 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003886158475426279, + "loss": 2.7391, + "theoretical_loss": 3.747826652750465, + "tokens_seen": 760872960 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003886058174523571, + "loss": 2.834, + "theoretical_loss": 3.74779436341011, + "tokens_seen": 760938496 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038859578736208627, + "loss": 2.9348, + "theoretical_loss": 3.7477620776291447, + "tokens_seen": 761004032 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038858575727181545, + "loss": 3.0346, + "theoretical_loss": 3.7477297954068707, + "tokens_seen": 761069568 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038857572718154463, + "loss": 2.8209, + "theoretical_loss": 3.747697516742589, + "tokens_seen": 761135104 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003885656970912738, + "loss": 2.5717, + "theoretical_loss": 3.7476652416356018, + "tokens_seen": 761200640 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038855566700100305, + "loss": 2.5643, + "theoretical_loss": 3.7476329700852107, + "tokens_seen": 761266176 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003885456369107322, + "loss": 3.0551, + "theoretical_loss": 3.7476007020907165, + "tokens_seen": 761331712 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003885356068204614, + "loss": 3.0309, + "theoretical_loss": 3.747568437651423, + "tokens_seen": 761397248 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003885255767301906, + "loss": 2.6769, + "theoretical_loss": 3.7475361767666318, + "tokens_seen": 761462784 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003885155466399198, + "loss": 2.8207, + "theoretical_loss": 3.747503919435645, + "tokens_seen": 761528320 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038850551654964896, + "loss": 2.938, + "theoretical_loss": 3.747471665657766, + "tokens_seen": 761593856 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038849548645937814, + "loss": 2.7413, + "theoretical_loss": 3.7474394154322974, + "tokens_seen": 761659392 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003884854563691073, + "loss": 2.8387, + "theoretical_loss": 3.747407168758542, + "tokens_seen": 761724928 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038847542627883655, + "loss": 2.8607, + "theoretical_loss": 3.747374925635804, + "tokens_seen": 761790464 + }, + { + "epoch": 0.23, + "objective/train/docs_used": 439620, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.1359565258026123, + "objective/train/theoretical_loss": 3.7473426860633863, + "objective/train/tokens_used": 782316000, + "theoretical_loss": 3.7473426860633863, + "tokens_seen": 761856000 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003884653961885657, + "loss": 2.7198, + "theoretical_loss": 3.7473426860633863, + "tokens_seen": 761856000 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003884553660982949, + "loss": 2.9167, + "theoretical_loss": 3.747310450040593, + "tokens_seen": 761921536 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038844533600802404, + "loss": 2.7874, + "theoretical_loss": 3.747278217566728, + "tokens_seen": 761987072 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003884353059177533, + "loss": 2.9116, + "theoretical_loss": 3.7472459886410947, + "tokens_seen": 762052608 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038842527582748246, + "loss": 2.8178, + "theoretical_loss": 3.747213763262998, + "tokens_seen": 762118144 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038841524573721164, + "loss": 2.8402, + "theoretical_loss": 3.7471815414317424, + "tokens_seen": 762183680 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003884052156469408, + "loss": 2.8903, + "theoretical_loss": 3.7471493231466324, + "tokens_seen": 762249216 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038839518555667, + "loss": 2.8428, + "theoretical_loss": 3.747117108406973, + "tokens_seen": 762314752 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003883851554663992, + "loss": 2.8152, + "theoretical_loss": 3.747084897212069, + "tokens_seen": 762380288 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003883751253761284, + "loss": 2.8922, + "theoretical_loss": 3.747052689561227, + "tokens_seen": 762445824 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038836509528585755, + "loss": 2.7047, + "theoretical_loss": 3.74702048545375, + "tokens_seen": 762511360 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003883550651955868, + "loss": 2.7495, + "theoretical_loss": 3.746988284888946, + "tokens_seen": 762576896 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038834503510531596, + "loss": 2.5864, + "theoretical_loss": 3.7469560878661197, + "tokens_seen": 762642432 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038833500501504514, + "loss": 2.8335, + "theoretical_loss": 3.746923894384578, + "tokens_seen": 762707968 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003883249749247743, + "loss": 2.8557, + "theoretical_loss": 3.7468917044436267, + "tokens_seen": 762773504 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003883149448345035, + "loss": 2.8453, + "theoretical_loss": 3.7468595180425712, + "tokens_seen": 762839040 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003883049147442327, + "loss": 2.7749, + "theoretical_loss": 3.74682733518072, + "tokens_seen": 762904576 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003882948846539619, + "loss": 2.724, + "theoretical_loss": 3.746795155857379, + "tokens_seen": 762970112 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038828485456369105, + "loss": 2.7312, + "theoretical_loss": 3.746762980071855, + "tokens_seen": 763035648 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003882748244734203, + "loss": 2.8081, + "theoretical_loss": 3.7467308078234565, + "tokens_seen": 763101184 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003882647943831494, + "loss": 2.9996, + "theoretical_loss": 3.7466986391114894, + "tokens_seen": 763166720 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038825476429287865, + "loss": 2.8645, + "theoretical_loss": 3.746666473935262, + "tokens_seen": 763232256 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038824473420260783, + "loss": 2.8234, + "theoretical_loss": 3.7466343122940815, + "tokens_seen": 763297792 + }, + { + "epoch": 0.23, + "learning_rate": 0.000388234704112337, + "loss": 3.0056, + "theoretical_loss": 3.7466021541872574, + "tokens_seen": 763363328 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003882246740220662, + "loss": 2.7229, + "theoretical_loss": 3.7465699996140964, + "tokens_seen": 763428864 + }, + { + "epoch": 0.23, + "objective/train/docs_used": 440741, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.247100353240967, + "objective/train/theoretical_loss": 3.746537848573908, + "objective/train/tokens_used": 783954400, + "theoretical_loss": 3.746537848573908, + "tokens_seen": 763494400 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038821464393179537, + "loss": 3.1927, + "theoretical_loss": 3.746537848573908, + "tokens_seen": 763494400 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038820461384152455, + "loss": 2.7403, + "theoretical_loss": 3.746505701066001, + "tokens_seen": 763559936 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003881945837512538, + "loss": 2.7913, + "theoretical_loss": 3.7464735570896823, + "tokens_seen": 763625472 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003881845536609829, + "loss": 2.8844, + "theoretical_loss": 3.746441416644263, + "tokens_seen": 763691008 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038817452357071215, + "loss": 2.5666, + "theoretical_loss": 3.7464092797290505, + "tokens_seen": 763756544 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038816449348044133, + "loss": 2.8838, + "theoretical_loss": 3.746377146343356, + "tokens_seen": 763822080 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003881544633901705, + "loss": 2.825, + "theoretical_loss": 3.746345016486488, + "tokens_seen": 763887616 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003881444332998997, + "loss": 2.8369, + "theoretical_loss": 3.7463128901577565, + "tokens_seen": 763953152 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003881344032096289, + "loss": 2.7512, + "theoretical_loss": 3.7462807673564713, + "tokens_seen": 764018688 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003881243731193581, + "loss": 2.7089, + "theoretical_loss": 3.746248648081943, + "tokens_seen": 764084224 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003881143430290873, + "loss": 2.6583, + "theoretical_loss": 3.7462165323334813, + "tokens_seen": 764149760 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038810431293881647, + "loss": 2.8282, + "theoretical_loss": 3.746184420110397, + "tokens_seen": 764215296 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038809428284854565, + "loss": 2.716, + "theoretical_loss": 3.7461523114120014, + "tokens_seen": 764280832 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038808425275827483, + "loss": 2.8924, + "theoretical_loss": 3.746120206237605, + "tokens_seen": 764346368 + }, + { + "epoch": 0.23, + "learning_rate": 0.000388074222668004, + "loss": 2.8413, + "theoretical_loss": 3.7460881045865184, + "tokens_seen": 764411904 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038806419257773325, + "loss": 2.7744, + "theoretical_loss": 3.746056006458054, + "tokens_seen": 764477440 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003880541624874624, + "loss": 2.8587, + "theoretical_loss": 3.7460239118515224, + "tokens_seen": 764542976 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003880441323971916, + "loss": 2.882, + "theoretical_loss": 3.745991820766236, + "tokens_seen": 764608512 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003880341023069208, + "loss": 2.7141, + "theoretical_loss": 3.745959733201506, + "tokens_seen": 764674048 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038802407221665, + "loss": 2.739, + "theoretical_loss": 3.7459276491566453, + "tokens_seen": 764739584 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038801404212637916, + "loss": 2.7218, + "theoretical_loss": 3.7458955686309654, + "tokens_seen": 764805120 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038800401203610834, + "loss": 2.7415, + "theoretical_loss": 3.74586349162378, + "tokens_seen": 764870656 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003879939819458375, + "loss": 2.8643, + "theoretical_loss": 3.7458314181344, + "tokens_seen": 764936192 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038798395185556675, + "loss": 2.8903, + "theoretical_loss": 3.74579934816214, + "tokens_seen": 765001728 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003879739217652959, + "loss": 3.0037, + "theoretical_loss": 3.7457672817063115, + "tokens_seen": 765067264 + }, + { + "epoch": 0.23, + "objective/train/docs_used": 441342, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.1323585510253906, + "objective/train/theoretical_loss": 3.745735218766229, + "objective/train/tokens_used": 785592800, + "theoretical_loss": 3.745735218766229, + "tokens_seen": 765132800 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003879638916750251, + "loss": 2.8674, + "theoretical_loss": 3.745735218766229, + "tokens_seen": 765132800 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038795386158475424, + "loss": 2.8657, + "theoretical_loss": 3.745703159341205, + "tokens_seen": 765198336 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003879438314944835, + "loss": 2.8353, + "theoretical_loss": 3.745671103430554, + "tokens_seen": 765263872 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038793380140421266, + "loss": 3.0763, + "theoretical_loss": 3.7456390510335895, + "tokens_seen": 765329408 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038792377131394184, + "loss": 2.6409, + "theoretical_loss": 3.7456070021496255, + "tokens_seen": 765394944 + }, + { + "epoch": 0.23, + "learning_rate": 0.000387913741223671, + "loss": 2.6812, + "theoretical_loss": 3.7455749567779764, + "tokens_seen": 765460480 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003879037111334002, + "loss": 3.0034, + "theoretical_loss": 3.745542914917956, + "tokens_seen": 765526016 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003878936810431294, + "loss": 2.6862, + "theoretical_loss": 3.74551087656888, + "tokens_seen": 765591552 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003878836509528586, + "loss": 2.8509, + "theoretical_loss": 3.745478841730062, + "tokens_seen": 765657088 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038787362086258775, + "loss": 2.8291, + "theoretical_loss": 3.7454468104008174, + "tokens_seen": 765722624 + }, + { + "epoch": 0.23, + "learning_rate": 0.000387863590772317, + "loss": 2.7332, + "theoretical_loss": 3.7454147825804616, + "tokens_seen": 765788160 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038785356068204616, + "loss": 2.7787, + "theoretical_loss": 3.74538275826831, + "tokens_seen": 765853696 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038784353059177534, + "loss": 2.9419, + "theoretical_loss": 3.745350737463678, + "tokens_seen": 765919232 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003878335005015045, + "loss": 2.7719, + "theoretical_loss": 3.7453187201658817, + "tokens_seen": 765984768 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003878234704112337, + "loss": 2.6319, + "theoretical_loss": 3.7452867063742366, + "tokens_seen": 766050304 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003878134403209629, + "loss": 2.9275, + "theoretical_loss": 3.7452546960880593, + "tokens_seen": 766115840 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003878034102306921, + "loss": 2.744, + "theoretical_loss": 3.745222689306665, + "tokens_seen": 766181376 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038779338014042125, + "loss": 2.9147, + "theoretical_loss": 3.745190686029372, + "tokens_seen": 766246912 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003877833500501505, + "loss": 2.9835, + "theoretical_loss": 3.745158686255496, + "tokens_seen": 766312448 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003877733199598796, + "loss": 2.9616, + "theoretical_loss": 3.7451266899843536, + "tokens_seen": 766377984 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038776328986960885, + "loss": 3.0834, + "theoretical_loss": 3.7450946972152623, + "tokens_seen": 766443520 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038775325977933803, + "loss": 2.6281, + "theoretical_loss": 3.7450627079475396, + "tokens_seen": 766509056 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003877432296890672, + "loss": 2.7475, + "theoretical_loss": 3.745030722180503, + "tokens_seen": 766574592 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003877331995987964, + "loss": 2.716, + "theoretical_loss": 3.74499873991347, + "tokens_seen": 766640128 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038772316950852557, + "loss": 2.7099, + "theoretical_loss": 3.7449667611457578, + "tokens_seen": 766705664 + }, + { + "epoch": 0.23, + "objective/train/docs_used": 442736, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.9325008392333984, + "objective/train/theoretical_loss": 3.744934785876686, + "objective/train/tokens_used": 787231200, + "theoretical_loss": 3.744934785876686, + "tokens_seen": 766771200 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038771313941825475, + "loss": 2.8828, + "theoretical_loss": 3.744934785876686, + "tokens_seen": 766771200 + }, + { + "epoch": 0.23, + "learning_rate": 0.000387703109327984, + "loss": 2.8792, + "theoretical_loss": 3.744902814105571, + "tokens_seen": 766836736 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003876930792377131, + "loss": 2.7328, + "theoretical_loss": 3.7448708458317332, + "tokens_seen": 766902272 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038768304914744235, + "loss": 2.9503, + "theoretical_loss": 3.7448388810544895, + "tokens_seen": 766967808 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038767301905717153, + "loss": 2.8759, + "theoretical_loss": 3.7448069197731595, + "tokens_seen": 767033344 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003876629889669007, + "loss": 2.9557, + "theoretical_loss": 3.744774961987063, + "tokens_seen": 767098880 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003876529588766299, + "loss": 2.7001, + "theoretical_loss": 3.7447430076955177, + "tokens_seen": 767164416 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003876429287863591, + "loss": 2.9324, + "theoretical_loss": 3.7447110568978434, + "tokens_seen": 767229952 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038763289869608826, + "loss": 2.8332, + "theoretical_loss": 3.7446791095933603, + "tokens_seen": 767295488 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003876228686058175, + "loss": 2.7886, + "theoretical_loss": 3.7446471657813882, + "tokens_seen": 767361024 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003876128385155466, + "loss": 2.9733, + "theoretical_loss": 3.744615225461246, + "tokens_seen": 767426560 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038760280842527585, + "loss": 3.0808, + "theoretical_loss": 3.744583288632255, + "tokens_seen": 767492096 + }, + { + "epoch": 0.23, + "learning_rate": 0.000387592778335005, + "loss": 2.8579, + "theoretical_loss": 3.7445513552937353, + "tokens_seen": 767557632 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003875827482447342, + "loss": 3.0008, + "theoretical_loss": 3.7445194254450067, + "tokens_seen": 767623168 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003875727181544634, + "loss": 2.7894, + "theoretical_loss": 3.7444874990853907, + "tokens_seen": 767688704 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003875626880641926, + "loss": 3.145, + "theoretical_loss": 3.744455576214208, + "tokens_seen": 767754240 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038755265797392176, + "loss": 2.9183, + "theoretical_loss": 3.7444236568307803, + "tokens_seen": 767819776 + }, + { + "epoch": 0.23, + "learning_rate": 0.000387542627883651, + "loss": 3.0067, + "theoretical_loss": 3.7443917409344274, + "tokens_seen": 767885312 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003875325977933801, + "loss": 2.903, + "theoretical_loss": 3.7443598285244724, + "tokens_seen": 767950848 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038752256770310936, + "loss": 2.8136, + "theoretical_loss": 3.7443279196002357, + "tokens_seen": 768016384 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003875125376128385, + "loss": 3.2548, + "theoretical_loss": 3.74429601416104, + "tokens_seen": 768081920 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003875025075225677, + "loss": 2.9378, + "theoretical_loss": 3.7442641122062073, + "tokens_seen": 768147456 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003874924774322969, + "loss": 2.9538, + "theoretical_loss": 3.7442322137350597, + "tokens_seen": 768212992 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003874824473420261, + "loss": 2.8472, + "theoretical_loss": 3.7442003187469193, + "tokens_seen": 768278528 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038747241725175526, + "loss": 2.7871, + "theoretical_loss": 3.7441684272411093, + "tokens_seen": 768344064 + }, + { + "epoch": 0.23, + "objective/train/docs_used": 443303, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.842745065689087, + "objective/train/theoretical_loss": 3.744136539216952, + "objective/train/tokens_used": 788869600, + "theoretical_loss": 3.744136539216952, + "tokens_seen": 768409600 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038746238716148444, + "loss": 2.7866, + "theoretical_loss": 3.744136539216952, + "tokens_seen": 768409600 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003874523570712136, + "loss": 3.1254, + "theoretical_loss": 3.744104654673771, + "tokens_seen": 768475136 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038744232698094286, + "loss": 2.8055, + "theoretical_loss": 3.7440727736108887, + "tokens_seen": 768540672 + }, + { + "epoch": 0.23, + "learning_rate": 0.000387432296890672, + "loss": 2.6904, + "theoretical_loss": 3.744040896027629, + "tokens_seen": 768606208 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003874222668004012, + "loss": 2.8213, + "theoretical_loss": 3.7440090219233157, + "tokens_seen": 768671744 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038741223671013035, + "loss": 2.6849, + "theoretical_loss": 3.7439771512972726, + "tokens_seen": 768737280 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003874022066198596, + "loss": 2.9572, + "theoretical_loss": 3.7439452841488228, + "tokens_seen": 768802816 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038739217652958876, + "loss": 2.5857, + "theoretical_loss": 3.7439134204772913, + "tokens_seen": 768868352 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038738214643931795, + "loss": 2.9874, + "theoretical_loss": 3.7438815602820017, + "tokens_seen": 768933888 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003873721163490472, + "loss": 2.9715, + "theoretical_loss": 3.743849703562279, + "tokens_seen": 768999424 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038736208625877636, + "loss": 2.7895, + "theoretical_loss": 3.743817850317448, + "tokens_seen": 769064960 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038735205616850554, + "loss": 2.8146, + "theoretical_loss": 3.7437860005468337, + "tokens_seen": 769130496 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003873420260782347, + "loss": 2.6504, + "theoretical_loss": 3.7437541542497605, + "tokens_seen": 769196032 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003873319959879639, + "loss": 2.7603, + "theoretical_loss": 3.7437223114255542, + "tokens_seen": 769261568 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003873219658976931, + "loss": 2.8894, + "theoretical_loss": 3.74369047207354, + "tokens_seen": 769327104 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003873119358074223, + "loss": 2.9627, + "theoretical_loss": 3.7436586361930444, + "tokens_seen": 769392640 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038730190571715145, + "loss": 2.9118, + "theoretical_loss": 3.743626803783392, + "tokens_seen": 769458176 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003872918756268807, + "loss": 2.8095, + "theoretical_loss": 3.7435949748439095, + "tokens_seen": 769523712 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003872818455366098, + "loss": 2.5044, + "theoretical_loss": 3.743563149373923, + "tokens_seen": 769589248 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038727181544633905, + "loss": 2.7143, + "theoretical_loss": 3.743531327372759, + "tokens_seen": 769654784 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038726178535606823, + "loss": 2.953, + "theoretical_loss": 3.743499508839744, + "tokens_seen": 769720320 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003872517552657974, + "loss": 2.476, + "theoretical_loss": 3.7434676937742046, + "tokens_seen": 769785856 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003872417251755266, + "loss": 2.8146, + "theoretical_loss": 3.743435882175468, + "tokens_seen": 769851392 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038723169508525577, + "loss": 2.7722, + "theoretical_loss": 3.7434040740428616, + "tokens_seen": 769916928 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038722166499498495, + "loss": 2.7919, + "theoretical_loss": 3.743372269375712, + "tokens_seen": 769982464 + }, + { + "epoch": 0.23, + "objective/train/docs_used": 444724, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.631140947341919, + "objective/train/theoretical_loss": 3.7433404681733475, + "objective/train/tokens_used": 790508000, + "theoretical_loss": 3.7433404681733475, + "tokens_seen": 770048000 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003872116349047142, + "loss": 2.7009, + "theoretical_loss": 3.7433404681733475, + "tokens_seen": 770048000 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003872016048144433, + "loss": 2.854, + "theoretical_loss": 3.7433086704350957, + "tokens_seen": 770113536 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038719157472417255, + "loss": 2.6488, + "theoretical_loss": 3.7432768761602837, + "tokens_seen": 770179072 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038718154463390173, + "loss": 3.0284, + "theoretical_loss": 3.743245085348241, + "tokens_seen": 770244608 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003871715145436309, + "loss": 2.7356, + "theoretical_loss": 3.7432132979982944, + "tokens_seen": 770310144 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003871614844533601, + "loss": 2.766, + "theoretical_loss": 3.7431815141097733, + "tokens_seen": 770375680 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003871514543630893, + "loss": 2.7333, + "theoretical_loss": 3.7431497336820065, + "tokens_seen": 770441216 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038714142427281846, + "loss": 2.8861, + "theoretical_loss": 3.743117956714322, + "tokens_seen": 770506752 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003871313941825477, + "loss": 2.7393, + "theoretical_loss": 3.7430861832060494, + "tokens_seen": 770572288 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003871213640922768, + "loss": 2.8737, + "theoretical_loss": 3.7430544131565178, + "tokens_seen": 770637824 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038711133400200605, + "loss": 2.8225, + "theoretical_loss": 3.7430226465650573, + "tokens_seen": 770703360 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003871013039117352, + "loss": 3.0635, + "theoretical_loss": 3.7429908834309957, + "tokens_seen": 770768896 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003870912738214644, + "loss": 2.9715, + "theoretical_loss": 3.742959123753665, + "tokens_seen": 770834432 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003870812437311936, + "loss": 2.6296, + "theoretical_loss": 3.7429273675323933, + "tokens_seen": 770899968 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003870712136409228, + "loss": 2.8425, + "theoretical_loss": 3.742895614766512, + "tokens_seen": 770965504 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038706118355065196, + "loss": 3.0425, + "theoretical_loss": 3.7428638654553508, + "tokens_seen": 771031040 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003870511534603812, + "loss": 2.747, + "theoretical_loss": 3.74283211959824, + "tokens_seen": 771096576 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003870411233701103, + "loss": 2.7323, + "theoretical_loss": 3.7428003771945115, + "tokens_seen": 771162112 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038703109327983956, + "loss": 2.7093, + "theoretical_loss": 3.7427686382434953, + "tokens_seen": 771227648 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003870210631895687, + "loss": 2.763, + "theoretical_loss": 3.742736902744522, + "tokens_seen": 771293184 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003870110330992979, + "loss": 2.8928, + "theoretical_loss": 3.742705170696924, + "tokens_seen": 771358720 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003870010030090271, + "loss": 2.6726, + "theoretical_loss": 3.742673442100032, + "tokens_seen": 771424256 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003869909729187563, + "loss": 3.0833, + "theoretical_loss": 3.7426417169531776, + "tokens_seen": 771489792 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038698094282848546, + "loss": 2.8036, + "theoretical_loss": 3.7426099952556937, + "tokens_seen": 771555328 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038697091273821464, + "loss": 2.9711, + "theoretical_loss": 3.742578277006911, + "tokens_seen": 771620864 + }, + { + "epoch": 0.23, + "objective/train/docs_used": 445347, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7711522579193115, + "objective/train/theoretical_loss": 3.7425465622061624, + "objective/train/tokens_used": 792146400, + "theoretical_loss": 3.7425465622061624, + "tokens_seen": 771686400 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003869608826479438, + "loss": 2.7983, + "theoretical_loss": 3.7425465622061624, + "tokens_seen": 771686400 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038695085255767306, + "loss": 2.7203, + "theoretical_loss": 3.74251485085278, + "tokens_seen": 771751936 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003869408224674022, + "loss": 2.8102, + "theoretical_loss": 3.7424831429460967, + "tokens_seen": 771817472 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003869307923771314, + "loss": 2.851, + "theoretical_loss": 3.7424514384854453, + "tokens_seen": 771883008 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038692076228686055, + "loss": 2.8807, + "theoretical_loss": 3.742419737470158, + "tokens_seen": 771948544 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003869107321965898, + "loss": 2.6826, + "theoretical_loss": 3.742388039899569, + "tokens_seen": 772014080 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038690070210631896, + "loss": 2.9245, + "theoretical_loss": 3.742356345773011, + "tokens_seen": 772079616 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038689067201604815, + "loss": 2.9522, + "theoretical_loss": 3.742324655089817, + "tokens_seen": 772145152 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038688064192577733, + "loss": 2.7158, + "theoretical_loss": 3.742292967849322, + "tokens_seen": 772210688 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038687061183550656, + "loss": 2.7979, + "theoretical_loss": 3.742261284050859, + "tokens_seen": 772276224 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003868605817452357, + "loss": 2.9002, + "theoretical_loss": 3.7422296036937617, + "tokens_seen": 772341760 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003868505516549649, + "loss": 3.046, + "theoretical_loss": 3.7421979267773655, + "tokens_seen": 772407296 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038684052156469405, + "loss": 2.4043, + "theoretical_loss": 3.742166253301004, + "tokens_seen": 772472832 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003868304914744233, + "loss": 2.9941, + "theoretical_loss": 3.7421345832640123, + "tokens_seen": 772538368 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038682046138415247, + "loss": 2.6714, + "theoretical_loss": 3.7421029166657243, + "tokens_seen": 772603904 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038681043129388165, + "loss": 2.7524, + "theoretical_loss": 3.7420712535054763, + "tokens_seen": 772669440 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038680040120361083, + "loss": 2.7743, + "theoretical_loss": 3.742039593782602, + "tokens_seen": 772734976 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038679037111334, + "loss": 2.7827, + "theoretical_loss": 3.742007937496438, + "tokens_seen": 772800512 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003867803410230692, + "loss": 2.8948, + "theoretical_loss": 3.7419762846463196, + "tokens_seen": 772866048 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038677031093279843, + "loss": 2.816, + "theoretical_loss": 3.741944635231582, + "tokens_seen": 772931584 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038676028084252755, + "loss": 2.4571, + "theoretical_loss": 3.7419129892515617, + "tokens_seen": 772997120 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003867502507522568, + "loss": 2.7164, + "theoretical_loss": 3.7418813467055942, + "tokens_seen": 773062656 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003867402206619859, + "loss": 2.8688, + "theoretical_loss": 3.7418497075930164, + "tokens_seen": 773128192 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038673019057171515, + "loss": 2.8028, + "theoretical_loss": 3.741818071913164, + "tokens_seen": 773193728 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038672016048144433, + "loss": 2.9394, + "theoretical_loss": 3.7417864396653746, + "tokens_seen": 773259264 + }, + { + "epoch": 0.23, + "objective/train/docs_used": 446623, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7704248428344727, + "objective/train/theoretical_loss": 3.7417548108489846, + "objective/train/tokens_used": 793784800, + "theoretical_loss": 3.7417548108489846, + "tokens_seen": 773324800 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003867101303911735, + "loss": 2.6967, + "theoretical_loss": 3.7417548108489846, + "tokens_seen": 773324800 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003867001003009027, + "loss": 2.6305, + "theoretical_loss": 3.741723185463331, + "tokens_seen": 773390336 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038669007021063193, + "loss": 2.9816, + "theoretical_loss": 3.7416915635077506, + "tokens_seen": 773455872 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038668004012036106, + "loss": 2.7472, + "theoretical_loss": 3.7416599449815813, + "tokens_seen": 773521408 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003866700100300903, + "loss": 2.3922, + "theoretical_loss": 3.7416283298841604, + "tokens_seen": 773586944 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003866599799398194, + "loss": 2.9533, + "theoretical_loss": 3.7415967182148266, + "tokens_seen": 773652480 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038664994984954866, + "loss": 2.7841, + "theoretical_loss": 3.7415651099729166, + "tokens_seen": 773718016 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038663991975927784, + "loss": 2.8829, + "theoretical_loss": 3.7415335051577685, + "tokens_seen": 773783552 + }, + { + "epoch": 0.23, + "learning_rate": 0.000386629889669007, + "loss": 2.5103, + "theoretical_loss": 3.7415019037687216, + "tokens_seen": 773849088 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038661985957873625, + "loss": 2.9588, + "theoretical_loss": 3.7414703058051137, + "tokens_seen": 773914624 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003866098294884654, + "loss": 2.8515, + "theoretical_loss": 3.7414387112662837, + "tokens_seen": 773980160 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003865997993981946, + "loss": 2.6486, + "theoretical_loss": 3.74140712015157, + "tokens_seen": 774045696 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003865897693079238, + "loss": 2.8019, + "theoretical_loss": 3.741375532460313, + "tokens_seen": 774111232 + }, + { + "epoch": 0.23, + "learning_rate": 0.000386579739217653, + "loss": 2.5513, + "theoretical_loss": 3.74134394819185, + "tokens_seen": 774176768 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038656970912738216, + "loss": 3.2704, + "theoretical_loss": 3.741312367345522, + "tokens_seen": 774242304 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003865596790371114, + "loss": 2.6369, + "theoretical_loss": 3.7412807899206677, + "tokens_seen": 774307840 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003865496489468405, + "loss": 2.9951, + "theoretical_loss": 3.7412492159166275, + "tokens_seen": 774373376 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038653961885656976, + "loss": 2.87, + "theoretical_loss": 3.741217645332741, + "tokens_seen": 774438912 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003865295887662989, + "loss": 2.7223, + "theoretical_loss": 3.7411860781683477, + "tokens_seen": 774504448 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003865195586760281, + "loss": 2.9314, + "theoretical_loss": 3.741154514422789, + "tokens_seen": 774569984 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003865095285857573, + "loss": 2.673, + "theoretical_loss": 3.741122954095405, + "tokens_seen": 774635520 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003864994984954865, + "loss": 3.1206, + "theoretical_loss": 3.7410913971855364, + "tokens_seen": 774701056 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038648946840521566, + "loss": 2.7115, + "theoretical_loss": 3.7410598436925238, + "tokens_seen": 774766592 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038647943831494484, + "loss": 2.8988, + "theoretical_loss": 3.7410282936157087, + "tokens_seen": 774832128 + }, + { + "epoch": 0.23, + "learning_rate": 0.000386469408224674, + "loss": 2.7068, + "theoretical_loss": 3.7409967469544325, + "tokens_seen": 774897664 + }, + { + "epoch": 0.23, + "objective/train/docs_used": 447277, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.3530914783477783, + "objective/train/theoretical_loss": 3.7409652037080363, + "objective/train/tokens_used": 795423200, + "theoretical_loss": 3.7409652037080363, + "tokens_seen": 774963200 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038645937813440326, + "loss": 2.6654, + "theoretical_loss": 3.7409652037080363, + "tokens_seen": 774963200 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003864493480441324, + "loss": 2.9094, + "theoretical_loss": 3.740933663875861, + "tokens_seen": 775028736 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003864393179538616, + "loss": 2.6591, + "theoretical_loss": 3.7409021274572494, + "tokens_seen": 775094272 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038642928786359075, + "loss": 2.9436, + "theoretical_loss": 3.7408705944515432, + "tokens_seen": 775159808 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038641925777332, + "loss": 2.8075, + "theoretical_loss": 3.740839064858085, + "tokens_seen": 775225344 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038640922768304917, + "loss": 2.7154, + "theoretical_loss": 3.740807538676216, + "tokens_seen": 775290880 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038639919759277835, + "loss": 3.0003, + "theoretical_loss": 3.7407760159052796, + "tokens_seen": 775356416 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038638916750250753, + "loss": 2.7281, + "theoretical_loss": 3.740744496544618, + "tokens_seen": 775421952 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038637913741223676, + "loss": 2.7603, + "theoretical_loss": 3.7407129805935746, + "tokens_seen": 775487488 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003863691073219659, + "loss": 2.7168, + "theoretical_loss": 3.740681468051492, + "tokens_seen": 775553024 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003863590772316951, + "loss": 2.9653, + "theoretical_loss": 3.740649958917714, + "tokens_seen": 775618560 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038634904714142425, + "loss": 2.6224, + "theoretical_loss": 3.7406184531915834, + "tokens_seen": 775684096 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003863390170511535, + "loss": 2.9099, + "theoretical_loss": 3.7405869508724443, + "tokens_seen": 775749632 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038632898696088267, + "loss": 2.8636, + "theoretical_loss": 3.74055545195964, + "tokens_seen": 775815168 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038631895687061185, + "loss": 2.7785, + "theoretical_loss": 3.740523956452515, + "tokens_seen": 775880704 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038630892678034103, + "loss": 2.6417, + "theoretical_loss": 3.7404924643504134, + "tokens_seen": 775946240 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003862988966900702, + "loss": 2.8748, + "theoretical_loss": 3.740460975652679, + "tokens_seen": 776011776 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003862888665997994, + "loss": 2.75, + "theoretical_loss": 3.740429490358657, + "tokens_seen": 776077312 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038627883650952863, + "loss": 2.9841, + "theoretical_loss": 3.7403980084676918, + "tokens_seen": 776142848 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038626880641925775, + "loss": 2.8505, + "theoretical_loss": 3.740366529979128, + "tokens_seen": 776208384 + }, + { + "epoch": 0.24, + "learning_rate": 0.000386258776328987, + "loss": 2.7692, + "theoretical_loss": 3.740335054892311, + "tokens_seen": 776273920 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003862487462387161, + "loss": 2.6606, + "theoretical_loss": 3.740303583206586, + "tokens_seen": 776339456 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038623871614844535, + "loss": 2.7387, + "theoretical_loss": 3.7402721149212983, + "tokens_seen": 776404992 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038622868605817453, + "loss": 2.9118, + "theoretical_loss": 3.7402406500357936, + "tokens_seen": 776470528 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003862186559679037, + "loss": 2.5662, + "theoretical_loss": 3.740209188549418, + "tokens_seen": 776536064 + }, + { + "epoch": 0.24, + "objective/train/docs_used": 447858, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.4882426261901855, + "objective/train/theoretical_loss": 3.740177730461517, + "objective/train/tokens_used": 797061600, + "theoretical_loss": 3.740177730461517, + "tokens_seen": 776601600 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003862086258776329, + "loss": 2.6823, + "theoretical_loss": 3.740177730461517, + "tokens_seen": 776601600 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038619859578736213, + "loss": 2.5041, + "theoretical_loss": 3.740146275771437, + "tokens_seen": 776667136 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038618856569709126, + "loss": 2.6637, + "theoretical_loss": 3.740114824478524, + "tokens_seen": 776732672 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003861785356068205, + "loss": 2.7805, + "theoretical_loss": 3.740083376582125, + "tokens_seen": 776798208 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003861685055165496, + "loss": 2.7993, + "theoretical_loss": 3.7400519320815864, + "tokens_seen": 776863744 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038615847542627886, + "loss": 2.7728, + "theoretical_loss": 3.740020490976255, + "tokens_seen": 776929280 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038614844533600804, + "loss": 2.7663, + "theoretical_loss": 3.7399890532654783, + "tokens_seen": 776994816 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003861384152457372, + "loss": 2.528, + "theoretical_loss": 3.739957618948603, + "tokens_seen": 777060352 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003861283851554664, + "loss": 2.5452, + "theoretical_loss": 3.7399261880249766, + "tokens_seen": 777125888 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003861183550651956, + "loss": 2.7788, + "theoretical_loss": 3.7398947604939474, + "tokens_seen": 777191424 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038610832497492476, + "loss": 2.9369, + "theoretical_loss": 3.739863336354862, + "tokens_seen": 777256960 + }, + { + "epoch": 0.24, + "learning_rate": 0.000386098294884654, + "loss": 2.5308, + "theoretical_loss": 3.7398319156070694, + "tokens_seen": 777322496 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003860882647943831, + "loss": 2.9745, + "theoretical_loss": 3.739800498249917, + "tokens_seen": 777388032 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038607823470411236, + "loss": 2.8087, + "theoretical_loss": 3.7397690842827536, + "tokens_seen": 777453568 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003860682046138415, + "loss": 2.9107, + "theoretical_loss": 3.7397376737049277, + "tokens_seen": 777519104 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003860581745235707, + "loss": 2.9363, + "theoretical_loss": 3.7397062665157876, + "tokens_seen": 777584640 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003860481444332999, + "loss": 2.6218, + "theoretical_loss": 3.7396748627146827, + "tokens_seen": 777650176 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003860381143430291, + "loss": 2.6278, + "theoretical_loss": 3.739643462300961, + "tokens_seen": 777715712 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038602808425275826, + "loss": 2.6021, + "theoretical_loss": 3.739612065273973, + "tokens_seen": 777781248 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003860180541624875, + "loss": 2.5293, + "theoretical_loss": 3.7395806716330675, + "tokens_seen": 777846784 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003860080240722166, + "loss": 2.7086, + "theoretical_loss": 3.7395492813775943, + "tokens_seen": 777912320 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038599799398194586, + "loss": 2.7161, + "theoretical_loss": 3.7395178945069025, + "tokens_seen": 777977856 + }, + { + "epoch": 0.24, + "learning_rate": 0.000385987963891675, + "loss": 2.8818, + "theoretical_loss": 3.739486511020343, + "tokens_seen": 778043392 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003859779338014042, + "loss": 2.6829, + "theoretical_loss": 3.739455130917265, + "tokens_seen": 778108928 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003859679037111334, + "loss": 3.0368, + "theoretical_loss": 3.739423754197019, + "tokens_seen": 778174464 + }, + { + "epoch": 0.24, + "objective/train/docs_used": 448758, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.757201910018921, + "objective/train/theoretical_loss": 3.7393923808589564, + "objective/train/tokens_used": 798700000, + "theoretical_loss": 3.7393923808589564, + "tokens_seen": 778240000 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003859578736208626, + "loss": 2.6996, + "theoretical_loss": 3.7393923808589564, + "tokens_seen": 778240000 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038594784353059177, + "loss": 2.5437, + "theoretical_loss": 3.7393610109024267, + "tokens_seen": 778305536 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038593781344032095, + "loss": 2.6253, + "theoretical_loss": 3.7393296443267814, + "tokens_seen": 778371072 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038592778335005013, + "loss": 2.7438, + "theoretical_loss": 3.7392982811313713, + "tokens_seen": 778436608 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038591775325977937, + "loss": 2.8202, + "theoretical_loss": 3.739266921315547, + "tokens_seen": 778502144 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003859077231695085, + "loss": 2.8575, + "theoretical_loss": 3.739235564878661, + "tokens_seen": 778567680 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038589769307923773, + "loss": 2.8146, + "theoretical_loss": 3.7392042118200646, + "tokens_seen": 778633216 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038588766298896685, + "loss": 2.8834, + "theoretical_loss": 3.739172862139109, + "tokens_seen": 778698752 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003858776328986961, + "loss": 2.9581, + "theoretical_loss": 3.739141515835146, + "tokens_seen": 778764288 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003858676028084253, + "loss": 2.8166, + "theoretical_loss": 3.7391101729075285, + "tokens_seen": 778829824 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038585757271815445, + "loss": 2.7793, + "theoretical_loss": 3.7390788333556086, + "tokens_seen": 778895360 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003858475426278837, + "loss": 2.9614, + "theoretical_loss": 3.7390474971787375, + "tokens_seen": 778960896 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038583751253761287, + "loss": 2.9082, + "theoretical_loss": 3.7390161643762694, + "tokens_seen": 779026432 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038582748244734205, + "loss": 2.8853, + "theoretical_loss": 3.7389848349475567, + "tokens_seen": 779091968 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038581745235707123, + "loss": 2.8476, + "theoretical_loss": 3.738953508891952, + "tokens_seen": 779157504 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003858074222668004, + "loss": 2.8652, + "theoretical_loss": 3.7389221862088085, + "tokens_seen": 779223040 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003857973921765296, + "loss": 2.8059, + "theoretical_loss": 3.7388908668974796, + "tokens_seen": 779288576 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038578736208625883, + "loss": 2.9307, + "theoretical_loss": 3.738859550957319, + "tokens_seen": 779354112 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038577733199598796, + "loss": 2.779, + "theoretical_loss": 3.73882823838768, + "tokens_seen": 779419648 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003857673019057172, + "loss": 3.0866, + "theoretical_loss": 3.7387969291879175, + "tokens_seen": 779485184 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003857572718154463, + "loss": 2.8843, + "theoretical_loss": 3.7387656233573843, + "tokens_seen": 779550720 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038574724172517555, + "loss": 2.9377, + "theoretical_loss": 3.7387343208954356, + "tokens_seen": 779616256 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038573721163490473, + "loss": 2.8778, + "theoretical_loss": 3.7387030218014248, + "tokens_seen": 779681792 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003857271815446339, + "loss": 2.7067, + "theoretical_loss": 3.738671726074707, + "tokens_seen": 779747328 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003857171514543631, + "loss": 2.7502, + "theoretical_loss": 3.7386404337146373, + "tokens_seen": 779812864 + }, + { + "epoch": 0.24, + "objective/train/docs_used": 449318, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6445298194885254, + "objective/train/theoretical_loss": 3.73860914472057, + "objective/train/tokens_used": 800338400, + "theoretical_loss": 3.73860914472057, + "tokens_seen": 779878400 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038570712136409233, + "loss": 2.7979, + "theoretical_loss": 3.73860914472057, + "tokens_seen": 779878400 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038569709127382146, + "loss": 3.0481, + "theoretical_loss": 3.738577859091861, + "tokens_seen": 779943936 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003856870611835507, + "loss": 2.718, + "theoretical_loss": 3.738546576827865, + "tokens_seen": 780009472 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003856770310932798, + "loss": 2.637, + "theoretical_loss": 3.738515297927938, + "tokens_seen": 780075008 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038566700100300906, + "loss": 2.7855, + "theoretical_loss": 3.7384840223914346, + "tokens_seen": 780140544 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038565697091273824, + "loss": 2.8538, + "theoretical_loss": 3.7384527502177116, + "tokens_seen": 780206080 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003856469408224674, + "loss": 2.8157, + "theoretical_loss": 3.738421481406125, + "tokens_seen": 780271616 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003856369107321966, + "loss": 2.8484, + "theoretical_loss": 3.7383902159560307, + "tokens_seen": 780337152 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003856268806419258, + "loss": 2.9769, + "theoretical_loss": 3.7383589538667845, + "tokens_seen": 780402688 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038561685055165496, + "loss": 2.7299, + "theoretical_loss": 3.7383276951377438, + "tokens_seen": 780468224 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003856068204613842, + "loss": 2.6979, + "theoretical_loss": 3.7382964397682654, + "tokens_seen": 780533760 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003855967903711133, + "loss": 2.9098, + "theoretical_loss": 3.7382651877577056, + "tokens_seen": 780599296 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038558676028084256, + "loss": 2.9695, + "theoretical_loss": 3.7382339391054216, + "tokens_seen": 780664832 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003855767301905717, + "loss": 2.9, + "theoretical_loss": 3.738202693810771, + "tokens_seen": 780730368 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003855667001003009, + "loss": 2.9428, + "theoretical_loss": 3.738171451873111, + "tokens_seen": 780795904 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003855566700100301, + "loss": 3.0074, + "theoretical_loss": 3.738140213291799, + "tokens_seen": 780861440 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003855466399197593, + "loss": 2.6656, + "theoretical_loss": 3.7381089780661925, + "tokens_seen": 780926976 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038553660982948846, + "loss": 2.9564, + "theoretical_loss": 3.7380777461956503, + "tokens_seen": 780992512 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003855265797392177, + "loss": 2.8961, + "theoretical_loss": 3.73804651767953, + "tokens_seen": 781058048 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003855165496489468, + "loss": 2.8354, + "theoretical_loss": 3.738015292517191, + "tokens_seen": 781123584 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038550651955867606, + "loss": 2.7986, + "theoretical_loss": 3.7379840707079897, + "tokens_seen": 781189120 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003854964894684052, + "loss": 2.761, + "theoretical_loss": 3.7379528522512864, + "tokens_seen": 781254656 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003854864593781344, + "loss": 2.9208, + "theoretical_loss": 3.7379216371464388, + "tokens_seen": 781320192 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003854764292878636, + "loss": 2.9275, + "theoretical_loss": 3.7378904253928074, + "tokens_seen": 781385728 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003854663991975928, + "loss": 2.739, + "theoretical_loss": 3.73785921698975, + "tokens_seen": 781451264 + }, + { + "epoch": 0.24, + "objective/train/docs_used": 450726, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.5179219245910645, + "objective/train/theoretical_loss": 3.7378280119366267, + "objective/train/tokens_used": 801976800, + "theoretical_loss": 3.7378280119366267, + "tokens_seen": 781516800 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038545636910732197, + "loss": 2.7759, + "theoretical_loss": 3.7378280119366267, + "tokens_seen": 781516800 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038544633901705115, + "loss": 2.9004, + "theoretical_loss": 3.7377968102327968, + "tokens_seen": 781582336 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038543630892678033, + "loss": 2.7164, + "theoretical_loss": 3.7377656118776206, + "tokens_seen": 781647872 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038542627883650957, + "loss": 2.6685, + "theoretical_loss": 3.737734416870457, + "tokens_seen": 781713408 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003854162487462387, + "loss": 2.8602, + "theoretical_loss": 3.737703225210667, + "tokens_seen": 781778944 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038540621865596793, + "loss": 2.899, + "theoretical_loss": 3.7376720368976093, + "tokens_seen": 781844480 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038539618856569705, + "loss": 2.8819, + "theoretical_loss": 3.7376408519306463, + "tokens_seen": 781910016 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003853861584754263, + "loss": 2.7321, + "theoretical_loss": 3.7376096703091375, + "tokens_seen": 781975552 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038537612838515547, + "loss": 2.915, + "theoretical_loss": 3.737578492032444, + "tokens_seen": 782041088 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038536609829488465, + "loss": 2.9336, + "theoretical_loss": 3.7375473170999265, + "tokens_seen": 782106624 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038535606820461383, + "loss": 2.9101, + "theoretical_loss": 3.7375161455109467, + "tokens_seen": 782172160 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038534603811434307, + "loss": 2.967, + "theoretical_loss": 3.737484977264865, + "tokens_seen": 782237696 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003853360080240722, + "loss": 2.7346, + "theoretical_loss": 3.7374538123610432, + "tokens_seen": 782303232 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038532597793380143, + "loss": 2.9538, + "theoretical_loss": 3.7374226507988437, + "tokens_seen": 782368768 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038531594784353056, + "loss": 2.882, + "theoretical_loss": 3.7373914925776273, + "tokens_seen": 782434304 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003853059177532598, + "loss": 2.9094, + "theoretical_loss": 3.7373603376967566, + "tokens_seen": 782499840 + }, + { + "epoch": 0.24, + "learning_rate": 0.000385295887662989, + "loss": 2.9505, + "theoretical_loss": 3.737329186155593, + "tokens_seen": 782565376 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038528585757271816, + "loss": 2.4931, + "theoretical_loss": 3.7372980379535004, + "tokens_seen": 782630912 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038527582748244734, + "loss": 2.5951, + "theoretical_loss": 3.73726689308984, + "tokens_seen": 782696448 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003852657973921765, + "loss": 2.8429, + "theoretical_loss": 3.7372357515639747, + "tokens_seen": 782761984 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003852557673019057, + "loss": 3.0577, + "theoretical_loss": 3.737204613375268, + "tokens_seen": 782827520 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038524573721163493, + "loss": 2.6451, + "theoretical_loss": 3.7371734785230823, + "tokens_seen": 782893056 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038523570712136406, + "loss": 2.7056, + "theoretical_loss": 3.737142347006781, + "tokens_seen": 782958592 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003852256770310933, + "loss": 2.8869, + "theoretical_loss": 3.737111218825728, + "tokens_seen": 783024128 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003852156469408225, + "loss": 3.0201, + "theoretical_loss": 3.737080093979286, + "tokens_seen": 783089664 + }, + { + "epoch": 0.24, + "objective/train/docs_used": 451239, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.8194870948791504, + "objective/train/theoretical_loss": 3.7370489724668197, + "objective/train/tokens_used": 803615200, + "theoretical_loss": 3.7370489724668197, + "tokens_seen": 783155200 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038520561685055166, + "loss": 2.7156, + "theoretical_loss": 3.7370489724668197, + "tokens_seen": 783155200 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038519558676028084, + "loss": 2.7471, + "theoretical_loss": 3.737017854287692, + "tokens_seen": 783220736 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038518555667001, + "loss": 2.9162, + "theoretical_loss": 3.7369867394412677, + "tokens_seen": 783286272 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003851755265797392, + "loss": 2.9033, + "theoretical_loss": 3.7369556279269105, + "tokens_seen": 783351808 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038516549648946844, + "loss": 2.7594, + "theoretical_loss": 3.736924519743986, + "tokens_seen": 783417344 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038515546639919756, + "loss": 2.804, + "theoretical_loss": 3.736893414891858, + "tokens_seen": 783482880 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003851454363089268, + "loss": 2.8118, + "theoretical_loss": 3.7368623133698913, + "tokens_seen": 783548416 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003851354062186559, + "loss": 2.925, + "theoretical_loss": 3.7368312151774505, + "tokens_seen": 783613952 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038512537612838516, + "loss": 2.9767, + "theoretical_loss": 3.7368001203139016, + "tokens_seen": 783679488 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003851153460381144, + "loss": 2.9752, + "theoretical_loss": 3.7367690287786095, + "tokens_seen": 783745024 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003851053159478435, + "loss": 2.914, + "theoretical_loss": 3.73673794057094, + "tokens_seen": 783810560 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038509528585757276, + "loss": 3.1163, + "theoretical_loss": 3.736706855690258, + "tokens_seen": 783876096 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003850852557673019, + "loss": 2.7536, + "theoretical_loss": 3.7366757741359304, + "tokens_seen": 783941632 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003850752256770311, + "loss": 2.7883, + "theoretical_loss": 3.736644695907322, + "tokens_seen": 784007168 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003850651955867603, + "loss": 2.7815, + "theoretical_loss": 3.7366136210038006, + "tokens_seen": 784072704 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003850551654964895, + "loss": 2.8785, + "theoretical_loss": 3.736582549424731, + "tokens_seen": 784138240 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038504513540621866, + "loss": 2.53, + "theoretical_loss": 3.73655148116948, + "tokens_seen": 784203776 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003850351053159479, + "loss": 2.9741, + "theoretical_loss": 3.7365204162374153, + "tokens_seen": 784269312 + }, + { + "epoch": 0.24, + "learning_rate": 0.000385025075225677, + "loss": 2.9405, + "theoretical_loss": 3.736489354627903, + "tokens_seen": 784334848 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038501504513540626, + "loss": 2.8066, + "theoretical_loss": 3.7364582963403103, + "tokens_seen": 784400384 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003850050150451354, + "loss": 2.7352, + "theoretical_loss": 3.7364272413740047, + "tokens_seen": 784465920 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003849949849548646, + "loss": 2.9877, + "theoretical_loss": 3.736396189728353, + "tokens_seen": 784531456 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003849849548645938, + "loss": 2.7233, + "theoretical_loss": 3.7363651414027235, + "tokens_seen": 784596992 + }, + { + "epoch": 0.24, + "learning_rate": 0.000384974924774323, + "loss": 3.0919, + "theoretical_loss": 3.7363340963964835, + "tokens_seen": 784662528 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038496489468405217, + "loss": 2.9516, + "theoretical_loss": 3.7363030547090013, + "tokens_seen": 784728064 + }, + { + "epoch": 0.24, + "objective/train/docs_used": 452482, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.414663553237915, + "objective/train/theoretical_loss": 3.736272016339644, + "objective/train/tokens_used": 805253600, + "theoretical_loss": 3.736272016339644, + "tokens_seen": 784793600 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038495486459378135, + "loss": 2.7874, + "theoretical_loss": 3.736272016339644, + "tokens_seen": 784793600 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038494483450351053, + "loss": 2.8594, + "theoretical_loss": 3.7362409812877813, + "tokens_seen": 784859136 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038493480441323977, + "loss": 2.9864, + "theoretical_loss": 3.7362099495527805, + "tokens_seen": 784924672 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003849247743229689, + "loss": 2.731, + "theoretical_loss": 3.736178921134011, + "tokens_seen": 784990208 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038491474423269813, + "loss": 2.8092, + "theoretical_loss": 3.7361478960308414, + "tokens_seen": 785055744 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038490471414242725, + "loss": 2.6741, + "theoretical_loss": 3.73611687424264, + "tokens_seen": 785121280 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003848946840521565, + "loss": 2.7089, + "theoretical_loss": 3.736085855768777, + "tokens_seen": 785186816 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038488465396188567, + "loss": 2.7263, + "theoretical_loss": 3.736054840608621, + "tokens_seen": 785252352 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038487462387161485, + "loss": 2.75, + "theoretical_loss": 3.7360238287615415, + "tokens_seen": 785317888 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038486459378134403, + "loss": 3.2369, + "theoretical_loss": 3.7359928202269086, + "tokens_seen": 785383424 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038485456369107327, + "loss": 2.6572, + "theoretical_loss": 3.735961815004092, + "tokens_seen": 785448960 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003848445336008024, + "loss": 2.9687, + "theoretical_loss": 3.735930813092461, + "tokens_seen": 785514496 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038483450351053163, + "loss": 2.7153, + "theoretical_loss": 3.7358998144913866, + "tokens_seen": 785580032 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038482447342026076, + "loss": 2.6444, + "theoretical_loss": 3.735868819200239, + "tokens_seen": 785645568 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038481444332999, + "loss": 2.9512, + "theoretical_loss": 3.7358378272183885, + "tokens_seen": 785711104 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003848044132397192, + "loss": 2.6275, + "theoretical_loss": 3.735806838545206, + "tokens_seen": 785776640 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038479438314944836, + "loss": 2.6375, + "theoretical_loss": 3.735775853180062, + "tokens_seen": 785842176 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038478435305917754, + "loss": 2.9309, + "theoretical_loss": 3.735744871122328, + "tokens_seen": 785907712 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003847743229689067, + "loss": 2.7808, + "theoretical_loss": 3.7357138923713746, + "tokens_seen": 785973248 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003847642928786359, + "loss": 2.5466, + "theoretical_loss": 3.7356829169265744, + "tokens_seen": 786038784 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038475426278836513, + "loss": 2.6364, + "theoretical_loss": 3.735651944787297, + "tokens_seen": 786104320 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038474423269809426, + "loss": 2.9098, + "theoretical_loss": 3.7356209759529158, + "tokens_seen": 786169856 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003847342026078235, + "loss": 2.7638, + "theoretical_loss": 3.7355900104228024, + "tokens_seen": 786235392 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003847241725175527, + "loss": 2.9346, + "theoretical_loss": 3.735559048196328, + "tokens_seen": 786300928 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038471414242728186, + "loss": 2.7102, + "theoretical_loss": 3.735528089272866, + "tokens_seen": 786366464 + }, + { + "epoch": 0.24, + "objective/train/docs_used": 453141, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.3242013454437256, + "objective/train/theoretical_loss": 3.735497133651788, + "objective/train/tokens_used": 806892000, + "theoretical_loss": 3.735497133651788, + "tokens_seen": 786432000 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038470411233701104, + "loss": 3.0289, + "theoretical_loss": 3.735497133651788, + "tokens_seen": 786432000 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003846940822467402, + "loss": 2.6384, + "theoretical_loss": 3.735466181332467, + "tokens_seen": 786497536 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003846840521564694, + "loss": 2.8651, + "theoretical_loss": 3.735435232314275, + "tokens_seen": 786563072 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038467402206619864, + "loss": 2.706, + "theoretical_loss": 3.7354042865965864, + "tokens_seen": 786628608 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038466399197592776, + "loss": 2.6943, + "theoretical_loss": 3.7353733441787726, + "tokens_seen": 786694144 + }, + { + "epoch": 0.24, + "learning_rate": 0.000384653961885657, + "loss": 2.6813, + "theoretical_loss": 3.735342405060208, + "tokens_seen": 786759680 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003846439317953861, + "loss": 2.6564, + "theoretical_loss": 3.7353114692402656, + "tokens_seen": 786825216 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038463390170511536, + "loss": 2.9555, + "theoretical_loss": 3.7352805367183195, + "tokens_seen": 786890752 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038462387161484454, + "loss": 2.6092, + "theoretical_loss": 3.7352496074937425, + "tokens_seen": 786956288 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003846138415245737, + "loss": 2.9268, + "theoretical_loss": 3.735218681565909, + "tokens_seen": 787021824 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003846038114343029, + "loss": 2.8071, + "theoretical_loss": 3.7351877589341944, + "tokens_seen": 787087360 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003845937813440321, + "loss": 2.6341, + "theoretical_loss": 3.7351568395979706, + "tokens_seen": 787152896 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038458375125376127, + "loss": 2.9803, + "theoretical_loss": 3.7351259235566143, + "tokens_seen": 787218432 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003845737211634905, + "loss": 2.6547, + "theoretical_loss": 3.7350950108094985, + "tokens_seen": 787283968 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038456369107321963, + "loss": 2.9937, + "theoretical_loss": 3.7350641013559986, + "tokens_seen": 787349504 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038455366098294886, + "loss": 2.8297, + "theoretical_loss": 3.73503319519549, + "tokens_seen": 787415040 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038454363089267805, + "loss": 2.7971, + "theoretical_loss": 3.7350022923273474, + "tokens_seen": 787480576 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038453360080240723, + "loss": 2.7743, + "theoretical_loss": 3.7349713927509463, + "tokens_seen": 787546112 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003845235707121364, + "loss": 2.6843, + "theoretical_loss": 3.7349404964656614, + "tokens_seen": 787611648 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003845135406218656, + "loss": 2.9205, + "theoretical_loss": 3.7349096034708698, + "tokens_seen": 787677184 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038450351053159477, + "loss": 2.905, + "theoretical_loss": 3.734878713765946, + "tokens_seen": 787742720 + }, + { + "epoch": 0.24, + "learning_rate": 0.000384493480441324, + "loss": 2.8789, + "theoretical_loss": 3.734847827350267, + "tokens_seen": 787808256 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038448345035105313, + "loss": 2.4386, + "theoretical_loss": 3.7348169442232084, + "tokens_seen": 787873792 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038447342026078237, + "loss": 2.8427, + "theoretical_loss": 3.7347860643841466, + "tokens_seen": 787939328 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003844633901705115, + "loss": 2.6606, + "theoretical_loss": 3.7347551878324587, + "tokens_seen": 788004864 + }, + { + "epoch": 0.24, + "objective/train/docs_used": 453953, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.755188226699829, + "objective/train/theoretical_loss": 3.73472431456752, + "objective/train/tokens_used": 808530400, + "theoretical_loss": 3.73472431456752, + "tokens_seen": 788070400 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038445336008024073, + "loss": 2.7235, + "theoretical_loss": 3.73472431456752, + "tokens_seen": 788070400 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003844433299899699, + "loss": 2.6044, + "theoretical_loss": 3.7346934445887086, + "tokens_seen": 788135936 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003844332998996991, + "loss": 2.9236, + "theoretical_loss": 3.734662577895401, + "tokens_seen": 788201472 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003844232698094283, + "loss": 2.897, + "theoretical_loss": 3.7346317144869747, + "tokens_seen": 788267008 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038441323971915745, + "loss": 2.7827, + "theoretical_loss": 3.7346008543628066, + "tokens_seen": 788332544 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038440320962888664, + "loss": 2.9331, + "theoretical_loss": 3.734569997522275, + "tokens_seen": 788398080 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038439317953861587, + "loss": 2.572, + "theoretical_loss": 3.734539143964757, + "tokens_seen": 788463616 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038438314944834505, + "loss": 2.6605, + "theoretical_loss": 3.73450829368963, + "tokens_seen": 788529152 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038437311935807423, + "loss": 2.7112, + "theoretical_loss": 3.734477446696273, + "tokens_seen": 788594688 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038436308926780347, + "loss": 2.7998, + "theoretical_loss": 3.7344466029840637, + "tokens_seen": 788660224 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003843530591775326, + "loss": 2.7712, + "theoretical_loss": 3.7344157625523806, + "tokens_seen": 788725760 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038434302908726183, + "loss": 2.4592, + "theoretical_loss": 3.7343849254006027, + "tokens_seen": 788791296 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038433299899699096, + "loss": 2.4868, + "theoretical_loss": 3.7343540915281075, + "tokens_seen": 788856832 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003843229689067202, + "loss": 2.7333, + "theoretical_loss": 3.734323260934275, + "tokens_seen": 788922368 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003843129388164494, + "loss": 2.6985, + "theoretical_loss": 3.734292433618484, + "tokens_seen": 788987904 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038430290872617856, + "loss": 2.7423, + "theoretical_loss": 3.7342616095801136, + "tokens_seen": 789053440 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038429287863590774, + "loss": 3.1071, + "theoretical_loss": 3.734230788818543, + "tokens_seen": 789118976 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003842828485456369, + "loss": 2.8433, + "theoretical_loss": 3.734199971333152, + "tokens_seen": 789184512 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003842728184553661, + "loss": 2.8953, + "theoretical_loss": 3.73416915712332, + "tokens_seen": 789250048 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038426278836509533, + "loss": 2.8477, + "theoretical_loss": 3.734138346188428, + "tokens_seen": 789315584 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038425275827482446, + "loss": 2.9378, + "theoretical_loss": 3.734107538527854, + "tokens_seen": 789381120 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003842427281845537, + "loss": 2.702, + "theoretical_loss": 3.734076734140981, + "tokens_seen": 789446656 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003842326980942829, + "loss": 2.7565, + "theoretical_loss": 3.7340459330271867, + "tokens_seen": 789512192 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038422266800401206, + "loss": 2.711, + "theoretical_loss": 3.7340151351858535, + "tokens_seen": 789577728 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038421263791374124, + "loss": 2.8371, + "theoretical_loss": 3.7339843406163613, + "tokens_seen": 789643264 + }, + { + "epoch": 0.24, + "objective/train/docs_used": 455438, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.3390536308288574, + "objective/train/theoretical_loss": 3.733953549318091, + "objective/train/tokens_used": 810168800, + "theoretical_loss": 3.733953549318091, + "tokens_seen": 789708800 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003842026078234704, + "loss": 3.0268, + "theoretical_loss": 3.733953549318091, + "tokens_seen": 789708800 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003841925777331996, + "loss": 2.8495, + "theoretical_loss": 3.733922761290424, + "tokens_seen": 789774336 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038418254764292884, + "loss": 3.0089, + "theoretical_loss": 3.733891976532742, + "tokens_seen": 789839872 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038417251755265796, + "loss": 2.9107, + "theoretical_loss": 3.733861195044425, + "tokens_seen": 789905408 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003841624874623872, + "loss": 2.7299, + "theoretical_loss": 3.7338304168248566, + "tokens_seen": 789970944 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003841524573721163, + "loss": 2.7247, + "theoretical_loss": 3.733799641873417, + "tokens_seen": 790036480 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038414242728184556, + "loss": 2.7984, + "theoretical_loss": 3.733768870189488, + "tokens_seen": 790102016 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038413239719157474, + "loss": 2.9539, + "theoretical_loss": 3.7337381017724525, + "tokens_seen": 790167552 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003841223671013039, + "loss": 2.595, + "theoretical_loss": 3.733707336621692, + "tokens_seen": 790233088 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003841123370110331, + "loss": 2.8535, + "theoretical_loss": 3.7336765747365903, + "tokens_seen": 790298624 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003841023069207623, + "loss": 2.7557, + "theoretical_loss": 3.7336458161165287, + "tokens_seen": 790364160 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038409227683049147, + "loss": 2.7271, + "theoretical_loss": 3.73361506076089, + "tokens_seen": 790429696 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003840822467402207, + "loss": 2.7742, + "theoretical_loss": 3.7335843086690583, + "tokens_seen": 790495232 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038407221664994983, + "loss": 2.7921, + "theoretical_loss": 3.7335535598404155, + "tokens_seen": 790560768 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038406218655967907, + "loss": 2.8368, + "theoretical_loss": 3.733522814274345, + "tokens_seen": 790626304 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038405215646940825, + "loss": 2.6845, + "theoretical_loss": 3.7334920719702307, + "tokens_seen": 790691840 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038404212637913743, + "loss": 2.9464, + "theoretical_loss": 3.733461332927456, + "tokens_seen": 790757376 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003840320962888666, + "loss": 2.7344, + "theoretical_loss": 3.7334305971454045, + "tokens_seen": 790822912 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003840220661985958, + "loss": 2.7365, + "theoretical_loss": 3.7333998646234603, + "tokens_seen": 790888448 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038401203610832497, + "loss": 2.7644, + "theoretical_loss": 3.7333691353610075, + "tokens_seen": 790953984 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003840020060180542, + "loss": 2.9412, + "theoretical_loss": 3.7333384093574304, + "tokens_seen": 791019520 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038399197592778333, + "loss": 2.839, + "theoretical_loss": 3.7333076866121138, + "tokens_seen": 791085056 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038398194583751257, + "loss": 2.9947, + "theoretical_loss": 3.733276967124441, + "tokens_seen": 791150592 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003839719157472417, + "loss": 2.6287, + "theoretical_loss": 3.7332462508937985, + "tokens_seen": 791216128 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038396188565697093, + "loss": 2.8851, + "theoretical_loss": 3.7332155379195697, + "tokens_seen": 791281664 + }, + { + "epoch": 0.24, + "objective/train/docs_used": 456184, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.8457651138305664, + "objective/train/theoretical_loss": 3.733184828201141, + "objective/train/tokens_used": 811807200, + "theoretical_loss": 3.733184828201141, + "tokens_seen": 791347200 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003839518555667001, + "loss": 2.7688, + "theoretical_loss": 3.733184828201141, + "tokens_seen": 791347200 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003839418254764293, + "loss": 2.7556, + "theoretical_loss": 3.733154121737897, + "tokens_seen": 791412736 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003839317953861585, + "loss": 2.9923, + "theoretical_loss": 3.733123418529223, + "tokens_seen": 791478272 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038392176529588765, + "loss": 2.796, + "theoretical_loss": 3.733092718574505, + "tokens_seen": 791543808 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038391173520561684, + "loss": 2.8118, + "theoretical_loss": 3.7330620218731285, + "tokens_seen": 791609344 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038390170511534607, + "loss": 2.6598, + "theoretical_loss": 3.7330313284244796, + "tokens_seen": 791674880 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003838916750250752, + "loss": 2.7317, + "theoretical_loss": 3.7330006382279444, + "tokens_seen": 791740416 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038388164493480443, + "loss": 2.9346, + "theoretical_loss": 3.732969951282909, + "tokens_seen": 791805952 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003838716148445336, + "loss": 2.4842, + "theoretical_loss": 3.73293926758876, + "tokens_seen": 791871488 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003838615847542628, + "loss": 2.7078, + "theoretical_loss": 3.732908587144884, + "tokens_seen": 791937024 + }, + { + "epoch": 0.24, + "learning_rate": 0.000383851554663992, + "loss": 2.7329, + "theoretical_loss": 3.732877909950668, + "tokens_seen": 792002560 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038384152457372116, + "loss": 2.8099, + "theoretical_loss": 3.7328472360054983, + "tokens_seen": 792068096 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038383149448345034, + "loss": 2.7633, + "theoretical_loss": 3.7328165653087626, + "tokens_seen": 792133632 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003838214643931796, + "loss": 2.8658, + "theoretical_loss": 3.7327858978598476, + "tokens_seen": 792199168 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003838114343029087, + "loss": 2.5538, + "theoretical_loss": 3.7327552336581418, + "tokens_seen": 792264704 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038380140421263794, + "loss": 2.4853, + "theoretical_loss": 3.732724572703032, + "tokens_seen": 792330240 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038379137412236706, + "loss": 2.8076, + "theoretical_loss": 3.732693914993906, + "tokens_seen": 792395776 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003837813440320963, + "loss": 2.6861, + "theoretical_loss": 3.732663260530151, + "tokens_seen": 792461312 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003837713139418255, + "loss": 2.6156, + "theoretical_loss": 3.732632609311157, + "tokens_seen": 792526848 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038376128385155466, + "loss": 2.7139, + "theoretical_loss": 3.7326019613363104, + "tokens_seen": 792592384 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038375125376128384, + "loss": 2.7267, + "theoretical_loss": 3.732571316605001, + "tokens_seen": 792657920 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003837412236710131, + "loss": 2.715, + "theoretical_loss": 3.7325406751166166, + "tokens_seen": 792723456 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003837311935807422, + "loss": 2.806, + "theoretical_loss": 3.732510036870546, + "tokens_seen": 792788992 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038372116349047144, + "loss": 2.626, + "theoretical_loss": 3.7324794018661787, + "tokens_seen": 792854528 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038371113340020057, + "loss": 2.7776, + "theoretical_loss": 3.732448770102903, + "tokens_seen": 792920064 + }, + { + "epoch": 0.24, + "objective/train/docs_used": 457386, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.824763774871826, + "objective/train/theoretical_loss": 3.7324181415801094, + "objective/train/tokens_used": 813445600, + "theoretical_loss": 3.7324181415801094, + "tokens_seen": 792985600 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003837011033099298, + "loss": 2.7304, + "theoretical_loss": 3.7324181415801094, + "tokens_seen": 792985600 + }, + { + "epoch": 0.24, + "learning_rate": 0.000383691073219659, + "loss": 2.6632, + "theoretical_loss": 3.7323875162971856, + "tokens_seen": 793051136 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038368104312938816, + "loss": 2.8227, + "theoretical_loss": 3.7323568942535226, + "tokens_seen": 793116672 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038367101303911735, + "loss": 2.6893, + "theoretical_loss": 3.73232627544851, + "tokens_seen": 793182208 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003836609829488465, + "loss": 2.7768, + "theoretical_loss": 3.7322956598815367, + "tokens_seen": 793247744 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003836509528585757, + "loss": 2.7864, + "theoretical_loss": 3.7322650475519934, + "tokens_seen": 793313280 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038364092276830494, + "loss": 2.9134, + "theoretical_loss": 3.7322344384592707, + "tokens_seen": 793378816 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003836308926780341, + "loss": 2.7699, + "theoretical_loss": 3.7322038326027593, + "tokens_seen": 793444352 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003836208625877633, + "loss": 2.9116, + "theoretical_loss": 3.7321732299818486, + "tokens_seen": 793509888 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003836108324974925, + "loss": 2.8314, + "theoretical_loss": 3.73214263059593, + "tokens_seen": 793575424 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038360080240722167, + "loss": 2.68, + "theoretical_loss": 3.7321120344443948, + "tokens_seen": 793640960 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003835907723169509, + "loss": 2.8643, + "theoretical_loss": 3.732081441526634, + "tokens_seen": 793706496 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038358074222668003, + "loss": 2.6718, + "theoretical_loss": 3.732050851842038, + "tokens_seen": 793772032 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038357071213640927, + "loss": 2.7618, + "theoretical_loss": 3.7320202653899983, + "tokens_seen": 793837568 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038356068204613845, + "loss": 2.5797, + "theoretical_loss": 3.7319896821699077, + "tokens_seen": 793903104 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038355065195586763, + "loss": 2.7854, + "theoretical_loss": 3.731959102181157, + "tokens_seen": 793968640 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003835406218655968, + "loss": 2.9217, + "theoretical_loss": 3.7319285254231382, + "tokens_seen": 794034176 + }, + { + "epoch": 0.24, + "learning_rate": 0.000383530591775326, + "loss": 2.8997, + "theoretical_loss": 3.7318979518952435, + "tokens_seen": 794099712 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038352056168505517, + "loss": 2.6546, + "theoretical_loss": 3.7318673815968655, + "tokens_seen": 794165248 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003835105315947844, + "loss": 2.7056, + "theoretical_loss": 3.7318368145273952, + "tokens_seen": 794230784 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038350050150451353, + "loss": 2.7048, + "theoretical_loss": 3.731806250686227, + "tokens_seen": 794296320 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038349047141424277, + "loss": 2.9824, + "theoretical_loss": 3.7317756900727526, + "tokens_seen": 794361856 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003834804413239719, + "loss": 2.5525, + "theoretical_loss": 3.731745132686365, + "tokens_seen": 794427392 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038347041123370113, + "loss": 2.7937, + "theoretical_loss": 3.731714578526457, + "tokens_seen": 794492928 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003834603811434303, + "loss": 2.6491, + "theoretical_loss": 3.7316840275924226, + "tokens_seen": 794558464 + }, + { + "epoch": 0.24, + "objective/train/docs_used": 458050, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.27347731590271, + "objective/train/theoretical_loss": 3.7316534798836543, + "objective/train/tokens_used": 815084000, + "theoretical_loss": 3.7316534798836543, + "tokens_seen": 794624000 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003834503510531595, + "loss": 2.9552, + "theoretical_loss": 3.7316534798836543, + "tokens_seen": 794624000 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003834403209628887, + "loss": 2.9178, + "theoretical_loss": 3.731622935399547, + "tokens_seen": 794689536 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038343029087261786, + "loss": 2.5572, + "theoretical_loss": 3.7315923941394926, + "tokens_seen": 794755072 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038342026078234704, + "loss": 2.663, + "theoretical_loss": 3.7315618561028865, + "tokens_seen": 794820608 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038341023069207627, + "loss": 2.8133, + "theoretical_loss": 3.731531321289121, + "tokens_seen": 794886144 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003834002006018054, + "loss": 2.6359, + "theoretical_loss": 3.7315007896975927, + "tokens_seen": 794951680 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038339017051153463, + "loss": 2.6272, + "theoretical_loss": 3.731470261327694, + "tokens_seen": 795017216 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003833801404212638, + "loss": 2.7887, + "theoretical_loss": 3.73143973617882, + "tokens_seen": 795082752 + }, + { + "epoch": 0.24, + "learning_rate": 0.000383370110330993, + "loss": 2.9156, + "theoretical_loss": 3.731409214250366, + "tokens_seen": 795148288 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003833600802407222, + "loss": 2.7545, + "theoretical_loss": 3.731378695541726, + "tokens_seen": 795213824 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038335005015045136, + "loss": 2.8655, + "theoretical_loss": 3.7313481800522954, + "tokens_seen": 795279360 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038334002006018054, + "loss": 2.9734, + "theoretical_loss": 3.73131766778147, + "tokens_seen": 795344896 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003833299899699098, + "loss": 2.6632, + "theoretical_loss": 3.7312871587286436, + "tokens_seen": 795410432 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003833199598796389, + "loss": 2.745, + "theoretical_loss": 3.7312566528932134, + "tokens_seen": 795475968 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038330992978936814, + "loss": 2.5258, + "theoretical_loss": 3.7312261502745736, + "tokens_seen": 795541504 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038329989969909726, + "loss": 2.4173, + "theoretical_loss": 3.731195650872121, + "tokens_seen": 795607040 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003832898696088265, + "loss": 2.8674, + "theoretical_loss": 3.731165154685252, + "tokens_seen": 795672576 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003832798395185557, + "loss": 2.9002, + "theoretical_loss": 3.7311346617133614, + "tokens_seen": 795738112 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038326980942828486, + "loss": 2.7418, + "theoretical_loss": 3.731104171955847, + "tokens_seen": 795803648 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038325977933801404, + "loss": 2.6526, + "theoretical_loss": 3.731073685412104, + "tokens_seen": 795869184 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003832497492477433, + "loss": 2.6165, + "theoretical_loss": 3.73104320208153, + "tokens_seen": 795934720 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003832397191574724, + "loss": 2.9952, + "theoretical_loss": 3.7310127219635207, + "tokens_seen": 796000256 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038322968906720164, + "loss": 2.7528, + "theoretical_loss": 3.730982245057474, + "tokens_seen": 796065792 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038321965897693077, + "loss": 2.7264, + "theoretical_loss": 3.730951771362787, + "tokens_seen": 796131328 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038320962888666, + "loss": 2.6342, + "theoretical_loss": 3.7309213008788573, + "tokens_seen": 796196864 + }, + { + "epoch": 0.24, + "objective/train/docs_used": 459137, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.0809667110443115, + "objective/train/theoretical_loss": 3.7308908336050814, + "objective/train/tokens_used": 816722400, + "theoretical_loss": 3.7308908336050814, + "tokens_seen": 796262400 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003831995987963892, + "loss": 2.9868, + "theoretical_loss": 3.7308908336050814, + "tokens_seen": 796262400 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038318956870611836, + "loss": 2.8558, + "theoretical_loss": 3.730860369540858, + "tokens_seen": 796327936 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038317953861584755, + "loss": 2.8122, + "theoretical_loss": 3.7308299086855836, + "tokens_seen": 796393472 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003831695085255767, + "loss": 2.9047, + "theoretical_loss": 3.7307994510386577, + "tokens_seen": 796459008 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003831594784353059, + "loss": 2.9204, + "theoretical_loss": 3.730768996599477, + "tokens_seen": 796524544 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038314944834503514, + "loss": 2.6743, + "theoretical_loss": 3.7307385453674407, + "tokens_seen": 796590080 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038313941825476427, + "loss": 2.6638, + "theoretical_loss": 3.730708097341947, + "tokens_seen": 796655616 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003831293881644935, + "loss": 2.8741, + "theoretical_loss": 3.7306776525223944, + "tokens_seen": 796721152 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038311935807422263, + "loss": 2.6751, + "theoretical_loss": 3.730647210908182, + "tokens_seen": 796786688 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038310932798395187, + "loss": 2.6047, + "theoretical_loss": 3.730616772498708, + "tokens_seen": 796852224 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038309929789368105, + "loss": 2.8187, + "theoretical_loss": 3.7305863372933725, + "tokens_seen": 796917760 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038308926780341023, + "loss": 2.7018, + "theoretical_loss": 3.730555905291574, + "tokens_seen": 796983296 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003830792377131394, + "loss": 2.5456, + "theoretical_loss": 3.730525476492712, + "tokens_seen": 797048832 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038306920762286865, + "loss": 2.4481, + "theoretical_loss": 3.7304950508961863, + "tokens_seen": 797114368 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003830591775325978, + "loss": 2.5922, + "theoretical_loss": 3.7304646285013963, + "tokens_seen": 797179904 + }, + { + "epoch": 0.24, + "learning_rate": 0.000383049147442327, + "loss": 2.8382, + "theoretical_loss": 3.7304342093077425, + "tokens_seen": 797245440 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038303911735205614, + "loss": 2.7758, + "theoretical_loss": 3.7304037933146246, + "tokens_seen": 797310976 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038302908726178537, + "loss": 2.8337, + "theoretical_loss": 3.7303733805214425, + "tokens_seen": 797376512 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038301905717151455, + "loss": 2.7851, + "theoretical_loss": 3.7303429709275973, + "tokens_seen": 797442048 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038300902708124373, + "loss": 2.6157, + "theoretical_loss": 3.730312564532489, + "tokens_seen": 797507584 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003829989969909729, + "loss": 2.9748, + "theoretical_loss": 3.7302821613355186, + "tokens_seen": 797573120 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003829889669007021, + "loss": 2.5618, + "theoretical_loss": 3.7302517613360866, + "tokens_seen": 797638656 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003829789368104313, + "loss": 2.602, + "theoretical_loss": 3.7302213645335947, + "tokens_seen": 797704192 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003829689067201605, + "loss": 2.4782, + "theoretical_loss": 3.7301909709274437, + "tokens_seen": 797769728 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038295887662988964, + "loss": 2.6567, + "theoretical_loss": 3.730160580517035, + "tokens_seen": 797835264 + }, + { + "epoch": 0.24, + "objective/train/docs_used": 459607, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.440809726715088, + "objective/train/theoretical_loss": 3.73013019330177, + "objective/train/tokens_used": 818360800, + "theoretical_loss": 3.73013019330177, + "tokens_seen": 797900800 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003829488465396189, + "loss": 2.9558, + "theoretical_loss": 3.73013019330177, + "tokens_seen": 797900800 + }, + { + "epoch": 0.24, + "learning_rate": 0.000382938816449348, + "loss": 2.8444, + "theoretical_loss": 3.7300998092810502, + "tokens_seen": 797966336 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038292878635907724, + "loss": 2.6959, + "theoretical_loss": 3.7300694284542786, + "tokens_seen": 798031872 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003829187562688064, + "loss": 2.9396, + "theoretical_loss": 3.730039050820855, + "tokens_seen": 798097408 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003829087261785356, + "loss": 2.6352, + "theoretical_loss": 3.7300086763801836, + "tokens_seen": 798162944 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003828986960882648, + "loss": 2.8298, + "theoretical_loss": 3.729978305131666, + "tokens_seen": 798228480 + }, + { + "epoch": 0.24, + "learning_rate": 0.000382888665997994, + "loss": 2.7631, + "theoretical_loss": 3.729947937074705, + "tokens_seen": 798294016 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003828786359077232, + "loss": 2.7292, + "theoretical_loss": 3.7299175722087026, + "tokens_seen": 798359552 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003828686058174524, + "loss": 2.6145, + "theoretical_loss": 3.729887210533062, + "tokens_seen": 798425088 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038285857572718156, + "loss": 2.9868, + "theoretical_loss": 3.7298568520471855, + "tokens_seen": 798490624 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038284854563691074, + "loss": 2.6827, + "theoretical_loss": 3.7298264967504773, + "tokens_seen": 798556160 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038283851554664, + "loss": 2.8812, + "theoretical_loss": 3.72979614464234, + "tokens_seen": 798621696 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003828284854563691, + "loss": 2.5892, + "theoretical_loss": 3.7297657957221784, + "tokens_seen": 798687232 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038281845536609834, + "loss": 2.5783, + "theoretical_loss": 3.729735449989394, + "tokens_seen": 798752768 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038280842527582746, + "loss": 2.778, + "theoretical_loss": 3.729705107443391, + "tokens_seen": 798818304 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003827983951855567, + "loss": 2.7237, + "theoretical_loss": 3.7296747680835747, + "tokens_seen": 798883840 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003827883650952859, + "loss": 2.8592, + "theoretical_loss": 3.7296444319093487, + "tokens_seen": 798949376 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038277833500501506, + "loss": 2.8053, + "theoretical_loss": 3.7296140989201163, + "tokens_seen": 799014912 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038276830491474424, + "loss": 2.7555, + "theoretical_loss": 3.7295837691152824, + "tokens_seen": 799080448 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003827582748244735, + "loss": 2.6763, + "theoretical_loss": 3.729553442494252, + "tokens_seen": 799145984 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003827482447342026, + "loss": 2.7164, + "theoretical_loss": 3.72952311905643, + "tokens_seen": 799211520 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038273821464393184, + "loss": 2.7938, + "theoretical_loss": 3.7294927988012203, + "tokens_seen": 799277056 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038272818455366097, + "loss": 2.7576, + "theoretical_loss": 3.729462481728029, + "tokens_seen": 799342592 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003827181544633902, + "loss": 2.8337, + "theoretical_loss": 3.7294321678362596, + "tokens_seen": 799408128 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003827081243731194, + "loss": 2.8638, + "theoretical_loss": 3.72940185712532, + "tokens_seen": 799473664 + }, + { + "epoch": 0.24, + "objective/train/docs_used": 460885, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7605931758880615, + "objective/train/theoretical_loss": 3.729371549594614, + "objective/train/tokens_used": 819999200, + "theoretical_loss": 3.729371549594614, + "tokens_seen": 799539200 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038269809428284856, + "loss": 2.8523, + "theoretical_loss": 3.729371549594614, + "tokens_seen": 799539200 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038268806419257775, + "loss": 2.787, + "theoretical_loss": 3.729341245243547, + "tokens_seen": 799604736 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003826780341023069, + "loss": 2.822, + "theoretical_loss": 3.7293109440715266, + "tokens_seen": 799670272 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003826680040120361, + "loss": 2.7464, + "theoretical_loss": 3.729280646077957, + "tokens_seen": 799735808 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038265797392176534, + "loss": 2.7615, + "theoretical_loss": 3.7292503512622455, + "tokens_seen": 799801344 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038264794383149447, + "loss": 2.5737, + "theoretical_loss": 3.729220059623798, + "tokens_seen": 799866880 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003826379137412237, + "loss": 2.9582, + "theoretical_loss": 3.729189771162021, + "tokens_seen": 799932416 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038262788365095283, + "loss": 2.8976, + "theoretical_loss": 3.7291594858763215, + "tokens_seen": 799997952 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038261785356068207, + "loss": 2.6914, + "theoretical_loss": 3.7291292037661057, + "tokens_seen": 800063488 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038260782347041125, + "loss": 2.6876, + "theoretical_loss": 3.7290989248307813, + "tokens_seen": 800129024 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038259779338014043, + "loss": 2.8446, + "theoretical_loss": 3.729068649069754, + "tokens_seen": 800194560 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003825877632898696, + "loss": 2.7704, + "theoretical_loss": 3.729038376482433, + "tokens_seen": 800260096 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038257773319959885, + "loss": 2.9293, + "theoretical_loss": 3.729008107068225, + "tokens_seen": 800325632 + }, + { + "epoch": 0.24, + "learning_rate": 0.000382567703109328, + "loss": 2.7572, + "theoretical_loss": 3.728977840826537, + "tokens_seen": 800391168 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003825576730190572, + "loss": 2.7362, + "theoretical_loss": 3.7289475777567773, + "tokens_seen": 800456704 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038254764292878634, + "loss": 2.9103, + "theoretical_loss": 3.728917317858354, + "tokens_seen": 800522240 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038253761283851557, + "loss": 2.8537, + "theoretical_loss": 3.7288870611306746, + "tokens_seen": 800587776 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038252758274824475, + "loss": 2.8837, + "theoretical_loss": 3.7288568075731474, + "tokens_seen": 800653312 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038251755265797393, + "loss": 2.6206, + "theoretical_loss": 3.7288265571851817, + "tokens_seen": 800718848 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003825075225677031, + "loss": 2.799, + "theoretical_loss": 3.728796309966185, + "tokens_seen": 800784384 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003824974924774323, + "loss": 2.7506, + "theoretical_loss": 3.7287660659155666, + "tokens_seen": 800849920 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003824874623871615, + "loss": 2.4744, + "theoretical_loss": 3.7287358250327354, + "tokens_seen": 800915456 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003824774322968907, + "loss": 2.8077, + "theoretical_loss": 3.7287055873171, + "tokens_seen": 800980992 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038246740220661984, + "loss": 2.7581, + "theoretical_loss": 3.7286753527680703, + "tokens_seen": 801046528 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003824573721163491, + "loss": 2.6891, + "theoretical_loss": 3.7286451213850547, + "tokens_seen": 801112064 + }, + { + "epoch": 0.24, + "objective/train/docs_used": 461399, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6303045749664307, + "objective/train/theoretical_loss": 3.7286148931674634, + "objective/train/tokens_used": 821637600, + "theoretical_loss": 3.7286148931674634, + "tokens_seen": 801177600 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003824473420260782, + "loss": 2.8272, + "theoretical_loss": 3.7286148931674634, + "tokens_seen": 801177600 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038243731193580744, + "loss": 2.6568, + "theoretical_loss": 3.728584668114706, + "tokens_seen": 801243136 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003824272818455366, + "loss": 2.7691, + "theoretical_loss": 3.728554446226192, + "tokens_seen": 801308672 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003824172517552658, + "loss": 2.9675, + "theoretical_loss": 3.7285242275013317, + "tokens_seen": 801374208 + }, + { + "epoch": 0.24, + "learning_rate": 0.000382407221664995, + "loss": 2.7534, + "theoretical_loss": 3.728494011939535, + "tokens_seen": 801439744 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003823971915747242, + "loss": 2.5691, + "theoretical_loss": 3.7284637995402123, + "tokens_seen": 801505280 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038238716148445334, + "loss": 2.7221, + "theoretical_loss": 3.7284335903027745, + "tokens_seen": 801570816 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003823771313941826, + "loss": 2.6693, + "theoretical_loss": 3.728403384226632, + "tokens_seen": 801636352 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003823671013039117, + "loss": 2.8721, + "theoretical_loss": 3.728373181311195, + "tokens_seen": 801701888 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038235707121364094, + "loss": 2.6978, + "theoretical_loss": 3.728342981555875, + "tokens_seen": 801767424 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003823470411233701, + "loss": 2.8393, + "theoretical_loss": 3.7283127849600826, + "tokens_seen": 801832960 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003823370110330993, + "loss": 2.8734, + "theoretical_loss": 3.7282825915232296, + "tokens_seen": 801898496 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003823269809428285, + "loss": 2.9252, + "theoretical_loss": 3.728252401244727, + "tokens_seen": 801964032 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038231695085255766, + "loss": 2.9603, + "theoretical_loss": 3.7282222141239867, + "tokens_seen": 802029568 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038230692076228685, + "loss": 2.6426, + "theoretical_loss": 3.7281920301604208, + "tokens_seen": 802095104 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003822968906720161, + "loss": 2.5739, + "theoretical_loss": 3.72816184935344, + "tokens_seen": 802160640 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003822868605817452, + "loss": 2.7026, + "theoretical_loss": 3.728131671702457, + "tokens_seen": 802226176 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038227683049147444, + "loss": 2.6668, + "theoretical_loss": 3.728101497206884, + "tokens_seen": 802291712 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038226680040120357, + "loss": 2.8829, + "theoretical_loss": 3.728071325866134, + "tokens_seen": 802357248 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003822567703109328, + "loss": 2.8478, + "theoretical_loss": 3.7280411576796184, + "tokens_seen": 802422784 + }, + { + "epoch": 0.24, + "learning_rate": 0.000382246740220662, + "loss": 2.8425, + "theoretical_loss": 3.7280109926467504, + "tokens_seen": 802488320 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038223671013039117, + "loss": 2.901, + "theoretical_loss": 3.727980830766943, + "tokens_seen": 802553856 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038222668004012035, + "loss": 2.8118, + "theoretical_loss": 3.7279506720396083, + "tokens_seen": 802619392 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003822166499498496, + "loss": 2.6534, + "theoretical_loss": 3.7279205164641604, + "tokens_seen": 802684928 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003822066198595787, + "loss": 2.7905, + "theoretical_loss": 3.7278903640400127, + "tokens_seen": 802750464 + }, + { + "epoch": 0.24, + "objective/train/docs_used": 462084, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.8370156288146973, + "objective/train/theoretical_loss": 3.7278602147665776, + "objective/train/tokens_used": 823276000, + "theoretical_loss": 3.7278602147665776, + "tokens_seen": 802816000 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038219658976930795, + "loss": 2.7819, + "theoretical_loss": 3.7278602147665776, + "tokens_seen": 802816000 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003821865596790371, + "loss": 2.6514, + "theoretical_loss": 3.7278300686432697, + "tokens_seen": 802881536 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003821765295887663, + "loss": 2.7551, + "theoretical_loss": 3.7277999256695025, + "tokens_seen": 802947072 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003821664994984955, + "loss": 2.8025, + "theoretical_loss": 3.7277697858446897, + "tokens_seen": 803012608 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038215646940822467, + "loss": 2.8291, + "theoretical_loss": 3.727739649168245, + "tokens_seen": 803078144 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038214643931795385, + "loss": 2.8268, + "theoretical_loss": 3.7277095156395843, + "tokens_seen": 803143680 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038213640922768303, + "loss": 2.7043, + "theoretical_loss": 3.7276793852581203, + "tokens_seen": 803209216 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038212637913741227, + "loss": 2.7298, + "theoretical_loss": 3.727649258023268, + "tokens_seen": 803274752 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038211634904714145, + "loss": 2.7222, + "theoretical_loss": 3.7276191339344424, + "tokens_seen": 803340288 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038210631895687063, + "loss": 2.7668, + "theoretical_loss": 3.727589012991058, + "tokens_seen": 803405824 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003820962888665998, + "loss": 2.7802, + "theoretical_loss": 3.72755889519253, + "tokens_seen": 803471360 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038208625877632905, + "loss": 2.9252, + "theoretical_loss": 3.7275287805382744, + "tokens_seen": 803536896 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003820762286860582, + "loss": 2.893, + "theoretical_loss": 3.7274986690277045, + "tokens_seen": 803602432 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003820661985957874, + "loss": 2.814, + "theoretical_loss": 3.727468560660238, + "tokens_seen": 803667968 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038205616850551654, + "loss": 2.6523, + "theoretical_loss": 3.727438455435289, + "tokens_seen": 803733504 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038204613841524577, + "loss": 2.6127, + "theoretical_loss": 3.727408353352274, + "tokens_seen": 803799040 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038203610832497495, + "loss": 2.8578, + "theoretical_loss": 3.727378254410609, + "tokens_seen": 803864576 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038202607823470413, + "loss": 2.4825, + "theoretical_loss": 3.72734815860971, + "tokens_seen": 803930112 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003820160481444333, + "loss": 2.8532, + "theoretical_loss": 3.7273180659489933, + "tokens_seen": 803995648 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003820060180541625, + "loss": 2.9371, + "theoretical_loss": 3.727287976427875, + "tokens_seen": 804061184 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003819959879638917, + "loss": 2.9495, + "theoretical_loss": 3.727257890045772, + "tokens_seen": 804126720 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003819859578736209, + "loss": 2.7006, + "theoretical_loss": 3.7272278068021008, + "tokens_seen": 804192256 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038197592778335004, + "loss": 2.622, + "theoretical_loss": 3.7271977266962786, + "tokens_seen": 804257792 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003819658976930793, + "loss": 2.7428, + "theoretical_loss": 3.727167649727723, + "tokens_seen": 804323328 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003819558676028084, + "loss": 2.6612, + "theoretical_loss": 3.7271375758958496, + "tokens_seen": 804388864 + }, + { + "epoch": 0.24, + "objective/train/docs_used": 463313, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7193517684936523, + "objective/train/theoretical_loss": 3.7271075052000775, + "objective/train/tokens_used": 824914400, + "theoretical_loss": 3.7271075052000775, + "tokens_seen": 804454400 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038194583751253764, + "loss": 2.7059, + "theoretical_loss": 3.7271075052000775, + "tokens_seen": 804454400 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003819358074222668, + "loss": 2.7236, + "theoretical_loss": 3.7270774376398226, + "tokens_seen": 804519936 + }, + { + "epoch": 0.24, + "learning_rate": 0.000381925777331996, + "loss": 2.68, + "theoretical_loss": 3.727047373214504, + "tokens_seen": 804585472 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003819157472417252, + "loss": 2.8922, + "theoretical_loss": 3.727017311923539, + "tokens_seen": 804651008 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003819057171514544, + "loss": 2.5737, + "theoretical_loss": 3.726987253766345, + "tokens_seen": 804716544 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038189568706118354, + "loss": 2.652, + "theoretical_loss": 3.7269571987423413, + "tokens_seen": 804782080 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003818856569709128, + "loss": 2.8843, + "theoretical_loss": 3.726927146850945, + "tokens_seen": 804847616 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003818756268806419, + "loss": 2.6961, + "theoretical_loss": 3.726897098091576, + "tokens_seen": 804913152 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038186559679037114, + "loss": 2.6404, + "theoretical_loss": 3.7268670524636507, + "tokens_seen": 804978688 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003818555667001003, + "loss": 2.8394, + "theoretical_loss": 3.7268370099665904, + "tokens_seen": 805044224 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003818455366098295, + "loss": 2.9171, + "theoretical_loss": 3.7268069705998124, + "tokens_seen": 805109760 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003818355065195587, + "loss": 2.9469, + "theoretical_loss": 3.7267769343627357, + "tokens_seen": 805175296 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038182547642928786, + "loss": 2.7711, + "theoretical_loss": 3.726746901254781, + "tokens_seen": 805240832 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038181544633901705, + "loss": 2.9103, + "theoretical_loss": 3.726716871275366, + "tokens_seen": 805306368 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003818054162487463, + "loss": 2.6158, + "theoretical_loss": 3.726686844423911, + "tokens_seen": 805371904 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003817953861584754, + "loss": 2.9479, + "theoretical_loss": 3.726656820699836, + "tokens_seen": 805437440 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038178535606820464, + "loss": 2.8636, + "theoretical_loss": 3.72662680010256, + "tokens_seen": 805502976 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038177532597793377, + "loss": 2.8337, + "theoretical_loss": 3.726596782631504, + "tokens_seen": 805568512 + }, + { + "epoch": 0.24, + "learning_rate": 0.000381765295887663, + "loss": 2.6895, + "theoretical_loss": 3.7265667682860872, + "tokens_seen": 805634048 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003817552657973922, + "loss": 2.7216, + "theoretical_loss": 3.7265367570657304, + "tokens_seen": 805699584 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038174523570712137, + "loss": 2.8951, + "theoretical_loss": 3.7265067489698547, + "tokens_seen": 805765120 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038173520561685055, + "loss": 2.9073, + "theoretical_loss": 3.726476743997879, + "tokens_seen": 805830656 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003817251755265798, + "loss": 2.5678, + "theoretical_loss": 3.7264467421492258, + "tokens_seen": 805896192 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003817151454363089, + "loss": 2.8267, + "theoretical_loss": 3.7264167434233153, + "tokens_seen": 805961728 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038170511534603815, + "loss": 2.9198, + "theoretical_loss": 3.7263867478195687, + "tokens_seen": 806027264 + }, + { + "epoch": 0.24, + "objective/train/docs_used": 464591, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.681321859359741, + "objective/train/theoretical_loss": 3.726356755337407, + "objective/train/tokens_used": 826552800, + "theoretical_loss": 3.726356755337407, + "tokens_seen": 806092800 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003816950852557673, + "loss": 2.9198, + "theoretical_loss": 3.726356755337407, + "tokens_seen": 806092800 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003816850551654965, + "loss": 2.9046, + "theoretical_loss": 3.7263267659762516, + "tokens_seen": 806158336 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003816750250752257, + "loss": 2.8976, + "theoretical_loss": 3.7262967797355246, + "tokens_seen": 806223872 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038166499498495487, + "loss": 3.1646, + "theoretical_loss": 3.7262667966146474, + "tokens_seen": 806289408 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038165496489468405, + "loss": 2.9905, + "theoretical_loss": 3.7262368166130413, + "tokens_seen": 806354944 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038164493480441323, + "loss": 2.9842, + "theoretical_loss": 3.7262068397301293, + "tokens_seen": 806420480 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003816349047141424, + "loss": 2.6123, + "theoretical_loss": 3.726176865965333, + "tokens_seen": 806486016 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038162487462387165, + "loss": 2.761, + "theoretical_loss": 3.7261468953180747, + "tokens_seen": 806551552 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003816148445336008, + "loss": 2.6813, + "theoretical_loss": 3.7261169277877766, + "tokens_seen": 806617088 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038160481444333, + "loss": 3.0582, + "theoretical_loss": 3.7260869633738625, + "tokens_seen": 806682624 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038159478435305914, + "loss": 2.9763, + "theoretical_loss": 3.7260570020757537, + "tokens_seen": 806748160 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003815847542627884, + "loss": 2.8691, + "theoretical_loss": 3.7260270438928744, + "tokens_seen": 806813696 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038157472417251756, + "loss": 2.7222, + "theoretical_loss": 3.7259970888246468, + "tokens_seen": 806879232 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038156469408224674, + "loss": 2.9275, + "theoretical_loss": 3.7259671368704943, + "tokens_seen": 806944768 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003815546639919759, + "loss": 2.8785, + "theoretical_loss": 3.7259371880298406, + "tokens_seen": 807010304 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038154463390170515, + "loss": 3.024, + "theoretical_loss": 3.7259072423021085, + "tokens_seen": 807075840 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003815346038114343, + "loss": 2.6428, + "theoretical_loss": 3.725877299686723, + "tokens_seen": 807141376 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003815245737211635, + "loss": 2.6675, + "theoretical_loss": 3.7258473601831072, + "tokens_seen": 807206912 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038151454363089264, + "loss": 2.8815, + "theoretical_loss": 3.725817423790685, + "tokens_seen": 807272448 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003815045135406219, + "loss": 2.653, + "theoretical_loss": 3.72578749050888, + "tokens_seen": 807337984 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038149448345035106, + "loss": 2.7669, + "theoretical_loss": 3.7257575603371187, + "tokens_seen": 807403520 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038148445336008024, + "loss": 2.7232, + "theoretical_loss": 3.7257276332748233, + "tokens_seen": 807469056 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003814744232698094, + "loss": 2.5689, + "theoretical_loss": 3.725697709321419, + "tokens_seen": 807534592 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003814643931795386, + "loss": 2.7556, + "theoretical_loss": 3.725667788476331, + "tokens_seen": 807600128 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003814543630892678, + "loss": 2.7213, + "theoretical_loss": 3.7256378707389843, + "tokens_seen": 807665664 + }, + { + "epoch": 0.24, + "objective/train/docs_used": 465345, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.9825339317321777, + "objective/train/theoretical_loss": 3.725607956108804, + "objective/train/tokens_used": 828191200, + "theoretical_loss": 3.725607956108804, + "tokens_seen": 807731200 + }, + { + "epoch": 0.24, + "learning_rate": 0.000381444332998997, + "loss": 2.8661, + "theoretical_loss": 3.725607956108804, + "tokens_seen": 807731200 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038143430290872614, + "loss": 2.8337, + "theoretical_loss": 3.7255780445852142, + "tokens_seen": 807796736 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003814242728184554, + "loss": 2.8943, + "theoretical_loss": 3.7255481361676415, + "tokens_seen": 807862272 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038141424272818456, + "loss": 2.9147, + "theoretical_loss": 3.725518230855511, + "tokens_seen": 807927808 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038140421263791374, + "loss": 2.7559, + "theoretical_loss": 3.7254883286482485, + "tokens_seen": 807993344 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003813941825476429, + "loss": 2.659, + "theoretical_loss": 3.72545842954528, + "tokens_seen": 808058880 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003813841524573721, + "loss": 2.9294, + "theoretical_loss": 3.7254285335460313, + "tokens_seen": 808124416 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038137412236710134, + "loss": 3.0047, + "theoretical_loss": 3.725398640649929, + "tokens_seen": 808189952 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003813640922768305, + "loss": 2.5914, + "theoretical_loss": 3.725368750856398, + "tokens_seen": 808255488 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003813540621865597, + "loss": 2.6359, + "theoretical_loss": 3.7253388641648666, + "tokens_seen": 808321024 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003813440320962889, + "loss": 2.5223, + "theoretical_loss": 3.72530898057476, + "tokens_seen": 808386560 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038133400200601806, + "loss": 2.7451, + "theoretical_loss": 3.7252791000855057, + "tokens_seen": 808452096 + }, + { + "epoch": 0.25, + "learning_rate": 0.00038132397191574725, + "loss": 2.7495, + "theoretical_loss": 3.7252492226965304, + "tokens_seen": 808517632 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003813139418254765, + "loss": 2.7817, + "theoretical_loss": 3.7252193484072613, + "tokens_seen": 808583168 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003813039117352056, + "loss": 2.9757, + "theoretical_loss": 3.7251894772171257, + "tokens_seen": 808648704 + }, + { + "epoch": 0.25, + "learning_rate": 0.00038129388164493484, + "loss": 2.7256, + "theoretical_loss": 3.72515960912555, + "tokens_seen": 808714240 + }, + { + "epoch": 0.25, + "learning_rate": 0.00038128385155466397, + "loss": 2.965, + "theoretical_loss": 3.725129744131963, + "tokens_seen": 808779776 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003812738214643932, + "loss": 2.4397, + "theoretical_loss": 3.725099882235792, + "tokens_seen": 808845312 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003812637913741224, + "loss": 2.7893, + "theoretical_loss": 3.725070023436465, + "tokens_seen": 808910848 + }, + { + "epoch": 0.25, + "learning_rate": 0.00038125376128385157, + "loss": 2.8748, + "theoretical_loss": 3.7250401677334093, + "tokens_seen": 808976384 + }, + { + "epoch": 0.25, + "learning_rate": 0.00038124373119358075, + "loss": 2.8941, + "theoretical_loss": 3.7250103151260534, + "tokens_seen": 809041920 + }, + { + "epoch": 0.25, + "learning_rate": 0.00038123370110331, + "loss": 2.7271, + "theoretical_loss": 3.7249804656138257, + "tokens_seen": 809107456 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003812236710130391, + "loss": 3.0164, + "theoretical_loss": 3.7249506191961546, + "tokens_seen": 809172992 + }, + { + "epoch": 0.25, + "learning_rate": 0.00038121364092276835, + "loss": 3.0852, + "theoretical_loss": 3.7249207758724685, + "tokens_seen": 809238528 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003812036108324975, + "loss": 2.8475, + "theoretical_loss": 3.7248909356421964, + "tokens_seen": 809304064 + }, + { + "epoch": 0.25, + "objective/train/docs_used": 466409, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.475501537322998, + "objective/train/theoretical_loss": 3.724861098504767, + "objective/train/tokens_used": 829829600, + "theoretical_loss": 3.724861098504767, + "tokens_seen": 809369600 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003811935807422267, + "loss": 2.6672, + "theoretical_loss": 3.724861098504767, + "tokens_seen": 809369600 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003811835506519559, + "loss": 2.5982, + "theoretical_loss": 3.7248312644596098, + "tokens_seen": 809435136 + }, + { + "epoch": 0.25, + "learning_rate": 0.00038117352056168507, + "loss": 2.8001, + "theoretical_loss": 3.724801433506153, + "tokens_seen": 809500672 + }, + { + "epoch": 0.25, + "learning_rate": 0.00038116349047141425, + "loss": 2.7989, + "theoretical_loss": 3.724771605643827, + "tokens_seen": 809566208 + }, + { + "epoch": 0.25, + "learning_rate": 0.00038115346038114343, + "loss": 2.8945, + "theoretical_loss": 3.7247417808720606, + "tokens_seen": 809631744 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003811434302908726, + "loss": 2.9368, + "theoretical_loss": 3.7247119591902837, + "tokens_seen": 809697280 + }, + { + "epoch": 0.25, + "learning_rate": 0.00038113340020060185, + "loss": 2.7546, + "theoretical_loss": 3.7246821405979267, + "tokens_seen": 809762816 + }, + { + "epoch": 0.25, + "learning_rate": 0.000381123370110331, + "loss": 2.917, + "theoretical_loss": 3.724652325094418, + "tokens_seen": 809828352 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003811133400200602, + "loss": 2.9113, + "theoretical_loss": 3.724622512679189, + "tokens_seen": 809893888 + }, + { + "epoch": 0.25, + "learning_rate": 0.00038110330992978934, + "loss": 2.5927, + "theoretical_loss": 3.7245927033516697, + "tokens_seen": 809959424 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003810932798395186, + "loss": 2.9581, + "theoretical_loss": 3.7245628971112907, + "tokens_seen": 810024960 + }, + { + "epoch": 0.25, + "learning_rate": 0.00038108324974924776, + "loss": 2.9134, + "theoretical_loss": 3.724533093957482, + "tokens_seen": 810090496 + }, + { + "epoch": 0.25, + "learning_rate": 0.00038107321965897694, + "loss": 2.7128, + "theoretical_loss": 3.7245032938896743, + "tokens_seen": 810156032 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003810631895687061, + "loss": 2.7845, + "theoretical_loss": 3.724473496907299, + "tokens_seen": 810221568 + }, + { + "epoch": 0.25, + "learning_rate": 0.00038105315947843535, + "loss": 3.0035, + "theoretical_loss": 3.724443703009787, + "tokens_seen": 810287104 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003810431293881645, + "loss": 2.853, + "theoretical_loss": 3.724413912196569, + "tokens_seen": 810352640 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003810330992978937, + "loss": 2.6847, + "theoretical_loss": 3.7243841244670772, + "tokens_seen": 810418176 + }, + { + "epoch": 0.25, + "learning_rate": 0.00038102306920762284, + "loss": 2.6511, + "theoretical_loss": 3.724354339820742, + "tokens_seen": 810483712 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003810130391173521, + "loss": 2.498, + "theoretical_loss": 3.7243245582569955, + "tokens_seen": 810549248 + }, + { + "epoch": 0.25, + "learning_rate": 0.00038100300902708126, + "loss": 2.6705, + "theoretical_loss": 3.7242947797752697, + "tokens_seen": 810614784 + }, + { + "epoch": 0.25, + "learning_rate": 0.00038099297893681044, + "loss": 2.8583, + "theoretical_loss": 3.7242650043749963, + "tokens_seen": 810680320 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003809829488465396, + "loss": 3.0044, + "theoretical_loss": 3.724235232055607, + "tokens_seen": 810745856 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003809729187562688, + "loss": 2.9007, + "theoretical_loss": 3.7242054628165344, + "tokens_seen": 810811392 + }, + { + "epoch": 0.25, + "learning_rate": 0.000380962888665998, + "loss": 2.8004, + "theoretical_loss": 3.724175696657211, + "tokens_seen": 810876928 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003809528585757272, + "loss": 2.5691, + "theoretical_loss": 3.724145933577069, + "tokens_seen": 810942464 + }, + { + "epoch": 0.25, + "objective/train/docs_used": 466961, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.863931894302368, + "objective/train/theoretical_loss": 3.724116173575542, + "objective/train/tokens_used": 831468000, + "theoretical_loss": 3.724116173575542, + "tokens_seen": 811008000 + }, + { + "epoch": 0.25, + "learning_rate": 0.00038094282848545635, + "loss": 2.8083, + "theoretical_loss": 3.724116173575542, + "tokens_seen": 811008000 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003809327983951856, + "loss": 2.8974, + "theoretical_loss": 3.724086416652061, + "tokens_seen": 811073536 + }, + { + "epoch": 0.25, + "learning_rate": 0.00038092276830491476, + "loss": 2.8481, + "theoretical_loss": 3.7240566628060607, + "tokens_seen": 811139072 + }, + { + "epoch": 0.25, + "learning_rate": 0.00038091273821464394, + "loss": 2.7612, + "theoretical_loss": 3.7240269120369733, + "tokens_seen": 811204608 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003809027081243731, + "loss": 2.8735, + "theoretical_loss": 3.7239971643442322, + "tokens_seen": 811270144 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003808926780341023, + "loss": 2.5263, + "theoretical_loss": 3.7239674197272707, + "tokens_seen": 811335680 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003808826479438315, + "loss": 2.726, + "theoretical_loss": 3.7239376781855222, + "tokens_seen": 811401216 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003808726178535607, + "loss": 2.8175, + "theoretical_loss": 3.723907939718422, + "tokens_seen": 811466752 + }, + { + "epoch": 0.25, + "learning_rate": 0.00038086258776328985, + "loss": 2.6567, + "theoretical_loss": 3.723878204325402, + "tokens_seen": 811532288 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003808525576730191, + "loss": 2.9273, + "theoretical_loss": 3.7238484720058973, + "tokens_seen": 811597824 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003808425275827482, + "loss": 2.9332, + "theoretical_loss": 3.7238187427593417, + "tokens_seen": 811663360 + }, + { + "epoch": 0.25, + "learning_rate": 0.00038083249749247745, + "loss": 3.1031, + "theoretical_loss": 3.7237890165851693, + "tokens_seen": 811728896 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003808224674022066, + "loss": 2.6412, + "theoretical_loss": 3.723759293482815, + "tokens_seen": 811794432 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003808124373119358, + "loss": 2.8472, + "theoretical_loss": 3.7237295734517133, + "tokens_seen": 811859968 + }, + { + "epoch": 0.25, + "learning_rate": 0.000380802407221665, + "loss": 2.9537, + "theoretical_loss": 3.7236998564912986, + "tokens_seen": 811925504 + }, + { + "epoch": 0.25, + "learning_rate": 0.00038079237713139417, + "loss": 3.0141, + "theoretical_loss": 3.723670142601007, + "tokens_seen": 811991040 + }, + { + "epoch": 0.25, + "learning_rate": 0.00038078234704112335, + "loss": 3.0396, + "theoretical_loss": 3.7236404317802716, + "tokens_seen": 812056576 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003807723169508526, + "loss": 2.7821, + "theoretical_loss": 3.7236107240285294, + "tokens_seen": 812122112 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003807622868605817, + "loss": 2.8403, + "theoretical_loss": 3.723581019345215, + "tokens_seen": 812187648 + }, + { + "epoch": 0.25, + "learning_rate": 0.00038075225677031095, + "loss": 2.6732, + "theoretical_loss": 3.723551317729764, + "tokens_seen": 812253184 + }, + { + "epoch": 0.25, + "learning_rate": 0.00038074222668004013, + "loss": 2.9227, + "theoretical_loss": 3.723521619181612, + "tokens_seen": 812318720 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003807321965897693, + "loss": 2.7628, + "theoretical_loss": 3.7234919237001947, + "tokens_seen": 812384256 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003807221664994985, + "loss": 2.9749, + "theoretical_loss": 3.723462231284949, + "tokens_seen": 812449792 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003807121364092277, + "loss": 2.9245, + "theoretical_loss": 3.72343254193531, + "tokens_seen": 812515328 + }, + { + "epoch": 0.25, + "learning_rate": 0.00038070210631895685, + "loss": 2.8561, + "theoretical_loss": 3.7234028556507135, + "tokens_seen": 812580864 + }, + { + "epoch": 0.25, + "objective/train/docs_used": 468484, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.5587408542633057, + "objective/train/theoretical_loss": 3.7233731724305974, + "objective/train/tokens_used": 833106400, + "theoretical_loss": 3.7233731724305974, + "tokens_seen": 812646400 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003806920762286861, + "loss": 2.6753, + "theoretical_loss": 3.7233731724305974, + "tokens_seen": 812646400 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003806820461384152, + "loss": 2.9501, + "theoretical_loss": 3.723343492274397, + "tokens_seen": 812711936 + }, + { + "epoch": 0.25, + "learning_rate": 0.00038067201604814445, + "loss": 2.9443, + "theoretical_loss": 3.72331381518155, + "tokens_seen": 812777472 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003806619859578736, + "loss": 2.7951, + "theoretical_loss": 3.7232841411514923, + "tokens_seen": 812843008 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003806519558676028, + "loss": 2.8142, + "theoretical_loss": 3.7232544701836616, + "tokens_seen": 812908544 + }, + { + "epoch": 0.25, + "learning_rate": 0.000380641925777332, + "loss": 2.7173, + "theoretical_loss": 3.7232248022774943, + "tokens_seen": 812974080 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003806318956870612, + "loss": 2.8971, + "theoretical_loss": 3.723195137432429, + "tokens_seen": 813039616 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003806218655967904, + "loss": 2.9372, + "theoretical_loss": 3.7231654756479022, + "tokens_seen": 813105152 + }, + { + "epoch": 0.25, + "learning_rate": 0.00038061183550651954, + "loss": 2.8287, + "theoretical_loss": 3.723135816923352, + "tokens_seen": 813170688 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003806018054162488, + "loss": 2.823, + "theoretical_loss": 3.7231061612582153, + "tokens_seen": 813236224 + }, + { + "epoch": 0.25, + "learning_rate": 0.00038059177532597796, + "loss": 2.8925, + "theoretical_loss": 3.7230765086519306, + "tokens_seen": 813301760 + }, + { + "epoch": 0.25, + "learning_rate": 0.00038058174523570714, + "loss": 2.5347, + "theoretical_loss": 3.723046859103936, + "tokens_seen": 813367296 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003805717151454363, + "loss": 2.865, + "theoretical_loss": 3.7230172126136694, + "tokens_seen": 813432832 + }, + { + "epoch": 0.25, + "learning_rate": 0.00038056168505516555, + "loss": 3.1556, + "theoretical_loss": 3.7229875691805696, + "tokens_seen": 813498368 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003805516549648947, + "loss": 2.8899, + "theoretical_loss": 3.7229579288040746, + "tokens_seen": 813563904 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003805416248746239, + "loss": 3.0065, + "theoretical_loss": 3.7229282914836235, + "tokens_seen": 813629440 + }, + { + "epoch": 0.25, + "learning_rate": 0.00038053159478435304, + "loss": 2.9856, + "theoretical_loss": 3.7228986572186553, + "tokens_seen": 813694976 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003805215646940823, + "loss": 2.6003, + "theoretical_loss": 3.722869026008608, + "tokens_seen": 813760512 + }, + { + "epoch": 0.25, + "learning_rate": 0.00038051153460381146, + "loss": 2.6746, + "theoretical_loss": 3.722839397852921, + "tokens_seen": 813826048 + }, + { + "epoch": 0.25, + "learning_rate": 0.00038050150451354064, + "loss": 2.9529, + "theoretical_loss": 3.7228097727510336, + "tokens_seen": 813891584 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003804914744232698, + "loss": 2.6184, + "theoretical_loss": 3.722780150702386, + "tokens_seen": 813957120 + }, + { + "epoch": 0.25, + "learning_rate": 0.000380481444332999, + "loss": 2.8946, + "theoretical_loss": 3.7227505317064162, + "tokens_seen": 814022656 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003804714142427282, + "loss": 2.8289, + "theoretical_loss": 3.722720915762565, + "tokens_seen": 814088192 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003804613841524574, + "loss": 2.8214, + "theoretical_loss": 3.722691302870272, + "tokens_seen": 814153728 + }, + { + "epoch": 0.25, + "learning_rate": 0.00038045135406218655, + "loss": 2.9829, + "theoretical_loss": 3.7226616930289773, + "tokens_seen": 814219264 + }, + { + "epoch": 0.25, + "objective/train/docs_used": 469177, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.438084125518799, + "objective/train/theoretical_loss": 3.7226320862381206, + "objective/train/tokens_used": 834744800, + "theoretical_loss": 3.7226320862381206, + "tokens_seen": 814284800 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003804413239719158, + "loss": 2.7828, + "theoretical_loss": 3.7226320862381206, + "tokens_seen": 814284800 + }, + { + "epoch": 0.25, + "learning_rate": 0.00038043129388164496, + "loss": 2.7419, + "theoretical_loss": 3.722602482497143, + "tokens_seen": 814350336 + }, + { + "epoch": 0.25, + "learning_rate": 0.00038042126379137414, + "loss": 2.8036, + "theoretical_loss": 3.7225728818054833, + "tokens_seen": 814415872 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003804112337011033, + "loss": 2.7697, + "theoretical_loss": 3.7225432841625836, + "tokens_seen": 814481408 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003804012036108325, + "loss": 2.9989, + "theoretical_loss": 3.722513689567884, + "tokens_seen": 814546944 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003803911735205617, + "loss": 2.8279, + "theoretical_loss": 3.722484098020826, + "tokens_seen": 814612480 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003803811434302909, + "loss": 2.6422, + "theoretical_loss": 3.7224545095208494, + "tokens_seen": 814678016 + }, + { + "epoch": 0.25, + "learning_rate": 0.00038037111334002005, + "loss": 2.6757, + "theoretical_loss": 3.722424924067396, + "tokens_seen": 814743552 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003803610832497493, + "loss": 2.965, + "theoretical_loss": 3.722395341659907, + "tokens_seen": 814809088 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003803510531594784, + "loss": 2.558, + "theoretical_loss": 3.722365762297825, + "tokens_seen": 814874624 + }, + { + "epoch": 0.25, + "learning_rate": 0.00038034102306920765, + "loss": 2.9279, + "theoretical_loss": 3.72233618598059, + "tokens_seen": 814940160 + }, + { + "epoch": 0.25, + "learning_rate": 0.00038033099297893683, + "loss": 2.8192, + "theoretical_loss": 3.7223066127076443, + "tokens_seen": 815005696 + }, + { + "epoch": 0.25, + "learning_rate": 0.000380320962888666, + "loss": 3.1204, + "theoretical_loss": 3.7222770424784297, + "tokens_seen": 815071232 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003803109327983952, + "loss": 2.7783, + "theoretical_loss": 3.722247475292388, + "tokens_seen": 815136768 + }, + { + "epoch": 0.25, + "learning_rate": 0.00038030090270812437, + "loss": 2.7501, + "theoretical_loss": 3.7222179111489626, + "tokens_seen": 815202304 + }, + { + "epoch": 0.25, + "learning_rate": 0.00038029087261785355, + "loss": 2.9108, + "theoretical_loss": 3.7221883500475945, + "tokens_seen": 815267840 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003802808425275828, + "loss": 2.8168, + "theoretical_loss": 3.7221587919877264, + "tokens_seen": 815333376 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003802708124373119, + "loss": 2.9246, + "theoretical_loss": 3.7221292369688017, + "tokens_seen": 815398912 + }, + { + "epoch": 0.25, + "learning_rate": 0.00038026078234704115, + "loss": 2.8356, + "theoretical_loss": 3.722099684990263, + "tokens_seen": 815464448 + }, + { + "epoch": 0.25, + "learning_rate": 0.00038025075225677033, + "loss": 2.8169, + "theoretical_loss": 3.722070136051552, + "tokens_seen": 815529984 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003802407221664995, + "loss": 2.9016, + "theoretical_loss": 3.7220405901521127, + "tokens_seen": 815595520 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003802306920762287, + "loss": 3.0235, + "theoretical_loss": 3.7220110472913888, + "tokens_seen": 815661056 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003802206619859579, + "loss": 2.8894, + "theoretical_loss": 3.721981507468823, + "tokens_seen": 815726592 + }, + { + "epoch": 0.25, + "learning_rate": 0.00038021063189568705, + "loss": 2.9853, + "theoretical_loss": 3.7219519706838584, + "tokens_seen": 815792128 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003802006018054163, + "loss": 2.6642, + "theoretical_loss": 3.72192243693594, + "tokens_seen": 815857664 + }, + { + "epoch": 0.25, + "objective/train/docs_used": 469961, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.436821460723877, + "objective/train/theoretical_loss": 3.7218929062245105, + "objective/train/tokens_used": 836383200, + "theoretical_loss": 3.7218929062245105, + "tokens_seen": 815923200 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003801905717151454, + "loss": 2.7565, + "theoretical_loss": 3.7218929062245105, + "tokens_seen": 815923200 + }, + { + "epoch": 0.25, + "learning_rate": 0.00038018054162487465, + "loss": 3.0414, + "theoretical_loss": 3.7218633785490143, + "tokens_seen": 815988736 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003801705115346038, + "loss": 3.0368, + "theoretical_loss": 3.721833853908895, + "tokens_seen": 816054272 + }, + { + "epoch": 0.25, + "learning_rate": 0.000380160481444333, + "loss": 2.7468, + "theoretical_loss": 3.721804332303597, + "tokens_seen": 816119808 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003801504513540622, + "loss": 2.631, + "theoretical_loss": 3.7217748137325652, + "tokens_seen": 816185344 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003801404212637914, + "loss": 2.7969, + "theoretical_loss": 3.721745298195244, + "tokens_seen": 816250880 + }, + { + "epoch": 0.25, + "learning_rate": 0.00038013039117352056, + "loss": 2.7758, + "theoretical_loss": 3.7217157856910776, + "tokens_seen": 816316416 + }, + { + "epoch": 0.25, + "learning_rate": 0.00038012036108324974, + "loss": 2.8613, + "theoretical_loss": 3.7216862762195113, + "tokens_seen": 816381952 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003801103309929789, + "loss": 2.8809, + "theoretical_loss": 3.72165676977999, + "tokens_seen": 816447488 + }, + { + "epoch": 0.25, + "learning_rate": 0.00038010030090270816, + "loss": 2.8271, + "theoretical_loss": 3.721627266371958, + "tokens_seen": 816513024 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003800902708124373, + "loss": 2.7916, + "theoretical_loss": 3.721597765994862, + "tokens_seen": 816578560 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003800802407221665, + "loss": 2.9009, + "theoretical_loss": 3.721568268648146, + "tokens_seen": 816644096 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003800702106318957, + "loss": 2.712, + "theoretical_loss": 3.7215387743312567, + "tokens_seen": 816709632 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003800601805416249, + "loss": 2.8925, + "theoretical_loss": 3.7215092830436394, + "tokens_seen": 816775168 + }, + { + "epoch": 0.25, + "learning_rate": 0.00038005015045135406, + "loss": 2.7361, + "theoretical_loss": 3.72147979478474, + "tokens_seen": 816840704 + }, + { + "epoch": 0.25, + "learning_rate": 0.00038004012036108324, + "loss": 2.8324, + "theoretical_loss": 3.7214503095540037, + "tokens_seen": 816906240 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003800300902708124, + "loss": 2.6997, + "theoretical_loss": 3.721420827350878, + "tokens_seen": 816971776 + }, + { + "epoch": 0.25, + "learning_rate": 0.00038002006018054166, + "loss": 2.7312, + "theoretical_loss": 3.7213913481748087, + "tokens_seen": 817037312 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003800100300902708, + "loss": 2.757, + "theoretical_loss": 3.7213618720252413, + "tokens_seen": 817102848 + }, + { + "epoch": 0.25, + "learning_rate": 0.00038, + "loss": 2.8326, + "theoretical_loss": 3.7213323989016236, + "tokens_seen": 817168384 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037998996990972915, + "loss": 2.786, + "theoretical_loss": 3.721302928803402, + "tokens_seen": 817233920 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003799799398194584, + "loss": 3.0037, + "theoretical_loss": 3.7212734617300223, + "tokens_seen": 817299456 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037996990972918756, + "loss": 2.7487, + "theoretical_loss": 3.7212439976809337, + "tokens_seen": 817364992 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037995987963891675, + "loss": 3.0527, + "theoretical_loss": 3.721214536655581, + "tokens_seen": 817430528 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003799498495486459, + "loss": 2.7506, + "theoretical_loss": 3.721185078653413, + "tokens_seen": 817496064 + }, + { + "epoch": 0.25, + "objective/train/docs_used": 471203, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.720289468765259, + "objective/train/theoretical_loss": 3.7211556236738765, + "objective/train/tokens_used": 838021600, + "theoretical_loss": 3.7211556236738765, + "tokens_seen": 817561600 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037993981945837516, + "loss": 2.9551, + "theoretical_loss": 3.7211556236738765, + "tokens_seen": 817561600 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003799297893681043, + "loss": 2.514, + "theoretical_loss": 3.7211261717164197, + "tokens_seen": 817627136 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003799197592778335, + "loss": 2.717, + "theoretical_loss": 3.72109672278049, + "tokens_seen": 817692672 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037990972918756265, + "loss": 2.7485, + "theoretical_loss": 3.7210672768655355, + "tokens_seen": 817758208 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003798996990972919, + "loss": 2.8915, + "theoretical_loss": 3.7210378339710033, + "tokens_seen": 817823744 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037988966900702107, + "loss": 2.6508, + "theoretical_loss": 3.7210083940963425, + "tokens_seen": 817889280 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037987963891675025, + "loss": 2.6459, + "theoretical_loss": 3.7209789572410013, + "tokens_seen": 817954816 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003798696088264795, + "loss": 2.7947, + "theoretical_loss": 3.7209495234044283, + "tokens_seen": 818020352 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003798595787362086, + "loss": 2.8681, + "theoretical_loss": 3.7209200925860717, + "tokens_seen": 818085888 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037984954864593785, + "loss": 2.7129, + "theoretical_loss": 3.7208906647853803, + "tokens_seen": 818151424 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037983951855566703, + "loss": 2.6855, + "theoretical_loss": 3.720861240001803, + "tokens_seen": 818216960 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003798294884653962, + "loss": 2.6894, + "theoretical_loss": 3.720831818234789, + "tokens_seen": 818282496 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003798194583751254, + "loss": 2.6454, + "theoretical_loss": 3.7208023994837878, + "tokens_seen": 818348032 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037980942828485457, + "loss": 2.8986, + "theoretical_loss": 3.7207729837482484, + "tokens_seen": 818413568 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037979939819458375, + "loss": 2.558, + "theoretical_loss": 3.7207435710276195, + "tokens_seen": 818479104 + }, + { + "epoch": 0.25, + "learning_rate": 0.000379789368104313, + "loss": 2.7917, + "theoretical_loss": 3.720714161321352, + "tokens_seen": 818544640 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003797793380140421, + "loss": 3.03, + "theoretical_loss": 3.7206847546288957, + "tokens_seen": 818610176 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037976930792377135, + "loss": 2.9142, + "theoretical_loss": 3.7206553509496993, + "tokens_seen": 818675712 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037975927783350053, + "loss": 2.9113, + "theoretical_loss": 3.720625950283214, + "tokens_seen": 818741248 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003797492477432297, + "loss": 2.9142, + "theoretical_loss": 3.720596552628889, + "tokens_seen": 818806784 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003797392176529589, + "loss": 2.7678, + "theoretical_loss": 3.720567157986175, + "tokens_seen": 818872320 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003797291875626881, + "loss": 2.5686, + "theoretical_loss": 3.7205377663545236, + "tokens_seen": 818937856 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037971915747241725, + "loss": 2.8941, + "theoretical_loss": 3.720508377733384, + "tokens_seen": 819003392 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003797091273821465, + "loss": 3.1667, + "theoretical_loss": 3.7204789921222075, + "tokens_seen": 819068928 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003796990972918756, + "loss": 3.0322, + "theoretical_loss": 3.720449609520445, + "tokens_seen": 819134464 + }, + { + "debugging/Self-BLEU-5": 0.39739847440424436, + "debugging/distinct-1-grams": 0.8281364151245951, + "debugging/distinct-2-grams": 0.9743107396331714, + "debugging/entropy-1-grams": 5.929594633984304, + "debugging/entropy-2-grams": 6.683211367437131, + "debugging/length": 458.0769230769231, + "debugging/num_segments": 13, + "debugging/score": 0.00995348841418095, + "debugging/score_std": 0.010782348819564497, + "epoch": 0.25, + "objective/train/docs_used": 471797, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.1465864181518555, + "objective/train/theoretical_loss": 3.7204202299275475, + "objective/train/tokens_used": 839660000, + "theoretical_loss": 3.7204202299275475, + "tokens_seen": 819200000 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037968906720160485, + "loss": 3.0293, + "theoretical_loss": 3.7204202299275475, + "tokens_seen": 819200000 + }, + { + "epoch": 0.25, + "learning_rate": 0.000379679037111334, + "loss": 2.6222, + "theoretical_loss": 3.7203908533429666, + "tokens_seen": 819265536 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003796690070210632, + "loss": 2.869, + "theoretical_loss": 3.7203614797661535, + "tokens_seen": 819331072 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003796589769307924, + "loss": 2.9278, + "theoretical_loss": 3.720332109196559, + "tokens_seen": 819396608 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003796489468405216, + "loss": 2.6401, + "theoretical_loss": 3.7203027416336356, + "tokens_seen": 819462144 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037963891675025076, + "loss": 2.7348, + "theoretical_loss": 3.7202733770768344, + "tokens_seen": 819527680 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037962888665997994, + "loss": 2.6264, + "theoretical_loss": 3.7202440155256076, + "tokens_seen": 819593216 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003796188565697091, + "loss": 2.6029, + "theoretical_loss": 3.720214656979408, + "tokens_seen": 819658752 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037960882647943836, + "loss": 2.6693, + "theoretical_loss": 3.720185301437687, + "tokens_seen": 819724288 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003795987963891675, + "loss": 2.8733, + "theoretical_loss": 3.720155948899897, + "tokens_seen": 819789824 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003795887662988967, + "loss": 2.882, + "theoretical_loss": 3.720126599365491, + "tokens_seen": 819855360 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003795787362086259, + "loss": 2.6293, + "theoretical_loss": 3.720097252833921, + "tokens_seen": 819920896 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003795687061183551, + "loss": 2.6756, + "theoretical_loss": 3.72006790930464, + "tokens_seen": 819986432 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037955867602808426, + "loss": 2.9535, + "theoretical_loss": 3.720038568777101, + "tokens_seen": 820051968 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037954864593781344, + "loss": 2.7353, + "theoretical_loss": 3.7200092312507573, + "tokens_seen": 820117504 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003795386158475426, + "loss": 2.5285, + "theoretical_loss": 3.719979896725062, + "tokens_seen": 820183040 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037952858575727186, + "loss": 2.8983, + "theoretical_loss": 3.7199505651994684, + "tokens_seen": 820248576 + }, + { + "epoch": 0.25, + "learning_rate": 0.000379518555667001, + "loss": 2.7694, + "theoretical_loss": 3.71992123667343, + "tokens_seen": 820314112 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003795085255767302, + "loss": 2.6455, + "theoretical_loss": 3.7198919111464006, + "tokens_seen": 820379648 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037949849548645935, + "loss": 2.7241, + "theoretical_loss": 3.7198625886178336, + "tokens_seen": 820445184 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003794884653961886, + "loss": 2.9106, + "theoretical_loss": 3.719833269087183, + "tokens_seen": 820510720 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037947843530591776, + "loss": 2.8053, + "theoretical_loss": 3.719803952553904, + "tokens_seen": 820576256 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037946840521564695, + "loss": 2.8141, + "theoretical_loss": 3.719774639017449, + "tokens_seen": 820641792 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003794583751253761, + "loss": 2.9375, + "theoretical_loss": 3.7197453284772735, + "tokens_seen": 820707328 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037944834503510536, + "loss": 2.6048, + "theoretical_loss": 3.719716020932832, + "tokens_seen": 820772864 + }, + { + "epoch": 0.25, + "objective/train/docs_used": 473281, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.5523409843444824, + "objective/train/theoretical_loss": 3.719686716383579, + "objective/train/tokens_used": 841298400, + "theoretical_loss": 3.719686716383579, + "tokens_seen": 820838400 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003794383149448345, + "loss": 2.645, + "theoretical_loss": 3.719686716383579, + "tokens_seen": 820838400 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003794282848545637, + "loss": 2.6967, + "theoretical_loss": 3.719657414828969, + "tokens_seen": 820903936 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037941825476429285, + "loss": 2.6471, + "theoretical_loss": 3.7196281162684572, + "tokens_seen": 820969472 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003794082246740221, + "loss": 2.6156, + "theoretical_loss": 3.7195988207014983, + "tokens_seen": 821035008 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037939819458375127, + "loss": 2.6701, + "theoretical_loss": 3.7195695281275487, + "tokens_seen": 821100544 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037938816449348045, + "loss": 2.7966, + "theoretical_loss": 3.7195402385460623, + "tokens_seen": 821166080 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037937813440320963, + "loss": 2.9775, + "theoretical_loss": 3.7195109519564955, + "tokens_seen": 821231616 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003793681043129388, + "loss": 2.9072, + "theoretical_loss": 3.719481668358304, + "tokens_seen": 821297152 + }, + { + "epoch": 0.25, + "learning_rate": 0.000379358074222668, + "loss": 2.6984, + "theoretical_loss": 3.7194523877509424, + "tokens_seen": 821362688 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037934804413239723, + "loss": 2.932, + "theoretical_loss": 3.719423110133868, + "tokens_seen": 821428224 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037933801404212635, + "loss": 2.7462, + "theoretical_loss": 3.719393835506537, + "tokens_seen": 821493760 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003793279839518556, + "loss": 3.0567, + "theoretical_loss": 3.719364563868404, + "tokens_seen": 821559296 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003793179538615847, + "loss": 3.0147, + "theoretical_loss": 3.719335295218927, + "tokens_seen": 821624832 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037930792377131395, + "loss": 2.7422, + "theoretical_loss": 3.719306029557562, + "tokens_seen": 821690368 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037929789368104313, + "loss": 3.0031, + "theoretical_loss": 3.719276766883765, + "tokens_seen": 821755904 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003792878635907723, + "loss": 2.8267, + "theoretical_loss": 3.719247507196994, + "tokens_seen": 821821440 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003792778335005015, + "loss": 2.9874, + "theoretical_loss": 3.719218250496705, + "tokens_seen": 821886976 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037926780341023073, + "loss": 2.7502, + "theoretical_loss": 3.7191889967823557, + "tokens_seen": 821952512 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037925777331995986, + "loss": 2.8464, + "theoretical_loss": 3.7191597460534025, + "tokens_seen": 822018048 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003792477432296891, + "loss": 2.9135, + "theoretical_loss": 3.7191304983093034, + "tokens_seen": 822083584 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003792377131394182, + "loss": 2.7345, + "theoretical_loss": 3.719101253549516, + "tokens_seen": 822149120 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037922768304914746, + "loss": 2.8132, + "theoretical_loss": 3.719072011773497, + "tokens_seen": 822214656 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037921765295887664, + "loss": 2.8488, + "theoretical_loss": 3.719042772980705, + "tokens_seen": 822280192 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003792076228686058, + "loss": 2.7564, + "theoretical_loss": 3.719013537170598, + "tokens_seen": 822345728 + }, + { + "epoch": 0.25, + "learning_rate": 0.000379197592778335, + "loss": 2.7797, + "theoretical_loss": 3.718984304342634, + "tokens_seen": 822411264 + }, + { + "epoch": 0.25, + "objective/train/docs_used": 473825, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6765310764312744, + "objective/train/theoretical_loss": 3.7189550744962707, + "objective/train/tokens_used": 842936800, + "theoretical_loss": 3.7189550744962707, + "tokens_seen": 822476800 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003791875626880642, + "loss": 2.7306, + "theoretical_loss": 3.7189550744962707, + "tokens_seen": 822476800 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037917753259779336, + "loss": 2.6735, + "theoretical_loss": 3.718925847630967, + "tokens_seen": 822542336 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003791675025075226, + "loss": 2.7865, + "theoretical_loss": 3.7188966237461814, + "tokens_seen": 822607872 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003791574724172517, + "loss": 2.9325, + "theoretical_loss": 3.718867402841372, + "tokens_seen": 822673408 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037914744232698096, + "loss": 2.9893, + "theoretical_loss": 3.718838184915998, + "tokens_seen": 822738944 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003791374122367101, + "loss": 2.7943, + "theoretical_loss": 3.718808969969518, + "tokens_seen": 822804480 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003791273821464393, + "loss": 2.8677, + "theoretical_loss": 3.718779758001392, + "tokens_seen": 822870016 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037911735205616856, + "loss": 2.8259, + "theoretical_loss": 3.7187505490110775, + "tokens_seen": 822935552 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003791073219658977, + "loss": 2.8174, + "theoretical_loss": 3.7187213429980357, + "tokens_seen": 823001088 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003790972918756269, + "loss": 2.6053, + "theoretical_loss": 3.7186921399617248, + "tokens_seen": 823066624 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003790872617853561, + "loss": 2.7584, + "theoretical_loss": 3.7186629399016047, + "tokens_seen": 823132160 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003790772316950853, + "loss": 2.7113, + "theoretical_loss": 3.7186337428171354, + "tokens_seen": 823197696 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037906720160481446, + "loss": 2.6834, + "theoretical_loss": 3.7186045487077766, + "tokens_seen": 823263232 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037905717151454364, + "loss": 2.828, + "theoretical_loss": 3.7185753575729885, + "tokens_seen": 823328768 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003790471414242728, + "loss": 2.7322, + "theoretical_loss": 3.7185461694122313, + "tokens_seen": 823394304 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037903711133400206, + "loss": 2.9045, + "theoretical_loss": 3.7185169842249652, + "tokens_seen": 823459840 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003790270812437312, + "loss": 2.6431, + "theoretical_loss": 3.7184878020106504, + "tokens_seen": 823525376 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003790170511534604, + "loss": 2.9183, + "theoretical_loss": 3.718458622768748, + "tokens_seen": 823590912 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037900702106318955, + "loss": 2.5405, + "theoretical_loss": 3.718429446498719, + "tokens_seen": 823656448 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003789969909729188, + "loss": 2.7086, + "theoretical_loss": 3.7184002732000234, + "tokens_seen": 823721984 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037898696088264796, + "loss": 2.8996, + "theoretical_loss": 3.7183711028721227, + "tokens_seen": 823787520 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037897693079237715, + "loss": 2.8497, + "theoretical_loss": 3.7183419355144784, + "tokens_seen": 823853056 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003789669007021063, + "loss": 2.7715, + "theoretical_loss": 3.718312771126551, + "tokens_seen": 823918592 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037895687061183556, + "loss": 2.8566, + "theoretical_loss": 3.718283609707803, + "tokens_seen": 823984128 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003789468405215647, + "loss": 2.7357, + "theoretical_loss": 3.718254451257695, + "tokens_seen": 824049664 + }, + { + "epoch": 0.25, + "objective/train/docs_used": 475209, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.867845296859741, + "objective/train/theoretical_loss": 3.7182252957756896, + "objective/train/tokens_used": 844575200, + "theoretical_loss": 3.7182252957756896, + "tokens_seen": 824115200 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003789368104312939, + "loss": 2.7451, + "theoretical_loss": 3.7182252957756896, + "tokens_seen": 824115200 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037892678034102305, + "loss": 2.6186, + "theoretical_loss": 3.718196143261248, + "tokens_seen": 824180736 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003789167502507523, + "loss": 2.8387, + "theoretical_loss": 3.718166993713832, + "tokens_seen": 824246272 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037890672016048147, + "loss": 2.9795, + "theoretical_loss": 3.7181378471329047, + "tokens_seen": 824311808 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037889669007021065, + "loss": 3.0422, + "theoretical_loss": 3.7181087035179283, + "tokens_seen": 824377344 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037888665997993983, + "loss": 2.6585, + "theoretical_loss": 3.718079562868364, + "tokens_seen": 824442880 + }, + { + "epoch": 0.25, + "learning_rate": 0.000378876629889669, + "loss": 2.9867, + "theoretical_loss": 3.7180504251836757, + "tokens_seen": 824508416 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003788665997993982, + "loss": 2.8445, + "theoretical_loss": 3.7180212904633256, + "tokens_seen": 824573952 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037885656970912743, + "loss": 2.8434, + "theoretical_loss": 3.7179921587067764, + "tokens_seen": 824639488 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037884653961885655, + "loss": 2.8683, + "theoretical_loss": 3.7179630299134914, + "tokens_seen": 824705024 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003788365095285858, + "loss": 2.9044, + "theoretical_loss": 3.7179339040829333, + "tokens_seen": 824770560 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003788264794383149, + "loss": 2.8239, + "theoretical_loss": 3.7179047812145662, + "tokens_seen": 824836096 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037881644934804415, + "loss": 2.734, + "theoretical_loss": 3.7178756613078527, + "tokens_seen": 824901632 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037880641925777333, + "loss": 2.8099, + "theoretical_loss": 3.717846544362257, + "tokens_seen": 824967168 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003787963891675025, + "loss": 2.7108, + "theoretical_loss": 3.717817430377242, + "tokens_seen": 825032704 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003787863590772317, + "loss": 2.7949, + "theoretical_loss": 3.7177883193522723, + "tokens_seen": 825098240 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037877632898696093, + "loss": 2.7448, + "theoretical_loss": 3.717759211286811, + "tokens_seen": 825163776 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037876629889669006, + "loss": 2.724, + "theoretical_loss": 3.7177301061803236, + "tokens_seen": 825229312 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003787562688064193, + "loss": 2.7376, + "theoretical_loss": 3.717701004032273, + "tokens_seen": 825294848 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003787462387161484, + "loss": 2.6008, + "theoretical_loss": 3.7176719048421245, + "tokens_seen": 825360384 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037873620862587766, + "loss": 2.9699, + "theoretical_loss": 3.717642808609342, + "tokens_seen": 825425920 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037872617853560684, + "loss": 2.7785, + "theoretical_loss": 3.717613715333391, + "tokens_seen": 825491456 + }, + { + "epoch": 0.25, + "learning_rate": 0.000378716148445336, + "loss": 2.6975, + "theoretical_loss": 3.7175846250137345, + "tokens_seen": 825556992 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003787061183550652, + "loss": 2.753, + "theoretical_loss": 3.7175555376498397, + "tokens_seen": 825622528 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003786960882647944, + "loss": 2.6361, + "theoretical_loss": 3.7175264532411703, + "tokens_seen": 825688064 + }, + { + "epoch": 0.25, + "objective/train/docs_used": 475848, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.252037286758423, + "objective/train/theoretical_loss": 3.717497371787192, + "objective/train/tokens_used": 846213600, + "theoretical_loss": 3.717497371787192, + "tokens_seen": 825753600 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037868605817452356, + "loss": 2.8955, + "theoretical_loss": 3.717497371787192, + "tokens_seen": 825753600 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003786760280842528, + "loss": 2.7124, + "theoretical_loss": 3.7174682932873706, + "tokens_seen": 825819136 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003786659979939819, + "loss": 2.7698, + "theoretical_loss": 3.7174392177411706, + "tokens_seen": 825884672 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037865596790371116, + "loss": 2.9954, + "theoretical_loss": 3.717410145148058, + "tokens_seen": 825950208 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003786459378134403, + "loss": 2.9345, + "theoretical_loss": 3.717381075507499, + "tokens_seen": 826015744 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003786359077231695, + "loss": 2.721, + "theoretical_loss": 3.717352008818959, + "tokens_seen": 826081280 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003786258776328987, + "loss": 2.9506, + "theoretical_loss": 3.7173229450819045, + "tokens_seen": 826146816 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003786158475426279, + "loss": 2.7117, + "theoretical_loss": 3.7172938842958017, + "tokens_seen": 826212352 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037860581745235706, + "loss": 2.9471, + "theoretical_loss": 3.7172648264601165, + "tokens_seen": 826277888 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003785957873620863, + "loss": 2.4551, + "theoretical_loss": 3.7172357715743156, + "tokens_seen": 826343424 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003785857572718154, + "loss": 2.7674, + "theoretical_loss": 3.7172067196378658, + "tokens_seen": 826408960 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037857572718154466, + "loss": 2.7768, + "theoretical_loss": 3.7171776706502335, + "tokens_seen": 826474496 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003785656970912738, + "loss": 2.6606, + "theoretical_loss": 3.717148624610886, + "tokens_seen": 826540032 + }, + { + "epoch": 0.25, + "learning_rate": 0.000378555667001003, + "loss": 2.8131, + "theoretical_loss": 3.71711958151929, + "tokens_seen": 826605568 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003785456369107322, + "loss": 2.7155, + "theoretical_loss": 3.717090541374913, + "tokens_seen": 826671104 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003785356068204614, + "loss": 2.8148, + "theoretical_loss": 3.7170615041772224, + "tokens_seen": 826736640 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037852557673019057, + "loss": 2.9009, + "theoretical_loss": 3.7170324699256847, + "tokens_seen": 826802176 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037851554663991975, + "loss": 2.9417, + "theoretical_loss": 3.717003438619769, + "tokens_seen": 826867712 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037850551654964893, + "loss": 2.6606, + "theoretical_loss": 3.7169744102589415, + "tokens_seen": 826933248 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037849548645937816, + "loss": 2.7263, + "theoretical_loss": 3.7169453848426715, + "tokens_seen": 826998784 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003784854563691073, + "loss": 2.915, + "theoretical_loss": 3.7169163623704256, + "tokens_seen": 827064320 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003784754262788365, + "loss": 2.6336, + "theoretical_loss": 3.716887342841673, + "tokens_seen": 827129856 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037846539618856565, + "loss": 2.6644, + "theoretical_loss": 3.716858326255881, + "tokens_seen": 827195392 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003784553660982949, + "loss": 2.7585, + "theoretical_loss": 3.7168293126125196, + "tokens_seen": 827260928 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037844533600802407, + "loss": 2.6152, + "theoretical_loss": 3.7168003019110554, + "tokens_seen": 827326464 + }, + { + "epoch": 0.25, + "objective/train/docs_used": 477214, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7780086994171143, + "objective/train/theoretical_loss": 3.7167712941509587, + "objective/train/tokens_used": 847852000, + "theoretical_loss": 3.7167712941509587, + "tokens_seen": 827392000 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037843530591775325, + "loss": 2.8789, + "theoretical_loss": 3.7167712941509587, + "tokens_seen": 827392000 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037842527582748243, + "loss": 2.8497, + "theoretical_loss": 3.7167422893316977, + "tokens_seen": 827457536 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037841524573721167, + "loss": 2.9238, + "theoretical_loss": 3.716713287452741, + "tokens_seen": 827523072 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003784052156469408, + "loss": 2.9967, + "theoretical_loss": 3.7166842885135583, + "tokens_seen": 827588608 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037839518555667003, + "loss": 2.9686, + "theoretical_loss": 3.716655292513619, + "tokens_seen": 827654144 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037838515546639916, + "loss": 2.9128, + "theoretical_loss": 3.7166262994523915, + "tokens_seen": 827719680 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003783751253761284, + "loss": 2.9078, + "theoretical_loss": 3.7165973093293463, + "tokens_seen": 827785216 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037836509528585763, + "loss": 2.7404, + "theoretical_loss": 3.7165683221439525, + "tokens_seen": 827850752 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037835506519558675, + "loss": 2.904, + "theoretical_loss": 3.7165393378956804, + "tokens_seen": 827916288 + }, + { + "epoch": 0.25, + "learning_rate": 0.000378345035105316, + "loss": 2.9332, + "theoretical_loss": 3.716510356583999, + "tokens_seen": 827981824 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003783350050150451, + "loss": 2.6782, + "theoretical_loss": 3.71648137820838, + "tokens_seen": 828047360 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037832497492477435, + "loss": 3.0075, + "theoretical_loss": 3.716452402768292, + "tokens_seen": 828112896 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037831494483450353, + "loss": 2.8148, + "theoretical_loss": 3.716423430263206, + "tokens_seen": 828178432 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003783049147442327, + "loss": 2.865, + "theoretical_loss": 3.7163944606925927, + "tokens_seen": 828243968 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003782948846539619, + "loss": 2.8557, + "theoretical_loss": 3.716365494055922, + "tokens_seen": 828309504 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037828485456369113, + "loss": 2.7615, + "theoretical_loss": 3.716336530352666, + "tokens_seen": 828375040 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037827482447342026, + "loss": 2.7653, + "theoretical_loss": 3.716307569582294, + "tokens_seen": 828440576 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003782647943831495, + "loss": 2.9005, + "theoretical_loss": 3.716278611744278, + "tokens_seen": 828506112 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003782547642928786, + "loss": 2.7502, + "theoretical_loss": 3.7162496568380887, + "tokens_seen": 828571648 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037824473420260786, + "loss": 2.8899, + "theoretical_loss": 3.716220704863198, + "tokens_seen": 828637184 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037823470411233704, + "loss": 2.7639, + "theoretical_loss": 3.716191755819077, + "tokens_seen": 828702720 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003782246740220662, + "loss": 2.8715, + "theoretical_loss": 3.716162809705197, + "tokens_seen": 828768256 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003782146439317954, + "loss": 2.9205, + "theoretical_loss": 3.71613386652103, + "tokens_seen": 828833792 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003782046138415246, + "loss": 2.8994, + "theoretical_loss": 3.7161049262660484, + "tokens_seen": 828899328 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037819458375125376, + "loss": 2.9348, + "theoretical_loss": 3.7160759889397235, + "tokens_seen": 828964864 + }, + { + "epoch": 0.25, + "objective/train/docs_used": 478014, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.5984835624694824, + "objective/train/theoretical_loss": 3.7160470545415274, + "objective/train/tokens_used": 849490400, + "theoretical_loss": 3.7160470545415274, + "tokens_seen": 829030400 + }, + { + "epoch": 0.25, + "learning_rate": 0.000378184553660983, + "loss": 2.863, + "theoretical_loss": 3.7160470545415274, + "tokens_seen": 829030400 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003781745235707121, + "loss": 2.9671, + "theoretical_loss": 3.7160181230709326, + "tokens_seen": 829095936 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037816449348044136, + "loss": 2.6685, + "theoretical_loss": 3.7159891945274115, + "tokens_seen": 829161472 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003781544633901705, + "loss": 2.823, + "theoretical_loss": 3.715960268910437, + "tokens_seen": 829227008 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003781444332998997, + "loss": 2.3189, + "theoretical_loss": 3.715931346219481, + "tokens_seen": 829292544 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003781344032096289, + "loss": 2.8531, + "theoretical_loss": 3.715902426454017, + "tokens_seen": 829358080 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003781243731193581, + "loss": 2.872, + "theoretical_loss": 3.715873509613517, + "tokens_seen": 829423616 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037811434302908726, + "loss": 2.7485, + "theoretical_loss": 3.7158445956974555, + "tokens_seen": 829489152 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003781043129388165, + "loss": 2.5401, + "theoretical_loss": 3.715815684705304, + "tokens_seen": 829554688 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003780942828485456, + "loss": 2.5818, + "theoretical_loss": 3.715786776636538, + "tokens_seen": 829620224 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037808425275827486, + "loss": 2.909, + "theoretical_loss": 3.715757871490629, + "tokens_seen": 829685760 + }, + { + "epoch": 0.25, + "learning_rate": 0.000378074222668004, + "loss": 2.7356, + "theoretical_loss": 3.715728969267052, + "tokens_seen": 829751296 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003780641925777332, + "loss": 2.778, + "theoretical_loss": 3.71570006996528, + "tokens_seen": 829816832 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003780541624874624, + "loss": 2.8658, + "theoretical_loss": 3.715671173584787, + "tokens_seen": 829882368 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003780441323971916, + "loss": 2.8879, + "theoretical_loss": 3.7156422801250475, + "tokens_seen": 829947904 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037803410230692077, + "loss": 2.8451, + "theoretical_loss": 3.7156133895855357, + "tokens_seen": 830013440 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037802407221664995, + "loss": 2.8986, + "theoretical_loss": 3.7155845019657248, + "tokens_seen": 830078976 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037801404212637913, + "loss": 2.8178, + "theoretical_loss": 3.7155556172650903, + "tokens_seen": 830144512 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037800401203610836, + "loss": 2.8951, + "theoretical_loss": 3.715526735483107, + "tokens_seen": 830210048 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003779939819458375, + "loss": 2.8205, + "theoretical_loss": 3.7154978566192485, + "tokens_seen": 830275584 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037798395185556673, + "loss": 2.6757, + "theoretical_loss": 3.715468980672991, + "tokens_seen": 830341120 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037797392176529585, + "loss": 2.6971, + "theoretical_loss": 3.715440107643808, + "tokens_seen": 830406656 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003779638916750251, + "loss": 2.7445, + "theoretical_loss": 3.7154112375311765, + "tokens_seen": 830472192 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037795386158475427, + "loss": 2.7242, + "theoretical_loss": 3.71538237033457, + "tokens_seen": 830537728 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037794383149448345, + "loss": 2.9805, + "theoretical_loss": 3.7153535060534644, + "tokens_seen": 830603264 + }, + { + "epoch": 0.25, + "objective/train/docs_used": 479315, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.948397397994995, + "objective/train/theoretical_loss": 3.7153246446873363, + "objective/train/tokens_used": 851128800, + "theoretical_loss": 3.7153246446873363, + "tokens_seen": 830668800 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037793380140421263, + "loss": 2.83, + "theoretical_loss": 3.7153246446873363, + "tokens_seen": 830668800 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037792377131394187, + "loss": 2.7171, + "theoretical_loss": 3.71529578623566, + "tokens_seen": 830734336 + }, + { + "epoch": 0.25, + "learning_rate": 0.000377913741223671, + "loss": 2.6456, + "theoretical_loss": 3.715266930697912, + "tokens_seen": 830799872 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037790371113340023, + "loss": 2.7705, + "theoretical_loss": 3.715238078073568, + "tokens_seen": 830865408 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037789368104312936, + "loss": 2.746, + "theoretical_loss": 3.715209228362104, + "tokens_seen": 830930944 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003778836509528586, + "loss": 2.7864, + "theoretical_loss": 3.715180381562997, + "tokens_seen": 830996480 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003778736208625878, + "loss": 2.7519, + "theoretical_loss": 3.7151515376757223, + "tokens_seen": 831062016 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037786359077231695, + "loss": 2.6448, + "theoretical_loss": 3.7151226966997566, + "tokens_seen": 831127552 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037785356068204614, + "loss": 2.5626, + "theoretical_loss": 3.715093858634577, + "tokens_seen": 831193088 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003778435305917753, + "loss": 2.7188, + "theoretical_loss": 3.71506502347966, + "tokens_seen": 831258624 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003778335005015045, + "loss": 2.7343, + "theoretical_loss": 3.715036191234483, + "tokens_seen": 831324160 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037782347041123373, + "loss": 2.9133, + "theoretical_loss": 3.715007361898522, + "tokens_seen": 831389696 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037781344032096286, + "loss": 2.9304, + "theoretical_loss": 3.7149785354712552, + "tokens_seen": 831455232 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003778034102306921, + "loss": 2.7971, + "theoretical_loss": 3.7149497119521593, + "tokens_seen": 831520768 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003777933801404213, + "loss": 2.9132, + "theoretical_loss": 3.7149208913407117, + "tokens_seen": 831586304 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037778335005015046, + "loss": 2.7449, + "theoretical_loss": 3.71489207363639, + "tokens_seen": 831651840 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037777331995987964, + "loss": 2.6659, + "theoretical_loss": 3.714863258838672, + "tokens_seen": 831717376 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003777632898696088, + "loss": 2.8595, + "theoretical_loss": 3.7148344469470356, + "tokens_seen": 831782912 + }, + { + "epoch": 0.25, + "learning_rate": 0.000377753259779338, + "loss": 2.6014, + "theoretical_loss": 3.714805637960959, + "tokens_seen": 831848448 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037774322968906724, + "loss": 2.8584, + "theoretical_loss": 3.71477683187992, + "tokens_seen": 831913984 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037773319959879636, + "loss": 2.7756, + "theoretical_loss": 3.7147480287033967, + "tokens_seen": 831979520 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003777231695085256, + "loss": 2.8441, + "theoretical_loss": 3.7147192284308677, + "tokens_seen": 832045056 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003777131394182547, + "loss": 2.8047, + "theoretical_loss": 3.7146904310618116, + "tokens_seen": 832110592 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037770310932798396, + "loss": 2.6909, + "theoretical_loss": 3.7146616365957073, + "tokens_seen": 832176128 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037769307923771314, + "loss": 2.8779, + "theoretical_loss": 3.7146328450320327, + "tokens_seen": 832241664 + }, + { + "epoch": 0.25, + "objective/train/docs_used": 479893, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.72455096244812, + "objective/train/theoretical_loss": 3.714604056370267, + "objective/train/tokens_used": 852767200, + "theoretical_loss": 3.714604056370267, + "tokens_seen": 832307200 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003776830491474423, + "loss": 2.5358, + "theoretical_loss": 3.714604056370267, + "tokens_seen": 832307200 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003776730190571715, + "loss": 2.8052, + "theoretical_loss": 3.71457527060989, + "tokens_seen": 832372736 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003776629889669007, + "loss": 2.7639, + "theoretical_loss": 3.7145464877503804, + "tokens_seen": 832438272 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037765295887662987, + "loss": 2.5208, + "theoretical_loss": 3.7145177077912175, + "tokens_seen": 832503808 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003776429287863591, + "loss": 2.7992, + "theoretical_loss": 3.714488930731881, + "tokens_seen": 832569344 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037763289869608823, + "loss": 2.6932, + "theoretical_loss": 3.7144601565718496, + "tokens_seen": 832634880 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037762286860581746, + "loss": 2.8102, + "theoretical_loss": 3.714431385310604, + "tokens_seen": 832700416 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003776128385155467, + "loss": 2.4143, + "theoretical_loss": 3.7144026169476243, + "tokens_seen": 832765952 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003776028084252758, + "loss": 2.6744, + "theoretical_loss": 3.714373851482389, + "tokens_seen": 832831488 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037759277833500506, + "loss": 2.9008, + "theoretical_loss": 3.7143450889143796, + "tokens_seen": 832897024 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003775827482447342, + "loss": 2.7262, + "theoretical_loss": 3.7143163292430765, + "tokens_seen": 832962560 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003775727181544634, + "loss": 2.7464, + "theoretical_loss": 3.714287572467959, + "tokens_seen": 833028096 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003775626880641926, + "loss": 2.6984, + "theoretical_loss": 3.714258818588508, + "tokens_seen": 833093632 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003775526579739218, + "loss": 2.8371, + "theoretical_loss": 3.7142300676042046, + "tokens_seen": 833159168 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037754262788365097, + "loss": 2.7608, + "theoretical_loss": 3.7142013195145287, + "tokens_seen": 833224704 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037753259779338015, + "loss": 2.5257, + "theoretical_loss": 3.7141725743189626, + "tokens_seen": 833290240 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037752256770310933, + "loss": 2.5048, + "theoretical_loss": 3.714143832016987, + "tokens_seen": 833355776 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037751253761283857, + "loss": 2.9318, + "theoretical_loss": 3.7141150926080817, + "tokens_seen": 833421312 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003775025075225677, + "loss": 2.6175, + "theoretical_loss": 3.7140863560917294, + "tokens_seen": 833486848 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037749247743229693, + "loss": 2.6619, + "theoretical_loss": 3.7140576224674113, + "tokens_seen": 833552384 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037748244734202605, + "loss": 2.684, + "theoretical_loss": 3.714028891734609, + "tokens_seen": 833617920 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003774724172517553, + "loss": 2.6805, + "theoretical_loss": 3.714000163892804, + "tokens_seen": 833683456 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037746238716148447, + "loss": 2.5642, + "theoretical_loss": 3.7139714389414786, + "tokens_seen": 833748992 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037745235707121365, + "loss": 2.7728, + "theoretical_loss": 3.7139427168801147, + "tokens_seen": 833814528 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037744232698094283, + "loss": 2.5656, + "theoretical_loss": 3.713913997708194, + "tokens_seen": 833880064 + }, + { + "epoch": 0.25, + "objective/train/docs_used": 480416, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.9075164794921875, + "objective/train/theoretical_loss": 3.713885281425199, + "objective/train/tokens_used": 854405600, + "theoretical_loss": 3.713885281425199, + "tokens_seen": 833945600 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037743229689067207, + "loss": 2.7669, + "theoretical_loss": 3.713885281425199, + "tokens_seen": 833945600 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003774222668004012, + "loss": 3.0215, + "theoretical_loss": 3.7138565680306117, + "tokens_seen": 834011136 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037741223671013043, + "loss": 2.8353, + "theoretical_loss": 3.7138278575239156, + "tokens_seen": 834076672 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037740220661985956, + "loss": 2.7592, + "theoretical_loss": 3.7137991499045926, + "tokens_seen": 834142208 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003773921765295888, + "loss": 2.6786, + "theoretical_loss": 3.713770445172126, + "tokens_seen": 834207744 + }, + { + "epoch": 0.25, + "learning_rate": 0.000377382146439318, + "loss": 2.7909, + "theoretical_loss": 3.7137417433259987, + "tokens_seen": 834273280 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037737211634904715, + "loss": 2.7859, + "theoretical_loss": 3.7137130443656936, + "tokens_seen": 834338816 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037736208625877634, + "loss": 2.7291, + "theoretical_loss": 3.713684348290693, + "tokens_seen": 834404352 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003773520561685055, + "loss": 2.5042, + "theoretical_loss": 3.713655655100482, + "tokens_seen": 834469888 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003773420260782347, + "loss": 2.5504, + "theoretical_loss": 3.713626964794543, + "tokens_seen": 834535424 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037733199598796393, + "loss": 2.7253, + "theoretical_loss": 3.7135982773723595, + "tokens_seen": 834600960 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037732196589769306, + "loss": 2.6987, + "theoretical_loss": 3.713569592833416, + "tokens_seen": 834666496 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003773119358074223, + "loss": 2.7279, + "theoretical_loss": 3.713540911177195, + "tokens_seen": 834732032 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003773019057171515, + "loss": 2.5244, + "theoretical_loss": 3.713512232403182, + "tokens_seen": 834797568 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037729187562688066, + "loss": 2.8636, + "theoretical_loss": 3.7134835565108606, + "tokens_seen": 834863104 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037728184553660984, + "loss": 2.4281, + "theoretical_loss": 3.7134548834997148, + "tokens_seen": 834928640 + }, + { + "epoch": 0.25, + "learning_rate": 0.000377271815446339, + "loss": 2.952, + "theoretical_loss": 3.713426213369229, + "tokens_seen": 834994176 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003772617853560682, + "loss": 2.7504, + "theoretical_loss": 3.7133975461188884, + "tokens_seen": 835059712 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037725175526579744, + "loss": 2.7077, + "theoretical_loss": 3.7133688817481767, + "tokens_seen": 835125248 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037724172517552656, + "loss": 2.8029, + "theoretical_loss": 3.7133402202565793, + "tokens_seen": 835190784 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003772316950852558, + "loss": 2.9033, + "theoretical_loss": 3.713311561643581, + "tokens_seen": 835256320 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003772216649949849, + "loss": 2.7576, + "theoretical_loss": 3.7132829059086667, + "tokens_seen": 835321856 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037721163490471416, + "loss": 2.7458, + "theoretical_loss": 3.713254253051322, + "tokens_seen": 835387392 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037720160481444334, + "loss": 2.6747, + "theoretical_loss": 3.7132256030710318, + "tokens_seen": 835452928 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003771915747241725, + "loss": 2.7307, + "theoretical_loss": 3.713196955967282, + "tokens_seen": 835518464 + }, + { + "epoch": 0.25, + "objective/train/docs_used": 481156, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6389381885528564, + "objective/train/theoretical_loss": 3.713168311739558, + "objective/train/tokens_used": 856044000, + "theoretical_loss": 3.713168311739558, + "tokens_seen": 835584000 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003771815446339017, + "loss": 2.6815, + "theoretical_loss": 3.713168311739558, + "tokens_seen": 835584000 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003771715145436309, + "loss": 2.7607, + "theoretical_loss": 3.7131396703873456, + "tokens_seen": 835649536 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037716148445336007, + "loss": 2.9889, + "theoretical_loss": 3.71311103191013, + "tokens_seen": 835715072 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003771514543630893, + "loss": 2.8039, + "theoretical_loss": 3.713082396307398, + "tokens_seen": 835780608 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037714142427281843, + "loss": 2.5874, + "theoretical_loss": 3.7130537635786354, + "tokens_seen": 835846144 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037713139418254766, + "loss": 2.7142, + "theoretical_loss": 3.7130251337233284, + "tokens_seen": 835911680 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037712136409227685, + "loss": 2.5693, + "theoretical_loss": 3.7129965067409634, + "tokens_seen": 835977216 + }, + { + "epoch": 0.25, + "learning_rate": 0.000377111334002006, + "loss": 2.5506, + "theoretical_loss": 3.712967882631027, + "tokens_seen": 836042752 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003771013039117352, + "loss": 2.9731, + "theoretical_loss": 3.7129392613930063, + "tokens_seen": 836108288 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003770912738214644, + "loss": 2.6701, + "theoretical_loss": 3.712910643026387, + "tokens_seen": 836173824 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037708124373119357, + "loss": 2.826, + "theoretical_loss": 3.712882027530657, + "tokens_seen": 836239360 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003770712136409228, + "loss": 2.8721, + "theoretical_loss": 3.712853414905303, + "tokens_seen": 836304896 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037706118355065193, + "loss": 2.7509, + "theoretical_loss": 3.712824805149812, + "tokens_seen": 836370432 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037705115346038117, + "loss": 2.8738, + "theoretical_loss": 3.7127961982636712, + "tokens_seen": 836435968 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003770411233701103, + "loss": 2.8609, + "theoretical_loss": 3.712767594246369, + "tokens_seen": 836501504 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037703109327983953, + "loss": 2.958, + "theoretical_loss": 3.7127389930973917, + "tokens_seen": 836567040 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003770210631895687, + "loss": 2.814, + "theoretical_loss": 3.712710394816228, + "tokens_seen": 836632576 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003770110330992979, + "loss": 2.6435, + "theoretical_loss": 3.7126817994023655, + "tokens_seen": 836698112 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003770010030090271, + "loss": 2.711, + "theoretical_loss": 3.7126532068552915, + "tokens_seen": 836763648 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037699097291875625, + "loss": 2.6342, + "theoretical_loss": 3.7126246171744945, + "tokens_seen": 836829184 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037698094282848544, + "loss": 2.6623, + "theoretical_loss": 3.712596030359463, + "tokens_seen": 836894720 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037697091273821467, + "loss": 2.5908, + "theoretical_loss": 3.712567446409685, + "tokens_seen": 836960256 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003769608826479438, + "loss": 2.6082, + "theoretical_loss": 3.7125388653246496, + "tokens_seen": 837025792 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037695085255767303, + "loss": 2.7685, + "theoretical_loss": 3.7125102871038447, + "tokens_seen": 837091328 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003769408224674022, + "loss": 2.7819, + "theoretical_loss": 3.7124817117467597, + "tokens_seen": 837156864 + }, + { + "epoch": 0.25, + "objective/train/docs_used": 482509, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.589763879776001, + "objective/train/theoretical_loss": 3.712453139252883, + "objective/train/tokens_used": 857682400, + "theoretical_loss": 3.712453139252883, + "tokens_seen": 837222400 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003769307923771314, + "loss": 2.8144, + "theoretical_loss": 3.712453139252883, + "tokens_seen": 837222400 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003769207622868606, + "loss": 2.7271, + "theoretical_loss": 3.712424569621703, + "tokens_seen": 837287936 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037691073219658976, + "loss": 2.9557, + "theoretical_loss": 3.712396002852711, + "tokens_seen": 837353472 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037690070210631894, + "loss": 2.7258, + "theoretical_loss": 3.712367438945394, + "tokens_seen": 837419008 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003768906720160482, + "loss": 2.9476, + "theoretical_loss": 3.712338877899242, + "tokens_seen": 837484544 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003768806419257773, + "loss": 2.5781, + "theoretical_loss": 3.7123103197137457, + "tokens_seen": 837550080 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037687061183550654, + "loss": 3.016, + "theoretical_loss": 3.7122817643883934, + "tokens_seen": 837615616 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003768605817452357, + "loss": 2.6656, + "theoretical_loss": 3.7122532119226754, + "tokens_seen": 837681152 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003768505516549649, + "loss": 2.7251, + "theoretical_loss": 3.712224662316082, + "tokens_seen": 837746688 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037684052156469413, + "loss": 2.9027, + "theoretical_loss": 3.7121961155681027, + "tokens_seen": 837812224 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037683049147442326, + "loss": 2.6199, + "theoretical_loss": 3.712167571678228, + "tokens_seen": 837877760 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003768204613841525, + "loss": 2.556, + "theoretical_loss": 3.712139030645948, + "tokens_seen": 837943296 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003768104312938817, + "loss": 2.7624, + "theoretical_loss": 3.7121104924707535, + "tokens_seen": 838008832 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037680040120361086, + "loss": 2.9084, + "theoretical_loss": 3.712081957152135, + "tokens_seen": 838074368 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037679037111334004, + "loss": 2.7672, + "theoretical_loss": 3.712053424689583, + "tokens_seen": 838139904 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003767803410230692, + "loss": 2.6437, + "theoretical_loss": 3.712024895082588, + "tokens_seen": 838205440 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003767703109327984, + "loss": 2.6487, + "theoretical_loss": 3.711996368330642, + "tokens_seen": 838270976 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037676028084252764, + "loss": 2.8424, + "theoretical_loss": 3.7119678444332354, + "tokens_seen": 838336512 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037675025075225676, + "loss": 2.8112, + "theoretical_loss": 3.71193932338986, + "tokens_seen": 838402048 + }, + { + "epoch": 0.25, + "learning_rate": 0.000376740220661986, + "loss": 2.6406, + "theoretical_loss": 3.711910805200006, + "tokens_seen": 838467584 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003767301905717151, + "loss": 2.7997, + "theoretical_loss": 3.7118822898631665, + "tokens_seen": 838533120 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037672016048144436, + "loss": 2.8544, + "theoretical_loss": 3.711853777378832, + "tokens_seen": 838598656 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037671013039117354, + "loss": 2.8369, + "theoretical_loss": 3.7118252677464945, + "tokens_seen": 838664192 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003767001003009027, + "loss": 2.8148, + "theoretical_loss": 3.7117967609656457, + "tokens_seen": 838729728 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003766900702106319, + "loss": 2.8279, + "theoretical_loss": 3.7117682570357786, + "tokens_seen": 838795264 + }, + { + "epoch": 0.25, + "objective/train/docs_used": 483094, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.937314510345459, + "objective/train/theoretical_loss": 3.7117397559563843, + "objective/train/tokens_used": 859320800, + "theoretical_loss": 3.7117397559563843, + "tokens_seen": 838860800 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003766800401203611, + "loss": 2.7754, + "theoretical_loss": 3.7117397559563843, + "tokens_seen": 838860800 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037667001003009027, + "loss": 2.7252, + "theoretical_loss": 3.7117112577269555, + "tokens_seen": 838926336 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003766599799398195, + "loss": 2.5698, + "theoretical_loss": 3.7116827623469844, + "tokens_seen": 838991872 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037664994984954863, + "loss": 2.6802, + "theoretical_loss": 3.7116542698159636, + "tokens_seen": 839057408 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037663991975927786, + "loss": 2.9108, + "theoretical_loss": 3.711625780133386, + "tokens_seen": 839122944 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037662988966900705, + "loss": 2.8595, + "theoretical_loss": 3.711597293298744, + "tokens_seen": 839188480 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003766198595787362, + "loss": 2.2739, + "theoretical_loss": 3.7115688093115313, + "tokens_seen": 839254016 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003766098294884654, + "loss": 2.7229, + "theoretical_loss": 3.7115403281712407, + "tokens_seen": 839319552 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003765997993981946, + "loss": 3.009, + "theoretical_loss": 3.7115118498773647, + "tokens_seen": 839385088 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037658976930792377, + "loss": 2.8501, + "theoretical_loss": 3.711483374429397, + "tokens_seen": 839450624 + }, + { + "epoch": 0.25, + "learning_rate": 0.000376579739217653, + "loss": 2.8848, + "theoretical_loss": 3.7114549018268317, + "tokens_seen": 839516160 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037656970912738213, + "loss": 2.6962, + "theoretical_loss": 3.7114264320691612, + "tokens_seen": 839581696 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037655967903711137, + "loss": 2.7418, + "theoretical_loss": 3.7113979651558804, + "tokens_seen": 839647232 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003765496489468405, + "loss": 2.7751, + "theoretical_loss": 3.7113695010864824, + "tokens_seen": 839712768 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037653961885656973, + "loss": 2.7137, + "theoretical_loss": 3.7113410398604616, + "tokens_seen": 839778304 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003765295887662989, + "loss": 2.6322, + "theoretical_loss": 3.711312581477311, + "tokens_seen": 839843840 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003765195586760281, + "loss": 2.5883, + "theoretical_loss": 3.7112841259365266, + "tokens_seen": 839909376 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003765095285857573, + "loss": 2.8788, + "theoretical_loss": 3.7112556732376016, + "tokens_seen": 839974912 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037649949849548645, + "loss": 2.8393, + "theoretical_loss": 3.711227223380031, + "tokens_seen": 840040448 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037648946840521564, + "loss": 2.6418, + "theoretical_loss": 3.7111987763633083, + "tokens_seen": 840105984 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037647943831494487, + "loss": 2.6834, + "theoretical_loss": 3.71117033218693, + "tokens_seen": 840171520 + }, + { + "epoch": 0.25, + "learning_rate": 0.000376469408224674, + "loss": 2.8044, + "theoretical_loss": 3.71114189085039, + "tokens_seen": 840237056 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037645937813440323, + "loss": 2.7565, + "theoretical_loss": 3.7111134523531835, + "tokens_seen": 840302592 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003764493480441324, + "loss": 2.9472, + "theoretical_loss": 3.711085016694805, + "tokens_seen": 840368128 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003764393179538616, + "loss": 3.0445, + "theoretical_loss": 3.7110565838747505, + "tokens_seen": 840433664 + }, + { + "epoch": 0.25, + "objective/train/docs_used": 483926, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6501119136810303, + "objective/train/theoretical_loss": 3.7110281538925154, + "objective/train/tokens_used": 860959200, + "theoretical_loss": 3.7110281538925154, + "tokens_seen": 840499200 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003764292878635908, + "loss": 2.8262, + "theoretical_loss": 3.7110281538925154, + "tokens_seen": 840499200 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037641925777331996, + "loss": 2.8583, + "theoretical_loss": 3.7109997267475947, + "tokens_seen": 840564736 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037640922768304914, + "loss": 2.8843, + "theoretical_loss": 3.710971302439485, + "tokens_seen": 840630272 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003763991975927784, + "loss": 2.9312, + "theoretical_loss": 3.710942880967681, + "tokens_seen": 840695808 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003763891675025075, + "loss": 2.7536, + "theoretical_loss": 3.710914462331679, + "tokens_seen": 840761344 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037637913741223674, + "loss": 2.6136, + "theoretical_loss": 3.710886046530975, + "tokens_seen": 840826880 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037636910732196586, + "loss": 2.7608, + "theoretical_loss": 3.710857633565066, + "tokens_seen": 840892416 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003763590772316951, + "loss": 2.6974, + "theoretical_loss": 3.7108292234334472, + "tokens_seen": 840957952 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003763490471414243, + "loss": 3.0106, + "theoretical_loss": 3.710800816135615, + "tokens_seen": 841023488 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037633901705115346, + "loss": 2.7758, + "theoretical_loss": 3.7107724116710665, + "tokens_seen": 841089024 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037632898696088264, + "loss": 2.6683, + "theoretical_loss": 3.7107440100392988, + "tokens_seen": 841154560 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003763189568706119, + "loss": 2.5873, + "theoretical_loss": 3.7107156112398076, + "tokens_seen": 841220096 + }, + { + "epoch": 0.25, + "learning_rate": 0.000376308926780341, + "loss": 2.554, + "theoretical_loss": 3.710687215272091, + "tokens_seen": 841285632 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037629889669007024, + "loss": 2.8508, + "theoretical_loss": 3.7106588221356445, + "tokens_seen": 841351168 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037628886659979937, + "loss": 2.8778, + "theoretical_loss": 3.7106304318299674, + "tokens_seen": 841416704 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003762788365095286, + "loss": 2.5122, + "theoretical_loss": 3.7106020443545553, + "tokens_seen": 841482240 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003762688064192578, + "loss": 2.7504, + "theoretical_loss": 3.7105736597089063, + "tokens_seen": 841547776 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037625877632898696, + "loss": 2.8645, + "theoretical_loss": 3.7105452778925176, + "tokens_seen": 841613312 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037624874623871615, + "loss": 2.9699, + "theoretical_loss": 3.7105168989048876, + "tokens_seen": 841678848 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003762387161484453, + "loss": 2.9787, + "theoretical_loss": 3.7104885227455138, + "tokens_seen": 841744384 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003762286860581745, + "loss": 2.8408, + "theoretical_loss": 3.710460149413894, + "tokens_seen": 841809920 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037621865596790374, + "loss": 2.9447, + "theoretical_loss": 3.7104317789095265, + "tokens_seen": 841875456 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037620862587763287, + "loss": 2.7839, + "theoretical_loss": 3.7104034112319093, + "tokens_seen": 841940992 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003761985957873621, + "loss": 2.684, + "theoretical_loss": 3.710375046380541, + "tokens_seen": 842006528 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037618856569709123, + "loss": 2.9564, + "theoretical_loss": 3.7103466843549207, + "tokens_seen": 842072064 + }, + { + "epoch": 0.26, + "objective/train/docs_used": 484283, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.63257098197937, + "objective/train/theoretical_loss": 3.710318325154545, + "objective/train/tokens_used": 862597600, + "theoretical_loss": 3.710318325154545, + "tokens_seen": 842137600 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037617853560682047, + "loss": 2.7945, + "theoretical_loss": 3.710318325154545, + "tokens_seen": 842137600 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037616850551654965, + "loss": 2.9286, + "theoretical_loss": 3.7102899687789153, + "tokens_seen": 842203136 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037615847542627883, + "loss": 2.8154, + "theoretical_loss": 3.710261615227528, + "tokens_seen": 842268672 + }, + { + "epoch": 0.26, + "learning_rate": 0.000376148445336008, + "loss": 2.7707, + "theoretical_loss": 3.710233264499884, + "tokens_seen": 842334208 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037613841524573725, + "loss": 2.8861, + "theoretical_loss": 3.7102049165954814, + "tokens_seen": 842399744 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037612838515546637, + "loss": 2.6791, + "theoretical_loss": 3.710176571513819, + "tokens_seen": 842465280 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003761183550651956, + "loss": 2.7677, + "theoretical_loss": 3.710148229254398, + "tokens_seen": 842530816 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003761083249749248, + "loss": 2.9692, + "theoretical_loss": 3.710119889816716, + "tokens_seen": 842596352 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037609829488465397, + "loss": 2.7547, + "theoretical_loss": 3.7100915532002734, + "tokens_seen": 842661888 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003760882647943832, + "loss": 2.7988, + "theoretical_loss": 3.71006321940457, + "tokens_seen": 842727424 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037607823470411233, + "loss": 2.492, + "theoretical_loss": 3.7100348884291057, + "tokens_seen": 842792960 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037606820461384157, + "loss": 2.6905, + "theoretical_loss": 3.7100065602733805, + "tokens_seen": 842858496 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003760581745235707, + "loss": 2.9726, + "theoretical_loss": 3.7099782349368944, + "tokens_seen": 842924032 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037604814443329993, + "loss": 3.0046, + "theoretical_loss": 3.7099499124191473, + "tokens_seen": 842989568 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003760381143430291, + "loss": 2.6587, + "theoretical_loss": 3.709921592719641, + "tokens_seen": 843055104 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003760280842527583, + "loss": 2.8433, + "theoretical_loss": 3.7098932758378744, + "tokens_seen": 843120640 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003760180541624875, + "loss": 2.9229, + "theoretical_loss": 3.7098649617733486, + "tokens_seen": 843186176 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037600802407221665, + "loss": 2.8573, + "theoretical_loss": 3.709836650525565, + "tokens_seen": 843251712 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037599799398194584, + "loss": 2.5957, + "theoretical_loss": 3.709808342094024, + "tokens_seen": 843317248 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037598796389167507, + "loss": 2.8011, + "theoretical_loss": 3.709780036478227, + "tokens_seen": 843382784 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003759779338014042, + "loss": 2.7544, + "theoretical_loss": 3.7097517336776744, + "tokens_seen": 843448320 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037596790371113343, + "loss": 2.777, + "theoretical_loss": 3.7097234336918685, + "tokens_seen": 843513856 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003759578736208626, + "loss": 2.6848, + "theoretical_loss": 3.7096951365203097, + "tokens_seen": 843579392 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003759478435305918, + "loss": 2.7687, + "theoretical_loss": 3.7096668421625, + "tokens_seen": 843644928 + }, + { + "epoch": 0.26, + "learning_rate": 0.000375937813440321, + "loss": 2.7165, + "theoretical_loss": 3.7096385506179415, + "tokens_seen": 843710464 + }, + { + "epoch": 0.26, + "objective/train/docs_used": 485567, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.034116268157959, + "objective/train/theoretical_loss": 3.7096102618861355, + "objective/train/tokens_used": 864236000, + "theoretical_loss": 3.7096102618861355, + "tokens_seen": 843776000 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037592778335005016, + "loss": 2.7523, + "theoretical_loss": 3.7096102618861355, + "tokens_seen": 843776000 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037591775325977934, + "loss": 2.9924, + "theoretical_loss": 3.709581975966584, + "tokens_seen": 843841536 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003759077231695086, + "loss": 2.8846, + "theoretical_loss": 3.709553692858789, + "tokens_seen": 843907072 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003758976930792377, + "loss": 2.7054, + "theoretical_loss": 3.709525412562253, + "tokens_seen": 843972608 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037588766298896694, + "loss": 2.7483, + "theoretical_loss": 3.709497135076478, + "tokens_seen": 844038144 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037587763289869606, + "loss": 2.8091, + "theoretical_loss": 3.709468860400966, + "tokens_seen": 844103680 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003758676028084253, + "loss": 2.7727, + "theoretical_loss": 3.7094405885352204, + "tokens_seen": 844169216 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003758575727181545, + "loss": 2.9192, + "theoretical_loss": 3.7094123194787434, + "tokens_seen": 844234752 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037584754262788366, + "loss": 2.9508, + "theoretical_loss": 3.709384053231038, + "tokens_seen": 844300288 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037583751253761284, + "loss": 2.8175, + "theoretical_loss": 3.7093557897916067, + "tokens_seen": 844365824 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003758274824473421, + "loss": 2.9061, + "theoretical_loss": 3.7093275291599532, + "tokens_seen": 844431360 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003758174523570712, + "loss": 2.827, + "theoretical_loss": 3.7092992713355804, + "tokens_seen": 844496896 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037580742226680044, + "loss": 2.6839, + "theoretical_loss": 3.7092710163179916, + "tokens_seen": 844562432 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037579739217652957, + "loss": 2.9399, + "theoretical_loss": 3.7092427641066905, + "tokens_seen": 844627968 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003757873620862588, + "loss": 2.8295, + "theoretical_loss": 3.7092145147011806, + "tokens_seen": 844693504 + }, + { + "epoch": 0.26, + "learning_rate": 0.000375777331995988, + "loss": 2.8256, + "theoretical_loss": 3.709186268100965, + "tokens_seen": 844759040 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037576730190571716, + "loss": 2.7098, + "theoretical_loss": 3.7091580243055478, + "tokens_seen": 844824576 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037575727181544635, + "loss": 2.8911, + "theoretical_loss": 3.709129783314433, + "tokens_seen": 844890112 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003757472417251755, + "loss": 2.7197, + "theoretical_loss": 3.709101545127125, + "tokens_seen": 844955648 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003757372116349047, + "loss": 2.7875, + "theoretical_loss": 3.709073309743128, + "tokens_seen": 845021184 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037572718154463394, + "loss": 2.9257, + "theoretical_loss": 3.709045077161946, + "tokens_seen": 845086720 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037571715145436307, + "loss": 2.9092, + "theoretical_loss": 3.7090168473830833, + "tokens_seen": 845152256 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003757071213640923, + "loss": 2.8247, + "theoretical_loss": 3.708988620406045, + "tokens_seen": 845217792 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037569709127382143, + "loss": 2.8192, + "theoretical_loss": 3.708960396230335, + "tokens_seen": 845283328 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037568706118355067, + "loss": 2.8663, + "theoretical_loss": 3.7089321748554593, + "tokens_seen": 845348864 + }, + { + "epoch": 0.26, + "objective/train/docs_used": 486205, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.5688910484313965, + "objective/train/theoretical_loss": 3.7089039562809223, + "objective/train/tokens_used": 865874400, + "theoretical_loss": 3.7089039562809223, + "tokens_seen": 845414400 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037567703109327985, + "loss": 2.9665, + "theoretical_loss": 3.7089039562809223, + "tokens_seen": 845414400 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037566700100300903, + "loss": 2.8285, + "theoretical_loss": 3.7088757405062283, + "tokens_seen": 845479936 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003756569709127382, + "loss": 2.8734, + "theoretical_loss": 3.7088475275308834, + "tokens_seen": 845545472 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037564694082246745, + "loss": 2.8305, + "theoretical_loss": 3.708819317354393, + "tokens_seen": 845611008 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003756369107321966, + "loss": 2.6505, + "theoretical_loss": 3.708791109976262, + "tokens_seen": 845676544 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003756268806419258, + "loss": 2.8463, + "theoretical_loss": 3.7087629053959965, + "tokens_seen": 845742080 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037561685055165494, + "loss": 2.6872, + "theoretical_loss": 3.7087347036131018, + "tokens_seen": 845807616 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037560682046138417, + "loss": 2.8028, + "theoretical_loss": 3.7087065046270844, + "tokens_seen": 845873152 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037559679037111335, + "loss": 2.8813, + "theoretical_loss": 3.708678308437449, + "tokens_seen": 845938688 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037558676028084253, + "loss": 2.5661, + "theoretical_loss": 3.708650115043703, + "tokens_seen": 846004224 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003755767301905717, + "loss": 2.7154, + "theoretical_loss": 3.7086219244453518, + "tokens_seen": 846069760 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003755667001003009, + "loss": 2.6549, + "theoretical_loss": 3.7085937366419017, + "tokens_seen": 846135296 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003755566700100301, + "loss": 2.8352, + "theoretical_loss": 3.7085655516328604, + "tokens_seen": 846200832 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003755466399197593, + "loss": 2.6337, + "theoretical_loss": 3.7085373694177326, + "tokens_seen": 846266368 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037553660982948844, + "loss": 2.6274, + "theoretical_loss": 3.7085091899960263, + "tokens_seen": 846331904 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003755265797392177, + "loss": 2.8273, + "theoretical_loss": 3.7084810133672486, + "tokens_seen": 846397440 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003755165496489468, + "loss": 2.8605, + "theoretical_loss": 3.708452839530905, + "tokens_seen": 846462976 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037550651955867604, + "loss": 2.7203, + "theoretical_loss": 3.7084246684865043, + "tokens_seen": 846528512 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003754964894684052, + "loss": 3.0279, + "theoretical_loss": 3.708396500233552, + "tokens_seen": 846594048 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003754864593781344, + "loss": 2.6735, + "theoretical_loss": 3.708368334771557, + "tokens_seen": 846659584 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003754764292878636, + "loss": 2.9077, + "theoretical_loss": 3.7083401721000255, + "tokens_seen": 846725120 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003754663991975928, + "loss": 2.7853, + "theoretical_loss": 3.708312012218466, + "tokens_seen": 846790656 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037545636910732194, + "loss": 2.9915, + "theoretical_loss": 3.708283855126386, + "tokens_seen": 846856192 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003754463390170512, + "loss": 2.8206, + "theoretical_loss": 3.708255700823293, + "tokens_seen": 846921728 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003754363089267803, + "loss": 3.1542, + "theoretical_loss": 3.7082275493086954, + "tokens_seen": 846987264 + }, + { + "epoch": 0.26, + "objective/train/docs_used": 487507, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.239950656890869, + "objective/train/theoretical_loss": 3.708199400582101, + "objective/train/tokens_used": 867512800, + "theoretical_loss": 3.708199400582101, + "tokens_seen": 847052800 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037542627883650954, + "loss": 2.5364, + "theoretical_loss": 3.708199400582101, + "tokens_seen": 847052800 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003754162487462387, + "loss": 2.6937, + "theoretical_loss": 3.708171254643018, + "tokens_seen": 847118336 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003754062186559679, + "loss": 2.9156, + "theoretical_loss": 3.708143111490955, + "tokens_seen": 847183872 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003753961885656971, + "loss": 2.7701, + "theoretical_loss": 3.70811497112542, + "tokens_seen": 847249408 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037538615847542626, + "loss": 2.8338, + "theoretical_loss": 3.708086833545922, + "tokens_seen": 847314944 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037537612838515544, + "loss": 2.5574, + "theoretical_loss": 3.70805869875197, + "tokens_seen": 847380480 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003753660982948847, + "loss": 2.6441, + "theoretical_loss": 3.708030566743072, + "tokens_seen": 847446016 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037535606820461386, + "loss": 2.5009, + "theoretical_loss": 3.7080024375187373, + "tokens_seen": 847511552 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037534603811434304, + "loss": 2.671, + "theoretical_loss": 3.7079743110784755, + "tokens_seen": 847577088 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003753360080240723, + "loss": 2.7851, + "theoretical_loss": 3.707946187421795, + "tokens_seen": 847642624 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003753259779338014, + "loss": 2.8752, + "theoretical_loss": 3.707918066548206, + "tokens_seen": 847708160 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037531594784353064, + "loss": 2.9542, + "theoretical_loss": 3.707889948457217, + "tokens_seen": 847773696 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037530591775325977, + "loss": 2.8483, + "theoretical_loss": 3.7078618331483386, + "tokens_seen": 847839232 + }, + { + "epoch": 0.26, + "learning_rate": 0.000375295887662989, + "loss": 2.6418, + "theoretical_loss": 3.7078337206210796, + "tokens_seen": 847904768 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003752858575727182, + "loss": 2.8301, + "theoretical_loss": 3.70780561087495, + "tokens_seen": 847970304 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037527582748244736, + "loss": 2.6341, + "theoretical_loss": 3.7077775039094605, + "tokens_seen": 848035840 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037526579739217655, + "loss": 2.8033, + "theoretical_loss": 3.70774939972412, + "tokens_seen": 848101376 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003752557673019057, + "loss": 2.8091, + "theoretical_loss": 3.7077212983184404, + "tokens_seen": 848166912 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003752457372116349, + "loss": 2.7047, + "theoretical_loss": 3.70769319969193, + "tokens_seen": 848232448 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037523570712136414, + "loss": 2.7661, + "theoretical_loss": 3.707665103844101, + "tokens_seen": 848297984 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037522567703109327, + "loss": 2.7185, + "theoretical_loss": 3.7076370107744627, + "tokens_seen": 848363520 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003752156469408225, + "loss": 2.7279, + "theoretical_loss": 3.7076089204825267, + "tokens_seen": 848429056 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037520561685055163, + "loss": 2.8495, + "theoretical_loss": 3.7075808329678033, + "tokens_seen": 848494592 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037519558676028087, + "loss": 2.6283, + "theoretical_loss": 3.707552748229803, + "tokens_seen": 848560128 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037518555667001005, + "loss": 3.0296, + "theoretical_loss": 3.7075246662680383, + "tokens_seen": 848625664 + }, + { + "epoch": 0.26, + "objective/train/docs_used": 487917, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.736583948135376, + "objective/train/theoretical_loss": 3.7074965870820193, + "objective/train/tokens_used": 869151200, + "theoretical_loss": 3.7074965870820193, + "tokens_seen": 848691200 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037517552657973923, + "loss": 2.8361, + "theoretical_loss": 3.7074965870820193, + "tokens_seen": 848691200 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003751654964894684, + "loss": 2.8143, + "theoretical_loss": 3.7074685106712577, + "tokens_seen": 848756736 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037515546639919765, + "loss": 2.9672, + "theoretical_loss": 3.7074404370352645, + "tokens_seen": 848822272 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003751454363089268, + "loss": 2.7714, + "theoretical_loss": 3.707412366173552, + "tokens_seen": 848887808 + }, + { + "epoch": 0.26, + "learning_rate": 0.000375135406218656, + "loss": 3.0198, + "theoretical_loss": 3.707384298085631, + "tokens_seen": 848953344 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037512537612838514, + "loss": 2.9873, + "theoretical_loss": 3.7073562327710143, + "tokens_seen": 849018880 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037511534603811437, + "loss": 2.7627, + "theoretical_loss": 3.7073281702292133, + "tokens_seen": 849084416 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037510531594784355, + "loss": 2.7514, + "theoretical_loss": 3.70730011045974, + "tokens_seen": 849149952 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037509528585757273, + "loss": 2.7946, + "theoretical_loss": 3.707272053462107, + "tokens_seen": 849215488 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003750852557673019, + "loss": 2.8556, + "theoretical_loss": 3.7072439992358257, + "tokens_seen": 849281024 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003750752256770311, + "loss": 2.938, + "theoretical_loss": 3.7072159477804094, + "tokens_seen": 849346560 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003750651955867603, + "loss": 2.7911, + "theoretical_loss": 3.707187899095371, + "tokens_seen": 849412096 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003750551654964895, + "loss": 2.7059, + "theoretical_loss": 3.707159853180222, + "tokens_seen": 849477632 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037504513540621864, + "loss": 2.8188, + "theoretical_loss": 3.7071318100344754, + "tokens_seen": 849543168 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003750351053159479, + "loss": 2.8715, + "theoretical_loss": 3.707103769657645, + "tokens_seen": 849608704 + }, + { + "epoch": 0.26, + "learning_rate": 0.000375025075225677, + "loss": 2.7367, + "theoretical_loss": 3.707075732049244, + "tokens_seen": 849674240 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037501504513540624, + "loss": 2.7322, + "theoretical_loss": 3.7070476972087842, + "tokens_seen": 849739776 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003750050150451354, + "loss": 2.8684, + "theoretical_loss": 3.7070196651357796, + "tokens_seen": 849805312 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003749949849548646, + "loss": 3.0278, + "theoretical_loss": 3.706991635829744, + "tokens_seen": 849870848 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003749849548645938, + "loss": 2.7415, + "theoretical_loss": 3.70696360929019, + "tokens_seen": 849936384 + }, + { + "epoch": 0.26, + "learning_rate": 0.000374974924774323, + "loss": 2.9185, + "theoretical_loss": 3.706935585516632, + "tokens_seen": 850001920 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037496489468405214, + "loss": 2.9055, + "theoretical_loss": 3.7069075645085845, + "tokens_seen": 850067456 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003749548645937814, + "loss": 2.9717, + "theoretical_loss": 3.7068795462655597, + "tokens_seen": 850132992 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003749448345035105, + "loss": 2.7955, + "theoretical_loss": 3.7068515307870733, + "tokens_seen": 850198528 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037493480441323974, + "loss": 2.7515, + "theoretical_loss": 3.706823518072638, + "tokens_seen": 850264064 + }, + { + "epoch": 0.26, + "objective/train/docs_used": 489068, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.2560830116271973, + "objective/train/theoretical_loss": 3.706795508121769, + "objective/train/tokens_used": 870789600, + "theoretical_loss": 3.706795508121769, + "tokens_seen": 850329600 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003749247743229689, + "loss": 2.9076, + "theoretical_loss": 3.706795508121769, + "tokens_seen": 850329600 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003749147442326981, + "loss": 2.8592, + "theoretical_loss": 3.7067675009339807, + "tokens_seen": 850395136 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003749047141424273, + "loss": 2.8987, + "theoretical_loss": 3.7067394965087868, + "tokens_seen": 850460672 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037489468405215646, + "loss": 2.854, + "theoretical_loss": 3.7067114948457025, + "tokens_seen": 850526208 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037488465396188564, + "loss": 2.8154, + "theoretical_loss": 3.706683495944243, + "tokens_seen": 850591744 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003748746238716149, + "loss": 2.5316, + "theoretical_loss": 3.7066554998039227, + "tokens_seen": 850657280 + }, + { + "epoch": 0.26, + "learning_rate": 0.000374864593781344, + "loss": 2.9971, + "theoretical_loss": 3.7066275064242564, + "tokens_seen": 850722816 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037485456369107324, + "loss": 2.7268, + "theoretical_loss": 3.70659951580476, + "tokens_seen": 850788352 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037484453360080237, + "loss": 2.81, + "theoretical_loss": 3.706571527944948, + "tokens_seen": 850853888 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003748345035105316, + "loss": 3.0429, + "theoretical_loss": 3.7065435428443365, + "tokens_seen": 850919424 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003748244734202608, + "loss": 2.7975, + "theoretical_loss": 3.70651556050244, + "tokens_seen": 850984960 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037481444332998997, + "loss": 2.715, + "theoretical_loss": 3.706487580918775, + "tokens_seen": 851050496 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037480441323971915, + "loss": 2.8544, + "theoretical_loss": 3.7064596040928572, + "tokens_seen": 851116032 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003747943831494484, + "loss": 2.8276, + "theoretical_loss": 3.706431630024202, + "tokens_seen": 851181568 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003747843530591775, + "loss": 2.857, + "theoretical_loss": 3.7064036587123255, + "tokens_seen": 851247104 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037477432296890675, + "loss": 2.682, + "theoretical_loss": 3.7063756901567437, + "tokens_seen": 851312640 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037476429287863587, + "loss": 2.9653, + "theoretical_loss": 3.7063477243569736, + "tokens_seen": 851378176 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003747542627883651, + "loss": 2.7685, + "theoretical_loss": 3.706319761312531, + "tokens_seen": 851443712 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003747442326980943, + "loss": 2.8965, + "theoretical_loss": 3.7062918010229327, + "tokens_seen": 851509248 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037473420260782347, + "loss": 2.8369, + "theoretical_loss": 3.7062638434876947, + "tokens_seen": 851574784 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037472417251755265, + "loss": 2.8508, + "theoretical_loss": 3.706235888706334, + "tokens_seen": 851640320 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037471414242728183, + "loss": 2.8058, + "theoretical_loss": 3.706207936678368, + "tokens_seen": 851705856 + }, + { + "epoch": 0.26, + "learning_rate": 0.000374704112337011, + "loss": 2.8529, + "theoretical_loss": 3.7061799874033126, + "tokens_seen": 851771392 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037469408224674025, + "loss": 2.913, + "theoretical_loss": 3.706152040880686, + "tokens_seen": 851836928 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003746840521564694, + "loss": 2.8107, + "theoretical_loss": 3.7061240971100045, + "tokens_seen": 851902464 + }, + { + "epoch": 0.26, + "objective/train/docs_used": 489617, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.3579962253570557, + "objective/train/theoretical_loss": 3.7060961560907857, + "objective/train/tokens_used": 872428000, + "theoretical_loss": 3.7060961560907857, + "tokens_seen": 851968000 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003746740220661986, + "loss": 2.8388, + "theoretical_loss": 3.7060961560907857, + "tokens_seen": 851968000 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037466399197592774, + "loss": 2.7719, + "theoretical_loss": 3.7060682178225477, + "tokens_seen": 852033536 + }, + { + "epoch": 0.26, + "learning_rate": 0.000374653961885657, + "loss": 2.6374, + "theoretical_loss": 3.7060402823048073, + "tokens_seen": 852099072 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037464393179538615, + "loss": 2.8291, + "theoretical_loss": 3.7060123495370827, + "tokens_seen": 852164608 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037464393179538615, + "loss": 2.8391, + "theoretical_loss": 3.705984419518891, + "tokens_seen": 852230144 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037463390170511534, + "loss": 2.8275, + "theoretical_loss": 3.7059564922497517, + "tokens_seen": 852295680 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037462387161484457, + "loss": 2.7043, + "theoretical_loss": 3.7059285677291807, + "tokens_seen": 852361216 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037461384152457375, + "loss": 3.0243, + "theoretical_loss": 3.705900645956698, + "tokens_seen": 852426752 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037460381143430293, + "loss": 2.9705, + "theoretical_loss": 3.705872726931821, + "tokens_seen": 852492288 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003745937813440321, + "loss": 2.8295, + "theoretical_loss": 3.7058448106540682, + "tokens_seen": 852557824 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003745837512537613, + "loss": 2.8645, + "theoretical_loss": 3.7058168971229586, + "tokens_seen": 852623360 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003745737211634905, + "loss": 2.9133, + "theoretical_loss": 3.7057889863380105, + "tokens_seen": 852688896 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003745636910732197, + "loss": 2.8723, + "theoretical_loss": 3.7057610782987433, + "tokens_seen": 852754432 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037455366098294884, + "loss": 2.7858, + "theoretical_loss": 3.7057331730046745, + "tokens_seen": 852819968 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003745436308926781, + "loss": 2.6476, + "theoretical_loss": 3.705705270455325, + "tokens_seen": 852885504 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003745336008024072, + "loss": 2.8696, + "theoretical_loss": 3.7056773706502124, + "tokens_seen": 852951040 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037452357071213644, + "loss": 2.947, + "theoretical_loss": 3.7056494735888563, + "tokens_seen": 853016576 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003745135406218656, + "loss": 2.9173, + "theoretical_loss": 3.705621579270777, + "tokens_seen": 853082112 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003745035105315948, + "loss": 2.8841, + "theoretical_loss": 3.705593687695493, + "tokens_seen": 853147648 + }, + { + "epoch": 0.26, + "learning_rate": 0.000374493480441324, + "loss": 2.9694, + "theoretical_loss": 3.7055657988625246, + "tokens_seen": 853213184 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003744834503510532, + "loss": 2.6993, + "theoretical_loss": 3.705537912771391, + "tokens_seen": 853278720 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037447342026078234, + "loss": 2.9448, + "theoretical_loss": 3.7055100294216126, + "tokens_seen": 853344256 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003744633901705116, + "loss": 2.7496, + "theoretical_loss": 3.7054821488127097, + "tokens_seen": 853409792 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003744533600802407, + "loss": 2.9158, + "theoretical_loss": 3.705454270944201, + "tokens_seen": 853475328 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037444332998996994, + "loss": 2.74, + "theoretical_loss": 3.7054263958156084, + "tokens_seen": 853540864 + }, + { + "epoch": 0.26, + "objective/train/docs_used": 491051, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7283437252044678, + "objective/train/theoretical_loss": 3.7053985234264513, + "objective/train/tokens_used": 874066400, + "theoretical_loss": 3.7053985234264513, + "tokens_seen": 853606400 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003744332998996991, + "loss": 2.9607, + "theoretical_loss": 3.7053985234264513, + "tokens_seen": 853606400 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003744232698094283, + "loss": 2.834, + "theoretical_loss": 3.7053706537762503, + "tokens_seen": 853671936 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003744132397191575, + "loss": 2.9558, + "theoretical_loss": 3.7053427868645263, + "tokens_seen": 853737472 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037440320962888666, + "loss": 2.8922, + "theoretical_loss": 3.7053149226908, + "tokens_seen": 853803008 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037439317953861585, + "loss": 2.7847, + "theoretical_loss": 3.7052870612545927, + "tokens_seen": 853868544 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003743831494483451, + "loss": 2.8502, + "theoretical_loss": 3.705259202555424, + "tokens_seen": 853934080 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003743731193580742, + "loss": 2.8177, + "theoretical_loss": 3.705231346592816, + "tokens_seen": 853999616 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037436308926780344, + "loss": 2.7293, + "theoretical_loss": 3.70520349336629, + "tokens_seen": 854065152 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037435305917753257, + "loss": 2.6179, + "theoretical_loss": 3.705175642875367, + "tokens_seen": 854130688 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003743430290872618, + "loss": 2.4667, + "theoretical_loss": 3.7051477951195677, + "tokens_seen": 854196224 + }, + { + "epoch": 0.26, + "learning_rate": 0.000374332998996991, + "loss": 2.7943, + "theoretical_loss": 3.7051199500984158, + "tokens_seen": 854261760 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037432296890672017, + "loss": 2.8424, + "theoretical_loss": 3.705092107811431, + "tokens_seen": 854327296 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037431293881644935, + "loss": 3.0088, + "theoretical_loss": 3.705064268258136, + "tokens_seen": 854392832 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003743029087261786, + "loss": 2.9495, + "theoretical_loss": 3.705036431438052, + "tokens_seen": 854458368 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003742928786359077, + "loss": 2.831, + "theoretical_loss": 3.7050085973507025, + "tokens_seen": 854523904 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037428284854563695, + "loss": 2.9274, + "theoretical_loss": 3.704980765995608, + "tokens_seen": 854589440 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037427281845536607, + "loss": 2.7783, + "theoretical_loss": 3.7049529373722923, + "tokens_seen": 854654976 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003742627883650953, + "loss": 2.7983, + "theoretical_loss": 3.704925111480277, + "tokens_seen": 854720512 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003742527582748245, + "loss": 3.0592, + "theoretical_loss": 3.704897288319084, + "tokens_seen": 854786048 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037424272818455367, + "loss": 2.8542, + "theoretical_loss": 3.704869467888237, + "tokens_seen": 854851584 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037423269809428285, + "loss": 2.9677, + "theoretical_loss": 3.7048416501872588, + "tokens_seen": 854917120 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037422266800401203, + "loss": 2.9847, + "theoretical_loss": 3.7048138352156723, + "tokens_seen": 854982656 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003742126379137412, + "loss": 2.8619, + "theoretical_loss": 3.7047860229729994, + "tokens_seen": 855048192 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037420260782347045, + "loss": 2.8803, + "theoretical_loss": 3.7047582134587644, + "tokens_seen": 855113728 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003741925777331996, + "loss": 2.8714, + "theoretical_loss": 3.70473040667249, + "tokens_seen": 855179264 + }, + { + "epoch": 0.26, + "objective/train/docs_used": 491675, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.9956138134002686, + "objective/train/theoretical_loss": 3.7047026026137, + "objective/train/tokens_used": 875704800, + "theoretical_loss": 3.7047026026137, + "tokens_seen": 855244800 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003741825476429288, + "loss": 2.8065, + "theoretical_loss": 3.7047026026137, + "tokens_seen": 855244800 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037417251755265794, + "loss": 2.8809, + "theoretical_loss": 3.7046748012819175, + "tokens_seen": 855310336 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003741624874623872, + "loss": 2.7697, + "theoretical_loss": 3.7046470026766665, + "tokens_seen": 855375872 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037415245737211635, + "loss": 2.5825, + "theoretical_loss": 3.70461920679747, + "tokens_seen": 855441408 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037414242728184554, + "loss": 2.8265, + "theoretical_loss": 3.704591413643853, + "tokens_seen": 855506944 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003741323971915747, + "loss": 2.6858, + "theoretical_loss": 3.7045636232153383, + "tokens_seen": 855572480 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037412236710130395, + "loss": 2.6346, + "theoretical_loss": 3.704535835511451, + "tokens_seen": 855638016 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003741123370110331, + "loss": 2.7306, + "theoretical_loss": 3.7045080505317145, + "tokens_seen": 855703552 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003741023069207623, + "loss": 2.9207, + "theoretical_loss": 3.7044802682756535, + "tokens_seen": 855769088 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037409227683049144, + "loss": 2.8617, + "theoretical_loss": 3.7044524887427928, + "tokens_seen": 855834624 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003740822467402207, + "loss": 2.8432, + "theoretical_loss": 3.7044247119326568, + "tokens_seen": 855900160 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037407221664994986, + "loss": 2.6436, + "theoretical_loss": 3.7043969378447694, + "tokens_seen": 855965696 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037406218655967904, + "loss": 3.0327, + "theoretical_loss": 3.7043691664786564, + "tokens_seen": 856031232 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003740521564694082, + "loss": 2.7302, + "theoretical_loss": 3.7043413978338426, + "tokens_seen": 856096768 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003740421263791374, + "loss": 2.8374, + "theoretical_loss": 3.7043136319098524, + "tokens_seen": 856162304 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003740320962888666, + "loss": 2.7484, + "theoretical_loss": 3.7042858687062115, + "tokens_seen": 856227840 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003740220661985958, + "loss": 3.0545, + "theoretical_loss": 3.7042581082224446, + "tokens_seen": 856293376 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037401203610832494, + "loss": 2.8792, + "theoretical_loss": 3.7042303504580785, + "tokens_seen": 856358912 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003740020060180542, + "loss": 2.608, + "theoretical_loss": 3.704202595412637, + "tokens_seen": 856424448 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037399197592778336, + "loss": 2.9725, + "theoretical_loss": 3.7041748430856467, + "tokens_seen": 856489984 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037398194583751254, + "loss": 2.9319, + "theoretical_loss": 3.7041470934766334, + "tokens_seen": 856555520 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003739719157472417, + "loss": 2.7796, + "theoretical_loss": 3.7041193465851228, + "tokens_seen": 856621056 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003739618856569709, + "loss": 2.7348, + "theoretical_loss": 3.704091602410641, + "tokens_seen": 856686592 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003739518555667001, + "loss": 2.5723, + "theoretical_loss": 3.704063860952714, + "tokens_seen": 856752128 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003739418254764293, + "loss": 2.8593, + "theoretical_loss": 3.704036122210868, + "tokens_seen": 856817664 + }, + { + "epoch": 0.26, + "objective/train/docs_used": 492132, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.098113536834717, + "objective/train/theoretical_loss": 3.704008386184629, + "objective/train/tokens_used": 877343200, + "theoretical_loss": 3.704008386184629, + "tokens_seen": 856883200 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037393179538615845, + "loss": 2.8961, + "theoretical_loss": 3.704008386184629, + "tokens_seen": 856883200 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003739217652958877, + "loss": 2.9332, + "theoretical_loss": 3.7039806528735246, + "tokens_seen": 856948736 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003739117352056168, + "loss": 2.915, + "theoretical_loss": 3.7039529222770806, + "tokens_seen": 857014272 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037390170511534605, + "loss": 2.6133, + "theoretical_loss": 3.7039251943948237, + "tokens_seen": 857079808 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003738916750250752, + "loss": 2.9508, + "theoretical_loss": 3.703897469226281, + "tokens_seen": 857145344 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003738816449348044, + "loss": 2.8363, + "theoretical_loss": 3.7038697467709794, + "tokens_seen": 857210880 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037387161484453364, + "loss": 3.0791, + "theoretical_loss": 3.703842027028445, + "tokens_seen": 857276416 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037386158475426277, + "loss": 2.6076, + "theoretical_loss": 3.703814309998207, + "tokens_seen": 857341952 + }, + { + "epoch": 0.26, + "learning_rate": 0.000373851554663992, + "loss": 2.9578, + "theoretical_loss": 3.703786595679791, + "tokens_seen": 857407488 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003738415245737212, + "loss": 2.8467, + "theoretical_loss": 3.703758884072725, + "tokens_seen": 857473024 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037383149448345037, + "loss": 2.7648, + "theoretical_loss": 3.7037311751765367, + "tokens_seen": 857538560 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037382146439317955, + "loss": 2.9627, + "theoretical_loss": 3.7037034689907538, + "tokens_seen": 857604096 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003738114343029088, + "loss": 2.8932, + "theoretical_loss": 3.7036757655149044, + "tokens_seen": 857669632 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003738014042126379, + "loss": 2.7421, + "theoretical_loss": 3.7036480647485153, + "tokens_seen": 857735168 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037379137412236715, + "loss": 2.8282, + "theoretical_loss": 3.7036203666911147, + "tokens_seen": 857800704 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037378134403209627, + "loss": 2.9661, + "theoretical_loss": 3.703592671342232, + "tokens_seen": 857866240 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003737713139418255, + "loss": 2.7685, + "theoretical_loss": 3.703564978701394, + "tokens_seen": 857931776 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003737612838515547, + "loss": 3.0257, + "theoretical_loss": 3.70353728876813, + "tokens_seen": 857997312 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037375125376128387, + "loss": 2.7994, + "theoretical_loss": 3.703509601541968, + "tokens_seen": 858062848 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037374122367101305, + "loss": 2.7497, + "theoretical_loss": 3.703481917022437, + "tokens_seen": 858128384 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037373119358074223, + "loss": 2.9627, + "theoretical_loss": 3.703454235209066, + "tokens_seen": 858193920 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003737211634904714, + "loss": 2.8593, + "theoretical_loss": 3.7034265561013826, + "tokens_seen": 858259456 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037371113340020065, + "loss": 2.9183, + "theoretical_loss": 3.7033988796989172, + "tokens_seen": 858324992 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003737011033099298, + "loss": 2.9556, + "theoretical_loss": 3.703371206001198, + "tokens_seen": 858390528 + }, + { + "epoch": 0.26, + "learning_rate": 0.000373691073219659, + "loss": 2.8297, + "theoretical_loss": 3.703343535007754, + "tokens_seen": 858456064 + }, + { + "epoch": 0.26, + "objective/train/docs_used": 493247, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.112765312194824, + "objective/train/theoretical_loss": 3.7033158667181154, + "objective/train/tokens_used": 878981600, + "theoretical_loss": 3.7033158667181154, + "tokens_seen": 858521600 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037368104312938814, + "loss": 2.9878, + "theoretical_loss": 3.7033158667181154, + "tokens_seen": 858521600 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003736710130391174, + "loss": 2.7534, + "theoretical_loss": 3.7032882011318113, + "tokens_seen": 858587136 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037366098294884655, + "loss": 2.9326, + "theoretical_loss": 3.703260538248371, + "tokens_seen": 858652672 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037365095285857574, + "loss": 2.8341, + "theoretical_loss": 3.7032328780673245, + "tokens_seen": 858718208 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003736409227683049, + "loss": 2.9647, + "theoretical_loss": 3.7032052205882007, + "tokens_seen": 858783744 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037363089267803415, + "loss": 2.8486, + "theoretical_loss": 3.703177565810531, + "tokens_seen": 858849280 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003736208625877633, + "loss": 2.7936, + "theoretical_loss": 3.7031499137338444, + "tokens_seen": 858914816 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003736108324974925, + "loss": 2.8375, + "theoretical_loss": 3.7031222643576713, + "tokens_seen": 858980352 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037360080240722164, + "loss": 3.0489, + "theoretical_loss": 3.7030946176815425, + "tokens_seen": 859045888 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003735907723169509, + "loss": 2.9826, + "theoretical_loss": 3.703066973704987, + "tokens_seen": 859111424 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037358074222668006, + "loss": 2.7309, + "theoretical_loss": 3.7030393324275366, + "tokens_seen": 859176960 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037357071213640924, + "loss": 2.9133, + "theoretical_loss": 3.7030116938487208, + "tokens_seen": 859242496 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003735606820461384, + "loss": 2.5959, + "theoretical_loss": 3.702984057968072, + "tokens_seen": 859308032 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003735506519558676, + "loss": 2.6811, + "theoretical_loss": 3.702956424785119, + "tokens_seen": 859373568 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003735406218655968, + "loss": 2.7229, + "theoretical_loss": 3.7029287942993943, + "tokens_seen": 859439104 + }, + { + "epoch": 0.26, + "learning_rate": 0.000373530591775326, + "loss": 2.4896, + "theoretical_loss": 3.7029011665104283, + "tokens_seen": 859504640 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037352056168505514, + "loss": 2.9649, + "theoretical_loss": 3.7028735414177527, + "tokens_seen": 859570176 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003735105315947844, + "loss": 3.0232, + "theoretical_loss": 3.7028459190208984, + "tokens_seen": 859635712 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037350050150451356, + "loss": 2.8368, + "theoretical_loss": 3.7028182993193965, + "tokens_seen": 859701248 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037349047141424274, + "loss": 2.9919, + "theoretical_loss": 3.70279068231278, + "tokens_seen": 859766784 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003734804413239719, + "loss": 2.7665, + "theoretical_loss": 3.7027630680005785, + "tokens_seen": 859832320 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003734704112337011, + "loss": 3.0523, + "theoretical_loss": 3.702735456382325, + "tokens_seen": 859897856 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003734603811434303, + "loss": 2.8408, + "theoretical_loss": 3.7027078474575514, + "tokens_seen": 859963392 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003734503510531595, + "loss": 2.7355, + "theoretical_loss": 3.7026802412257895, + "tokens_seen": 860028928 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037344032096288865, + "loss": 2.7705, + "theoretical_loss": 3.7026526376865716, + "tokens_seen": 860094464 + }, + { + "epoch": 0.26, + "objective/train/docs_used": 493898, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.9468815326690674, + "objective/train/theoretical_loss": 3.7026250368394296, + "objective/train/tokens_used": 880620000, + "theoretical_loss": 3.7026250368394296, + "tokens_seen": 860160000 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003734302908726179, + "loss": 2.8218, + "theoretical_loss": 3.7026250368394296, + "tokens_seen": 860160000 + }, + { + "epoch": 0.26, + "learning_rate": 0.000373420260782347, + "loss": 2.8481, + "theoretical_loss": 3.7025974386838962, + "tokens_seen": 860225536 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037341023069207625, + "loss": 2.8674, + "theoretical_loss": 3.7025698432195044, + "tokens_seen": 860291072 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003734002006018054, + "loss": 2.5807, + "theoretical_loss": 3.7025422504457857, + "tokens_seen": 860356608 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003733901705115346, + "loss": 2.7884, + "theoretical_loss": 3.702514660362273, + "tokens_seen": 860422144 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003733801404212638, + "loss": 2.8059, + "theoretical_loss": 3.7024870729684998, + "tokens_seen": 860487680 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037337011033099297, + "loss": 2.7357, + "theoretical_loss": 3.7024594882639987, + "tokens_seen": 860553216 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037336008024072215, + "loss": 2.9344, + "theoretical_loss": 3.702431906248303, + "tokens_seen": 860618752 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003733500501504514, + "loss": 2.8927, + "theoretical_loss": 3.702404326920945, + "tokens_seen": 860684288 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003733400200601805, + "loss": 2.7569, + "theoretical_loss": 3.7023767502814593, + "tokens_seen": 860749824 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037332998996990975, + "loss": 2.6888, + "theoretical_loss": 3.7023491763293785, + "tokens_seen": 860815360 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037331995987963893, + "loss": 2.9474, + "theoretical_loss": 3.7023216050642365, + "tokens_seen": 860880896 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003733099297893681, + "loss": 3.0112, + "theoretical_loss": 3.7022940364855668, + "tokens_seen": 860946432 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003732998996990973, + "loss": 2.772, + "theoretical_loss": 3.702266470592903, + "tokens_seen": 861011968 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003732898696088265, + "loss": 2.9853, + "theoretical_loss": 3.7022389073857793, + "tokens_seen": 861077504 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037327983951855565, + "loss": 3.023, + "theoretical_loss": 3.70221134686373, + "tokens_seen": 861143040 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003732698094282849, + "loss": 2.8904, + "theoretical_loss": 3.7021837890262876, + "tokens_seen": 861208576 + }, + { + "epoch": 0.26, + "learning_rate": 0.000373259779338014, + "loss": 3.2002, + "theoretical_loss": 3.7021562338729885, + "tokens_seen": 861274112 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037324974924774325, + "loss": 2.8144, + "theoretical_loss": 3.702128681403366, + "tokens_seen": 861339648 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003732397191574724, + "loss": 2.9047, + "theoretical_loss": 3.702101131616954, + "tokens_seen": 861405184 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003732296890672016, + "loss": 2.6084, + "theoretical_loss": 3.702073584513288, + "tokens_seen": 861470720 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003732196589769308, + "loss": 2.6156, + "theoretical_loss": 3.7020460400919024, + "tokens_seen": 861536256 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037320962888666, + "loss": 2.8346, + "theoretical_loss": 3.702018498352332, + "tokens_seen": 861601792 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037319959879638916, + "loss": 2.583, + "theoretical_loss": 3.701990959294112, + "tokens_seen": 861667328 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037318956870611834, + "loss": 2.7866, + "theoretical_loss": 3.7019634229167773, + "tokens_seen": 861732864 + }, + { + "epoch": 0.26, + "objective/train/docs_used": 495406, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.9691171646118164, + "objective/train/theoretical_loss": 3.701935889219863, + "objective/train/tokens_used": 882258400, + "theoretical_loss": 3.701935889219863, + "tokens_seen": 861798400 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003731795386158475, + "loss": 2.8741, + "theoretical_loss": 3.701935889219863, + "tokens_seen": 861798400 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037316950852557675, + "loss": 2.6792, + "theoretical_loss": 3.7019083582029038, + "tokens_seen": 861863936 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003731594784353059, + "loss": 2.7861, + "theoretical_loss": 3.701880829865436, + "tokens_seen": 861929472 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003731494483450351, + "loss": 3.0307, + "theoretical_loss": 3.7018533042069945, + "tokens_seen": 861995008 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003731394182547643, + "loss": 2.5197, + "theoretical_loss": 3.7018257812271154, + "tokens_seen": 862060544 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003731293881644935, + "loss": 2.8458, + "theoretical_loss": 3.701798260925335, + "tokens_seen": 862126080 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003731193580742227, + "loss": 2.8067, + "theoretical_loss": 3.7017707433011875, + "tokens_seen": 862191616 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037310932798395184, + "loss": 2.7739, + "theoretical_loss": 3.7017432283542098, + "tokens_seen": 862257152 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003730992978936811, + "loss": 2.8473, + "theoretical_loss": 3.701715716083938, + "tokens_seen": 862322688 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037308926780341026, + "loss": 2.9643, + "theoretical_loss": 3.7016882064899086, + "tokens_seen": 862388224 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037307923771313944, + "loss": 2.8955, + "theoretical_loss": 3.701660699571657, + "tokens_seen": 862453760 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003730692076228686, + "loss": 2.7289, + "theoretical_loss": 3.701633195328721, + "tokens_seen": 862519296 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003730591775325978, + "loss": 2.647, + "theoretical_loss": 3.701605693760636, + "tokens_seen": 862584832 + }, + { + "epoch": 0.26, + "learning_rate": 0.000373049147442327, + "loss": 2.8633, + "theoretical_loss": 3.701578194866939, + "tokens_seen": 862650368 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003730391173520562, + "loss": 2.888, + "theoretical_loss": 3.701550698647167, + "tokens_seen": 862715904 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037302908726178534, + "loss": 2.8041, + "theoretical_loss": 3.701523205100856, + "tokens_seen": 862781440 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003730190571715146, + "loss": 2.9025, + "theoretical_loss": 3.7014957142275446, + "tokens_seen": 862846976 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037300902708124376, + "loss": 2.8147, + "theoretical_loss": 3.701468226026769, + "tokens_seen": 862912512 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037299899699097294, + "loss": 2.6801, + "theoretical_loss": 3.7014407404980663, + "tokens_seen": 862978048 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003729889669007021, + "loss": 2.7239, + "theoretical_loss": 3.7014132576409744, + "tokens_seen": 863043584 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003729789368104313, + "loss": 2.7872, + "theoretical_loss": 3.7013857774550303, + "tokens_seen": 863109120 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003729689067201605, + "loss": 2.8891, + "theoretical_loss": 3.7013582999397716, + "tokens_seen": 863174656 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003729588766298897, + "loss": 2.8552, + "theoretical_loss": 3.701330825094736, + "tokens_seen": 863240192 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037294884653961885, + "loss": 2.8673, + "theoretical_loss": 3.701303352919462, + "tokens_seen": 863305728 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003729388164493481, + "loss": 2.8706, + "theoretical_loss": 3.701275883413487, + "tokens_seen": 863371264 + }, + { + "epoch": 0.26, + "objective/train/docs_used": 496136, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.812168836593628, + "objective/train/theoretical_loss": 3.7012484165763486, + "objective/train/tokens_used": 883896800, + "theoretical_loss": 3.7012484165763486, + "tokens_seen": 863436800 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003729287863590772, + "loss": 2.7602, + "theoretical_loss": 3.7012484165763486, + "tokens_seen": 863436800 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037291875626880645, + "loss": 2.7689, + "theoretical_loss": 3.701220952407586, + "tokens_seen": 863502336 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003729087261785356, + "loss": 2.8315, + "theoretical_loss": 3.7011934909067365, + "tokens_seen": 863567872 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003728986960882648, + "loss": 2.8815, + "theoretical_loss": 3.7011660320733393, + "tokens_seen": 863633408 + }, + { + "epoch": 0.26, + "learning_rate": 0.000372888665997994, + "loss": 2.7016, + "theoretical_loss": 3.701138575906932, + "tokens_seen": 863698944 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037287863590772317, + "loss": 2.8726, + "theoretical_loss": 3.7011111224070543, + "tokens_seen": 863764480 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037286860581745235, + "loss": 2.7695, + "theoretical_loss": 3.701083671573244, + "tokens_seen": 863830016 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003728585757271816, + "loss": 2.7044, + "theoretical_loss": 3.7010562234050406, + "tokens_seen": 863895552 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003728485456369107, + "loss": 2.7632, + "theoretical_loss": 3.701028777901983, + "tokens_seen": 863961088 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037283851554663995, + "loss": 2.6853, + "theoretical_loss": 3.7010013350636095, + "tokens_seen": 864026624 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037282848545636913, + "loss": 2.8147, + "theoretical_loss": 3.7009738948894597, + "tokens_seen": 864092160 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003728184553660983, + "loss": 2.847, + "theoretical_loss": 3.7009464573790742, + "tokens_seen": 864157696 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003728084252758275, + "loss": 2.7875, + "theoretical_loss": 3.7009190225319903, + "tokens_seen": 864223232 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003727983951855567, + "loss": 3.0252, + "theoretical_loss": 3.7008915903477497, + "tokens_seen": 864288768 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037278836509528585, + "loss": 2.7908, + "theoretical_loss": 3.70086416082589, + "tokens_seen": 864354304 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003727783350050151, + "loss": 2.8075, + "theoretical_loss": 3.7008367339659523, + "tokens_seen": 864419840 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003727683049147442, + "loss": 2.6622, + "theoretical_loss": 3.700809309767476, + "tokens_seen": 864485376 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037275827482447345, + "loss": 2.8426, + "theoretical_loss": 3.7007818882300016, + "tokens_seen": 864550912 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003727482447342026, + "loss": 2.8616, + "theoretical_loss": 3.7007544693530683, + "tokens_seen": 864616448 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003727382146439318, + "loss": 2.7731, + "theoretical_loss": 3.7007270531362173, + "tokens_seen": 864681984 + }, + { + "epoch": 0.26, + "learning_rate": 0.000372728184553661, + "loss": 2.5837, + "theoretical_loss": 3.700699639578988, + "tokens_seen": 864747520 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003727181544633902, + "loss": 2.6354, + "theoretical_loss": 3.700672228680922, + "tokens_seen": 864813056 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037270812437311936, + "loss": 2.7553, + "theoretical_loss": 3.700644820441559, + "tokens_seen": 864878592 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037269809428284854, + "loss": 2.8721, + "theoretical_loss": 3.70061741486044, + "tokens_seen": 864944128 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003726880641925777, + "loss": 2.8739, + "theoretical_loss": 3.7005900119371056, + "tokens_seen": 865009664 + }, + { + "epoch": 0.26, + "objective/train/docs_used": 497267, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.783201217651367, + "objective/train/theoretical_loss": 3.7005626116710966, + "objective/train/tokens_used": 885535200, + "theoretical_loss": 3.7005626116710966, + "tokens_seen": 865075200 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037267803410230696, + "loss": 2.9654, + "theoretical_loss": 3.7005626116710966, + "tokens_seen": 865075200 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003726680040120361, + "loss": 2.7701, + "theoretical_loss": 3.700535214061955, + "tokens_seen": 865140736 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003726579739217653, + "loss": 2.8873, + "theoretical_loss": 3.7005078191092204, + "tokens_seen": 865206272 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003726479438314945, + "loss": 2.7537, + "theoretical_loss": 3.7004804268124354, + "tokens_seen": 865271808 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003726379137412237, + "loss": 2.9551, + "theoretical_loss": 3.700453037171141, + "tokens_seen": 865337344 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037262788365095286, + "loss": 2.7721, + "theoretical_loss": 3.700425650184878, + "tokens_seen": 865402880 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037261785356068204, + "loss": 2.8258, + "theoretical_loss": 3.7003982658531887, + "tokens_seen": 865468416 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003726078234704112, + "loss": 2.7859, + "theoretical_loss": 3.700370884175615, + "tokens_seen": 865533952 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037259779338014046, + "loss": 2.8092, + "theoretical_loss": 3.7003435051516975, + "tokens_seen": 865599488 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003725877632898696, + "loss": 2.8098, + "theoretical_loss": 3.7003161287809796, + "tokens_seen": 865665024 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003725777331995988, + "loss": 2.792, + "theoretical_loss": 3.700288755063003, + "tokens_seen": 865730560 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037256770310932795, + "loss": 2.7111, + "theoretical_loss": 3.7002613839973097, + "tokens_seen": 865796096 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003725576730190572, + "loss": 2.8779, + "theoretical_loss": 3.7002340155834412, + "tokens_seen": 865861632 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037254764292878636, + "loss": 2.7156, + "theoretical_loss": 3.7002066498209416, + "tokens_seen": 865927168 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037253761283851554, + "loss": 2.7595, + "theoretical_loss": 3.7001792867093517, + "tokens_seen": 865992704 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003725275827482447, + "loss": 2.7425, + "theoretical_loss": 3.700151926248215, + "tokens_seen": 866058240 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037251755265797396, + "loss": 3.0333, + "theoretical_loss": 3.700124568437074, + "tokens_seen": 866123776 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003725075225677031, + "loss": 2.8532, + "theoretical_loss": 3.7000972132754715, + "tokens_seen": 866189312 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003724974924774323, + "loss": 2.8958, + "theoretical_loss": 3.7000698607629507, + "tokens_seen": 866254848 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037248746238716145, + "loss": 2.9084, + "theoretical_loss": 3.7000425108990553, + "tokens_seen": 866320384 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003724774322968907, + "loss": 2.942, + "theoretical_loss": 3.700015163683327, + "tokens_seen": 866385920 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037246740220661987, + "loss": 2.8284, + "theoretical_loss": 3.69998781911531, + "tokens_seen": 866451456 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037245737211634905, + "loss": 2.6576, + "theoretical_loss": 3.6999604771945473, + "tokens_seen": 866516992 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037244734202607823, + "loss": 2.9581, + "theoretical_loss": 3.699933137920583, + "tokens_seen": 866582528 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003724373119358074, + "loss": 2.6941, + "theoretical_loss": 3.6999058012929606, + "tokens_seen": 866648064 + }, + { + "epoch": 0.26, + "objective/train/docs_used": 497986, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.763414144515991, + "objective/train/theoretical_loss": 3.6998784673112235, + "objective/train/tokens_used": 887173600, + "theoretical_loss": 3.6998784673112235, + "tokens_seen": 866713600 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003724272818455366, + "loss": 3.0337, + "theoretical_loss": 3.6998784673112235, + "tokens_seen": 866713600 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003724172517552658, + "loss": 2.7935, + "theoretical_loss": 3.6998511359749164, + "tokens_seen": 866779136 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037240722166499495, + "loss": 2.7057, + "theoretical_loss": 3.6998238072835816, + "tokens_seen": 866844672 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003723971915747242, + "loss": 2.8765, + "theoretical_loss": 3.699796481236765, + "tokens_seen": 866910208 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003723871614844533, + "loss": 2.875, + "theoretical_loss": 3.6997691578340097, + "tokens_seen": 866975744 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037237713139418255, + "loss": 2.9095, + "theoretical_loss": 3.6997418370748605, + "tokens_seen": 867041280 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003723671013039118, + "loss": 2.7935, + "theoretical_loss": 3.699714518958862, + "tokens_seen": 867106816 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003723570712136409, + "loss": 2.7506, + "theoretical_loss": 3.699687203485558, + "tokens_seen": 867172352 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037234704112337015, + "loss": 2.8592, + "theoretical_loss": 3.699659890654494, + "tokens_seen": 867237888 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037233701103309933, + "loss": 2.7621, + "theoretical_loss": 3.699632580465215, + "tokens_seen": 867303424 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003723269809428285, + "loss": 2.9497, + "theoretical_loss": 3.699605272917264, + "tokens_seen": 867368960 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003723169508525577, + "loss": 2.8824, + "theoretical_loss": 3.699577968010188, + "tokens_seen": 867434496 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003723069207622869, + "loss": 2.8687, + "theoretical_loss": 3.6995506657435313, + "tokens_seen": 867500032 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037229689067201605, + "loss": 2.9361, + "theoretical_loss": 3.699523366116839, + "tokens_seen": 867565568 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003722868605817453, + "loss": 2.7619, + "theoretical_loss": 3.6994960691296574, + "tokens_seen": 867631104 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003722768304914744, + "loss": 2.8569, + "theoretical_loss": 3.6994687747815305, + "tokens_seen": 867696640 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037226680040120365, + "loss": 2.6999, + "theoretical_loss": 3.699441483072005, + "tokens_seen": 867762176 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003722567703109328, + "loss": 2.7415, + "theoretical_loss": 3.6994141940006253, + "tokens_seen": 867827712 + }, + { + "epoch": 0.26, + "learning_rate": 0.000372246740220662, + "loss": 2.8466, + "theoretical_loss": 3.6993869075669386, + "tokens_seen": 867893248 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003722367101303912, + "loss": 3.0524, + "theoretical_loss": 3.6993596237704907, + "tokens_seen": 867958784 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003722266800401204, + "loss": 2.8413, + "theoretical_loss": 3.699332342610827, + "tokens_seen": 868024320 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037221664994984956, + "loss": 2.6778, + "theoretical_loss": 3.6993050640874934, + "tokens_seen": 868089856 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037220661985957874, + "loss": 2.7706, + "theoretical_loss": 3.6992777882000363, + "tokens_seen": 868155392 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003721965897693079, + "loss": 2.751, + "theoretical_loss": 3.6992505149480026, + "tokens_seen": 868220928 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037218655967903716, + "loss": 3.0361, + "theoretical_loss": 3.6992232443309385, + "tokens_seen": 868286464 + }, + { + "epoch": 0.26, + "objective/train/docs_used": 499247, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7614128589630127, + "objective/train/theoretical_loss": 3.69919597634839, + "objective/train/tokens_used": 888812000, + "theoretical_loss": 3.69919597634839, + "tokens_seen": 868352000 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003721765295887663, + "loss": 2.8363, + "theoretical_loss": 3.69919597634839, + "tokens_seen": 868352000 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003721664994984955, + "loss": 2.7163, + "theoretical_loss": 3.699168710999905, + "tokens_seen": 868417536 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003721564694082247, + "loss": 2.8723, + "theoretical_loss": 3.6991414482850296, + "tokens_seen": 868483072 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003721464393179539, + "loss": 3.0012, + "theoretical_loss": 3.69911418820331, + "tokens_seen": 868548608 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037213640922768306, + "loss": 2.7633, + "theoretical_loss": 3.6990869307542944, + "tokens_seen": 868614144 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037212637913741224, + "loss": 2.8347, + "theoretical_loss": 3.699059675937529, + "tokens_seen": 868679680 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003721163490471414, + "loss": 2.9251, + "theoretical_loss": 3.6990324237525622, + "tokens_seen": 868745216 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037210631895687066, + "loss": 2.8026, + "theoretical_loss": 3.6990051741989403, + "tokens_seen": 868810752 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003720962888665998, + "loss": 2.9692, + "theoretical_loss": 3.698977927276211, + "tokens_seen": 868876288 + }, + { + "epoch": 0.26, + "learning_rate": 0.000372086258776329, + "loss": 2.9805, + "theoretical_loss": 3.6989506829839227, + "tokens_seen": 868941824 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037207622868605815, + "loss": 2.7563, + "theoretical_loss": 3.6989234413216217, + "tokens_seen": 869007360 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003720661985957874, + "loss": 2.7176, + "theoretical_loss": 3.6988962022888563, + "tokens_seen": 869072896 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037205616850551656, + "loss": 2.8766, + "theoretical_loss": 3.698868965885175, + "tokens_seen": 869138432 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037204613841524575, + "loss": 2.7168, + "theoretical_loss": 3.6988417321101252, + "tokens_seen": 869203968 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003720361083249749, + "loss": 2.7755, + "theoretical_loss": 3.6988145009632563, + "tokens_seen": 869269504 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037202607823470416, + "loss": 2.9306, + "theoretical_loss": 3.6987872724441146, + "tokens_seen": 869335040 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003720160481444333, + "loss": 2.8736, + "theoretical_loss": 3.698760046552249, + "tokens_seen": 869400576 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003720060180541625, + "loss": 2.8208, + "theoretical_loss": 3.6987328232872096, + "tokens_seen": 869466112 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037199598796389165, + "loss": 2.7373, + "theoretical_loss": 3.698705602648543, + "tokens_seen": 869531648 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003719859578736209, + "loss": 2.5668, + "theoretical_loss": 3.698678384635799, + "tokens_seen": 869597184 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037197592778335007, + "loss": 2.7888, + "theoretical_loss": 3.698651169248526, + "tokens_seen": 869662720 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037196589769307925, + "loss": 3.1617, + "theoretical_loss": 3.698623956486273, + "tokens_seen": 869728256 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037195586760280843, + "loss": 2.9043, + "theoretical_loss": 3.6985967463485894, + "tokens_seen": 869793792 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003719458375125376, + "loss": 2.8108, + "theoretical_loss": 3.6985695388350233, + "tokens_seen": 869859328 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003719358074222668, + "loss": 2.9698, + "theoretical_loss": 3.698542333945125, + "tokens_seen": 869924864 + }, + { + "epoch": 0.26, + "objective/train/docs_used": 499921, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.9648022651672363, + "objective/train/theoretical_loss": 3.6985151316784437, + "objective/train/tokens_used": 890450400, + "theoretical_loss": 3.6985151316784437, + "tokens_seen": 869990400 + }, + { + "epoch": 0.26, + "learning_rate": 0.000371925777331996, + "loss": 2.906, + "theoretical_loss": 3.6985151316784437, + "tokens_seen": 869990400 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037191574724172515, + "loss": 2.8853, + "theoretical_loss": 3.6984879320345287, + "tokens_seen": 870055936 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003719057171514544, + "loss": 2.7727, + "theoretical_loss": 3.6984607350129295, + "tokens_seen": 870121472 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003718956870611835, + "loss": 2.6833, + "theoretical_loss": 3.6984335406131956, + "tokens_seen": 870187008 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037188565697091275, + "loss": 2.7478, + "theoretical_loss": 3.6984063488348777, + "tokens_seen": 870252544 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037187562688064193, + "loss": 2.925, + "theoretical_loss": 3.698379159677525, + "tokens_seen": 870318080 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003718655967903711, + "loss": 2.6439, + "theoretical_loss": 3.6983519731406878, + "tokens_seen": 870383616 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003718555667001003, + "loss": 2.7743, + "theoretical_loss": 3.6983247892239155, + "tokens_seen": 870449152 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037184553660982953, + "loss": 2.8373, + "theoretical_loss": 3.6982976079267598, + "tokens_seen": 870514688 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037183550651955866, + "loss": 2.7103, + "theoretical_loss": 3.69827042924877, + "tokens_seen": 870580224 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003718254764292879, + "loss": 2.8517, + "theoretical_loss": 3.6982432531894966, + "tokens_seen": 870645760 + }, + { + "epoch": 0.26, + "learning_rate": 0.000371815446339017, + "loss": 2.7498, + "theoretical_loss": 3.69821607974849, + "tokens_seen": 870711296 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037180541624874625, + "loss": 2.7331, + "theoretical_loss": 3.698188908925302, + "tokens_seen": 870776832 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037179538615847544, + "loss": 2.5632, + "theoretical_loss": 3.698161740719483, + "tokens_seen": 870842368 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003717853560682046, + "loss": 2.7987, + "theoretical_loss": 3.6981345751305827, + "tokens_seen": 870907904 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003717753259779338, + "loss": 2.884, + "theoretical_loss": 3.698107412158154, + "tokens_seen": 870973440 + }, + { + "epoch": 0.26, + "learning_rate": 0.000371765295887663, + "loss": 2.7593, + "theoretical_loss": 3.6980802518017466, + "tokens_seen": 871038976 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037175526579739216, + "loss": 2.7666, + "theoretical_loss": 3.6980530940609126, + "tokens_seen": 871104512 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003717452357071214, + "loss": 2.9345, + "theoretical_loss": 3.6980259389352033, + "tokens_seen": 871170048 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003717352056168505, + "loss": 2.7494, + "theoretical_loss": 3.6979987864241695, + "tokens_seen": 871235584 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037172517552657976, + "loss": 2.8881, + "theoretical_loss": 3.697971636527364, + "tokens_seen": 871301120 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003717151454363089, + "loss": 2.7955, + "theoretical_loss": 3.697944489244337, + "tokens_seen": 871366656 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003717051153460381, + "loss": 3.013, + "theoretical_loss": 3.697917344574641, + "tokens_seen": 871432192 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003716950852557673, + "loss": 2.9726, + "theoretical_loss": 3.6978902025178284, + "tokens_seen": 871497728 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003716850551654965, + "loss": 2.8281, + "theoretical_loss": 3.697863063073451, + "tokens_seen": 871563264 + }, + { + "epoch": 0.26, + "objective/train/docs_used": 501092, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.563028573989868, + "objective/train/theoretical_loss": 3.6978359262410603, + "objective/train/tokens_used": 892088800, + "theoretical_loss": 3.6978359262410603, + "tokens_seen": 871628800 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037167502507522566, + "loss": 2.7863, + "theoretical_loss": 3.6978359262410603, + "tokens_seen": 871628800 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003716649949849549, + "loss": 2.7729, + "theoretical_loss": 3.6978087920202087, + "tokens_seen": 871694336 + }, + { + "epoch": 0.26, + "learning_rate": 0.000371654964894684, + "loss": 2.9757, + "theoretical_loss": 3.6977816604104494, + "tokens_seen": 871759872 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037164493480441326, + "loss": 2.9726, + "theoretical_loss": 3.6977545314113343, + "tokens_seen": 871825408 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003716349047141424, + "loss": 3.0404, + "theoretical_loss": 3.697727405022416, + "tokens_seen": 871890944 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003716248746238716, + "loss": 2.714, + "theoretical_loss": 3.6977002812432476, + "tokens_seen": 871956480 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037161484453360086, + "loss": 2.737, + "theoretical_loss": 3.6976731600733808, + "tokens_seen": 872022016 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037160481444333, + "loss": 2.8957, + "theoretical_loss": 3.6976460415123693, + "tokens_seen": 872087552 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003715947843530592, + "loss": 2.9485, + "theoretical_loss": 3.6976189255597665, + "tokens_seen": 872153088 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037158475426278835, + "loss": 2.8853, + "theoretical_loss": 3.697591812215125, + "tokens_seen": 872218624 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003715747241725176, + "loss": 2.7241, + "theoretical_loss": 3.697564701477998, + "tokens_seen": 872284160 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037156469408224676, + "loss": 2.7491, + "theoretical_loss": 3.6975375933479393, + "tokens_seen": 872349696 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037155466399197595, + "loss": 2.7769, + "theoretical_loss": 3.6975104878245015, + "tokens_seen": 872415232 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003715446339017051, + "loss": 2.8209, + "theoretical_loss": 3.6974833849072395, + "tokens_seen": 872480768 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037153460381143436, + "loss": 3.1644, + "theoretical_loss": 3.6974562845957055, + "tokens_seen": 872546304 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003715245737211635, + "loss": 2.7953, + "theoretical_loss": 3.697429186889454, + "tokens_seen": 872611840 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003715145436308927, + "loss": 2.7351, + "theoretical_loss": 3.6974020917880397, + "tokens_seen": 872677376 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037150451354062185, + "loss": 2.7789, + "theoretical_loss": 3.697374999291015, + "tokens_seen": 872742912 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003714944834503511, + "loss": 2.5522, + "theoretical_loss": 3.6973479093979353, + "tokens_seen": 872808448 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037148445336008027, + "loss": 2.7093, + "theoretical_loss": 3.697320822108354, + "tokens_seen": 872873984 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037147442326980945, + "loss": 3.1133, + "theoretical_loss": 3.697293737421826, + "tokens_seen": 872939520 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037146439317953863, + "loss": 2.949, + "theoretical_loss": 3.697266655337906, + "tokens_seen": 873005056 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003714543630892678, + "loss": 2.9387, + "theoretical_loss": 3.6972395758561483, + "tokens_seen": 873070592 + }, + { + "epoch": 0.26, + "learning_rate": 0.000371444332998997, + "loss": 2.8648, + "theoretical_loss": 3.6972124989761066, + "tokens_seen": 873136128 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037143430290872623, + "loss": 2.8509, + "theoretical_loss": 3.697185424697337, + "tokens_seen": 873201664 + }, + { + "epoch": 0.26, + "objective/train/docs_used": 501627, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7185165882110596, + "objective/train/theoretical_loss": 3.6971583530193937, + "objective/train/tokens_used": 893727200, + "theoretical_loss": 3.6971583530193937, + "tokens_seen": 873267200 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037142427281845535, + "loss": 2.8085, + "theoretical_loss": 3.6971583530193937, + "tokens_seen": 873267200 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003714142427281846, + "loss": 2.783, + "theoretical_loss": 3.697131283941832, + "tokens_seen": 873332736 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003714042126379137, + "loss": 2.8057, + "theoretical_loss": 3.697104217464207, + "tokens_seen": 873398272 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037139418254764295, + "loss": 2.8369, + "theoretical_loss": 3.697077153586074, + "tokens_seen": 873463808 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037138415245737213, + "loss": 2.7862, + "theoretical_loss": 3.697050092306988, + "tokens_seen": 873529344 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003713741223671013, + "loss": 2.9526, + "theoretical_loss": 3.697023033626504, + "tokens_seen": 873594880 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003713640922768305, + "loss": 2.8106, + "theoretical_loss": 3.696995977544179, + "tokens_seen": 873660416 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037135406218655973, + "loss": 2.6615, + "theoretical_loss": 3.6969689240595676, + "tokens_seen": 873725952 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037134403209628886, + "loss": 2.9572, + "theoretical_loss": 3.6969418731722254, + "tokens_seen": 873791488 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003713340020060181, + "loss": 2.7489, + "theoretical_loss": 3.696914824881709, + "tokens_seen": 873857024 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003713239719157472, + "loss": 2.5914, + "theoretical_loss": 3.6968877791875743, + "tokens_seen": 873922560 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037131394182547645, + "loss": 2.9515, + "theoretical_loss": 3.6968607360893766, + "tokens_seen": 873988096 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037130391173520564, + "loss": 2.6652, + "theoretical_loss": 3.6968336955866725, + "tokens_seen": 874053632 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003712938816449348, + "loss": 2.6651, + "theoretical_loss": 3.6968066576790193, + "tokens_seen": 874119168 + }, + { + "epoch": 0.26, + "learning_rate": 0.000371283851554664, + "loss": 2.6886, + "theoretical_loss": 3.6967796223659724, + "tokens_seen": 874184704 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003712738214643932, + "loss": 2.7728, + "theoretical_loss": 3.6967525896470876, + "tokens_seen": 874250240 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037126379137412236, + "loss": 2.9061, + "theoretical_loss": 3.6967255595219233, + "tokens_seen": 874315776 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003712537612838516, + "loss": 2.8165, + "theoretical_loss": 3.6966985319900356, + "tokens_seen": 874381312 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003712437311935807, + "loss": 2.8578, + "theoretical_loss": 3.6966715070509806, + "tokens_seen": 874446848 + }, + { + "epoch": 0.27, + "learning_rate": 0.00037123370110330996, + "loss": 2.962, + "theoretical_loss": 3.696644484704316, + "tokens_seen": 874512384 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003712236710130391, + "loss": 2.7789, + "theoretical_loss": 3.6966174649495986, + "tokens_seen": 874577920 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003712136409227683, + "loss": 2.8779, + "theoretical_loss": 3.696590447786386, + "tokens_seen": 874643456 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003712036108324975, + "loss": 2.9332, + "theoretical_loss": 3.6965634332142345, + "tokens_seen": 874708992 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003711935807422267, + "loss": 2.9299, + "theoretical_loss": 3.696536421232703, + "tokens_seen": 874774528 + }, + { + "epoch": 0.27, + "learning_rate": 0.00037118355065195586, + "loss": 2.8413, + "theoretical_loss": 3.696509411841348, + "tokens_seen": 874840064 + }, + { + "epoch": 0.27, + "objective/train/docs_used": 502963, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.8603179454803467, + "objective/train/theoretical_loss": 3.6964824050397276, + "objective/train/tokens_used": 895365600, + "theoretical_loss": 3.6964824050397276, + "tokens_seen": 874905600 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003711735205616851, + "loss": 2.6602, + "theoretical_loss": 3.6964824050397276, + "tokens_seen": 874905600 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003711634904714142, + "loss": 2.8053, + "theoretical_loss": 3.696455400827398, + "tokens_seen": 874971136 + }, + { + "epoch": 0.27, + "learning_rate": 0.00037115346038114346, + "loss": 2.8541, + "theoretical_loss": 3.6964283992039197, + "tokens_seen": 875036672 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003711434302908726, + "loss": 2.5965, + "theoretical_loss": 3.6964014001688486, + "tokens_seen": 875102208 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003711334002006018, + "loss": 2.8251, + "theoretical_loss": 3.6963744037217436, + "tokens_seen": 875167744 + }, + { + "epoch": 0.27, + "learning_rate": 0.000371123370110331, + "loss": 2.6355, + "theoretical_loss": 3.6963474098621623, + "tokens_seen": 875233280 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003711133400200602, + "loss": 3.1402, + "theoretical_loss": 3.6963204185896643, + "tokens_seen": 875298816 + }, + { + "epoch": 0.27, + "learning_rate": 0.00037110330992978937, + "loss": 2.6303, + "theoretical_loss": 3.6962934299038057, + "tokens_seen": 875364352 + }, + { + "epoch": 0.27, + "learning_rate": 0.00037109327983951855, + "loss": 2.7982, + "theoretical_loss": 3.6962664438041473, + "tokens_seen": 875429888 + }, + { + "epoch": 0.27, + "learning_rate": 0.00037108324974924773, + "loss": 2.8101, + "theoretical_loss": 3.696239460290246, + "tokens_seen": 875495424 + }, + { + "epoch": 0.27, + "learning_rate": 0.00037107321965897696, + "loss": 2.6708, + "theoretical_loss": 3.6962124793616615, + "tokens_seen": 875560960 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003710631895687061, + "loss": 2.7798, + "theoretical_loss": 3.6961855010179527, + "tokens_seen": 875626496 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003710531594784353, + "loss": 2.8015, + "theoretical_loss": 3.696158525258678, + "tokens_seen": 875692032 + }, + { + "epoch": 0.27, + "learning_rate": 0.00037104312938816445, + "loss": 2.6974, + "theoretical_loss": 3.696131552083396, + "tokens_seen": 875757568 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003710330992978937, + "loss": 2.8251, + "theoretical_loss": 3.696104581491667, + "tokens_seen": 875823104 + }, + { + "epoch": 0.27, + "learning_rate": 0.00037102306920762287, + "loss": 2.8724, + "theoretical_loss": 3.696077613483049, + "tokens_seen": 875888640 + }, + { + "epoch": 0.27, + "learning_rate": 0.00037101303911735205, + "loss": 2.8984, + "theoretical_loss": 3.6960506480571027, + "tokens_seen": 875954176 + }, + { + "epoch": 0.27, + "learning_rate": 0.00037100300902708123, + "loss": 2.7193, + "theoretical_loss": 3.6960236852133868, + "tokens_seen": 876019712 + }, + { + "epoch": 0.27, + "learning_rate": 0.00037099297893681047, + "loss": 2.81, + "theoretical_loss": 3.6959967249514607, + "tokens_seen": 876085248 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003709829488465396, + "loss": 2.938, + "theoretical_loss": 3.6959697672708844, + "tokens_seen": 876150784 + }, + { + "epoch": 0.27, + "learning_rate": 0.00037097291875626883, + "loss": 2.777, + "theoretical_loss": 3.695942812171218, + "tokens_seen": 876216320 + }, + { + "epoch": 0.27, + "learning_rate": 0.00037096288866599796, + "loss": 2.7213, + "theoretical_loss": 3.6959158596520205, + "tokens_seen": 876281856 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003709528585757272, + "loss": 2.7665, + "theoretical_loss": 3.6958889097128527, + "tokens_seen": 876347392 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003709428284854564, + "loss": 2.6647, + "theoretical_loss": 3.6958619623532742, + "tokens_seen": 876412928 + }, + { + "epoch": 0.27, + "learning_rate": 0.00037093279839518555, + "loss": 2.8367, + "theoretical_loss": 3.6958350175728456, + "tokens_seen": 876478464 + }, + { + "epoch": 0.27, + "objective/train/docs_used": 503455, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.2828423976898193, + "objective/train/theoretical_loss": 3.695808075371127, + "objective/train/tokens_used": 897004000, + "theoretical_loss": 3.695808075371127, + "tokens_seen": 876544000 + }, + { + "epoch": 0.27, + "learning_rate": 0.00037092276830491474, + "loss": 2.8154, + "theoretical_loss": 3.695808075371127, + "tokens_seen": 876544000 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003709127382146439, + "loss": 2.5934, + "theoretical_loss": 3.695781135747679, + "tokens_seen": 876609536 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003709027081243731, + "loss": 2.6836, + "theoretical_loss": 3.695754198702062, + "tokens_seen": 876675072 + }, + { + "epoch": 0.27, + "learning_rate": 0.00037089267803410233, + "loss": 2.9929, + "theoretical_loss": 3.6957272642338372, + "tokens_seen": 876740608 + }, + { + "epoch": 0.27, + "learning_rate": 0.00037088264794383146, + "loss": 2.7955, + "theoretical_loss": 3.695700332342564, + "tokens_seen": 876806144 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003708726178535607, + "loss": 2.9093, + "theoretical_loss": 3.6956734030278047, + "tokens_seen": 876871680 + }, + { + "epoch": 0.27, + "learning_rate": 0.00037086258776328993, + "loss": 2.8079, + "theoretical_loss": 3.6956464762891192, + "tokens_seen": 876937216 + }, + { + "epoch": 0.27, + "learning_rate": 0.00037085255767301906, + "loss": 2.9845, + "theoretical_loss": 3.69561955212607, + "tokens_seen": 877002752 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003708425275827483, + "loss": 3.0518, + "theoretical_loss": 3.6955926305382167, + "tokens_seen": 877068288 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003708324974924774, + "loss": 2.7646, + "theoretical_loss": 3.6955657115251213, + "tokens_seen": 877133824 + }, + { + "epoch": 0.27, + "learning_rate": 0.00037082246740220665, + "loss": 2.6857, + "theoretical_loss": 3.695538795086345, + "tokens_seen": 877199360 + }, + { + "epoch": 0.27, + "learning_rate": 0.00037081243731193584, + "loss": 2.6975, + "theoretical_loss": 3.69551188122145, + "tokens_seen": 877264896 + }, + { + "epoch": 0.27, + "learning_rate": 0.000370802407221665, + "loss": 2.698, + "theoretical_loss": 3.695484969929997, + "tokens_seen": 877330432 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003707923771313942, + "loss": 2.8451, + "theoretical_loss": 3.695458061211548, + "tokens_seen": 877395968 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003707823470411234, + "loss": 2.7039, + "theoretical_loss": 3.6954311550656658, + "tokens_seen": 877461504 + }, + { + "epoch": 0.27, + "learning_rate": 0.00037077231695085256, + "loss": 2.8711, + "theoretical_loss": 3.6954042514919108, + "tokens_seen": 877527040 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003707622868605818, + "loss": 2.7042, + "theoretical_loss": 3.6953773504898457, + "tokens_seen": 877592576 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003707522567703109, + "loss": 2.8687, + "theoretical_loss": 3.6953504520590332, + "tokens_seen": 877658112 + }, + { + "epoch": 0.27, + "learning_rate": 0.00037074222668004016, + "loss": 2.8747, + "theoretical_loss": 3.695323556199035, + "tokens_seen": 877723648 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003707321965897693, + "loss": 2.9101, + "theoretical_loss": 3.6952966629094135, + "tokens_seen": 877789184 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003707221664994985, + "loss": 2.7926, + "theoretical_loss": 3.695269772189731, + "tokens_seen": 877854720 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003707121364092277, + "loss": 2.8982, + "theoretical_loss": 3.6952428840395504, + "tokens_seen": 877920256 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003707021063189569, + "loss": 3.0102, + "theoretical_loss": 3.6952159984584343, + "tokens_seen": 877985792 + }, + { + "epoch": 0.27, + "learning_rate": 0.00037069207622868606, + "loss": 2.7709, + "theoretical_loss": 3.6951891154459453, + "tokens_seen": 878051328 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003706820461384153, + "loss": 2.8038, + "theoretical_loss": 3.6951622350016473, + "tokens_seen": 878116864 + }, + { + "epoch": 0.27, + "objective/train/docs_used": 504719, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.4778032302856445, + "objective/train/theoretical_loss": 3.6951353571251015, + "objective/train/tokens_used": 898642400, + "theoretical_loss": 3.6951353571251015, + "tokens_seen": 878182400 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003706720160481444, + "loss": 2.7949, + "theoretical_loss": 3.6951353571251015, + "tokens_seen": 878182400 + }, + { + "epoch": 0.27, + "learning_rate": 0.00037066198595787366, + "loss": 2.9709, + "theoretical_loss": 3.695108481815873, + "tokens_seen": 878247936 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003706519558676028, + "loss": 2.5679, + "theoretical_loss": 3.6950816090735232, + "tokens_seen": 878313472 + }, + { + "epoch": 0.27, + "learning_rate": 0.000370641925777332, + "loss": 2.9241, + "theoretical_loss": 3.6950547388976167, + "tokens_seen": 878379008 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003706318956870612, + "loss": 2.8261, + "theoretical_loss": 3.6950278712877163, + "tokens_seen": 878444544 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003706218655967904, + "loss": 2.8396, + "theoretical_loss": 3.6950010062433862, + "tokens_seen": 878510080 + }, + { + "epoch": 0.27, + "learning_rate": 0.00037061183550651957, + "loss": 2.7498, + "theoretical_loss": 3.694974143764189, + "tokens_seen": 878575616 + }, + { + "epoch": 0.27, + "learning_rate": 0.00037060180541624875, + "loss": 2.7468, + "theoretical_loss": 3.6949472838496895, + "tokens_seen": 878641152 + }, + { + "epoch": 0.27, + "learning_rate": 0.00037059177532597793, + "loss": 2.8425, + "theoretical_loss": 3.6949204264994515, + "tokens_seen": 878706688 + }, + { + "epoch": 0.27, + "learning_rate": 0.00037058174523570716, + "loss": 2.7483, + "theoretical_loss": 3.6948935717130382, + "tokens_seen": 878772224 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003705717151454363, + "loss": 2.9242, + "theoretical_loss": 3.6948667194900144, + "tokens_seen": 878837760 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003705616850551655, + "loss": 2.7845, + "theoretical_loss": 3.6948398698299436, + "tokens_seen": 878903296 + }, + { + "epoch": 0.27, + "learning_rate": 0.00037055165496489465, + "loss": 2.8626, + "theoretical_loss": 3.694813022732391, + "tokens_seen": 878968832 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003705416248746239, + "loss": 2.8063, + "theoretical_loss": 3.69478617819692, + "tokens_seen": 879034368 + }, + { + "epoch": 0.27, + "learning_rate": 0.00037053159478435307, + "loss": 2.7861, + "theoretical_loss": 3.694759336223096, + "tokens_seen": 879099904 + }, + { + "epoch": 0.27, + "learning_rate": 0.00037052156469408225, + "loss": 2.6931, + "theoretical_loss": 3.694732496810483, + "tokens_seen": 879165440 + }, + { + "epoch": 0.27, + "learning_rate": 0.00037051153460381143, + "loss": 2.8604, + "theoretical_loss": 3.694705659958646, + "tokens_seen": 879230976 + }, + { + "epoch": 0.27, + "learning_rate": 0.00037050150451354067, + "loss": 2.8954, + "theoretical_loss": 3.6946788256671494, + "tokens_seen": 879296512 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003704914744232698, + "loss": 2.7152, + "theoretical_loss": 3.694651993935559, + "tokens_seen": 879362048 + }, + { + "epoch": 0.27, + "learning_rate": 0.00037048144433299903, + "loss": 2.7373, + "theoretical_loss": 3.6946251647634396, + "tokens_seen": 879427584 + }, + { + "epoch": 0.27, + "learning_rate": 0.00037047141424272816, + "loss": 2.6329, + "theoretical_loss": 3.6945983381503558, + "tokens_seen": 879493120 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003704613841524574, + "loss": 2.7923, + "theoretical_loss": 3.6945715140958733, + "tokens_seen": 879558656 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003704513540621866, + "loss": 2.905, + "theoretical_loss": 3.694544692599557, + "tokens_seen": 879624192 + }, + { + "epoch": 0.27, + "learning_rate": 0.00037044132397191575, + "loss": 2.7994, + "theoretical_loss": 3.6945178736609723, + "tokens_seen": 879689728 + }, + { + "epoch": 0.27, + "learning_rate": 0.00037043129388164494, + "loss": 2.8164, + "theoretical_loss": 3.6944910572796856, + "tokens_seen": 879755264 + }, + { + "epoch": 0.27, + "objective/train/docs_used": 505361, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.905186891555786, + "objective/train/theoretical_loss": 3.694464243455262, + "objective/train/tokens_used": 900280800, + "theoretical_loss": 3.694464243455262, + "tokens_seen": 879820800 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003704212637913741, + "loss": 3.0382, + "theoretical_loss": 3.694464243455262, + "tokens_seen": 879820800 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003704112337011033, + "loss": 2.5987, + "theoretical_loss": 3.694437432187268, + "tokens_seen": 879886336 + }, + { + "epoch": 0.27, + "learning_rate": 0.00037040120361083253, + "loss": 2.9443, + "theoretical_loss": 3.694410623475268, + "tokens_seen": 879951872 + }, + { + "epoch": 0.27, + "learning_rate": 0.00037039117352056166, + "loss": 2.8105, + "theoretical_loss": 3.694383817318829, + "tokens_seen": 880017408 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003703811434302909, + "loss": 3.0475, + "theoretical_loss": 3.694357013717517, + "tokens_seen": 880082944 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003703711133400201, + "loss": 2.6094, + "theoretical_loss": 3.6943302126708986, + "tokens_seen": 880148480 + }, + { + "epoch": 0.27, + "learning_rate": 0.00037036108324974926, + "loss": 2.9415, + "theoretical_loss": 3.6943034141785396, + "tokens_seen": 880214016 + }, + { + "epoch": 0.27, + "learning_rate": 0.00037035105315947844, + "loss": 2.6509, + "theoretical_loss": 3.694276618240006, + "tokens_seen": 880279552 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003703410230692076, + "loss": 2.7283, + "theoretical_loss": 3.6942498248548654, + "tokens_seen": 880345088 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003703309929789368, + "loss": 2.9076, + "theoretical_loss": 3.6942230340226834, + "tokens_seen": 880410624 + }, + { + "epoch": 0.27, + "learning_rate": 0.00037032096288866604, + "loss": 2.9302, + "theoretical_loss": 3.6941962457430275, + "tokens_seen": 880476160 + }, + { + "epoch": 0.27, + "learning_rate": 0.00037031093279839516, + "loss": 2.9447, + "theoretical_loss": 3.6941694600154644, + "tokens_seen": 880541696 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003703009027081244, + "loss": 2.873, + "theoretical_loss": 3.6941426768395607, + "tokens_seen": 880607232 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003702908726178535, + "loss": 2.6248, + "theoretical_loss": 3.6941158962148837, + "tokens_seen": 880672768 + }, + { + "epoch": 0.27, + "learning_rate": 0.00037028084252758276, + "loss": 2.6296, + "theoretical_loss": 3.694089118141, + "tokens_seen": 880738304 + }, + { + "epoch": 0.27, + "learning_rate": 0.00037027081243731194, + "loss": 2.7801, + "theoretical_loss": 3.694062342617478, + "tokens_seen": 880803840 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003702607823470411, + "loss": 2.8051, + "theoretical_loss": 3.6940355696438845, + "tokens_seen": 880869376 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003702507522567703, + "loss": 2.8515, + "theoretical_loss": 3.6940087992197865, + "tokens_seen": 880934912 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003702407221664995, + "loss": 2.9465, + "theoretical_loss": 3.693982031344752, + "tokens_seen": 881000448 + }, + { + "epoch": 0.27, + "learning_rate": 0.00037023069207622867, + "loss": 2.8578, + "theoretical_loss": 3.6939552660183486, + "tokens_seen": 881065984 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003702206619859579, + "loss": 2.7477, + "theoretical_loss": 3.6939285032401443, + "tokens_seen": 881131520 + }, + { + "epoch": 0.27, + "learning_rate": 0.00037021063189568703, + "loss": 2.8042, + "theoretical_loss": 3.6939017430097074, + "tokens_seen": 881197056 + }, + { + "epoch": 0.27, + "learning_rate": 0.00037020060180541626, + "loss": 2.8769, + "theoretical_loss": 3.6938749853266044, + "tokens_seen": 881262592 + }, + { + "epoch": 0.27, + "learning_rate": 0.00037019057171514544, + "loss": 2.8885, + "theoretical_loss": 3.6938482301904054, + "tokens_seen": 881328128 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003701805416248746, + "loss": 3.0154, + "theoretical_loss": 3.693821477600677, + "tokens_seen": 881393664 + }, + { + "epoch": 0.27, + "objective/train/docs_used": 505856, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.063781261444092, + "objective/train/theoretical_loss": 3.693794727556988, + "objective/train/tokens_used": 901919200, + "theoretical_loss": 3.693794727556988, + "tokens_seen": 881459200 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003701705115346038, + "loss": 2.8133, + "theoretical_loss": 3.693794727556988, + "tokens_seen": 881459200 + }, + { + "epoch": 0.27, + "learning_rate": 0.000370160481444333, + "loss": 2.7577, + "theoretical_loss": 3.693767980058907, + "tokens_seen": 881524736 + }, + { + "epoch": 0.27, + "learning_rate": 0.00037015045135406217, + "loss": 2.7761, + "theoretical_loss": 3.693741235106003, + "tokens_seen": 881590272 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003701404212637914, + "loss": 2.8375, + "theoretical_loss": 3.6937144926978434, + "tokens_seen": 881655808 + }, + { + "epoch": 0.27, + "learning_rate": 0.00037013039117352053, + "loss": 2.8566, + "theoretical_loss": 3.693687752833998, + "tokens_seen": 881721344 + }, + { + "epoch": 0.27, + "learning_rate": 0.00037012036108324977, + "loss": 2.8672, + "theoretical_loss": 3.693661015514035, + "tokens_seen": 881786880 + }, + { + "epoch": 0.27, + "learning_rate": 0.00037011033099297895, + "loss": 2.9859, + "theoretical_loss": 3.693634280737524, + "tokens_seen": 881852416 + }, + { + "epoch": 0.27, + "learning_rate": 0.00037010030090270813, + "loss": 2.6077, + "theoretical_loss": 3.6936075485040334, + "tokens_seen": 881917952 + }, + { + "epoch": 0.27, + "learning_rate": 0.00037009027081243736, + "loss": 3.0172, + "theoretical_loss": 3.6935808188131327, + "tokens_seen": 881983488 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003700802407221665, + "loss": 2.7245, + "theoretical_loss": 3.6935540916643914, + "tokens_seen": 882049024 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003700702106318957, + "loss": 2.8024, + "theoretical_loss": 3.6935273670573787, + "tokens_seen": 882114560 + }, + { + "epoch": 0.27, + "learning_rate": 0.00037006018054162485, + "loss": 2.8266, + "theoretical_loss": 3.693500644991664, + "tokens_seen": 882180096 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003700501504513541, + "loss": 2.8805, + "theoretical_loss": 3.693473925466816, + "tokens_seen": 882245632 + }, + { + "epoch": 0.27, + "learning_rate": 0.00037004012036108327, + "loss": 2.8768, + "theoretical_loss": 3.693447208482406, + "tokens_seen": 882311168 + }, + { + "epoch": 0.27, + "learning_rate": 0.00037003009027081245, + "loss": 2.7009, + "theoretical_loss": 3.693420494038003, + "tokens_seen": 882376704 + }, + { + "epoch": 0.27, + "learning_rate": 0.00037002006018054163, + "loss": 2.612, + "theoretical_loss": 3.6933937821331764, + "tokens_seen": 882442240 + }, + { + "epoch": 0.27, + "learning_rate": 0.00037001003009027087, + "loss": 2.9051, + "theoretical_loss": 3.693367072767497, + "tokens_seen": 882507776 + }, + { + "epoch": 0.27, + "learning_rate": 0.00037, + "loss": 2.7787, + "theoretical_loss": 3.6933403659405344, + "tokens_seen": 882573312 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036998996990972923, + "loss": 2.8528, + "theoretical_loss": 3.6933136616518594, + "tokens_seen": 882638848 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036997993981945836, + "loss": 2.9384, + "theoretical_loss": 3.693286959901042, + "tokens_seen": 882704384 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003699699097291876, + "loss": 2.7413, + "theoretical_loss": 3.6932602606876523, + "tokens_seen": 882769920 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003699598796389168, + "loss": 2.7546, + "theoretical_loss": 3.693233564011261, + "tokens_seen": 882835456 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036994984954864595, + "loss": 2.8668, + "theoretical_loss": 3.693206869871439, + "tokens_seen": 882900992 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036993981945837514, + "loss": 2.9477, + "theoretical_loss": 3.6931801782677565, + "tokens_seen": 882966528 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003699297893681043, + "loss": 2.8264, + "theoretical_loss": 3.693153489199785, + "tokens_seen": 883032064 + }, + { + "epoch": 0.27, + "objective/train/docs_used": 506939, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.1572582721710205, + "objective/train/theoretical_loss": 3.693126802667095, + "objective/train/tokens_used": 903557600, + "theoretical_loss": 3.693126802667095, + "tokens_seen": 883097600 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003699197592778335, + "loss": 2.7755, + "theoretical_loss": 3.693126802667095, + "tokens_seen": 883097600 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036990972918756273, + "loss": 2.6641, + "theoretical_loss": 3.693100118669258, + "tokens_seen": 883163136 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036989969909729186, + "loss": 2.8984, + "theoretical_loss": 3.693073437205844, + "tokens_seen": 883228672 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003698896690070211, + "loss": 2.9732, + "theoretical_loss": 3.6930467582764255, + "tokens_seen": 883294208 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003698796389167503, + "loss": 2.8408, + "theoretical_loss": 3.6930200818805736, + "tokens_seen": 883359744 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036986960882647946, + "loss": 2.7351, + "theoretical_loss": 3.6929934080178595, + "tokens_seen": 883425280 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036985957873620864, + "loss": 2.9279, + "theoretical_loss": 3.692966736687855, + "tokens_seen": 883490816 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003698495486459378, + "loss": 2.6672, + "theoretical_loss": 3.692940067890131, + "tokens_seen": 883556352 + }, + { + "epoch": 0.27, + "learning_rate": 0.000369839518555667, + "loss": 2.707, + "theoretical_loss": 3.6929134016242604, + "tokens_seen": 883621888 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036982948846539624, + "loss": 2.7256, + "theoretical_loss": 3.6928867378898147, + "tokens_seen": 883687424 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036981945837512536, + "loss": 2.8815, + "theoretical_loss": 3.6928600766863653, + "tokens_seen": 883752960 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003698094282848546, + "loss": 2.6865, + "theoretical_loss": 3.6928334180134854, + "tokens_seen": 883818496 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003697993981945837, + "loss": 2.8095, + "theoretical_loss": 3.692806761870746, + "tokens_seen": 883884032 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036978936810431296, + "loss": 2.8405, + "theoretical_loss": 3.69278010825772, + "tokens_seen": 883949568 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036977933801404214, + "loss": 3.0252, + "theoretical_loss": 3.6927534571739797, + "tokens_seen": 884015104 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003697693079237713, + "loss": 3.1386, + "theoretical_loss": 3.692726808619098, + "tokens_seen": 884080640 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003697592778335005, + "loss": 2.9221, + "theoretical_loss": 3.6927001625926463, + "tokens_seen": 884146176 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003697492477432297, + "loss": 2.678, + "theoretical_loss": 3.692673519094199, + "tokens_seen": 884211712 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036973921765295887, + "loss": 2.7084, + "theoretical_loss": 3.6926468781233277, + "tokens_seen": 884277248 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003697291875626881, + "loss": 2.9438, + "theoretical_loss": 3.6926202396796057, + "tokens_seen": 884342784 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036971915747241723, + "loss": 3.0206, + "theoretical_loss": 3.692593603762606, + "tokens_seen": 884408320 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036970912738214646, + "loss": 2.6944, + "theoretical_loss": 3.692566970371902, + "tokens_seen": 884473856 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036969909729187565, + "loss": 3.16, + "theoretical_loss": 3.692540339507066, + "tokens_seen": 884539392 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003696890672016048, + "loss": 2.7482, + "theoretical_loss": 3.692513711167673, + "tokens_seen": 884604928 + }, + { + "epoch": 0.27, + "learning_rate": 0.000369679037111334, + "loss": 3.0536, + "theoretical_loss": 3.692487085353295, + "tokens_seen": 884670464 + }, + { + "epoch": 0.27, + "objective/train/docs_used": 507654, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.90828537940979, + "objective/train/theoretical_loss": 3.692460462063506, + "objective/train/tokens_used": 905196000, + "theoretical_loss": 3.692460462063506, + "tokens_seen": 884736000 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003696690070210632, + "loss": 3.0376, + "theoretical_loss": 3.692460462063506, + "tokens_seen": 884736000 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036965897693079237, + "loss": 2.6694, + "theoretical_loss": 3.6924338412978797, + "tokens_seen": 884801536 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003696489468405216, + "loss": 2.7914, + "theoretical_loss": 3.6924072230559895, + "tokens_seen": 884867072 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036963891675025073, + "loss": 2.8751, + "theoretical_loss": 3.6923806073374097, + "tokens_seen": 884932608 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036962888665997997, + "loss": 2.8812, + "theoretical_loss": 3.692353994141714, + "tokens_seen": 884998144 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003696188565697091, + "loss": 2.7668, + "theoretical_loss": 3.6923273834684767, + "tokens_seen": 885063680 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036960882647943833, + "loss": 2.3656, + "theoretical_loss": 3.6923007753172716, + "tokens_seen": 885129216 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003695987963891675, + "loss": 2.7527, + "theoretical_loss": 3.692274169687673, + "tokens_seen": 885194752 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003695887662988967, + "loss": 2.7054, + "theoretical_loss": 3.692247566579256, + "tokens_seen": 885260288 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036957873620862587, + "loss": 2.8328, + "theoretical_loss": 3.692220965991594, + "tokens_seen": 885325824 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036956870611835505, + "loss": 2.9855, + "theoretical_loss": 3.6921943679242624, + "tokens_seen": 885391360 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036955867602808424, + "loss": 2.7903, + "theoretical_loss": 3.692167772376836, + "tokens_seen": 885456896 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036954864593781347, + "loss": 2.854, + "theoretical_loss": 3.692141179348888, + "tokens_seen": 885522432 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003695386158475426, + "loss": 3.0357, + "theoretical_loss": 3.6921145888399955, + "tokens_seen": 885587968 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036952858575727183, + "loss": 3.0965, + "theoretical_loss": 3.692088000849732, + "tokens_seen": 885653504 + }, + { + "epoch": 0.27, + "learning_rate": 0.000369518555667001, + "loss": 2.9843, + "theoretical_loss": 3.692061415377673, + "tokens_seen": 885719040 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003695085255767302, + "loss": 2.8951, + "theoretical_loss": 3.692034832423394, + "tokens_seen": 885784576 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003694984954864594, + "loss": 2.8216, + "theoretical_loss": 3.6920082519864694, + "tokens_seen": 885850112 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036948846539618856, + "loss": 2.7319, + "theoretical_loss": 3.6919816740664757, + "tokens_seen": 885915648 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036947843530591774, + "loss": 2.8871, + "theoretical_loss": 3.691955098662988, + "tokens_seen": 885981184 + }, + { + "epoch": 0.27, + "learning_rate": 0.000369468405215647, + "loss": 2.7578, + "theoretical_loss": 3.6919285257755816, + "tokens_seen": 886046720 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003694583751253761, + "loss": 2.7712, + "theoretical_loss": 3.6919019554038326, + "tokens_seen": 886112256 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036944834503510534, + "loss": 2.7744, + "theoretical_loss": 3.6918753875473165, + "tokens_seen": 886177792 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036943831494483446, + "loss": 2.8194, + "theoretical_loss": 3.6918488222056096, + "tokens_seen": 886243328 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003694282848545637, + "loss": 2.8678, + "theoretical_loss": 3.691822259378287, + "tokens_seen": 886308864 + }, + { + "epoch": 0.27, + "objective/train/docs_used": 509172, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.2750673294067383, + "objective/train/theoretical_loss": 3.6917956990649263, + "objective/train/tokens_used": 906834400, + "theoretical_loss": 3.6917956990649263, + "tokens_seen": 886374400 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003694182547642929, + "loss": 2.8238, + "theoretical_loss": 3.6917956990649263, + "tokens_seen": 886374400 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036940822467402206, + "loss": 2.8604, + "theoretical_loss": 3.6917691412651026, + "tokens_seen": 886439936 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036939819458375124, + "loss": 2.8772, + "theoretical_loss": 3.6917425859783926, + "tokens_seen": 886505472 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003693881644934805, + "loss": 2.557, + "theoretical_loss": 3.691716033204373, + "tokens_seen": 886571008 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003693781344032096, + "loss": 3.0454, + "theoretical_loss": 3.6916894829426194, + "tokens_seen": 886636544 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036936810431293884, + "loss": 2.6236, + "theoretical_loss": 3.6916629351927095, + "tokens_seen": 886702080 + }, + { + "epoch": 0.27, + "learning_rate": 0.000369358074222668, + "loss": 2.9383, + "theoretical_loss": 3.69163638995422, + "tokens_seen": 886767616 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003693480441323972, + "loss": 2.7854, + "theoretical_loss": 3.6916098472267267, + "tokens_seen": 886833152 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036933801404212644, + "loss": 2.6747, + "theoretical_loss": 3.691583307009807, + "tokens_seen": 886898688 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036932798395185556, + "loss": 2.7366, + "theoretical_loss": 3.6915567693030384, + "tokens_seen": 886964224 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003693179538615848, + "loss": 2.7821, + "theoretical_loss": 3.691530234105998, + "tokens_seen": 887029760 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003693179538615848, + "loss": 2.7546, + "theoretical_loss": 3.6915037014182626, + "tokens_seen": 887095296 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003693179538615848, + "loss": 2.8864, + "theoretical_loss": 3.69147717123941, + "tokens_seen": 887160832 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003693179538615848, + "loss": 3.069, + "theoretical_loss": 3.6914506435690173, + "tokens_seen": 887226368 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003693079237713139, + "loss": 2.9132, + "theoretical_loss": 3.691424118406662, + "tokens_seen": 887291904 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036929789368104316, + "loss": 2.6238, + "theoretical_loss": 3.691397595751922, + "tokens_seen": 887357440 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036928786359077234, + "loss": 2.8279, + "theoretical_loss": 3.6913710756043754, + "tokens_seen": 887422976 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003692778335005015, + "loss": 2.5147, + "theoretical_loss": 3.6913445579635997, + "tokens_seen": 887488512 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003692678034102307, + "loss": 2.9267, + "theoretical_loss": 3.691318042829172, + "tokens_seen": 887554048 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003692577733199599, + "loss": 2.7007, + "theoretical_loss": 3.691291530200672, + "tokens_seen": 887619584 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036924774322968907, + "loss": 2.5644, + "theoretical_loss": 3.691265020077677, + "tokens_seen": 887685120 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003692377131394183, + "loss": 2.869, + "theoretical_loss": 3.6912385124597655, + "tokens_seen": 887750656 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036922768304914743, + "loss": 2.7087, + "theoretical_loss": 3.6912120073465156, + "tokens_seen": 887816192 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036921765295887666, + "loss": 2.8137, + "theoretical_loss": 3.6911855047375055, + "tokens_seen": 887881728 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036920762286860585, + "loss": 3.068, + "theoretical_loss": 3.691159004632315, + "tokens_seen": 887947264 + }, + { + "epoch": 0.27, + "objective/train/docs_used": 509747, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.978640556335449, + "objective/train/theoretical_loss": 3.691132507030521, + "objective/train/tokens_used": 908472800, + "theoretical_loss": 3.691132507030521, + "tokens_seen": 888012800 + }, + { + "epoch": 0.27, + "learning_rate": 0.000369197592778335, + "loss": 2.5756, + "theoretical_loss": 3.691132507030521, + "tokens_seen": 888012800 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003691875626880642, + "loss": 2.8994, + "theoretical_loss": 3.691106011931704, + "tokens_seen": 888078336 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003691775325977934, + "loss": 3.0209, + "theoretical_loss": 3.6910795193354415, + "tokens_seen": 888143872 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036916750250752257, + "loss": 2.7557, + "theoretical_loss": 3.6910530292413135, + "tokens_seen": 888209408 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003691574724172518, + "loss": 3.062, + "theoretical_loss": 3.691026541648899, + "tokens_seen": 888274944 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036914744232698093, + "loss": 2.8453, + "theoretical_loss": 3.691000056557777, + "tokens_seen": 888340480 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036913741223671017, + "loss": 2.9128, + "theoretical_loss": 3.6909735739675265, + "tokens_seen": 888406016 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003691273821464393, + "loss": 3.0104, + "theoretical_loss": 3.690947093877727, + "tokens_seen": 888471552 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036911735205616853, + "loss": 2.6928, + "theoretical_loss": 3.690920616287958, + "tokens_seen": 888537088 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003691073219658977, + "loss": 2.7791, + "theoretical_loss": 3.6908941411977993, + "tokens_seen": 888602624 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003690972918756269, + "loss": 2.8823, + "theoretical_loss": 3.6908676686068307, + "tokens_seen": 888668160 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003690872617853561, + "loss": 2.823, + "theoretical_loss": 3.690841198514632, + "tokens_seen": 888733696 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036907723169508525, + "loss": 2.8442, + "theoretical_loss": 3.6908147309207826, + "tokens_seen": 888799232 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036906720160481444, + "loss": 2.9522, + "theoretical_loss": 3.690788265824863, + "tokens_seen": 888864768 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036905717151454367, + "loss": 2.9753, + "theoretical_loss": 3.6907618032264526, + "tokens_seen": 888930304 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003690471414242728, + "loss": 2.7021, + "theoretical_loss": 3.6907353431251324, + "tokens_seen": 888995840 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036903711133400203, + "loss": 2.7741, + "theoretical_loss": 3.6907088855204826, + "tokens_seen": 889061376 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003690270812437312, + "loss": 2.6274, + "theoretical_loss": 3.6906824304120835, + "tokens_seen": 889126912 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003690170511534604, + "loss": 3.0553, + "theoretical_loss": 3.6906559777995156, + "tokens_seen": 889192448 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003690070210631896, + "loss": 2.6283, + "theoretical_loss": 3.690629527682359, + "tokens_seen": 889257984 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036899699097291876, + "loss": 2.7954, + "theoretical_loss": 3.690603080060195, + "tokens_seen": 889323520 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036898696088264794, + "loss": 2.7543, + "theoretical_loss": 3.690576634932605, + "tokens_seen": 889389056 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003689769307923772, + "loss": 2.7764, + "theoretical_loss": 3.6905501922991686, + "tokens_seen": 889454592 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003689669007021063, + "loss": 2.8612, + "theoretical_loss": 3.690523752159468, + "tokens_seen": 889520128 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036895687061183554, + "loss": 2.9613, + "theoretical_loss": 3.690497314513083, + "tokens_seen": 889585664 + }, + { + "epoch": 0.27, + "objective/train/docs_used": 511200, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.8089118003845215, + "objective/train/theoretical_loss": 3.690470879359596, + "objective/train/tokens_used": 910111200, + "theoretical_loss": 3.690470879359596, + "tokens_seen": 889651200 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036894684052156466, + "loss": 2.8821, + "theoretical_loss": 3.690470879359596, + "tokens_seen": 889651200 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003689368104312939, + "loss": 2.9391, + "theoretical_loss": 3.6904444466985877, + "tokens_seen": 889716736 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003689267803410231, + "loss": 3.0276, + "theoretical_loss": 3.69041801652964, + "tokens_seen": 889782272 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036891675025075226, + "loss": 3.0895, + "theoretical_loss": 3.6903915888523344, + "tokens_seen": 889847808 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036890672016048144, + "loss": 2.8435, + "theoretical_loss": 3.690365163666252, + "tokens_seen": 889913344 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003688966900702107, + "loss": 2.8127, + "theoretical_loss": 3.690338740970975, + "tokens_seen": 889978880 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003688866599799398, + "loss": 2.7233, + "theoretical_loss": 3.6903123207660844, + "tokens_seen": 890044416 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036887662988966904, + "loss": 2.6576, + "theoretical_loss": 3.6902859030511634, + "tokens_seen": 890109952 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036886659979939817, + "loss": 2.9554, + "theoretical_loss": 3.6902594878257933, + "tokens_seen": 890175488 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003688565697091274, + "loss": 2.7412, + "theoretical_loss": 3.6902330750895564, + "tokens_seen": 890241024 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003688465396188566, + "loss": 2.7963, + "theoretical_loss": 3.6902066648420346, + "tokens_seen": 890306560 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036883650952858576, + "loss": 2.9842, + "theoretical_loss": 3.690180257082811, + "tokens_seen": 890372096 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036882647943831494, + "loss": 2.9443, + "theoretical_loss": 3.690153851811467, + "tokens_seen": 890437632 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003688164493480441, + "loss": 2.635, + "theoretical_loss": 3.690127449027586, + "tokens_seen": 890503168 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003688064192577733, + "loss": 2.9812, + "theoretical_loss": 3.6901010487307504, + "tokens_seen": 890568704 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036879638916750254, + "loss": 2.6342, + "theoretical_loss": 3.6900746509205424, + "tokens_seen": 890634240 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036878635907723167, + "loss": 2.8255, + "theoretical_loss": 3.6900482555965457, + "tokens_seen": 890699776 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003687763289869609, + "loss": 2.5532, + "theoretical_loss": 3.6900218627583428, + "tokens_seen": 890765312 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036876629889669003, + "loss": 2.7264, + "theoretical_loss": 3.689995472405516, + "tokens_seen": 890830848 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036875626880641927, + "loss": 2.9268, + "theoretical_loss": 3.6899690845376503, + "tokens_seen": 890896384 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036874623871614845, + "loss": 2.4813, + "theoretical_loss": 3.689942699154327, + "tokens_seen": 890961920 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036873620862587763, + "loss": 2.8076, + "theoretical_loss": 3.6899163162551307, + "tokens_seen": 891027456 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003687261785356068, + "loss": 2.6772, + "theoretical_loss": 3.689889935839645, + "tokens_seen": 891092992 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036871614844533605, + "loss": 2.9321, + "theoretical_loss": 3.6898635579074517, + "tokens_seen": 891158528 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036870611835506517, + "loss": 2.9683, + "theoretical_loss": 3.6898371824581364, + "tokens_seen": 891224064 + }, + { + "epoch": 0.27, + "objective/train/docs_used": 511885, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.752180576324463, + "objective/train/theoretical_loss": 3.6898108094912816, + "objective/train/tokens_used": 911749600, + "theoretical_loss": 3.6898108094912816, + "tokens_seen": 891289600 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003686960882647944, + "loss": 2.8274, + "theoretical_loss": 3.6898108094912816, + "tokens_seen": 891289600 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036868605817452353, + "loss": 2.8465, + "theoretical_loss": 3.6897844390064716, + "tokens_seen": 891355136 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036867602808425277, + "loss": 2.7067, + "theoretical_loss": 3.6897580710032907, + "tokens_seen": 891420672 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036866599799398195, + "loss": 2.6537, + "theoretical_loss": 3.6897317054813223, + "tokens_seen": 891486208 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036865596790371113, + "loss": 2.836, + "theoretical_loss": 3.689705342440151, + "tokens_seen": 891551744 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003686459378134403, + "loss": 2.8201, + "theoretical_loss": 3.6896789818793607, + "tokens_seen": 891617280 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003686359077231695, + "loss": 2.7018, + "theoretical_loss": 3.689652623798536, + "tokens_seen": 891682816 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003686258776328987, + "loss": 2.8118, + "theoretical_loss": 3.689626268197261, + "tokens_seen": 891748352 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003686158475426279, + "loss": 3.0925, + "theoretical_loss": 3.689599915075121, + "tokens_seen": 891813888 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003686058174523571, + "loss": 2.5691, + "theoretical_loss": 3.6895735644317, + "tokens_seen": 891879424 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003685957873620863, + "loss": 2.8796, + "theoretical_loss": 3.6895472162665826, + "tokens_seen": 891944960 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036858575727181545, + "loss": 2.7907, + "theoretical_loss": 3.6895208705793534, + "tokens_seen": 892010496 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036857572718154464, + "loss": 2.9878, + "theoretical_loss": 3.6894945273695985, + "tokens_seen": 892076032 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036856569709127387, + "loss": 2.8022, + "theoretical_loss": 3.6894681866369026, + "tokens_seen": 892141568 + }, + { + "epoch": 0.27, + "learning_rate": 0.000368555667001003, + "loss": 2.8679, + "theoretical_loss": 3.6894418483808504, + "tokens_seen": 892207104 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036854563691073223, + "loss": 2.971, + "theoretical_loss": 3.689415512601027, + "tokens_seen": 892272640 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003685356068204614, + "loss": 2.5941, + "theoretical_loss": 3.6893891792970184, + "tokens_seen": 892338176 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003685255767301906, + "loss": 2.8152, + "theoretical_loss": 3.6893628484684093, + "tokens_seen": 892403712 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003685155466399198, + "loss": 2.6477, + "theoretical_loss": 3.689336520114786, + "tokens_seen": 892469248 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036850551654964896, + "loss": 2.7565, + "theoretical_loss": 3.689310194235733, + "tokens_seen": 892534784 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036849548645937814, + "loss": 2.7271, + "theoretical_loss": 3.6892838708308378, + "tokens_seen": 892600320 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003684854563691074, + "loss": 2.8833, + "theoretical_loss": 3.6892575498996845, + "tokens_seen": 892665856 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003684754262788365, + "loss": 2.6905, + "theoretical_loss": 3.6892312314418607, + "tokens_seen": 892731392 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036846539618856574, + "loss": 2.796, + "theoretical_loss": 3.689204915456951, + "tokens_seen": 892796928 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036845536609829486, + "loss": 2.8709, + "theoretical_loss": 3.689178601944542, + "tokens_seen": 892862464 + }, + { + "epoch": 0.27, + "objective/train/docs_used": 512575, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.490340232849121, + "objective/train/theoretical_loss": 3.68915229090422, + "objective/train/tokens_used": 913388000, + "theoretical_loss": 3.68915229090422, + "tokens_seen": 892928000 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003684453360080241, + "loss": 2.8201, + "theoretical_loss": 3.68915229090422, + "tokens_seen": 892928000 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003684353059177533, + "loss": 2.8133, + "theoretical_loss": 3.6891259823355718, + "tokens_seen": 892993536 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036842527582748246, + "loss": 2.6954, + "theoretical_loss": 3.689099676238183, + "tokens_seen": 893059072 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036841524573721164, + "loss": 2.7024, + "theoretical_loss": 3.6890733726116407, + "tokens_seen": 893124608 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003684052156469409, + "loss": 2.8217, + "theoretical_loss": 3.6890470714555317, + "tokens_seen": 893190144 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036839518555667, + "loss": 2.7649, + "theoretical_loss": 3.6890207727694415, + "tokens_seen": 893255680 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036838515546639924, + "loss": 2.8506, + "theoretical_loss": 3.688994476552959, + "tokens_seen": 893321216 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036837512537612837, + "loss": 2.8862, + "theoretical_loss": 3.6889681828056693, + "tokens_seen": 893386752 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003683650952858576, + "loss": 2.8592, + "theoretical_loss": 3.6889418915271603, + "tokens_seen": 893452288 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003683550651955868, + "loss": 2.8281, + "theoretical_loss": 3.688915602717019, + "tokens_seen": 893517824 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036834503510531596, + "loss": 2.7766, + "theoretical_loss": 3.688889316374833, + "tokens_seen": 893583360 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036833500501504514, + "loss": 2.7806, + "theoretical_loss": 3.688863032500189, + "tokens_seen": 893648896 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003683249749247743, + "loss": 2.8507, + "theoretical_loss": 3.6888367510926745, + "tokens_seen": 893714432 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003683149448345035, + "loss": 2.9942, + "theoretical_loss": 3.6888104721518777, + "tokens_seen": 893779968 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036830491474423274, + "loss": 2.7134, + "theoretical_loss": 3.6887841956773855, + "tokens_seen": 893845504 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036829488465396187, + "loss": 2.668, + "theoretical_loss": 3.688757921668786, + "tokens_seen": 893911040 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003682848545636911, + "loss": 2.9565, + "theoretical_loss": 3.688731650125667, + "tokens_seen": 893976576 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036827482447342023, + "loss": 2.741, + "theoretical_loss": 3.6887053810476162, + "tokens_seen": 894042112 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036826479438314947, + "loss": 3.0732, + "theoretical_loss": 3.6886791144342217, + "tokens_seen": 894107648 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036825476429287865, + "loss": 2.636, + "theoretical_loss": 3.688652850285072, + "tokens_seen": 894173184 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036824473420260783, + "loss": 2.9386, + "theoretical_loss": 3.688626588599755, + "tokens_seen": 894238720 + }, + { + "epoch": 0.27, + "learning_rate": 0.000368234704112337, + "loss": 2.8952, + "theoretical_loss": 3.6886003293778584, + "tokens_seen": 894304256 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036822467402206625, + "loss": 2.8394, + "theoretical_loss": 3.688574072618972, + "tokens_seen": 894369792 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036821464393179537, + "loss": 2.6754, + "theoretical_loss": 3.6885478183226836, + "tokens_seen": 894435328 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003682046138415246, + "loss": 2.9905, + "theoretical_loss": 3.688521566488582, + "tokens_seen": 894500864 + }, + { + "epoch": 0.27, + "objective/train/docs_used": 513630, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7166202068328857, + "objective/train/theoretical_loss": 3.6884953171162556, + "objective/train/tokens_used": 915026400, + "theoretical_loss": 3.6884953171162556, + "tokens_seen": 894566400 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036819458375125373, + "loss": 2.7905, + "theoretical_loss": 3.6884953171162556, + "tokens_seen": 894566400 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036818455366098297, + "loss": 2.7769, + "theoretical_loss": 3.688469070205293, + "tokens_seen": 894631936 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036817452357071215, + "loss": 2.9815, + "theoretical_loss": 3.688442825755284, + "tokens_seen": 894697472 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036816449348044133, + "loss": 2.6785, + "theoretical_loss": 3.688416583765817, + "tokens_seen": 894763008 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003681544633901705, + "loss": 2.8305, + "theoretical_loss": 3.688390344236481, + "tokens_seen": 894828544 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003681444332998997, + "loss": 2.7354, + "theoretical_loss": 3.688364107166866, + "tokens_seen": 894894080 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003681344032096289, + "loss": 2.9126, + "theoretical_loss": 3.6883378725565605, + "tokens_seen": 894959616 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003681243731193581, + "loss": 2.7539, + "theoretical_loss": 3.6883116404051544, + "tokens_seen": 895025152 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036811434302908724, + "loss": 2.9212, + "theoretical_loss": 3.6882854107122363, + "tokens_seen": 895090688 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003681043129388165, + "loss": 2.8102, + "theoretical_loss": 3.6882591834773972, + "tokens_seen": 895156224 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003680942828485456, + "loss": 2.8365, + "theoretical_loss": 3.688232958700226, + "tokens_seen": 895221760 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036808425275827484, + "loss": 2.8585, + "theoretical_loss": 3.688206736380313, + "tokens_seen": 895287296 + }, + { + "epoch": 0.27, + "learning_rate": 0.000368074222668004, + "loss": 2.8581, + "theoretical_loss": 3.688180516517247, + "tokens_seen": 895352832 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003680641925777332, + "loss": 2.8574, + "theoretical_loss": 3.6881542991106193, + "tokens_seen": 895418368 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003680541624874624, + "loss": 2.6744, + "theoretical_loss": 3.6881280841600192, + "tokens_seen": 895483904 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003680441323971916, + "loss": 2.8647, + "theoretical_loss": 3.6881018716650376, + "tokens_seen": 895549440 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036803410230692074, + "loss": 3.0142, + "theoretical_loss": 3.688075661625264, + "tokens_seen": 895614976 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036802407221665, + "loss": 2.8957, + "theoretical_loss": 3.6880494540402893, + "tokens_seen": 895680512 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003680140421263791, + "loss": 2.7923, + "theoretical_loss": 3.688023248909704, + "tokens_seen": 895746048 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036800401203610834, + "loss": 2.7383, + "theoretical_loss": 3.6879970462330975, + "tokens_seen": 895811584 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003679939819458375, + "loss": 2.9479, + "theoretical_loss": 3.687970846010063, + "tokens_seen": 895877120 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003679839518555667, + "loss": 2.9033, + "theoretical_loss": 3.6879446482401885, + "tokens_seen": 895942656 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003679739217652959, + "loss": 2.81, + "theoretical_loss": 3.6879184529230673, + "tokens_seen": 896008192 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036796389167502506, + "loss": 2.9878, + "theoretical_loss": 3.687892260058289, + "tokens_seen": 896073728 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036795386158475424, + "loss": 2.7548, + "theoretical_loss": 3.6878660696454446, + "tokens_seen": 896139264 + }, + { + "epoch": 0.27, + "objective/train/docs_used": 514378, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.803912401199341, + "objective/train/theoretical_loss": 3.6878398816841265, + "objective/train/tokens_used": 916664800, + "theoretical_loss": 3.6878398816841265, + "tokens_seen": 896204800 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003679438314944835, + "loss": 2.8542, + "theoretical_loss": 3.6878398816841265, + "tokens_seen": 896204800 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003679338014042126, + "loss": 2.9843, + "theoretical_loss": 3.6878136961739245, + "tokens_seen": 896270336 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036792377131394184, + "loss": 2.8706, + "theoretical_loss": 3.687787513114431, + "tokens_seen": 896335872 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036791374122367097, + "loss": 2.8079, + "theoretical_loss": 3.6877613325052367, + "tokens_seen": 896401408 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003679037111334002, + "loss": 2.93, + "theoretical_loss": 3.687735154345934, + "tokens_seen": 896466944 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003678936810431294, + "loss": 2.6771, + "theoretical_loss": 3.687708978636114, + "tokens_seen": 896532480 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036788365095285857, + "loss": 2.772, + "theoretical_loss": 3.6876828053753687, + "tokens_seen": 896598016 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036787362086258775, + "loss": 2.6444, + "theoretical_loss": 3.68765663456329, + "tokens_seen": 896663552 + }, + { + "epoch": 0.27, + "learning_rate": 0.000367863590772317, + "loss": 2.9177, + "theoretical_loss": 3.6876304661994697, + "tokens_seen": 896729088 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036785356068204616, + "loss": 2.8561, + "theoretical_loss": 3.6876043002834997, + "tokens_seen": 896794624 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036784353059177535, + "loss": 2.8075, + "theoretical_loss": 3.687578136814973, + "tokens_seen": 896860160 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003678335005015045, + "loss": 2.6639, + "theoretical_loss": 3.6875519757934807, + "tokens_seen": 896925696 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003678234704112337, + "loss": 2.9357, + "theoretical_loss": 3.6875258172186163, + "tokens_seen": 896991232 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036781344032096294, + "loss": 2.9033, + "theoretical_loss": 3.687499661089971, + "tokens_seen": 897056768 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036780341023069207, + "loss": 2.6011, + "theoretical_loss": 3.6874735074071383, + "tokens_seen": 897122304 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003677933801404213, + "loss": 2.7263, + "theoretical_loss": 3.687447356169711, + "tokens_seen": 897187840 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036778335005015043, + "loss": 3.0653, + "theoretical_loss": 3.687421207377281, + "tokens_seen": 897253376 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036777331995987967, + "loss": 2.8588, + "theoretical_loss": 3.6873950610294415, + "tokens_seen": 897318912 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036776328986960885, + "loss": 2.7804, + "theoretical_loss": 3.687368917125786, + "tokens_seen": 897384448 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036775325977933803, + "loss": 2.677, + "theoretical_loss": 3.6873427756659067, + "tokens_seen": 897449984 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003677432296890672, + "loss": 2.9144, + "theoretical_loss": 3.6873166366493972, + "tokens_seen": 897515520 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036773319959879645, + "loss": 2.7846, + "theoretical_loss": 3.687290500075851, + "tokens_seen": 897581056 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036772316950852557, + "loss": 2.8504, + "theoretical_loss": 3.687264365944861, + "tokens_seen": 897646592 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003677131394182548, + "loss": 2.578, + "theoretical_loss": 3.6872382342560206, + "tokens_seen": 897712128 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036770310932798393, + "loss": 2.5627, + "theoretical_loss": 3.687212105008923, + "tokens_seen": 897777664 + }, + { + "epoch": 0.27, + "objective/train/docs_used": 515835, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7880024909973145, + "objective/train/theoretical_loss": 3.6871859782031624, + "objective/train/tokens_used": 918303200, + "theoretical_loss": 3.6871859782031624, + "tokens_seen": 897843200 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036769307923771317, + "loss": 2.7384, + "theoretical_loss": 3.6871859782031624, + "tokens_seen": 897843200 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036768304914744235, + "loss": 2.6985, + "theoretical_loss": 3.687159853838333, + "tokens_seen": 897908736 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036767301905717153, + "loss": 2.8756, + "theoretical_loss": 3.6871337319140274, + "tokens_seen": 897974272 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003676629889669007, + "loss": 2.8032, + "theoretical_loss": 3.6871076124298403, + "tokens_seen": 898039808 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003676529588766299, + "loss": 2.7821, + "theoretical_loss": 3.6870814953853657, + "tokens_seen": 898105344 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003676429287863591, + "loss": 2.6431, + "theoretical_loss": 3.6870553807801976, + "tokens_seen": 898170880 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003676328986960883, + "loss": 2.7854, + "theoretical_loss": 3.68702926861393, + "tokens_seen": 898236416 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036762286860581744, + "loss": 2.7653, + "theoretical_loss": 3.687003158886158, + "tokens_seen": 898301952 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003676128385155467, + "loss": 2.848, + "theoretical_loss": 3.6869770515964744, + "tokens_seen": 898367488 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003676028084252758, + "loss": 2.7093, + "theoretical_loss": 3.6869509467444757, + "tokens_seen": 898433024 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036759277833500504, + "loss": 2.6989, + "theoretical_loss": 3.686924844329755, + "tokens_seen": 898498560 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003675827482447342, + "loss": 3.0803, + "theoretical_loss": 3.686898744351907, + "tokens_seen": 898564096 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003675727181544634, + "loss": 2.7536, + "theoretical_loss": 3.6868726468105275, + "tokens_seen": 898629632 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003675626880641926, + "loss": 2.6745, + "theoretical_loss": 3.6868465517052105, + "tokens_seen": 898695168 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003675526579739218, + "loss": 2.9158, + "theoretical_loss": 3.686820459035552, + "tokens_seen": 898760704 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036754262788365094, + "loss": 2.6783, + "theoretical_loss": 3.686794368801146, + "tokens_seen": 898826240 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003675325977933802, + "loss": 2.7622, + "theoretical_loss": 3.686768281001588, + "tokens_seen": 898891776 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003675225677031093, + "loss": 2.7379, + "theoretical_loss": 3.6867421956364734, + "tokens_seen": 898957312 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036751253761283854, + "loss": 3.0382, + "theoretical_loss": 3.6867161127053976, + "tokens_seen": 899022848 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003675025075225677, + "loss": 2.6423, + "theoretical_loss": 3.686690032207956, + "tokens_seen": 899088384 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003674924774322969, + "loss": 2.8587, + "theoretical_loss": 3.6866639541437447, + "tokens_seen": 899153920 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003674824473420261, + "loss": 2.5245, + "theoretical_loss": 3.686637878512358, + "tokens_seen": 899219456 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036747241725175526, + "loss": 2.9179, + "theoretical_loss": 3.686611805313393, + "tokens_seen": 899284992 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036746238716148444, + "loss": 2.6307, + "theoretical_loss": 3.686585734546445, + "tokens_seen": 899350528 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003674523570712137, + "loss": 2.9052, + "theoretical_loss": 3.68655966621111, + "tokens_seen": 899416064 + }, + { + "epoch": 0.27, + "objective/train/docs_used": 516375, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.893627643585205, + "objective/train/theoretical_loss": 3.6865336003069844, + "objective/train/tokens_used": 919941600, + "theoretical_loss": 3.6865336003069844, + "tokens_seen": 899481600 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003674423269809428, + "loss": 2.7848, + "theoretical_loss": 3.6865336003069844, + "tokens_seen": 899481600 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036743229689067204, + "loss": 2.7093, + "theoretical_loss": 3.6865075368336635, + "tokens_seen": 899547136 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036742226680040117, + "loss": 2.7385, + "theoretical_loss": 3.6864814757907443, + "tokens_seen": 899612672 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003674122367101304, + "loss": 2.848, + "theoretical_loss": 3.686455417177823, + "tokens_seen": 899678208 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003674022066198596, + "loss": 2.8324, + "theoretical_loss": 3.6864293609944956, + "tokens_seen": 899743744 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036739217652958877, + "loss": 2.9317, + "theoretical_loss": 3.6864033072403593, + "tokens_seen": 899809280 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036738214643931795, + "loss": 2.7793, + "theoretical_loss": 3.6863772559150103, + "tokens_seen": 899874816 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003673721163490472, + "loss": 2.8581, + "theoretical_loss": 3.6863512070180455, + "tokens_seen": 899940352 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003673620862587763, + "loss": 2.781, + "theoretical_loss": 3.6863251605490612, + "tokens_seen": 900005888 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036735205616850555, + "loss": 2.8814, + "theoretical_loss": 3.686299116507655, + "tokens_seen": 900071424 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036734202607823467, + "loss": 2.7308, + "theoretical_loss": 3.686273074893424, + "tokens_seen": 900136960 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003673319959879639, + "loss": 2.7207, + "theoretical_loss": 3.6862470357059642, + "tokens_seen": 900202496 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003673219658976931, + "loss": 2.7722, + "theoretical_loss": 3.6862209989448744, + "tokens_seen": 900268032 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036731193580742227, + "loss": 2.8756, + "theoretical_loss": 3.6861949646097507, + "tokens_seen": 900333568 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036730190571715145, + "loss": 2.5949, + "theoretical_loss": 3.686168932700191, + "tokens_seen": 900399104 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036729187562688063, + "loss": 2.8541, + "theoretical_loss": 3.6861429032157926, + "tokens_seen": 900464640 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003672818455366098, + "loss": 2.7952, + "theoretical_loss": 3.686116876156153, + "tokens_seen": 900530176 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036727181544633905, + "loss": 2.7865, + "theoretical_loss": 3.6860908515208703, + "tokens_seen": 900595712 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003672617853560682, + "loss": 2.8532, + "theoretical_loss": 3.686064829309542, + "tokens_seen": 900661248 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003672517552657974, + "loss": 2.7227, + "theoretical_loss": 3.686038809521766, + "tokens_seen": 900726784 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036724172517552654, + "loss": 3.0233, + "theoretical_loss": 3.6860127921571406, + "tokens_seen": 900792320 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036723169508525577, + "loss": 2.8306, + "theoretical_loss": 3.685986777215263, + "tokens_seen": 900857856 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036722166499498495, + "loss": 2.7808, + "theoretical_loss": 3.685960764695732, + "tokens_seen": 900923392 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036721163490471414, + "loss": 2.5455, + "theoretical_loss": 3.6859347545981462, + "tokens_seen": 900988928 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003672016048144433, + "loss": 2.7105, + "theoretical_loss": 3.6859087469221032, + "tokens_seen": 901054464 + }, + { + "epoch": 0.27, + "objective/train/docs_used": 517599, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.2461740970611572, + "objective/train/theoretical_loss": 3.685882741667202, + "objective/train/tokens_used": 921580000, + "theoretical_loss": 3.685882741667202, + "tokens_seen": 901120000 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036719157472417255, + "loss": 3.007, + "theoretical_loss": 3.685882741667202, + "tokens_seen": 901120000 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003671815446339017, + "loss": 2.6506, + "theoretical_loss": 3.6858567388330408, + "tokens_seen": 901185536 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003671715145436309, + "loss": 2.642, + "theoretical_loss": 3.6858307384192184, + "tokens_seen": 901251072 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036716148445336004, + "loss": 2.7307, + "theoretical_loss": 3.6858047404253336, + "tokens_seen": 901316608 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003671514543630893, + "loss": 2.7467, + "theoretical_loss": 3.6857787448509853, + "tokens_seen": 901382144 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036714142427281846, + "loss": 2.8234, + "theoretical_loss": 3.685752751695772, + "tokens_seen": 901447680 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036713139418254764, + "loss": 2.7196, + "theoretical_loss": 3.685726760959293, + "tokens_seen": 901513216 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003671213640922768, + "loss": 2.6506, + "theoretical_loss": 3.6857007726411477, + "tokens_seen": 901578752 + }, + { + "epoch": 0.27, + "learning_rate": 0.000367111334002006, + "loss": 2.6501, + "theoretical_loss": 3.685674786740935, + "tokens_seen": 901644288 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036710130391173524, + "loss": 2.88, + "theoretical_loss": 3.6856488032582546, + "tokens_seen": 901709824 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003670912738214644, + "loss": 2.936, + "theoretical_loss": 3.6856228221927054, + "tokens_seen": 901775360 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003670812437311936, + "loss": 2.6637, + "theoretical_loss": 3.6855968435438866, + "tokens_seen": 901840896 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003670712136409228, + "loss": 2.8739, + "theoretical_loss": 3.685570867311399, + "tokens_seen": 901906432 + }, + { + "epoch": 0.27, + "learning_rate": 0.000367061183550652, + "loss": 2.6983, + "theoretical_loss": 3.685544893494841, + "tokens_seen": 901971968 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036705115346038114, + "loss": 2.5314, + "theoretical_loss": 3.6855189220938134, + "tokens_seen": 902037504 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003670411233701104, + "loss": 2.7539, + "theoretical_loss": 3.6854929531079157, + "tokens_seen": 902103040 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003670310932798395, + "loss": 2.8021, + "theoretical_loss": 3.6854669865367473, + "tokens_seen": 902168576 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036702106318956874, + "loss": 2.8681, + "theoretical_loss": 3.6854410223799094, + "tokens_seen": 902234112 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003670110330992979, + "loss": 2.8169, + "theoretical_loss": 3.685415060637002, + "tokens_seen": 902299648 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003670010030090271, + "loss": 2.786, + "theoretical_loss": 3.6853891013076243, + "tokens_seen": 902365184 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003669909729187563, + "loss": 3.0737, + "theoretical_loss": 3.6853631443913772, + "tokens_seen": 902430720 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036698094282848546, + "loss": 3.0636, + "theoretical_loss": 3.6853371898878615, + "tokens_seen": 902496256 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036697091273821464, + "loss": 2.8614, + "theoretical_loss": 3.685311237796678, + "tokens_seen": 902561792 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003669608826479439, + "loss": 2.6871, + "theoretical_loss": 3.6852852881174263, + "tokens_seen": 902627328 + }, + { + "epoch": 0.27, + "learning_rate": 0.000366950852557673, + "loss": 2.4221, + "theoretical_loss": 3.685259340849708, + "tokens_seen": 902692864 + }, + { + "epoch": 0.27, + "objective/train/docs_used": 518319, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.9001624584198, + "objective/train/theoretical_loss": 3.685233395993124, + "objective/train/tokens_used": 923218400, + "theoretical_loss": 3.685233395993124, + "tokens_seen": 902758400 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036694082246740224, + "loss": 2.88, + "theoretical_loss": 3.685233395993124, + "tokens_seen": 902758400 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036693079237713137, + "loss": 2.5623, + "theoretical_loss": 3.6852074535472745, + "tokens_seen": 902823936 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003669207622868606, + "loss": 2.7665, + "theoretical_loss": 3.685181513511761, + "tokens_seen": 902889472 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003669107321965898, + "loss": 2.6844, + "theoretical_loss": 3.685155575886185, + "tokens_seen": 902955008 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036690070210631897, + "loss": 2.9818, + "theoretical_loss": 3.6851296406701466, + "tokens_seen": 903020544 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036689067201604815, + "loss": 2.5268, + "theoretical_loss": 3.6851037078632483, + "tokens_seen": 903086080 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003668806419257774, + "loss": 2.9177, + "theoretical_loss": 3.685077777465091, + "tokens_seen": 903151616 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003668706118355065, + "loss": 2.7528, + "theoretical_loss": 3.685051849475276, + "tokens_seen": 903217152 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036686058174523575, + "loss": 2.8439, + "theoretical_loss": 3.6850259238934053, + "tokens_seen": 903282688 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036685055165496487, + "loss": 2.6226, + "theoretical_loss": 3.6850000007190804, + "tokens_seen": 903348224 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003668405215646941, + "loss": 2.7517, + "theoretical_loss": 3.684974079951903, + "tokens_seen": 903413760 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003668304914744233, + "loss": 2.781, + "theoretical_loss": 3.684948161591475, + "tokens_seen": 903479296 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036682046138415247, + "loss": 2.7514, + "theoretical_loss": 3.6849222456373987, + "tokens_seen": 903544832 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036681043129388165, + "loss": 2.6797, + "theoretical_loss": 3.684896332089276, + "tokens_seen": 903610368 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036680040120361083, + "loss": 2.9297, + "theoretical_loss": 3.6848704209467087, + "tokens_seen": 903675904 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036679037111334, + "loss": 2.732, + "theoretical_loss": 3.6848445122093, + "tokens_seen": 903741440 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036678034102306925, + "loss": 2.8418, + "theoretical_loss": 3.684818605876651, + "tokens_seen": 903806976 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003667703109327984, + "loss": 2.8294, + "theoretical_loss": 3.6847927019483646, + "tokens_seen": 903872512 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003667602808425276, + "loss": 2.6889, + "theoretical_loss": 3.6847668004240437, + "tokens_seen": 903938048 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036675025075225674, + "loss": 2.7533, + "theoretical_loss": 3.6847409013032912, + "tokens_seen": 904003584 + }, + { + "epoch": 0.27, + "learning_rate": 0.000366740220661986, + "loss": 2.7057, + "theoretical_loss": 3.684715004585709, + "tokens_seen": 904069120 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036673019057171515, + "loss": 2.5578, + "theoretical_loss": 3.6846891102709005, + "tokens_seen": 904134656 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036672016048144434, + "loss": 2.7613, + "theoretical_loss": 3.684663218358468, + "tokens_seen": 904200192 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003667101303911735, + "loss": 2.6615, + "theoretical_loss": 3.684637328848016, + "tokens_seen": 904265728 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036670010030090275, + "loss": 2.8792, + "theoretical_loss": 3.684611441739145, + "tokens_seen": 904331264 + }, + { + "epoch": 0.27, + "objective/train/docs_used": 519704, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.466201066970825, + "objective/train/theoretical_loss": 3.684585557031461, + "objective/train/tokens_used": 924856800, + "theoretical_loss": 3.684585557031461, + "tokens_seen": 904396800 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003666900702106319, + "loss": 2.6301, + "theoretical_loss": 3.684585557031461, + "tokens_seen": 904396800 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003666800401203611, + "loss": 2.7932, + "theoretical_loss": 3.684559674724566, + "tokens_seen": 904462336 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036667001003009024, + "loss": 2.9781, + "theoretical_loss": 3.684533794818063, + "tokens_seen": 904527872 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003666599799398195, + "loss": 2.5802, + "theoretical_loss": 3.6845079173115565, + "tokens_seen": 904593408 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036664994984954866, + "loss": 2.6708, + "theoretical_loss": 3.684482042204649, + "tokens_seen": 904658944 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036663991975927784, + "loss": 2.9051, + "theoretical_loss": 3.6844561694969453, + "tokens_seen": 904724480 + }, + { + "epoch": 0.27, + "learning_rate": 0.000366629889669007, + "loss": 2.8858, + "theoretical_loss": 3.6844302991880484, + "tokens_seen": 904790016 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003666198595787362, + "loss": 2.5493, + "theoretical_loss": 3.684404431277562, + "tokens_seen": 904855552 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003666098294884654, + "loss": 2.7109, + "theoretical_loss": 3.6843785657650905, + "tokens_seen": 904921088 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003665997993981946, + "loss": 2.834, + "theoretical_loss": 3.684352702650238, + "tokens_seen": 904986624 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036658976930792374, + "loss": 2.8311, + "theoretical_loss": 3.684326841932609, + "tokens_seen": 905052160 + }, + { + "epoch": 0.27, + "learning_rate": 0.000366579739217653, + "loss": 2.7177, + "theoretical_loss": 3.684300983611807, + "tokens_seen": 905117696 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036656970912738216, + "loss": 2.7326, + "theoretical_loss": 3.684275127687436, + "tokens_seen": 905183232 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036655967903711134, + "loss": 2.7526, + "theoretical_loss": 3.684249274159102, + "tokens_seen": 905248768 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003665496489468405, + "loss": 2.9, + "theoretical_loss": 3.684223423026408, + "tokens_seen": 905314304 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003665396188565697, + "loss": 2.9285, + "theoretical_loss": 3.6841975742889588, + "tokens_seen": 905379840 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003665295887662989, + "loss": 2.7575, + "theoretical_loss": 3.68417172794636, + "tokens_seen": 905445376 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003665195586760281, + "loss": 2.4548, + "theoretical_loss": 3.684145883998216, + "tokens_seen": 905510912 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036650952858575725, + "loss": 2.6697, + "theoretical_loss": 3.684120042444132, + "tokens_seen": 905576448 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003664994984954865, + "loss": 2.7826, + "theoretical_loss": 3.6840942032837116, + "tokens_seen": 905641984 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003664894684052156, + "loss": 2.7671, + "theoretical_loss": 3.6840683665165614, + "tokens_seen": 905707520 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036647943831494484, + "loss": 2.7432, + "theoretical_loss": 3.6840425321422856, + "tokens_seen": 905773056 + }, + { + "epoch": 0.27, + "learning_rate": 0.000366469408224674, + "loss": 2.6764, + "theoretical_loss": 3.6840167001604907, + "tokens_seen": 905838592 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003664593781344032, + "loss": 2.7661, + "theoretical_loss": 3.683990870570781, + "tokens_seen": 905904128 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003664493480441324, + "loss": 2.5754, + "theoretical_loss": 3.683965043372762, + "tokens_seen": 905969664 + }, + { + "epoch": 0.27, + "objective/train/docs_used": 520483, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.963351249694824, + "objective/train/theoretical_loss": 3.6839392185660396, + "objective/train/tokens_used": 926495200, + "theoretical_loss": 3.6839392185660396, + "tokens_seen": 906035200 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036643931795386157, + "loss": 2.7973, + "theoretical_loss": 3.6839392185660396, + "tokens_seen": 906035200 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036642928786359075, + "loss": 2.8381, + "theoretical_loss": 3.6839133961502197, + "tokens_seen": 906100736 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036641925777332, + "loss": 2.956, + "theoretical_loss": 3.6838875761249072, + "tokens_seen": 906166272 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003664092276830491, + "loss": 2.7646, + "theoretical_loss": 3.6838617584897086, + "tokens_seen": 906231808 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036639919759277835, + "loss": 2.4827, + "theoretical_loss": 3.68383594324423, + "tokens_seen": 906297344 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036638916750250753, + "loss": 2.7972, + "theoretical_loss": 3.6838101303880766, + "tokens_seen": 906362880 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003663791374122367, + "loss": 2.7395, + "theoretical_loss": 3.6837843199208558, + "tokens_seen": 906428416 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003663691073219659, + "loss": 2.5347, + "theoretical_loss": 3.6837585118421723, + "tokens_seen": 906493952 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036635907723169507, + "loss": 2.7902, + "theoretical_loss": 3.683732706151633, + "tokens_seen": 906559488 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003663490471414243, + "loss": 2.8284, + "theoretical_loss": 3.6837069028488454, + "tokens_seen": 906625024 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003663390170511535, + "loss": 2.8932, + "theoretical_loss": 3.683681101933414, + "tokens_seen": 906690560 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036632898696088267, + "loss": 2.5995, + "theoretical_loss": 3.683655303404947, + "tokens_seen": 906756096 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036631895687061185, + "loss": 2.7817, + "theoretical_loss": 3.68362950726305, + "tokens_seen": 906821632 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036630892678034103, + "loss": 2.8316, + "theoretical_loss": 3.6836037135073307, + "tokens_seen": 906887168 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003662988966900702, + "loss": 2.6968, + "theoretical_loss": 3.6835779221373954, + "tokens_seen": 906952704 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036628886659979945, + "loss": 2.74, + "theoretical_loss": 3.683552133152851, + "tokens_seen": 907018240 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003662788365095286, + "loss": 2.8009, + "theoretical_loss": 3.683526346553305, + "tokens_seen": 907083776 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003662688064192578, + "loss": 2.4656, + "theoretical_loss": 3.683500562338364, + "tokens_seen": 907149312 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036625877632898694, + "loss": 2.8899, + "theoretical_loss": 3.6834747805076353, + "tokens_seen": 907214848 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003662487462387162, + "loss": 2.3673, + "theoretical_loss": 3.6834490010607266, + "tokens_seen": 907280384 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036623871614844535, + "loss": 2.7649, + "theoretical_loss": 3.683423223997245, + "tokens_seen": 907345920 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036622868605817454, + "loss": 2.8316, + "theoretical_loss": 3.6833974493167982, + "tokens_seen": 907411456 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003662186559679037, + "loss": 2.7229, + "theoretical_loss": 3.683371677018994, + "tokens_seen": 907476992 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036620862587763295, + "loss": 2.5909, + "theoretical_loss": 3.6833459071034396, + "tokens_seen": 907542528 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003661985957873621, + "loss": 2.8133, + "theoretical_loss": 3.6833201395697435, + "tokens_seen": 907608064 + }, + { + "epoch": 0.28, + "objective/train/docs_used": 521740, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.330822467803955, + "objective/train/theoretical_loss": 3.6832943744175126, + "objective/train/tokens_used": 928133600, + "theoretical_loss": 3.6832943744175126, + "tokens_seen": 907673600 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003661885656970913, + "loss": 2.6843, + "theoretical_loss": 3.6832943744175126, + "tokens_seen": 907673600 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036617853560682044, + "loss": 2.7572, + "theoretical_loss": 3.6832686116463558, + "tokens_seen": 907739136 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003661685055165497, + "loss": 2.7393, + "theoretical_loss": 3.683242851255881, + "tokens_seen": 907804672 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036615847542627886, + "loss": 2.5731, + "theoretical_loss": 3.6832170932456956, + "tokens_seen": 907870208 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036614844533600804, + "loss": 2.7692, + "theoretical_loss": 3.6831913376154084, + "tokens_seen": 907935744 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003661384152457372, + "loss": 2.9858, + "theoretical_loss": 3.683165584364628, + "tokens_seen": 908001280 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003661283851554664, + "loss": 2.6404, + "theoretical_loss": 3.6831398334929624, + "tokens_seen": 908066816 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003661183550651956, + "loss": 2.5995, + "theoretical_loss": 3.6831140850000206, + "tokens_seen": 908132352 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003661083249749248, + "loss": 2.8738, + "theoretical_loss": 3.683088338885411, + "tokens_seen": 908197888 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036609829488465394, + "loss": 2.7574, + "theoretical_loss": 3.6830625951487423, + "tokens_seen": 908263424 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003660882647943832, + "loss": 2.7257, + "theoretical_loss": 3.6830368537896225, + "tokens_seen": 908328960 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036607823470411236, + "loss": 2.573, + "theoretical_loss": 3.6830111148076625, + "tokens_seen": 908394496 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036606820461384154, + "loss": 2.5316, + "theoretical_loss": 3.6829853782024697, + "tokens_seen": 908460032 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003660581745235707, + "loss": 2.7599, + "theoretical_loss": 3.6829596439736534, + "tokens_seen": 908525568 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003660481444332999, + "loss": 2.8256, + "theoretical_loss": 3.6829339121208227, + "tokens_seen": 908591104 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003660381143430291, + "loss": 2.7485, + "theoretical_loss": 3.6829081826435877, + "tokens_seen": 908656640 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003660280842527583, + "loss": 2.6623, + "theoretical_loss": 3.682882455541557, + "tokens_seen": 908722176 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036601805416248745, + "loss": 2.5654, + "theoretical_loss": 3.6828567308143403, + "tokens_seen": 908787712 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003660080240722167, + "loss": 2.5748, + "theoretical_loss": 3.682831008461547, + "tokens_seen": 908853248 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003659979939819458, + "loss": 2.6967, + "theoretical_loss": 3.682805288482787, + "tokens_seen": 908918784 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036598796389167504, + "loss": 2.7635, + "theoretical_loss": 3.6827795708776705, + "tokens_seen": 908984320 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003659779338014042, + "loss": 2.597, + "theoretical_loss": 3.6827538556458057, + "tokens_seen": 909049856 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003659679037111334, + "loss": 2.8817, + "theoretical_loss": 3.6827281427868046, + "tokens_seen": 909115392 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003659578736208626, + "loss": 2.7728, + "theoretical_loss": 3.6827024323002755, + "tokens_seen": 909180928 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036594784353059177, + "loss": 2.7644, + "theoretical_loss": 3.6826767241858294, + "tokens_seen": 909246464 + }, + { + "epoch": 0.28, + "objective/train/docs_used": 522301, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.861933708190918, + "objective/train/theoretical_loss": 3.6826510184430763, + "objective/train/tokens_used": 929772000, + "theoretical_loss": 3.6826510184430763, + "tokens_seen": 909312000 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036593781344032095, + "loss": 2.8426, + "theoretical_loss": 3.6826510184430763, + "tokens_seen": 909312000 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003659277833500502, + "loss": 2.6762, + "theoretical_loss": 3.6826253150716264, + "tokens_seen": 909377536 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003659177532597793, + "loss": 2.8325, + "theoretical_loss": 3.6825996140710906, + "tokens_seen": 909443072 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036590772316950855, + "loss": 2.6841, + "theoretical_loss": 3.682573915441078, + "tokens_seen": 909508608 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036589769307923773, + "loss": 2.9036, + "theoretical_loss": 3.682548219181201, + "tokens_seen": 909574144 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003658876629889669, + "loss": 2.6666, + "theoretical_loss": 3.682522525291069, + "tokens_seen": 909639680 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003658776328986961, + "loss": 2.8857, + "theoretical_loss": 3.6824968337702932, + "tokens_seen": 909705216 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036586760280842527, + "loss": 2.548, + "theoretical_loss": 3.6824711446184843, + "tokens_seen": 909770752 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036585757271815445, + "loss": 2.8598, + "theoretical_loss": 3.6824454578352537, + "tokens_seen": 909836288 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003658475426278837, + "loss": 2.9294, + "theoretical_loss": 3.6824197734202118, + "tokens_seen": 909901824 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003658375125376128, + "loss": 2.7716, + "theoretical_loss": 3.6823940913729696, + "tokens_seen": 909967360 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036582748244734205, + "loss": 2.8176, + "theoretical_loss": 3.6823684116931386, + "tokens_seen": 910032896 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003658174523570712, + "loss": 2.9542, + "theoretical_loss": 3.682342734380331, + "tokens_seen": 910098432 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003658074222668004, + "loss": 2.6519, + "theoretical_loss": 3.6823170594341565, + "tokens_seen": 910163968 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003657973921765296, + "loss": 2.9219, + "theoretical_loss": 3.682291386854228, + "tokens_seen": 910229504 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003657873620862588, + "loss": 2.9855, + "theoretical_loss": 3.6822657166401562, + "tokens_seen": 910295040 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036577733199598796, + "loss": 2.8706, + "theoretical_loss": 3.682240048791553, + "tokens_seen": 910360576 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036576730190571714, + "loss": 2.5337, + "theoretical_loss": 3.6822143833080307, + "tokens_seen": 910426112 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003657572718154463, + "loss": 2.7488, + "theoretical_loss": 3.6821887201892007, + "tokens_seen": 910491648 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036574724172517555, + "loss": 2.7993, + "theoretical_loss": 3.682163059434674, + "tokens_seen": 910557184 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003657372116349047, + "loss": 3.0144, + "theoretical_loss": 3.6821374010440646, + "tokens_seen": 910622720 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003657271815446339, + "loss": 2.7353, + "theoretical_loss": 3.6821117450169836, + "tokens_seen": 910688256 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003657171514543631, + "loss": 2.8738, + "theoretical_loss": 3.682086091353043, + "tokens_seen": 910753792 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003657071213640923, + "loss": 2.5163, + "theoretical_loss": 3.6820604400518553, + "tokens_seen": 910819328 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036569709127382146, + "loss": 2.7064, + "theoretical_loss": 3.6820347911130327, + "tokens_seen": 910884864 + }, + { + "epoch": 0.28, + "objective/train/docs_used": 523753, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.4526991844177246, + "objective/train/theoretical_loss": 3.682009144536188, + "objective/train/tokens_used": 931410400, + "theoretical_loss": 3.682009144536188, + "tokens_seen": 910950400 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036568706118355064, + "loss": 2.4662, + "theoretical_loss": 3.682009144536188, + "tokens_seen": 910950400 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003656770310932798, + "loss": 2.6577, + "theoretical_loss": 3.6819835003209342, + "tokens_seen": 911015936 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036566700100300906, + "loss": 2.806, + "theoretical_loss": 3.681957858466883, + "tokens_seen": 911081472 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003656569709127382, + "loss": 2.7405, + "theoretical_loss": 3.681932218973648, + "tokens_seen": 911147008 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003656469408224674, + "loss": 2.7817, + "theoretical_loss": 3.681906581840842, + "tokens_seen": 911212544 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036563691073219655, + "loss": 2.821, + "theoretical_loss": 3.681880947068077, + "tokens_seen": 911278080 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003656268806419258, + "loss": 2.6878, + "theoretical_loss": 3.681855314654967, + "tokens_seen": 911343616 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036561685055165496, + "loss": 2.7065, + "theoretical_loss": 3.681829684601125, + "tokens_seen": 911409152 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036560682046138414, + "loss": 2.8371, + "theoretical_loss": 3.681804056906164, + "tokens_seen": 911474688 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003655967903711134, + "loss": 2.7411, + "theoretical_loss": 3.6817784315696978, + "tokens_seen": 911540224 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036558676028084256, + "loss": 2.5417, + "theoretical_loss": 3.6817528085913382, + "tokens_seen": 911605760 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036557673019057174, + "loss": 2.945, + "theoretical_loss": 3.681727187970701, + "tokens_seen": 911671296 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003655667001003009, + "loss": 2.6833, + "theoretical_loss": 3.6817015697073985, + "tokens_seen": 911736832 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003655566700100301, + "loss": 2.8285, + "theoretical_loss": 3.6816759538010446, + "tokens_seen": 911802368 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003655466399197593, + "loss": 2.9237, + "theoretical_loss": 3.6816503402512533, + "tokens_seen": 911867904 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003655366098294885, + "loss": 2.5917, + "theoretical_loss": 3.6816247290576376, + "tokens_seen": 911933440 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036552657973921765, + "loss": 2.8544, + "theoretical_loss": 3.681599120219812, + "tokens_seen": 911998976 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003655165496489469, + "loss": 2.4754, + "theoretical_loss": 3.681573513737391, + "tokens_seen": 912064512 + }, + { + "epoch": 0.28, + "learning_rate": 0.000365506519558676, + "loss": 2.7561, + "theoretical_loss": 3.6815479096099883, + "tokens_seen": 912130048 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036549648946840525, + "loss": 2.7312, + "theoretical_loss": 3.681522307837218, + "tokens_seen": 912195584 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003654864593781344, + "loss": 2.9873, + "theoretical_loss": 3.6814967084186945, + "tokens_seen": 912261120 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003654764292878636, + "loss": 2.832, + "theoretical_loss": 3.681471111354033, + "tokens_seen": 912326656 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003654663991975928, + "loss": 2.503, + "theoretical_loss": 3.681445516642847, + "tokens_seen": 912392192 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036545636910732197, + "loss": 2.6139, + "theoretical_loss": 3.681419924284751, + "tokens_seen": 912457728 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036544633901705115, + "loss": 2.9523, + "theoretical_loss": 3.68139433427936, + "tokens_seen": 912523264 + }, + { + "epoch": 0.28, + "objective/train/docs_used": 524575, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.1262991428375244, + "objective/train/theoretical_loss": 3.681368746626289, + "objective/train/tokens_used": 933048800, + "theoretical_loss": 3.681368746626289, + "tokens_seen": 912588800 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003654363089267804, + "loss": 2.8127, + "theoretical_loss": 3.681368746626289, + "tokens_seen": 912588800 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003654262788365095, + "loss": 2.7064, + "theoretical_loss": 3.6813431613251533, + "tokens_seen": 912654336 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036541624874623875, + "loss": 2.5479, + "theoretical_loss": 3.6813175783755665, + "tokens_seen": 912719872 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036540621865596793, + "loss": 2.8395, + "theoretical_loss": 3.6812919977771448, + "tokens_seen": 912785408 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003653961885656971, + "loss": 2.8457, + "theoretical_loss": 3.6812664195295026, + "tokens_seen": 912850944 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003653861584754263, + "loss": 2.8128, + "theoretical_loss": 3.6812408436322555, + "tokens_seen": 912916480 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036537612838515547, + "loss": 2.6614, + "theoretical_loss": 3.681215270085019, + "tokens_seen": 912982016 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036536609829488465, + "loss": 2.8133, + "theoretical_loss": 3.681189698887408, + "tokens_seen": 913047552 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003653560682046139, + "loss": 2.8808, + "theoretical_loss": 3.681164130039039, + "tokens_seen": 913113088 + }, + { + "epoch": 0.28, + "learning_rate": 0.000365346038114343, + "loss": 2.6667, + "theoretical_loss": 3.681138563539526, + "tokens_seen": 913178624 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036533600802407225, + "loss": 2.7505, + "theoretical_loss": 3.6811129993884864, + "tokens_seen": 913244160 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003653259779338014, + "loss": 2.909, + "theoretical_loss": 3.6810874375855342, + "tokens_seen": 913309696 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003653159478435306, + "loss": 2.7237, + "theoretical_loss": 3.6810618781302864, + "tokens_seen": 913375232 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003653059177532598, + "loss": 2.9173, + "theoretical_loss": 3.681036321022359, + "tokens_seen": 913440768 + }, + { + "epoch": 0.28, + "learning_rate": 0.000365295887662989, + "loss": 2.7719, + "theoretical_loss": 3.681010766261368, + "tokens_seen": 913506304 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036528585757271816, + "loss": 2.8569, + "theoretical_loss": 3.680985213846929, + "tokens_seen": 913571840 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036527582748244734, + "loss": 2.8825, + "theoretical_loss": 3.6809596637786584, + "tokens_seen": 913637376 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003652657973921765, + "loss": 2.843, + "theoretical_loss": 3.6809341160561724, + "tokens_seen": 913702912 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036525576730190575, + "loss": 2.7764, + "theoretical_loss": 3.6809085706790876, + "tokens_seen": 913768448 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003652457372116349, + "loss": 3.0254, + "theoretical_loss": 3.6808830276470212, + "tokens_seen": 913833984 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003652357071213641, + "loss": 2.9487, + "theoretical_loss": 3.6808574869595887, + "tokens_seen": 913899520 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003652256770310933, + "loss": 2.621, + "theoretical_loss": 3.6808319486164067, + "tokens_seen": 913965056 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003652156469408225, + "loss": 2.9152, + "theoretical_loss": 3.680806412617093, + "tokens_seen": 914030592 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036520561685055166, + "loss": 2.8548, + "theoretical_loss": 3.6807808789612633, + "tokens_seen": 914096128 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036519558676028084, + "loss": 2.6434, + "theoretical_loss": 3.680755347648536, + "tokens_seen": 914161664 + }, + { + "epoch": 0.28, + "objective/train/docs_used": 525722, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.747633218765259, + "objective/train/theoretical_loss": 3.680729818678526, + "objective/train/tokens_used": 934687200, + "theoretical_loss": 3.680729818678526, + "tokens_seen": 914227200 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036518555667001, + "loss": 2.9906, + "theoretical_loss": 3.680729818678526, + "tokens_seen": 914227200 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036517552657973926, + "loss": 2.758, + "theoretical_loss": 3.6807042920508533, + "tokens_seen": 914292736 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003651654964894684, + "loss": 2.6127, + "theoretical_loss": 3.6806787677651323, + "tokens_seen": 914358272 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003651554663991976, + "loss": 3.0192, + "theoretical_loss": 3.6806532458209817, + "tokens_seen": 914423808 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036514543630892675, + "loss": 2.6724, + "theoretical_loss": 3.680627726218019, + "tokens_seen": 914489344 + }, + { + "epoch": 0.28, + "learning_rate": 0.000365135406218656, + "loss": 2.7561, + "theoretical_loss": 3.680602208955861, + "tokens_seen": 914554880 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036512537612838516, + "loss": 2.6065, + "theoretical_loss": 3.6805766940341256, + "tokens_seen": 914620416 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036511534603811434, + "loss": 2.3515, + "theoretical_loss": 3.6805511814524308, + "tokens_seen": 914685952 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003651053159478435, + "loss": 2.6136, + "theoretical_loss": 3.6805256712103938, + "tokens_seen": 914751488 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036509528585757276, + "loss": 2.8695, + "theoretical_loss": 3.6805001633076326, + "tokens_seen": 914817024 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003650852557673019, + "loss": 3.06, + "theoretical_loss": 3.6804746577437655, + "tokens_seen": 914882560 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003650752256770311, + "loss": 2.751, + "theoretical_loss": 3.68044915451841, + "tokens_seen": 914948096 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036506519558676025, + "loss": 2.6374, + "theoretical_loss": 3.680423653631184, + "tokens_seen": 915013632 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003650551654964895, + "loss": 2.5336, + "theoretical_loss": 3.680398155081707, + "tokens_seen": 915079168 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036504513540621867, + "loss": 2.9417, + "theoretical_loss": 3.680372658869596, + "tokens_seen": 915144704 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036503510531594785, + "loss": 2.7407, + "theoretical_loss": 3.68034716499447, + "tokens_seen": 915210240 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036502507522567703, + "loss": 2.6684, + "theoretical_loss": 3.680321673455947, + "tokens_seen": 915275776 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003650150451354062, + "loss": 2.7381, + "theoretical_loss": 3.6802961842536464, + "tokens_seen": 915341312 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003650050150451354, + "loss": 2.7309, + "theoretical_loss": 3.6802706973871855, + "tokens_seen": 915406848 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003649949849548646, + "loss": 2.7568, + "theoretical_loss": 3.6802452128561844, + "tokens_seen": 915472384 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036498495486459375, + "loss": 2.9231, + "theoretical_loss": 3.680219730660261, + "tokens_seen": 915537920 + }, + { + "epoch": 0.28, + "learning_rate": 0.000364974924774323, + "loss": 2.975, + "theoretical_loss": 3.6801942507990346, + "tokens_seen": 915603456 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003649648946840521, + "loss": 2.8414, + "theoretical_loss": 3.680168773272124, + "tokens_seen": 915668992 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036495486459378135, + "loss": 2.588, + "theoretical_loss": 3.680143298079149, + "tokens_seen": 915734528 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036494483450351053, + "loss": 2.6386, + "theoretical_loss": 3.680117825219728, + "tokens_seen": 915800064 + }, + { + "epoch": 0.28, + "objective/train/docs_used": 526330, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.716949462890625, + "objective/train/theoretical_loss": 3.6800923546934805, + "objective/train/tokens_used": 936325600, + "theoretical_loss": 3.6800923546934805, + "tokens_seen": 915865600 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003649348044132397, + "loss": 2.7415, + "theoretical_loss": 3.6800923546934805, + "tokens_seen": 915865600 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003649247743229689, + "loss": 2.75, + "theoretical_loss": 3.680066886500026, + "tokens_seen": 915931136 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036491474423269813, + "loss": 2.7542, + "theoretical_loss": 3.6800414206389833, + "tokens_seen": 915996672 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036490471414242726, + "loss": 2.5804, + "theoretical_loss": 3.6800159571099735, + "tokens_seen": 916062208 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003648946840521565, + "loss": 2.7911, + "theoretical_loss": 3.679990495912614, + "tokens_seen": 916127744 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003648846539618856, + "loss": 2.6877, + "theoretical_loss": 3.679965037046527, + "tokens_seen": 916193280 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036487462387161485, + "loss": 2.9765, + "theoretical_loss": 3.67993958051133, + "tokens_seen": 916258816 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003648645937813441, + "loss": 2.8483, + "theoretical_loss": 3.6799141263066444, + "tokens_seen": 916324352 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003648545636910732, + "loss": 2.8322, + "theoretical_loss": 3.67988867443209, + "tokens_seen": 916389888 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036484453360080245, + "loss": 2.8384, + "theoretical_loss": 3.6798632248872862, + "tokens_seen": 916455424 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003648345035105316, + "loss": 2.5514, + "theoretical_loss": 3.679837777671854, + "tokens_seen": 916520960 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003648244734202608, + "loss": 2.9516, + "theoretical_loss": 3.6798123327854126, + "tokens_seen": 916586496 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036481444332999, + "loss": 2.6547, + "theoretical_loss": 3.679786890227583, + "tokens_seen": 916652032 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003648044132397192, + "loss": 2.7696, + "theoretical_loss": 3.6797614499979865, + "tokens_seen": 916717568 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036479438314944836, + "loss": 2.6881, + "theoretical_loss": 3.6797360120962423, + "tokens_seen": 916783104 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036478435305917754, + "loss": 2.9643, + "theoretical_loss": 3.679710576521971, + "tokens_seen": 916848640 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003647743229689067, + "loss": 2.5353, + "theoretical_loss": 3.679685143274794, + "tokens_seen": 916914176 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036476429287863595, + "loss": 2.8681, + "theoretical_loss": 3.679659712354332, + "tokens_seen": 916979712 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003647542627883651, + "loss": 2.746, + "theoretical_loss": 3.6796342837602056, + "tokens_seen": 917045248 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003647442326980943, + "loss": 2.8408, + "theoretical_loss": 3.6796088574920356, + "tokens_seen": 917110784 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003647342026078235, + "loss": 2.8872, + "theoretical_loss": 3.6795834335494444, + "tokens_seen": 917176320 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003647241725175527, + "loss": 2.7713, + "theoretical_loss": 3.679558011932051, + "tokens_seen": 917241856 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036471414242728186, + "loss": 2.8735, + "theoretical_loss": 3.679532592639478, + "tokens_seen": 917307392 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036470411233701104, + "loss": 2.6896, + "theoretical_loss": 3.6795071756713464, + "tokens_seen": 917372928 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003646940822467402, + "loss": 2.7078, + "theoretical_loss": 3.6794817610272776, + "tokens_seen": 917438464 + }, + { + "epoch": 0.28, + "objective/train/docs_used": 527026, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.855184316635132, + "objective/train/theoretical_loss": 3.6794563487068936, + "objective/train/tokens_used": 937964000, + "theoretical_loss": 3.6794563487068936, + "tokens_seen": 917504000 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036468405215646946, + "loss": 2.8446, + "theoretical_loss": 3.6794563487068936, + "tokens_seen": 917504000 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003646740220661986, + "loss": 2.4429, + "theoretical_loss": 3.679430938709815, + "tokens_seen": 917569536 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003646639919759278, + "loss": 2.778, + "theoretical_loss": 3.6794055310356644, + "tokens_seen": 917635072 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036465396188565695, + "loss": 2.5521, + "theoretical_loss": 3.6793801256840633, + "tokens_seen": 917700608 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003646439317953862, + "loss": 2.7088, + "theoretical_loss": 3.679354722654633, + "tokens_seen": 917766144 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036463390170511536, + "loss": 2.8609, + "theoretical_loss": 3.6793293219469962, + "tokens_seen": 917831680 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036462387161484454, + "loss": 2.7377, + "theoretical_loss": 3.6793039235607745, + "tokens_seen": 917897216 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003646138415245737, + "loss": 2.7706, + "theoretical_loss": 3.6792785274955904, + "tokens_seen": 917962752 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036460381143430296, + "loss": 2.892, + "theoretical_loss": 3.6792531337510654, + "tokens_seen": 918028288 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003645937813440321, + "loss": 2.713, + "theoretical_loss": 3.6792277423268223, + "tokens_seen": 918093824 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003645837512537613, + "loss": 2.7633, + "theoretical_loss": 3.6792023532224833, + "tokens_seen": 918159360 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036457372116349045, + "loss": 2.7275, + "theoretical_loss": 3.6791769664376712, + "tokens_seen": 918224896 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003645636910732197, + "loss": 2.801, + "theoretical_loss": 3.6791515819720084, + "tokens_seen": 918290432 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036455366098294887, + "loss": 2.5479, + "theoretical_loss": 3.679126199825118, + "tokens_seen": 918355968 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036454363089267805, + "loss": 2.8685, + "theoretical_loss": 3.6791008199966218, + "tokens_seen": 918421504 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036453360080240723, + "loss": 2.4333, + "theoretical_loss": 3.679075442486143, + "tokens_seen": 918487040 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003645235707121364, + "loss": 2.6963, + "theoretical_loss": 3.6790500672933044, + "tokens_seen": 918552576 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003645135406218656, + "loss": 2.5583, + "theoretical_loss": 3.6790246944177296, + "tokens_seen": 918618112 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003645035105315948, + "loss": 2.7709, + "theoretical_loss": 3.678999323859041, + "tokens_seen": 918683648 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036449348044132395, + "loss": 2.7644, + "theoretical_loss": 3.678973955616862, + "tokens_seen": 918749184 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003644834503510532, + "loss": 2.6089, + "theoretical_loss": 3.6789485896908163, + "tokens_seen": 918814720 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003644734202607823, + "loss": 2.9914, + "theoretical_loss": 3.678923226080527, + "tokens_seen": 918880256 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036446339017051155, + "loss": 2.7913, + "theoretical_loss": 3.6788978647856174, + "tokens_seen": 918945792 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036445336008024073, + "loss": 2.6105, + "theoretical_loss": 3.678872505805711, + "tokens_seen": 919011328 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003644433299899699, + "loss": 2.7992, + "theoretical_loss": 3.678847149140431, + "tokens_seen": 919076864 + }, + { + "epoch": 0.28, + "objective/train/docs_used": 528220, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.962761640548706, + "objective/train/theoretical_loss": 3.678821794789402, + "objective/train/tokens_used": 939602400, + "theoretical_loss": 3.678821794789402, + "tokens_seen": 919142400 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003644332998996991, + "loss": 2.7765, + "theoretical_loss": 3.678821794789402, + "tokens_seen": 919142400 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036442326980942833, + "loss": 2.7962, + "theoretical_loss": 3.6787964427522475, + "tokens_seen": 919207936 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036441323971915746, + "loss": 2.7402, + "theoretical_loss": 3.678771093028591, + "tokens_seen": 919273472 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003644032096288867, + "loss": 2.5261, + "theoretical_loss": 3.678745745618057, + "tokens_seen": 919339008 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003643931795386158, + "loss": 2.7013, + "theoretical_loss": 3.6787204005202687, + "tokens_seen": 919404544 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036438314944834505, + "loss": 2.7236, + "theoretical_loss": 3.678695057734852, + "tokens_seen": 919470080 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036437311935807424, + "loss": 2.8444, + "theoretical_loss": 3.678669717261429, + "tokens_seen": 919535616 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003643630892678034, + "loss": 2.9318, + "theoretical_loss": 3.6786443790996257, + "tokens_seen": 919601152 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003643530591775326, + "loss": 2.9149, + "theoretical_loss": 3.6786190432490655, + "tokens_seen": 919666688 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003643430290872618, + "loss": 2.6379, + "theoretical_loss": 3.6785937097093733, + "tokens_seen": 919732224 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036433299899699096, + "loss": 2.8626, + "theoretical_loss": 3.6785683784801737, + "tokens_seen": 919797760 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003643229689067202, + "loss": 2.8958, + "theoretical_loss": 3.678543049561091, + "tokens_seen": 919863296 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003643129388164493, + "loss": 2.7177, + "theoretical_loss": 3.6785177229517507, + "tokens_seen": 919928832 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036430290872617856, + "loss": 2.8285, + "theoretical_loss": 3.678492398651777, + "tokens_seen": 919994368 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003642928786359077, + "loss": 2.5929, + "theoretical_loss": 3.678467076660795, + "tokens_seen": 920059904 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003642828485456369, + "loss": 3.0074, + "theoretical_loss": 3.6784417569784296, + "tokens_seen": 920125440 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003642728184553661, + "loss": 2.6022, + "theoretical_loss": 3.6784164396043066, + "tokens_seen": 920190976 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003642627883650953, + "loss": 2.8088, + "theoretical_loss": 3.6783911245380505, + "tokens_seen": 920256512 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036425275827482446, + "loss": 2.7488, + "theoretical_loss": 3.678365811779287, + "tokens_seen": 920322048 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003642427281845537, + "loss": 2.7335, + "theoretical_loss": 3.6783405013276402, + "tokens_seen": 920387584 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003642326980942828, + "loss": 2.6515, + "theoretical_loss": 3.678315193182738, + "tokens_seen": 920453120 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036422266800401206, + "loss": 2.7407, + "theoretical_loss": 3.678289887344204, + "tokens_seen": 920518656 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003642126379137412, + "loss": 2.8832, + "theoretical_loss": 3.6782645838116643, + "tokens_seen": 920584192 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003642026078234704, + "loss": 3.0838, + "theoretical_loss": 3.6782392825847445, + "tokens_seen": 920649728 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003641925777331996, + "loss": 2.7762, + "theoretical_loss": 3.6782139836630714, + "tokens_seen": 920715264 + }, + { + "epoch": 0.28, + "objective/train/docs_used": 528886, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.666376829147339, + "objective/train/theoretical_loss": 3.6781886870462692, + "objective/train/tokens_used": 941240800, + "theoretical_loss": 3.6781886870462692, + "tokens_seen": 920780800 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003641825476429288, + "loss": 2.5754, + "theoretical_loss": 3.6781886870462692, + "tokens_seen": 920780800 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036417251755265797, + "loss": 2.7869, + "theoretical_loss": 3.678163392733966, + "tokens_seen": 920846336 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036416248746238715, + "loss": 2.7631, + "theoretical_loss": 3.6781381007257856, + "tokens_seen": 920911872 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036415245737211633, + "loss": 2.8442, + "theoretical_loss": 3.6781128110213555, + "tokens_seen": 920977408 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036414242728184556, + "loss": 2.8931, + "theoretical_loss": 3.6780875236203023, + "tokens_seen": 921042944 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003641323971915747, + "loss": 2.9151, + "theoretical_loss": 3.6780622385222514, + "tokens_seen": 921108480 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003641223671013039, + "loss": 2.7278, + "theoretical_loss": 3.6780369557268298, + "tokens_seen": 921174016 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036411233701103316, + "loss": 2.7701, + "theoretical_loss": 3.6780116752336633, + "tokens_seen": 921239552 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003641023069207623, + "loss": 2.7534, + "theoretical_loss": 3.6779863970423796, + "tokens_seen": 921305088 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003640922768304915, + "loss": 2.6984, + "theoretical_loss": 3.677961121152604, + "tokens_seen": 921370624 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036408224674022065, + "loss": 2.8571, + "theoretical_loss": 3.677935847563965, + "tokens_seen": 921436160 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003640722166499499, + "loss": 2.636, + "theoretical_loss": 3.677910576276088, + "tokens_seen": 921501696 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036406218655967907, + "loss": 2.9833, + "theoretical_loss": 3.6778853072886006, + "tokens_seen": 921567232 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036405215646940825, + "loss": 2.8031, + "theoretical_loss": 3.6778600406011295, + "tokens_seen": 921632768 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036404212637913743, + "loss": 2.7954, + "theoretical_loss": 3.6778347762133023, + "tokens_seen": 921698304 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003640320962888666, + "loss": 2.7699, + "theoretical_loss": 3.677809514124746, + "tokens_seen": 921763840 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003640220661985958, + "loss": 2.8652, + "theoretical_loss": 3.677784254335087, + "tokens_seen": 921829376 + }, + { + "epoch": 0.28, + "learning_rate": 0.000364012036108325, + "loss": 2.6666, + "theoretical_loss": 3.677758996843955, + "tokens_seen": 921894912 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036400200601805415, + "loss": 2.7428, + "theoretical_loss": 3.6777337416509743, + "tokens_seen": 921960448 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003639919759277834, + "loss": 2.9504, + "theoretical_loss": 3.6777084887557754, + "tokens_seen": 922025984 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003639819458375125, + "loss": 2.6849, + "theoretical_loss": 3.677683238157984, + "tokens_seen": 922091520 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036397191574724175, + "loss": 2.6537, + "theoretical_loss": 3.677657989857229, + "tokens_seen": 922157056 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036396188565697093, + "loss": 2.6335, + "theoretical_loss": 3.6776327438531373, + "tokens_seen": 922222592 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003639518555667001, + "loss": 3.0424, + "theoretical_loss": 3.677607500145337, + "tokens_seen": 922288128 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003639418254764293, + "loss": 2.8074, + "theoretical_loss": 3.6775822587334566, + "tokens_seen": 922353664 + }, + { + "epoch": 0.28, + "objective/train/docs_used": 530064, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7098605632781982, + "objective/train/theoretical_loss": 3.677557019617124, + "objective/train/tokens_used": 942879200, + "theoretical_loss": 3.677557019617124, + "tokens_seen": 922419200 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036393179538615853, + "loss": 2.6279, + "theoretical_loss": 3.677557019617124, + "tokens_seen": 922419200 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036392176529588766, + "loss": 2.592, + "theoretical_loss": 3.6775317827959673, + "tokens_seen": 922484736 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003639117352056169, + "loss": 2.4308, + "theoretical_loss": 3.6775065482696148, + "tokens_seen": 922550272 + }, + { + "epoch": 0.28, + "learning_rate": 0.000363901705115346, + "loss": 2.7474, + "theoretical_loss": 3.677481316037694, + "tokens_seen": 922615808 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036389167502507525, + "loss": 2.7808, + "theoretical_loss": 3.6774560860998347, + "tokens_seen": 922681344 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036388164493480444, + "loss": 2.6997, + "theoretical_loss": 3.6774308584556645, + "tokens_seen": 922746880 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003638716148445336, + "loss": 2.7791, + "theoretical_loss": 3.6774056331048124, + "tokens_seen": 922812416 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003638615847542628, + "loss": 2.6425, + "theoretical_loss": 3.677380410046907, + "tokens_seen": 922877952 + }, + { + "epoch": 0.28, + "learning_rate": 0.000363851554663992, + "loss": 2.6824, + "theoretical_loss": 3.6773551892815766, + "tokens_seen": 922943488 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036384152457372116, + "loss": 2.7759, + "theoretical_loss": 3.677329970808451, + "tokens_seen": 923009024 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003638314944834504, + "loss": 2.6437, + "theoretical_loss": 3.6773047546271584, + "tokens_seen": 923074560 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003638214643931795, + "loss": 2.7056, + "theoretical_loss": 3.677279540737328, + "tokens_seen": 923140096 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036381143430290876, + "loss": 2.6113, + "theoretical_loss": 3.677254329138589, + "tokens_seen": 923205632 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003638014042126379, + "loss": 2.5541, + "theoretical_loss": 3.67722911983057, + "tokens_seen": 923271168 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003637913741223671, + "loss": 2.8963, + "theoretical_loss": 3.6772039128129013, + "tokens_seen": 923336704 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003637813440320963, + "loss": 2.8287, + "theoretical_loss": 3.6771787080852123, + "tokens_seen": 923402240 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003637713139418255, + "loss": 2.8691, + "theoretical_loss": 3.677153505647132, + "tokens_seen": 923467776 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036376128385155466, + "loss": 2.68, + "theoretical_loss": 3.6771283054982886, + "tokens_seen": 923533312 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003637512537612839, + "loss": 2.5222, + "theoretical_loss": 3.677103107638314, + "tokens_seen": 923598848 + }, + { + "epoch": 0.28, + "learning_rate": 0.000363741223671013, + "loss": 2.7745, + "theoretical_loss": 3.677077912066837, + "tokens_seen": 923664384 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036373119358074226, + "loss": 2.6392, + "theoretical_loss": 3.677052718783487, + "tokens_seen": 923729920 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003637211634904714, + "loss": 2.7462, + "theoretical_loss": 3.6770275277878945, + "tokens_seen": 923795456 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003637111334002006, + "loss": 2.8846, + "theoretical_loss": 3.677002339079689, + "tokens_seen": 923860992 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003637011033099298, + "loss": 2.8912, + "theoretical_loss": 3.676977152658501, + "tokens_seen": 923926528 + }, + { + "epoch": 0.28, + "learning_rate": 0.000363691073219659, + "loss": 2.825, + "theoretical_loss": 3.6769519685239604, + "tokens_seen": 923992064 + }, + { + "epoch": 0.28, + "objective/train/docs_used": 530632, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.601571798324585, + "objective/train/theoretical_loss": 3.676926786675698, + "objective/train/tokens_used": 944517600, + "theoretical_loss": 3.676926786675698, + "tokens_seen": 924057600 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036368104312938817, + "loss": 2.6332, + "theoretical_loss": 3.676926786675698, + "tokens_seen": 924057600 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036367101303911735, + "loss": 2.874, + "theoretical_loss": 3.6769016071133427, + "tokens_seen": 924123136 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036366098294884653, + "loss": 2.7684, + "theoretical_loss": 3.6768764298365264, + "tokens_seen": 924188672 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036365095285857576, + "loss": 2.6473, + "theoretical_loss": 3.6768512548448786, + "tokens_seen": 924254208 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003636409227683049, + "loss": 2.7272, + "theoretical_loss": 3.6768260821380307, + "tokens_seen": 924319744 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003636308926780341, + "loss": 2.8026, + "theoretical_loss": 3.6768009117156124, + "tokens_seen": 924385280 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036362086258776325, + "loss": 2.6102, + "theoretical_loss": 3.6767757435772555, + "tokens_seen": 924450816 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003636108324974925, + "loss": 2.7822, + "theoretical_loss": 3.67675057772259, + "tokens_seen": 924516352 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036360080240722167, + "loss": 2.7543, + "theoretical_loss": 3.6767254141512478, + "tokens_seen": 924581888 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036359077231695085, + "loss": 2.5649, + "theoretical_loss": 3.6767002528628585, + "tokens_seen": 924647424 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036358074222668003, + "loss": 2.4652, + "theoretical_loss": 3.676675093857054, + "tokens_seen": 924712960 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036357071213640927, + "loss": 2.9225, + "theoretical_loss": 3.676649937133466, + "tokens_seen": 924778496 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003635606820461384, + "loss": 2.6959, + "theoretical_loss": 3.6766247826917247, + "tokens_seen": 924844032 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036355065195586763, + "loss": 3.0667, + "theoretical_loss": 3.676599630531462, + "tokens_seen": 924909568 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036354062186559676, + "loss": 2.6776, + "theoretical_loss": 3.6765744806523095, + "tokens_seen": 924975104 + }, + { + "epoch": 0.28, + "learning_rate": 0.000363530591775326, + "loss": 2.6993, + "theoretical_loss": 3.6765493330538987, + "tokens_seen": 925040640 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036352056168505517, + "loss": 2.9128, + "theoretical_loss": 3.6765241877358603, + "tokens_seen": 925106176 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036351053159478435, + "loss": 2.6651, + "theoretical_loss": 3.6764990446978274, + "tokens_seen": 925171712 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036350050150451353, + "loss": 2.7307, + "theoretical_loss": 3.6764739039394305, + "tokens_seen": 925237248 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003634904714142427, + "loss": 2.7176, + "theoretical_loss": 3.676448765460303, + "tokens_seen": 925302784 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003634804413239719, + "loss": 2.7382, + "theoretical_loss": 3.676423629260075, + "tokens_seen": 925368320 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036347041123370113, + "loss": 2.7205, + "theoretical_loss": 3.67639849533838, + "tokens_seen": 925433856 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036346038114343026, + "loss": 2.6491, + "theoretical_loss": 3.676373363694849, + "tokens_seen": 925499392 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003634503510531595, + "loss": 2.958, + "theoretical_loss": 3.676348234329115, + "tokens_seen": 925564928 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003634403209628886, + "loss": 2.8663, + "theoretical_loss": 3.67632310724081, + "tokens_seen": 925630464 + }, + { + "epoch": 0.28, + "objective/train/docs_used": 532017, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.860863447189331, + "objective/train/theoretical_loss": 3.6762979824295665, + "objective/train/tokens_used": 946156000, + "theoretical_loss": 3.6762979824295665, + "tokens_seen": 925696000 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036343029087261786, + "loss": 3.1014, + "theoretical_loss": 3.6762979824295665, + "tokens_seen": 925696000 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036342026078234704, + "loss": 2.8591, + "theoretical_loss": 3.6762728598950174, + "tokens_seen": 925761536 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003634102306920762, + "loss": 2.5145, + "theoretical_loss": 3.676247739636794, + "tokens_seen": 925827072 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003634002006018054, + "loss": 2.7806, + "theoretical_loss": 3.6762226216545297, + "tokens_seen": 925892608 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036339017051153464, + "loss": 2.7843, + "theoretical_loss": 3.676197505947858, + "tokens_seen": 925958144 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036338014042126376, + "loss": 2.8097, + "theoretical_loss": 3.67617239251641, + "tokens_seen": 926023680 + }, + { + "epoch": 0.28, + "learning_rate": 0.000363370110330993, + "loss": 2.6169, + "theoretical_loss": 3.67614728135982, + "tokens_seen": 926089216 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003633600802407222, + "loss": 2.6658, + "theoretical_loss": 3.6761221724777204, + "tokens_seen": 926154752 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036335005015045136, + "loss": 2.4906, + "theoretical_loss": 3.676097065869744, + "tokens_seen": 926220288 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003633400200601806, + "loss": 2.6409, + "theoretical_loss": 3.6760719615355244, + "tokens_seen": 926285824 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003633299899699097, + "loss": 2.6831, + "theoretical_loss": 3.6760468594746953, + "tokens_seen": 926351360 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036331995987963896, + "loss": 2.71, + "theoretical_loss": 3.676021759686889, + "tokens_seen": 926416896 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003633099297893681, + "loss": 2.8867, + "theoretical_loss": 3.6759966621717393, + "tokens_seen": 926482432 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003632998996990973, + "loss": 2.8472, + "theoretical_loss": 3.6759715669288795, + "tokens_seen": 926547968 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003632898696088265, + "loss": 2.8846, + "theoretical_loss": 3.6759464739579437, + "tokens_seen": 926613504 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003632798395185557, + "loss": 2.6244, + "theoretical_loss": 3.6759213832585647, + "tokens_seen": 926679040 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036326980942828486, + "loss": 2.6918, + "theoretical_loss": 3.675896294830377, + "tokens_seen": 926744576 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003632597793380141, + "loss": 2.796, + "theoretical_loss": 3.6758712086730148, + "tokens_seen": 926810112 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003632497492477432, + "loss": 2.7798, + "theoretical_loss": 3.6758461247861107, + "tokens_seen": 926875648 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036323971915747246, + "loss": 2.7911, + "theoretical_loss": 3.6758210431692993, + "tokens_seen": 926941184 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003632296890672016, + "loss": 2.8591, + "theoretical_loss": 3.6757959638222153, + "tokens_seen": 927006720 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003632196589769308, + "loss": 2.8225, + "theoretical_loss": 3.675770886744492, + "tokens_seen": 927072256 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036320962888666, + "loss": 2.8388, + "theoretical_loss": 3.675745811935764, + "tokens_seen": 927137792 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003631995987963892, + "loss": 2.683, + "theoretical_loss": 3.6757207393956657, + "tokens_seen": 927203328 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036318956870611837, + "loss": 2.6456, + "theoretical_loss": 3.675695669123832, + "tokens_seen": 927268864 + }, + { + "epoch": 0.28, + "objective/train/docs_used": 532753, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7925102710723877, + "objective/train/theoretical_loss": 3.6756706011198963, + "objective/train/tokens_used": 947794400, + "theoretical_loss": 3.6756706011198963, + "tokens_seen": 927334400 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036317953861584755, + "loss": 2.8508, + "theoretical_loss": 3.6756706011198963, + "tokens_seen": 927334400 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036316950852557673, + "loss": 2.7499, + "theoretical_loss": 3.6756455353834934, + "tokens_seen": 927399936 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036315947843530596, + "loss": 2.5865, + "theoretical_loss": 3.675620471914259, + "tokens_seen": 927465472 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003631494483450351, + "loss": 2.5718, + "theoretical_loss": 3.6755954107118267, + "tokens_seen": 927531008 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003631394182547643, + "loss": 2.461, + "theoretical_loss": 3.675570351775832, + "tokens_seen": 927596544 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036312938816449345, + "loss": 2.8066, + "theoretical_loss": 3.675545295105909, + "tokens_seen": 927662080 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003631193580742227, + "loss": 2.6691, + "theoretical_loss": 3.6755202407016943, + "tokens_seen": 927727616 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036310932798395187, + "loss": 2.8051, + "theoretical_loss": 3.675495188562822, + "tokens_seen": 927793152 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036309929789368105, + "loss": 2.761, + "theoretical_loss": 3.675470138688927, + "tokens_seen": 927858688 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036308926780341023, + "loss": 2.7486, + "theoretical_loss": 3.6754450910796446, + "tokens_seen": 927924224 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036307923771313947, + "loss": 2.7376, + "theoretical_loss": 3.6754200457346107, + "tokens_seen": 927989760 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003630692076228686, + "loss": 2.7517, + "theoretical_loss": 3.6753950026534605, + "tokens_seen": 928055296 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036305917753259783, + "loss": 2.8552, + "theoretical_loss": 3.6753699618358295, + "tokens_seen": 928120832 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036304914744232696, + "loss": 2.771, + "theoretical_loss": 3.675344923281353, + "tokens_seen": 928186368 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003630391173520562, + "loss": 2.9012, + "theoretical_loss": 3.6753198869896675, + "tokens_seen": 928251904 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036302908726178537, + "loss": 2.7665, + "theoretical_loss": 3.675294852960408, + "tokens_seen": 928317440 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036301905717151455, + "loss": 2.8342, + "theoretical_loss": 3.6752698211932104, + "tokens_seen": 928382976 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036300902708124374, + "loss": 2.3708, + "theoretical_loss": 3.675244791687711, + "tokens_seen": 928448512 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003629989969909729, + "loss": 2.7579, + "theoretical_loss": 3.675219764443546, + "tokens_seen": 928514048 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003629889669007021, + "loss": 2.9211, + "theoretical_loss": 3.6751947394603506, + "tokens_seen": 928579584 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036297893681043133, + "loss": 2.7871, + "theoretical_loss": 3.675169716737762, + "tokens_seen": 928645120 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036296890672016046, + "loss": 2.5428, + "theoretical_loss": 3.6751446962754155, + "tokens_seen": 928710656 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003629588766298897, + "loss": 2.7497, + "theoretical_loss": 3.6751196780729485, + "tokens_seen": 928776192 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003629488465396188, + "loss": 2.9076, + "theoretical_loss": 3.675094662129997, + "tokens_seen": 928841728 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036293881644934806, + "loss": 2.5568, + "theoretical_loss": 3.6750696484461978, + "tokens_seen": 928907264 + }, + { + "epoch": 0.28, + "objective/train/docs_used": 534166, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.451158046722412, + "objective/train/theoretical_loss": 3.6750446370211867, + "objective/train/tokens_used": 949432800, + "theoretical_loss": 3.6750446370211867, + "tokens_seen": 928972800 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036292878635907724, + "loss": 2.4071, + "theoretical_loss": 3.6750446370211867, + "tokens_seen": 928972800 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003629187562688064, + "loss": 2.793, + "theoretical_loss": 3.675019627854601, + "tokens_seen": 929038336 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003629087261785356, + "loss": 2.6165, + "theoretical_loss": 3.6749946209460775, + "tokens_seen": 929103872 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036289869608826484, + "loss": 2.6857, + "theoretical_loss": 3.6749696162952525, + "tokens_seen": 929169408 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036288866599799396, + "loss": 2.9299, + "theoretical_loss": 3.674944613901764, + "tokens_seen": 929234944 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003628786359077232, + "loss": 2.9702, + "theoretical_loss": 3.674919613765248, + "tokens_seen": 929300480 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003628686058174523, + "loss": 2.8139, + "theoretical_loss": 3.674894615885342, + "tokens_seen": 929366016 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036285857572718156, + "loss": 2.7847, + "theoretical_loss": 3.6748696202616835, + "tokens_seen": 929431552 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036284854563691074, + "loss": 2.7566, + "theoretical_loss": 3.6748446268939094, + "tokens_seen": 929497088 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003628385155466399, + "loss": 2.7795, + "theoretical_loss": 3.674819635781658, + "tokens_seen": 929562624 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003628284854563691, + "loss": 3.0679, + "theoretical_loss": 3.674794646924565, + "tokens_seen": 929628160 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003628184553660983, + "loss": 2.6807, + "theoretical_loss": 3.674769660322269, + "tokens_seen": 929693696 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036280842527582747, + "loss": 2.5206, + "theoretical_loss": 3.6747446759744076, + "tokens_seen": 929759232 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003627983951855567, + "loss": 2.7587, + "theoretical_loss": 3.6747196938806184, + "tokens_seen": 929824768 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036278836509528583, + "loss": 2.6132, + "theoretical_loss": 3.6746947140405393, + "tokens_seen": 929890304 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036277833500501506, + "loss": 2.7489, + "theoretical_loss": 3.6746697364538083, + "tokens_seen": 929955840 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036276830491474424, + "loss": 2.6141, + "theoretical_loss": 3.6746447611200628, + "tokens_seen": 930021376 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003627582748244734, + "loss": 2.5688, + "theoretical_loss": 3.6746197880389415, + "tokens_seen": 930086912 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003627482447342026, + "loss": 2.7299, + "theoretical_loss": 3.674594817210082, + "tokens_seen": 930152448 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003627382146439318, + "loss": 2.6896, + "theoretical_loss": 3.674569848633123, + "tokens_seen": 930217984 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036272818455366097, + "loss": 2.7041, + "theoretical_loss": 3.674544882307702, + "tokens_seen": 930283520 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003627181544633902, + "loss": 2.6152, + "theoretical_loss": 3.6745199182334582, + "tokens_seen": 930349056 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036270812437311933, + "loss": 2.741, + "theoretical_loss": 3.6744949564100295, + "tokens_seen": 930414592 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036269809428284857, + "loss": 2.6955, + "theoretical_loss": 3.674469996837055, + "tokens_seen": 930480128 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003626880641925777, + "loss": 2.8485, + "theoretical_loss": 3.674445039514173, + "tokens_seen": 930545664 + }, + { + "epoch": 0.28, + "objective/train/docs_used": 534638, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6016719341278076, + "objective/train/theoretical_loss": 3.6744200844410217, + "objective/train/tokens_used": 951071200, + "theoretical_loss": 3.6744200844410217, + "tokens_seen": 930611200 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036267803410230693, + "loss": 2.4705, + "theoretical_loss": 3.6744200844410217, + "tokens_seen": 930611200 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003626680040120361, + "loss": 2.5837, + "theoretical_loss": 3.674395131617241, + "tokens_seen": 930676736 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003626579739217653, + "loss": 2.7417, + "theoretical_loss": 3.674370181042469, + "tokens_seen": 930742272 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036264794383149447, + "loss": 2.6964, + "theoretical_loss": 3.6743452327163446, + "tokens_seen": 930807808 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036263791374122365, + "loss": 2.5772, + "theoretical_loss": 3.6743202866385074, + "tokens_seen": 930873344 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036262788365095283, + "loss": 2.8291, + "theoretical_loss": 3.6742953428085965, + "tokens_seen": 930938880 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036261785356068207, + "loss": 2.8288, + "theoretical_loss": 3.6742704012262504, + "tokens_seen": 931004416 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036260782347041125, + "loss": 2.9272, + "theoretical_loss": 3.6742454618911085, + "tokens_seen": 931069952 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036259779338014043, + "loss": 2.7464, + "theoretical_loss": 3.674220524802811, + "tokens_seen": 931135488 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036258776328986967, + "loss": 2.7301, + "theoretical_loss": 3.6741955899609966, + "tokens_seen": 931201024 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003625777331995988, + "loss": 2.9748, + "theoretical_loss": 3.674170657365305, + "tokens_seen": 931266560 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036256770310932803, + "loss": 2.5131, + "theoretical_loss": 3.674145727015376, + "tokens_seen": 931332096 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036255767301905716, + "loss": 2.7336, + "theoretical_loss": 3.6741207989108493, + "tokens_seen": 931397632 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003625476429287864, + "loss": 2.9497, + "theoretical_loss": 3.674095873051365, + "tokens_seen": 931463168 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003625376128385156, + "loss": 2.793, + "theoretical_loss": 3.6740709494365618, + "tokens_seen": 931528704 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036252758274824475, + "loss": 2.6329, + "theoretical_loss": 3.6740460280660807, + "tokens_seen": 931594240 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036251755265797394, + "loss": 2.7897, + "theoretical_loss": 3.6740211089395616, + "tokens_seen": 931659776 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003625075225677031, + "loss": 2.7532, + "theoretical_loss": 3.673996192056644, + "tokens_seen": 931725312 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003624974924774323, + "loss": 2.7117, + "theoretical_loss": 3.6739712774169693, + "tokens_seen": 931790848 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036248746238716153, + "loss": 2.5223, + "theoretical_loss": 3.673946365020177, + "tokens_seen": 931856384 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036247743229689066, + "loss": 2.7326, + "theoretical_loss": 3.6739214548659076, + "tokens_seen": 931921920 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003624674022066199, + "loss": 2.7159, + "theoretical_loss": 3.6738965469538014, + "tokens_seen": 931987456 + }, + { + "epoch": 0.28, + "learning_rate": 0.000362457372116349, + "loss": 2.7217, + "theoretical_loss": 3.6738716412834984, + "tokens_seen": 932052992 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036244734202607826, + "loss": 2.6772, + "theoretical_loss": 3.6738467378546407, + "tokens_seen": 932118528 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036243731193580744, + "loss": 2.5826, + "theoretical_loss": 3.6738218366668685, + "tokens_seen": 932184064 + }, + { + "epoch": 0.28, + "objective/train/docs_used": 535899, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.9818763732910156, + "objective/train/theoretical_loss": 3.673796937719821, + "objective/train/tokens_used": 952709600, + "theoretical_loss": 3.673796937719821, + "tokens_seen": 932249600 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003624272818455366, + "loss": 2.5143, + "theoretical_loss": 3.673796937719821, + "tokens_seen": 932249600 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003624172517552658, + "loss": 2.5288, + "theoretical_loss": 3.673772041013142, + "tokens_seen": 932315136 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036240722166499504, + "loss": 2.682, + "theoretical_loss": 3.6737471465464697, + "tokens_seen": 932380672 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036239719157472416, + "loss": 2.7498, + "theoretical_loss": 3.673722254319446, + "tokens_seen": 932446208 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003623871614844534, + "loss": 2.7886, + "theoretical_loss": 3.673697364331713, + "tokens_seen": 932511744 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003623771313941825, + "loss": 2.8033, + "theoretical_loss": 3.6736724765829107, + "tokens_seen": 932577280 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036236710130391176, + "loss": 2.5632, + "theoretical_loss": 3.6736475910726814, + "tokens_seen": 932642816 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036235707121364094, + "loss": 3.0061, + "theoretical_loss": 3.673622707800665, + "tokens_seen": 932708352 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003623470411233701, + "loss": 2.9585, + "theoretical_loss": 3.6735978267665046, + "tokens_seen": 932773888 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003623370110330993, + "loss": 2.633, + "theoretical_loss": 3.6735729479698405, + "tokens_seen": 932839424 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003623269809428285, + "loss": 2.9423, + "theoretical_loss": 3.6735480714103153, + "tokens_seen": 932904960 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036231695085255767, + "loss": 2.5462, + "theoretical_loss": 3.6735231970875697, + "tokens_seen": 932970496 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003623069207622869, + "loss": 2.7554, + "theoretical_loss": 3.6734983250012467, + "tokens_seen": 933036032 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036229689067201603, + "loss": 2.7184, + "theoretical_loss": 3.6734734551509867, + "tokens_seen": 933101568 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036228686058174526, + "loss": 2.8812, + "theoretical_loss": 3.6734485875364324, + "tokens_seen": 933167104 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036227683049147444, + "loss": 2.7033, + "theoretical_loss": 3.673423722157226, + "tokens_seen": 933232640 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003622668004012036, + "loss": 2.5564, + "theoretical_loss": 3.6733988590130098, + "tokens_seen": 933298176 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003622567703109328, + "loss": 2.8041, + "theoretical_loss": 3.673373998103425, + "tokens_seen": 933363712 + }, + { + "epoch": 0.28, + "learning_rate": 0.000362246740220662, + "loss": 2.6603, + "theoretical_loss": 3.6733491394281153, + "tokens_seen": 933429248 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036223671013039117, + "loss": 2.7134, + "theoretical_loss": 3.673324282986721, + "tokens_seen": 933494784 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003622266800401204, + "loss": 2.7395, + "theoretical_loss": 3.6732994287788867, + "tokens_seen": 933560320 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036221664994984953, + "loss": 2.6579, + "theoretical_loss": 3.673274576804254, + "tokens_seen": 933625856 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036220661985957877, + "loss": 2.7116, + "theoretical_loss": 3.6732497270624656, + "tokens_seen": 933691392 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003621965897693079, + "loss": 3.0569, + "theoretical_loss": 3.673224879553163, + "tokens_seen": 933756928 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036218655967903713, + "loss": 2.6772, + "theoretical_loss": 3.6732000342759914, + "tokens_seen": 933822464 + }, + { + "epoch": 0.28, + "objective/train/docs_used": 536516, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.9241724014282227, + "objective/train/theoretical_loss": 3.6731751912305914, + "objective/train/tokens_used": 954348000, + "theoretical_loss": 3.6731751912305914, + "tokens_seen": 933888000 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003621765295887663, + "loss": 2.6122, + "theoretical_loss": 3.6731751912305914, + "tokens_seen": 933888000 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003621664994984955, + "loss": 2.7369, + "theoretical_loss": 3.6731503504166074, + "tokens_seen": 933953536 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036215646940822467, + "loss": 2.7741, + "theoretical_loss": 3.673125511833682, + "tokens_seen": 934019072 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036214643931795385, + "loss": 2.522, + "theoretical_loss": 3.6731006754814572, + "tokens_seen": 934084608 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036213640922768303, + "loss": 2.4914, + "theoretical_loss": 3.673075841359578, + "tokens_seen": 934150144 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036212637913741227, + "loss": 2.6395, + "theoretical_loss": 3.6730510094676863, + "tokens_seen": 934215680 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003621163490471414, + "loss": 2.7544, + "theoretical_loss": 3.6730261798054262, + "tokens_seen": 934281216 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036210631895687063, + "loss": 2.8916, + "theoretical_loss": 3.6730013523724407, + "tokens_seen": 934346752 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003620962888665998, + "loss": 2.6909, + "theoretical_loss": 3.6729765271683736, + "tokens_seen": 934412288 + }, + { + "epoch": 0.28, + "learning_rate": 0.000362086258776329, + "loss": 2.8899, + "theoretical_loss": 3.6729517041928683, + "tokens_seen": 934477824 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003620762286860582, + "loss": 2.5944, + "theoretical_loss": 3.6729268834455686, + "tokens_seen": 934543360 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036206619859578736, + "loss": 2.4949, + "theoretical_loss": 3.6729020649261184, + "tokens_seen": 934608896 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036205616850551654, + "loss": 2.7071, + "theoretical_loss": 3.672877248634161, + "tokens_seen": 934674432 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003620461384152458, + "loss": 2.4787, + "theoretical_loss": 3.6728524345693403, + "tokens_seen": 934739968 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003620361083249749, + "loss": 2.7971, + "theoretical_loss": 3.672827622731301, + "tokens_seen": 934805504 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036202607823470414, + "loss": 2.6195, + "theoretical_loss": 3.672802813119687, + "tokens_seen": 934871040 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036201604814443326, + "loss": 2.9063, + "theoretical_loss": 3.672778005734142, + "tokens_seen": 934936576 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003620060180541625, + "loss": 2.6416, + "theoretical_loss": 3.672753200574311, + "tokens_seen": 935002112 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003619959879638917, + "loss": 2.7188, + "theoretical_loss": 3.672728397639837, + "tokens_seen": 935067648 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036198595787362086, + "loss": 2.7995, + "theoretical_loss": 3.672703596930366, + "tokens_seen": 935133184 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036197592778335004, + "loss": 2.8007, + "theoretical_loss": 3.672678798445541, + "tokens_seen": 935198720 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003619658976930792, + "loss": 2.7091, + "theoretical_loss": 3.6726540021850083, + "tokens_seen": 935264256 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003619558676028084, + "loss": 2.6932, + "theoretical_loss": 3.672629208148411, + "tokens_seen": 935329792 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036194583751253764, + "loss": 2.8356, + "theoretical_loss": 3.6726044163353944, + "tokens_seen": 935395328 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036193580742226677, + "loss": 2.917, + "theoretical_loss": 3.6725796267456032, + "tokens_seen": 935460864 + }, + { + "epoch": 0.28, + "objective/train/docs_used": 537462, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.794448137283325, + "objective/train/theoretical_loss": 3.672554839378683, + "objective/train/tokens_used": 955986400, + "theoretical_loss": 3.672554839378683, + "tokens_seen": 935526400 + }, + { + "epoch": 0.28, + "learning_rate": 0.000361925777331996, + "loss": 2.8461, + "theoretical_loss": 3.672554839378683, + "tokens_seen": 935526400 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003619157472417252, + "loss": 2.7371, + "theoretical_loss": 3.6725300542342776, + "tokens_seen": 935591936 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036190571715145436, + "loss": 2.7935, + "theoretical_loss": 3.6725052713120325, + "tokens_seen": 935657472 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036189568706118354, + "loss": 2.8188, + "theoretical_loss": 3.6724804906115933, + "tokens_seen": 935723008 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003618856569709127, + "loss": 2.6661, + "theoretical_loss": 3.672455712132605, + "tokens_seen": 935788544 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003618756268806419, + "loss": 2.8353, + "theoretical_loss": 3.672430935874713, + "tokens_seen": 935854080 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036186559679037114, + "loss": 2.9416, + "theoretical_loss": 3.6724061618375625, + "tokens_seen": 935919616 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003618555667001003, + "loss": 2.7742, + "theoretical_loss": 3.6723813900207984, + "tokens_seen": 935985152 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003618455366098295, + "loss": 2.3653, + "theoretical_loss": 3.6723566204240674, + "tokens_seen": 936050688 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003618355065195587, + "loss": 2.6616, + "theoretical_loss": 3.672331853047014, + "tokens_seen": 936116224 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036182547642928787, + "loss": 2.8664, + "theoretical_loss": 3.672307087889285, + "tokens_seen": 936181760 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003618154463390171, + "loss": 2.8516, + "theoretical_loss": 3.6722823249505256, + "tokens_seen": 936247296 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036180541624874623, + "loss": 2.7701, + "theoretical_loss": 3.6722575642303816, + "tokens_seen": 936312832 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036179538615847546, + "loss": 2.9558, + "theoretical_loss": 3.672232805728499, + "tokens_seen": 936378368 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036178535606820464, + "loss": 2.7585, + "theoretical_loss": 3.6722080494445244, + "tokens_seen": 936443904 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003617753259779338, + "loss": 2.9516, + "theoretical_loss": 3.672183295378103, + "tokens_seen": 936509440 + }, + { + "epoch": 0.28, + "learning_rate": 0.000361765295887663, + "loss": 2.8924, + "theoretical_loss": 3.672158543528882, + "tokens_seen": 936574976 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003617552657973922, + "loss": 2.4272, + "theoretical_loss": 3.6721337938965064, + "tokens_seen": 936640512 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036174523570712137, + "loss": 2.742, + "theoretical_loss": 3.672109046480624, + "tokens_seen": 936706048 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003617352056168506, + "loss": 2.9497, + "theoretical_loss": 3.67208430128088, + "tokens_seen": 936771584 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036172517552657973, + "loss": 2.7798, + "theoretical_loss": 3.672059558296922, + "tokens_seen": 936837120 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036171514543630897, + "loss": 2.6876, + "theoretical_loss": 3.672034817528396, + "tokens_seen": 936902656 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003617051153460381, + "loss": 2.9874, + "theoretical_loss": 3.6720100789749486, + "tokens_seen": 936968192 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036169508525576733, + "loss": 2.8712, + "theoretical_loss": 3.6719853426362263, + "tokens_seen": 937033728 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003616850551654965, + "loss": 2.7095, + "theoretical_loss": 3.6719606085118772, + "tokens_seen": 937099264 + }, + { + "epoch": 0.28, + "objective/train/docs_used": 538123, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.889359712600708, + "objective/train/theoretical_loss": 3.671935876601547, + "objective/train/tokens_used": 957624800, + "theoretical_loss": 3.671935876601547, + "tokens_seen": 937164800 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003616750250752257, + "loss": 2.9565, + "theoretical_loss": 3.671935876601547, + "tokens_seen": 937164800 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036166499498495487, + "loss": 2.8072, + "theoretical_loss": 3.671911146904883, + "tokens_seen": 937230336 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036165496489468405, + "loss": 2.5979, + "theoretical_loss": 3.671886419421533, + "tokens_seen": 937295872 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036164493480441323, + "loss": 2.7514, + "theoretical_loss": 3.6718616941511426, + "tokens_seen": 937361408 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036163490471414247, + "loss": 2.9155, + "theoretical_loss": 3.6718369710933607, + "tokens_seen": 937426944 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003616248746238716, + "loss": 2.8409, + "theoretical_loss": 3.671812250247834, + "tokens_seen": 937492480 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036161484453360083, + "loss": 2.5793, + "theoretical_loss": 3.6717875316142097, + "tokens_seen": 937558016 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036160481444333, + "loss": 2.9515, + "theoretical_loss": 3.6717628151921353, + "tokens_seen": 937623552 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003615947843530592, + "loss": 2.6389, + "theoretical_loss": 3.6717381009812593, + "tokens_seen": 937689088 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003615847542627884, + "loss": 2.6925, + "theoretical_loss": 3.671713388981228, + "tokens_seen": 937754624 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036157472417251756, + "loss": 2.7516, + "theoretical_loss": 3.67168867919169, + "tokens_seen": 937820160 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036156469408224674, + "loss": 2.7405, + "theoretical_loss": 3.671663971612293, + "tokens_seen": 937885696 + }, + { + "epoch": 0.28, + "learning_rate": 0.000361554663991976, + "loss": 2.7259, + "theoretical_loss": 3.6716392662426847, + "tokens_seen": 937951232 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003615446339017051, + "loss": 2.6363, + "theoretical_loss": 3.671614563082513, + "tokens_seen": 938016768 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036153460381143434, + "loss": 2.8252, + "theoretical_loss": 3.6715898621314262, + "tokens_seen": 938082304 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036152457372116346, + "loss": 2.6258, + "theoretical_loss": 3.6715651633890722, + "tokens_seen": 938147840 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003615145436308927, + "loss": 2.7713, + "theoretical_loss": 3.6715404668550997, + "tokens_seen": 938213376 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003615045135406219, + "loss": 2.8202, + "theoretical_loss": 3.671515772529157, + "tokens_seen": 938278912 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036149448345035106, + "loss": 2.9742, + "theoretical_loss": 3.671491080410892, + "tokens_seen": 938344448 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036148445336008024, + "loss": 2.7893, + "theoretical_loss": 3.671466390499953, + "tokens_seen": 938409984 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003614744232698094, + "loss": 2.8811, + "theoretical_loss": 3.6714417027959887, + "tokens_seen": 938475520 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003614643931795386, + "loss": 2.8897, + "theoretical_loss": 3.6714170172986487, + "tokens_seen": 938541056 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036145436308926784, + "loss": 2.5812, + "theoretical_loss": 3.6713923340075802, + "tokens_seen": 938606592 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036144433299899697, + "loss": 2.9987, + "theoretical_loss": 3.6713676529224335, + "tokens_seen": 938672128 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003614343029087262, + "loss": 2.8512, + "theoretical_loss": 3.671342974042856, + "tokens_seen": 938737664 + }, + { + "epoch": 0.28, + "objective/train/docs_used": 539283, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.176745891571045, + "objective/train/theoretical_loss": 3.6713182973684972, + "objective/train/tokens_used": 959263200, + "theoretical_loss": 3.6713182973684972, + "tokens_seen": 938803200 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003614242728184554, + "loss": 2.7534, + "theoretical_loss": 3.6713182973684972, + "tokens_seen": 938803200 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036141424272818456, + "loss": 2.9063, + "theoretical_loss": 3.6712936228990065, + "tokens_seen": 938868736 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036140421263791374, + "loss": 2.8095, + "theoretical_loss": 3.6712689506340332, + "tokens_seen": 938934272 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003613941825476429, + "loss": 2.8568, + "theoretical_loss": 3.6712442805732257, + "tokens_seen": 938999808 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003613841524573721, + "loss": 2.8056, + "theoretical_loss": 3.671219612716234, + "tokens_seen": 939065344 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036137412236710134, + "loss": 2.6866, + "theoretical_loss": 3.6711949470627063, + "tokens_seen": 939130880 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036136409227683047, + "loss": 2.8974, + "theoretical_loss": 3.6711702836122937, + "tokens_seen": 939196416 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003613540621865597, + "loss": 2.6707, + "theoretical_loss": 3.671145622364644, + "tokens_seen": 939261952 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036134403209628883, + "loss": 2.9055, + "theoretical_loss": 3.671120963319408, + "tokens_seen": 939327488 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036133400200601807, + "loss": 2.946, + "theoretical_loss": 3.671096306476235, + "tokens_seen": 939393024 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036132397191574725, + "loss": 2.7421, + "theoretical_loss": 3.6710716518347746, + "tokens_seen": 939458560 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036131394182547643, + "loss": 2.6472, + "theoretical_loss": 3.671046999394677, + "tokens_seen": 939524096 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003613039117352056, + "loss": 2.5051, + "theoretical_loss": 3.6710223491555913, + "tokens_seen": 939589632 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036129388164493484, + "loss": 2.7843, + "theoretical_loss": 3.6709977011171686, + "tokens_seen": 939655168 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036128385155466397, + "loss": 2.6564, + "theoretical_loss": 3.6709730552790583, + "tokens_seen": 939720704 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003612738214643932, + "loss": 2.8497, + "theoretical_loss": 3.670948411640911, + "tokens_seen": 939786240 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036126379137412233, + "loss": 2.6001, + "theoretical_loss": 3.670923770202376, + "tokens_seen": 939851776 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036125376128385157, + "loss": 2.6519, + "theoretical_loss": 3.6708991309631043, + "tokens_seen": 939917312 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036124373119358075, + "loss": 2.6925, + "theoretical_loss": 3.6708744939227462, + "tokens_seen": 939982848 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036123370110330993, + "loss": 3.0568, + "theoretical_loss": 3.6708498590809526, + "tokens_seen": 940048384 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003612236710130391, + "loss": 2.7712, + "theoretical_loss": 3.6708252264373735, + "tokens_seen": 940113920 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003612136409227683, + "loss": 2.7798, + "theoretical_loss": 3.6708005959916594, + "tokens_seen": 940179456 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003612036108324975, + "loss": 2.6836, + "theoretical_loss": 3.6707759677434613, + "tokens_seen": 940244992 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003611935807422267, + "loss": 2.834, + "theoretical_loss": 3.67075134169243, + "tokens_seen": 940310528 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036118355065195584, + "loss": 2.7921, + "theoretical_loss": 3.6707267178382166, + "tokens_seen": 940376064 + }, + { + "epoch": 0.28, + "objective/train/docs_used": 539932, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7997701168060303, + "objective/train/theoretical_loss": 3.6707020961804715, + "objective/train/tokens_used": 960901600, + "theoretical_loss": 3.6707020961804715, + "tokens_seen": 940441600 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036117352056168507, + "loss": 2.5112, + "theoretical_loss": 3.6707020961804715, + "tokens_seen": 940441600 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003611634904714142, + "loss": 2.5687, + "theoretical_loss": 3.670677476718846, + "tokens_seen": 940507136 + }, + { + "epoch": 0.29, + "learning_rate": 0.00036115346038114343, + "loss": 3.1154, + "theoretical_loss": 3.6706528594529915, + "tokens_seen": 940572672 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003611434302908726, + "loss": 2.7424, + "theoretical_loss": 3.6706282443825584, + "tokens_seen": 940638208 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003611334002006018, + "loss": 3.0416, + "theoretical_loss": 3.6706036315071993, + "tokens_seen": 940703744 + }, + { + "epoch": 0.29, + "learning_rate": 0.000361123370110331, + "loss": 2.7868, + "theoretical_loss": 3.670579020826564, + "tokens_seen": 940769280 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003611133400200602, + "loss": 2.7638, + "theoretical_loss": 3.670554412340305, + "tokens_seen": 940834816 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003611033099297894, + "loss": 2.6936, + "theoretical_loss": 3.670529806048073, + "tokens_seen": 940900352 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003610932798395186, + "loss": 2.6707, + "theoretical_loss": 3.6705052019495206, + "tokens_seen": 940965888 + }, + { + "epoch": 0.29, + "learning_rate": 0.00036108324974924776, + "loss": 2.6969, + "theoretical_loss": 3.6704806000442987, + "tokens_seen": 941031424 + }, + { + "epoch": 0.29, + "learning_rate": 0.00036107321965897694, + "loss": 2.9621, + "theoretical_loss": 3.6704560003320594, + "tokens_seen": 941096960 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003610631895687062, + "loss": 2.7309, + "theoretical_loss": 3.6704314028124543, + "tokens_seen": 941162496 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003610531594784353, + "loss": 2.403, + "theoretical_loss": 3.6704068074851355, + "tokens_seen": 941228032 + }, + { + "epoch": 0.29, + "learning_rate": 0.00036104312938816454, + "loss": 2.8384, + "theoretical_loss": 3.670382214349755, + "tokens_seen": 941293568 + }, + { + "epoch": 0.29, + "learning_rate": 0.00036103309929789366, + "loss": 2.7171, + "theoretical_loss": 3.6703576234059643, + "tokens_seen": 941359104 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003610230692076229, + "loss": 2.5923, + "theoretical_loss": 3.6703330346534155, + "tokens_seen": 941424640 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003610130391173521, + "loss": 2.6463, + "theoretical_loss": 3.6703084480917623, + "tokens_seen": 941490176 + }, + { + "epoch": 0.29, + "learning_rate": 0.00036100300902708126, + "loss": 2.9954, + "theoretical_loss": 3.670283863720656, + "tokens_seen": 941555712 + }, + { + "epoch": 0.29, + "learning_rate": 0.00036099297893681044, + "loss": 2.638, + "theoretical_loss": 3.6702592815397486, + "tokens_seen": 941621248 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003609829488465396, + "loss": 2.7118, + "theoretical_loss": 3.6702347015486927, + "tokens_seen": 941686784 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003609729187562688, + "loss": 2.6541, + "theoretical_loss": 3.670210123747142, + "tokens_seen": 941752320 + }, + { + "epoch": 0.29, + "learning_rate": 0.00036096288866599804, + "loss": 2.8693, + "theoretical_loss": 3.6701855481347474, + "tokens_seen": 941817856 + }, + { + "epoch": 0.29, + "learning_rate": 0.00036095285857572717, + "loss": 2.8277, + "theoretical_loss": 3.6701609747111625, + "tokens_seen": 941883392 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003609428284854564, + "loss": 2.6998, + "theoretical_loss": 3.6701364034760404, + "tokens_seen": 941948928 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003609327983951856, + "loss": 2.664, + "theoretical_loss": 3.6701118344290333, + "tokens_seen": 942014464 + }, + { + "epoch": 0.29, + "objective/train/docs_used": 541311, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6348373889923096, + "objective/train/theoretical_loss": 3.6700872675697944, + "objective/train/tokens_used": 962540000, + "theoretical_loss": 3.6700872675697944, + "tokens_seen": 942080000 + }, + { + "epoch": 0.29, + "learning_rate": 0.00036092276830491476, + "loss": 2.7517, + "theoretical_loss": 3.6700872675697944, + "tokens_seen": 942080000 + }, + { + "epoch": 0.29, + "learning_rate": 0.00036091273821464394, + "loss": 2.9109, + "theoretical_loss": 3.670062702897977, + "tokens_seen": 942145536 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003609027081243731, + "loss": 2.5914, + "theoretical_loss": 3.670038140413234, + "tokens_seen": 942211072 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003608926780341023, + "loss": 2.8503, + "theoretical_loss": 3.6700135801152176, + "tokens_seen": 942276608 + }, + { + "epoch": 0.29, + "learning_rate": 0.00036088264794383154, + "loss": 2.8134, + "theoretical_loss": 3.6699890220035822, + "tokens_seen": 942342144 + }, + { + "epoch": 0.29, + "learning_rate": 0.00036087261785356067, + "loss": 2.7346, + "theoretical_loss": 3.6699644660779818, + "tokens_seen": 942407680 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003608625877632899, + "loss": 2.6412, + "theoretical_loss": 3.669939912338068, + "tokens_seen": 942473216 + }, + { + "epoch": 0.29, + "learning_rate": 0.00036085255767301903, + "loss": 2.6587, + "theoretical_loss": 3.669915360783495, + "tokens_seen": 942538752 + }, + { + "epoch": 0.29, + "learning_rate": 0.00036084252758274827, + "loss": 2.9403, + "theoretical_loss": 3.6698908114139175, + "tokens_seen": 942604288 + }, + { + "epoch": 0.29, + "learning_rate": 0.00036083249749247745, + "loss": 2.7363, + "theoretical_loss": 3.6698662642289874, + "tokens_seen": 942669824 + }, + { + "epoch": 0.29, + "learning_rate": 0.00036082246740220663, + "loss": 2.8781, + "theoretical_loss": 3.6698417192283594, + "tokens_seen": 942735360 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003608124373119358, + "loss": 2.7321, + "theoretical_loss": 3.669817176411687, + "tokens_seen": 942800896 + }, + { + "epoch": 0.29, + "learning_rate": 0.00036080240722166505, + "loss": 2.5915, + "theoretical_loss": 3.6697926357786246, + "tokens_seen": 942866432 + }, + { + "epoch": 0.29, + "learning_rate": 0.00036079237713139417, + "loss": 3.0283, + "theoretical_loss": 3.6697680973288254, + "tokens_seen": 942931968 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003607823470411234, + "loss": 2.6407, + "theoretical_loss": 3.669743561061944, + "tokens_seen": 942997504 + }, + { + "epoch": 0.29, + "learning_rate": 0.00036077231695085253, + "loss": 2.668, + "theoretical_loss": 3.6697190269776345, + "tokens_seen": 943063040 + }, + { + "epoch": 0.29, + "learning_rate": 0.00036076228686058177, + "loss": 2.5075, + "theoretical_loss": 3.669694495075551, + "tokens_seen": 943128576 + }, + { + "epoch": 0.29, + "learning_rate": 0.00036075225677031095, + "loss": 2.6409, + "theoretical_loss": 3.6696699653553475, + "tokens_seen": 943194112 + }, + { + "epoch": 0.29, + "learning_rate": 0.00036074222668004013, + "loss": 2.9548, + "theoretical_loss": 3.669645437816678, + "tokens_seen": 943259648 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003607321965897693, + "loss": 2.7059, + "theoretical_loss": 3.6696209124591985, + "tokens_seen": 943325184 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003607221664994985, + "loss": 2.8471, + "theoretical_loss": 3.669596389282562, + "tokens_seen": 943390720 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003607121364092277, + "loss": 2.7851, + "theoretical_loss": 3.6695718682864236, + "tokens_seen": 943456256 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003607021063189569, + "loss": 2.8246, + "theoretical_loss": 3.6695473494704385, + "tokens_seen": 943521792 + }, + { + "epoch": 0.29, + "learning_rate": 0.00036069207622868604, + "loss": 3.0652, + "theoretical_loss": 3.669522832834261, + "tokens_seen": 943587328 + }, + { + "epoch": 0.29, + "learning_rate": 0.00036068204613841527, + "loss": 2.8489, + "theoretical_loss": 3.6694983183775447, + "tokens_seen": 943652864 + }, + { + "epoch": 0.29, + "objective/train/docs_used": 542104, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.591085433959961, + "objective/train/theoretical_loss": 3.6694738060999468, + "objective/train/tokens_used": 964178400, + "theoretical_loss": 3.6694738060999468, + "tokens_seen": 943718400 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003606720160481444, + "loss": 2.7783, + "theoretical_loss": 3.6694738060999468, + "tokens_seen": 943718400 + }, + { + "epoch": 0.29, + "learning_rate": 0.00036066198595787364, + "loss": 2.5525, + "theoretical_loss": 3.6694492960011207, + "tokens_seen": 943783936 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003606519558676028, + "loss": 2.675, + "theoretical_loss": 3.6694247880807223, + "tokens_seen": 943849472 + }, + { + "epoch": 0.29, + "learning_rate": 0.000360641925777332, + "loss": 2.7889, + "theoretical_loss": 3.6694002823384064, + "tokens_seen": 943915008 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003606318956870612, + "loss": 2.7005, + "theoretical_loss": 3.669375778773828, + "tokens_seen": 943980544 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003606218655967904, + "loss": 2.6766, + "theoretical_loss": 3.6693512773866424, + "tokens_seen": 944046080 + }, + { + "epoch": 0.29, + "learning_rate": 0.00036061183550651954, + "loss": 2.9746, + "theoretical_loss": 3.6693267781765058, + "tokens_seen": 944111616 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003606018054162488, + "loss": 2.6891, + "theoretical_loss": 3.6693022811430724, + "tokens_seen": 944177152 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003605917753259779, + "loss": 2.6278, + "theoretical_loss": 3.669277786285999, + "tokens_seen": 944242688 + }, + { + "epoch": 0.29, + "learning_rate": 0.00036058174523570714, + "loss": 3.0296, + "theoretical_loss": 3.6692532936049407, + "tokens_seen": 944308224 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003605717151454363, + "loss": 2.6093, + "theoretical_loss": 3.6692288030995526, + "tokens_seen": 944373760 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003605616850551655, + "loss": 2.5855, + "theoretical_loss": 3.669204314769491, + "tokens_seen": 944439296 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003605516549648947, + "loss": 2.8238, + "theoretical_loss": 3.6691798286144124, + "tokens_seen": 944504832 + }, + { + "epoch": 0.29, + "learning_rate": 0.00036054162487462386, + "loss": 2.7585, + "theoretical_loss": 3.669155344633971, + "tokens_seen": 944570368 + }, + { + "epoch": 0.29, + "learning_rate": 0.00036053159478435304, + "loss": 2.6959, + "theoretical_loss": 3.6691308628278247, + "tokens_seen": 944635904 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003605215646940823, + "loss": 2.819, + "theoretical_loss": 3.6691063831956283, + "tokens_seen": 944701440 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003605115346038114, + "loss": 2.9109, + "theoretical_loss": 3.669081905737039, + "tokens_seen": 944766976 + }, + { + "epoch": 0.29, + "learning_rate": 0.00036050150451354064, + "loss": 2.7573, + "theoretical_loss": 3.669057430451712, + "tokens_seen": 944832512 + }, + { + "epoch": 0.29, + "learning_rate": 0.00036049147442326977, + "loss": 2.8658, + "theoretical_loss": 3.6690329573393043, + "tokens_seen": 944898048 + }, + { + "epoch": 0.29, + "learning_rate": 0.000360481444332999, + "loss": 2.7504, + "theoretical_loss": 3.669008486399472, + "tokens_seen": 944963584 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003604714142427282, + "loss": 2.728, + "theoretical_loss": 3.668984017631872, + "tokens_seen": 945029120 + }, + { + "epoch": 0.29, + "learning_rate": 0.00036046138415245737, + "loss": 2.9178, + "theoretical_loss": 3.66895955103616, + "tokens_seen": 945094656 + }, + { + "epoch": 0.29, + "learning_rate": 0.00036045135406218655, + "loss": 2.936, + "theoretical_loss": 3.6689350866119934, + "tokens_seen": 945160192 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003604413239719158, + "loss": 2.9079, + "theoretical_loss": 3.668910624359029, + "tokens_seen": 945225728 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003604312938816449, + "loss": 2.5055, + "theoretical_loss": 3.668886164276923, + "tokens_seen": 945291264 + }, + { + "epoch": 0.29, + "objective/train/docs_used": 543471, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.5615577697753906, + "objective/train/theoretical_loss": 3.668861706365332, + "objective/train/tokens_used": 965816800, + "theoretical_loss": 3.668861706365332, + "tokens_seen": 945356800 + }, + { + "epoch": 0.29, + "learning_rate": 0.00036042126379137414, + "loss": 2.7528, + "theoretical_loss": 3.668861706365332, + "tokens_seen": 945356800 + }, + { + "epoch": 0.29, + "learning_rate": 0.00036041123370110327, + "loss": 2.716, + "theoretical_loss": 3.668837250623914, + "tokens_seen": 945422336 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003604012036108325, + "loss": 2.9373, + "theoretical_loss": 3.6688127970523254, + "tokens_seen": 945487872 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003603911735205617, + "loss": 2.8614, + "theoretical_loss": 3.6687883456502233, + "tokens_seen": 945553408 + }, + { + "epoch": 0.29, + "learning_rate": 0.00036038114343029087, + "loss": 2.6573, + "theoretical_loss": 3.668763896417265, + "tokens_seen": 945618944 + }, + { + "epoch": 0.29, + "learning_rate": 0.00036037111334002005, + "loss": 2.6876, + "theoretical_loss": 3.668739449353108, + "tokens_seen": 945684480 + }, + { + "epoch": 0.29, + "learning_rate": 0.00036036108324974923, + "loss": 2.7742, + "theoretical_loss": 3.6687150044574093, + "tokens_seen": 945750016 + }, + { + "epoch": 0.29, + "learning_rate": 0.00036035105315947847, + "loss": 2.8233, + "theoretical_loss": 3.6686905617298264, + "tokens_seen": 945815552 + }, + { + "epoch": 0.29, + "learning_rate": 0.00036034102306920765, + "loss": 2.8558, + "theoretical_loss": 3.6686661211700167, + "tokens_seen": 945881088 + }, + { + "epoch": 0.29, + "learning_rate": 0.00036033099297893683, + "loss": 2.763, + "theoretical_loss": 3.668641682777638, + "tokens_seen": 945946624 + }, + { + "epoch": 0.29, + "learning_rate": 0.000360320962888666, + "loss": 2.7301, + "theoretical_loss": 3.6686172465523477, + "tokens_seen": 946012160 + }, + { + "epoch": 0.29, + "learning_rate": 0.00036031093279839525, + "loss": 2.7846, + "theoretical_loss": 3.668592812493803, + "tokens_seen": 946077696 + }, + { + "epoch": 0.29, + "learning_rate": 0.00036030090270812437, + "loss": 2.8597, + "theoretical_loss": 3.668568380601663, + "tokens_seen": 946143232 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003602908726178536, + "loss": 2.6764, + "theoretical_loss": 3.668543950875585, + "tokens_seen": 946208768 + }, + { + "epoch": 0.29, + "learning_rate": 0.00036028084252758273, + "loss": 2.7721, + "theoretical_loss": 3.6685195233152266, + "tokens_seen": 946274304 + }, + { + "epoch": 0.29, + "learning_rate": 0.00036027081243731197, + "loss": 2.736, + "theoretical_loss": 3.668495097920246, + "tokens_seen": 946339840 + }, + { + "epoch": 0.29, + "learning_rate": 0.00036026078234704115, + "loss": 2.7736, + "theoretical_loss": 3.668470674690302, + "tokens_seen": 946405376 + }, + { + "epoch": 0.29, + "learning_rate": 0.00036025075225677033, + "loss": 2.7634, + "theoretical_loss": 3.668446253625051, + "tokens_seen": 946470912 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003602407221664995, + "loss": 2.7275, + "theoretical_loss": 3.6684218347241533, + "tokens_seen": 946536448 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003602306920762287, + "loss": 2.446, + "theoretical_loss": 3.6683974179872667, + "tokens_seen": 946601984 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003602206619859579, + "loss": 2.8372, + "theoretical_loss": 3.6683730034140485, + "tokens_seen": 946667520 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003602106318956871, + "loss": 2.8422, + "theoretical_loss": 3.668348591004159, + "tokens_seen": 946733056 + }, + { + "epoch": 0.29, + "learning_rate": 0.00036020060180541624, + "loss": 2.9924, + "theoretical_loss": 3.668324180757255, + "tokens_seen": 946798592 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003601905717151455, + "loss": 2.7312, + "theoretical_loss": 3.6682997726729965, + "tokens_seen": 946864128 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003601805416248746, + "loss": 2.9575, + "theoretical_loss": 3.6682753667510415, + "tokens_seen": 946929664 + }, + { + "epoch": 0.29, + "objective/train/docs_used": 544140, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.001873016357422, + "objective/train/theoretical_loss": 3.668250962991049, + "objective/train/tokens_used": 967455200, + "theoretical_loss": 3.668250962991049, + "tokens_seen": 946995200 + }, + { + "epoch": 0.29, + "learning_rate": 0.00036017051153460384, + "loss": 2.7523, + "theoretical_loss": 3.668250962991049, + "tokens_seen": 946995200 + }, + { + "epoch": 0.29, + "learning_rate": 0.000360160481444333, + "loss": 2.9606, + "theoretical_loss": 3.668226561392678, + "tokens_seen": 947060736 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003601504513540622, + "loss": 2.7982, + "theoretical_loss": 3.668202161955587, + "tokens_seen": 947126272 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003601404212637914, + "loss": 2.6962, + "theoretical_loss": 3.6681777646794362, + "tokens_seen": 947191808 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003601303911735206, + "loss": 2.8641, + "theoretical_loss": 3.668153369563883, + "tokens_seen": 947257344 + }, + { + "epoch": 0.29, + "learning_rate": 0.00036012036108324974, + "loss": 2.8521, + "theoretical_loss": 3.668128976608588, + "tokens_seen": 947322880 + }, + { + "epoch": 0.29, + "learning_rate": 0.000360110330992979, + "loss": 2.5949, + "theoretical_loss": 3.6681045858132104, + "tokens_seen": 947388416 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003601003009027081, + "loss": 2.9111, + "theoretical_loss": 3.6680801971774084, + "tokens_seen": 947453952 + }, + { + "epoch": 0.29, + "learning_rate": 0.00036009027081243734, + "loss": 2.7974, + "theoretical_loss": 3.668055810700843, + "tokens_seen": 947519488 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003600802407221665, + "loss": 2.7759, + "theoretical_loss": 3.6680314263831724, + "tokens_seen": 947585024 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003600702106318957, + "loss": 2.9445, + "theoretical_loss": 3.6680070442240567, + "tokens_seen": 947650560 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003600601805416249, + "loss": 2.7578, + "theoretical_loss": 3.667982664223156, + "tokens_seen": 947716096 + }, + { + "epoch": 0.29, + "learning_rate": 0.00036005015045135406, + "loss": 2.6143, + "theoretical_loss": 3.667958286380129, + "tokens_seen": 947781632 + }, + { + "epoch": 0.29, + "learning_rate": 0.00036004012036108324, + "loss": 2.6212, + "theoretical_loss": 3.6679339106946363, + "tokens_seen": 947847168 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003600300902708125, + "loss": 2.6951, + "theoretical_loss": 3.667909537166338, + "tokens_seen": 947912704 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003600200601805416, + "loss": 2.8008, + "theoretical_loss": 3.667885165794893, + "tokens_seen": 947978240 + }, + { + "epoch": 0.29, + "learning_rate": 0.00036001003009027084, + "loss": 2.9889, + "theoretical_loss": 3.667860796579962, + "tokens_seen": 948043776 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035999999999999997, + "loss": 2.6962, + "theoretical_loss": 3.6678364295212056, + "tokens_seen": 948109312 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003599899699097292, + "loss": 2.9913, + "theoretical_loss": 3.6678120646182837, + "tokens_seen": 948174848 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003599799398194584, + "loss": 2.8807, + "theoretical_loss": 3.6677877018708562, + "tokens_seen": 948240384 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035996990972918757, + "loss": 2.9048, + "theoretical_loss": 3.6677633412785835, + "tokens_seen": 948305920 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035995987963891675, + "loss": 2.9218, + "theoretical_loss": 3.667738982841126, + "tokens_seen": 948371456 + }, + { + "epoch": 0.29, + "learning_rate": 0.000359949849548646, + "loss": 2.7911, + "theoretical_loss": 3.667714626558145, + "tokens_seen": 948436992 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003599398194583751, + "loss": 2.803, + "theoretical_loss": 3.6676902724293003, + "tokens_seen": 948502528 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035992978936810434, + "loss": 2.7182, + "theoretical_loss": 3.6676659204542528, + "tokens_seen": 948568064 + }, + { + "epoch": 0.29, + "objective/train/docs_used": 544543, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7242817878723145, + "objective/train/theoretical_loss": 3.6676415706326635, + "objective/train/tokens_used": 969093600, + "theoretical_loss": 3.6676415706326635, + "tokens_seen": 948633600 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035991975927783347, + "loss": 2.7925, + "theoretical_loss": 3.6676415706326635, + "tokens_seen": 948633600 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003599097291875627, + "loss": 3.0767, + "theoretical_loss": 3.6676172229641923, + "tokens_seen": 948699136 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003598996990972919, + "loss": 2.8361, + "theoretical_loss": 3.667592877448501, + "tokens_seen": 948764672 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035988966900702107, + "loss": 2.7618, + "theoretical_loss": 3.66756853408525, + "tokens_seen": 948830208 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035987963891675025, + "loss": 2.6878, + "theoretical_loss": 3.667544192874101, + "tokens_seen": 948895744 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035986960882647943, + "loss": 2.6737, + "theoretical_loss": 3.6675198538147153, + "tokens_seen": 948961280 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003598595787362086, + "loss": 2.8607, + "theoretical_loss": 3.6674955169067527, + "tokens_seen": 949026816 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035984954864593785, + "loss": 2.9423, + "theoretical_loss": 3.6674711821498756, + "tokens_seen": 949092352 + }, + { + "epoch": 0.29, + "learning_rate": 0.000359839518555667, + "loss": 2.9599, + "theoretical_loss": 3.6674468495437456, + "tokens_seen": 949157888 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003598294884653962, + "loss": 2.7915, + "theoretical_loss": 3.667422519088023, + "tokens_seen": 949223424 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035981945837512534, + "loss": 2.9425, + "theoretical_loss": 3.66739819078237, + "tokens_seen": 949288960 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035980942828485457, + "loss": 2.8057, + "theoretical_loss": 3.667373864626448, + "tokens_seen": 949354496 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035979939819458375, + "loss": 2.6562, + "theoretical_loss": 3.6673495406199197, + "tokens_seen": 949420032 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035978936810431293, + "loss": 2.9329, + "theoretical_loss": 3.667325218762445, + "tokens_seen": 949485568 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003597793380140421, + "loss": 2.7893, + "theoretical_loss": 3.667300899053687, + "tokens_seen": 949551104 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035976930792377135, + "loss": 2.9241, + "theoretical_loss": 3.667276581493307, + "tokens_seen": 949616640 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003597592778335005, + "loss": 2.7459, + "theoretical_loss": 3.6672522660809674, + "tokens_seen": 949682176 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003597492477432297, + "loss": 2.7024, + "theoretical_loss": 3.6672279528163303, + "tokens_seen": 949747712 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035973921765295884, + "loss": 2.8421, + "theoretical_loss": 3.667203641699057, + "tokens_seen": 949813248 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003597291875626881, + "loss": 2.8211, + "theoretical_loss": 3.6671793327288107, + "tokens_seen": 949878784 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035971915747241726, + "loss": 2.7426, + "theoretical_loss": 3.667155025905253, + "tokens_seen": 949944320 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035970912738214644, + "loss": 2.8536, + "theoretical_loss": 3.6671307212280473, + "tokens_seen": 950009856 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003596990972918756, + "loss": 2.6099, + "theoretical_loss": 3.6671064186968545, + "tokens_seen": 950075392 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003596890672016048, + "loss": 2.8128, + "theoretical_loss": 3.6670821183113373, + "tokens_seen": 950140928 + }, + { + "epoch": 0.29, + "learning_rate": 0.000359679037111334, + "loss": 2.9275, + "theoretical_loss": 3.6670578200711597, + "tokens_seen": 950206464 + }, + { + "epoch": 0.29, + "objective/train/docs_used": 545942, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.060412883758545, + "objective/train/theoretical_loss": 3.667033523975983, + "objective/train/tokens_used": 970732000, + "theoretical_loss": 3.667033523975983, + "tokens_seen": 950272000 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003596690070210632, + "loss": 2.7022, + "theoretical_loss": 3.667033523975983, + "tokens_seen": 950272000 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035965897693079234, + "loss": 2.8493, + "theoretical_loss": 3.667009230025471, + "tokens_seen": 950337536 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003596489468405216, + "loss": 2.7911, + "theoretical_loss": 3.6669849382192847, + "tokens_seen": 950403072 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035963891675025076, + "loss": 2.8411, + "theoretical_loss": 3.666960648557089, + "tokens_seen": 950468608 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035962888665997994, + "loss": 2.7747, + "theoretical_loss": 3.6669363610385464, + "tokens_seen": 950534144 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003596188565697091, + "loss": 2.5739, + "theoretical_loss": 3.6669120756633187, + "tokens_seen": 950599680 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003596088264794383, + "loss": 2.7053, + "theoretical_loss": 3.66688779243107, + "tokens_seen": 950665216 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035959879638916754, + "loss": 2.7293, + "theoretical_loss": 3.6668635113414636, + "tokens_seen": 950730752 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003595887662988967, + "loss": 2.7006, + "theoretical_loss": 3.666839232394163, + "tokens_seen": 950796288 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003595787362086259, + "loss": 2.7755, + "theoretical_loss": 3.6668149555888307, + "tokens_seen": 950861824 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003595687061183551, + "loss": 2.7157, + "theoretical_loss": 3.6667906809251303, + "tokens_seen": 950927360 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035955867602808426, + "loss": 2.659, + "theoretical_loss": 3.666766408402726, + "tokens_seen": 950992896 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035954864593781344, + "loss": 2.7691, + "theoretical_loss": 3.666742138021281, + "tokens_seen": 951058432 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003595386158475427, + "loss": 2.7537, + "theoretical_loss": 3.6667178697804577, + "tokens_seen": 951123968 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003595285857572718, + "loss": 2.8242, + "theoretical_loss": 3.6666936036799216, + "tokens_seen": 951189504 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035951855566700104, + "loss": 2.626, + "theoretical_loss": 3.666669339719336, + "tokens_seen": 951255040 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035950852557673017, + "loss": 2.6648, + "theoretical_loss": 3.666645077898364, + "tokens_seen": 951320576 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003594984954864594, + "loss": 2.9503, + "theoretical_loss": 3.666620818216671, + "tokens_seen": 951386112 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003594884653961886, + "loss": 2.7229, + "theoretical_loss": 3.66659656067392, + "tokens_seen": 951451648 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035947843530591777, + "loss": 2.7978, + "theoretical_loss": 3.666572305269775, + "tokens_seen": 951517184 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035946840521564695, + "loss": 3.0448, + "theoretical_loss": 3.6665480520039004, + "tokens_seen": 951582720 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003594583751253762, + "loss": 2.7406, + "theoretical_loss": 3.666523800875961, + "tokens_seen": 951648256 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003594483450351053, + "loss": 2.7506, + "theoretical_loss": 3.66649955188562, + "tokens_seen": 951713792 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035943831494483454, + "loss": 2.7832, + "theoretical_loss": 3.6664753050325425, + "tokens_seen": 951779328 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035942828485456367, + "loss": 2.7767, + "theoretical_loss": 3.666451060316393, + "tokens_seen": 951844864 + }, + { + "epoch": 0.29, + "objective/train/docs_used": 546639, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7628283500671387, + "objective/train/theoretical_loss": 3.6664268177368364, + "objective/train/tokens_used": 972370400, + "theoretical_loss": 3.6664268177368364, + "tokens_seen": 951910400 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003594182547642929, + "loss": 3.1233, + "theoretical_loss": 3.6664268177368364, + "tokens_seen": 951910400 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003594082246740221, + "loss": 2.8236, + "theoretical_loss": 3.666402577293537, + "tokens_seen": 951975936 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035939819458375127, + "loss": 2.9356, + "theoretical_loss": 3.666378338986159, + "tokens_seen": 952041472 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035938816449348045, + "loss": 2.5928, + "theoretical_loss": 3.666354102814368, + "tokens_seen": 952107008 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035937813440320963, + "loss": 2.6931, + "theoretical_loss": 3.6663298687778276, + "tokens_seen": 952172544 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003593681043129388, + "loss": 2.802, + "theoretical_loss": 3.6663056368762046, + "tokens_seen": 952238080 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035935807422266805, + "loss": 2.7016, + "theoretical_loss": 3.6662814071091625, + "tokens_seen": 952303616 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003593480441323972, + "loss": 2.7144, + "theoretical_loss": 3.666257179476367, + "tokens_seen": 952369152 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003593380140421264, + "loss": 2.6437, + "theoretical_loss": 3.6662329539774827, + "tokens_seen": 952434688 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035932798395185554, + "loss": 2.5705, + "theoretical_loss": 3.6662087306121762, + "tokens_seen": 952500224 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035931795386158477, + "loss": 2.663, + "theoretical_loss": 3.6661845093801118, + "tokens_seen": 952565760 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035930792377131395, + "loss": 2.9316, + "theoretical_loss": 3.6661602902809545, + "tokens_seen": 952631296 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035929789368104313, + "loss": 2.8867, + "theoretical_loss": 3.6661360733143704, + "tokens_seen": 952696832 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003592878635907723, + "loss": 2.8251, + "theoretical_loss": 3.6661118584800247, + "tokens_seen": 952762368 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035927783350050155, + "loss": 2.8243, + "theoretical_loss": 3.6660876457775835, + "tokens_seen": 952827904 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003592678034102307, + "loss": 2.7348, + "theoretical_loss": 3.6660634352067123, + "tokens_seen": 952893440 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003592577733199599, + "loss": 2.7507, + "theoretical_loss": 3.6660392267670767, + "tokens_seen": 952958976 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035924774322968904, + "loss": 2.7946, + "theoretical_loss": 3.6660150204583424, + "tokens_seen": 953024512 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003592377131394183, + "loss": 2.736, + "theoretical_loss": 3.6659908162801758, + "tokens_seen": 953090048 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035922768304914746, + "loss": 2.7721, + "theoretical_loss": 3.665966614232243, + "tokens_seen": 953155584 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035921765295887664, + "loss": 2.585, + "theoretical_loss": 3.665942414314209, + "tokens_seen": 953221120 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003592076228686058, + "loss": 2.4884, + "theoretical_loss": 3.665918216525741, + "tokens_seen": 953286656 + }, + { + "epoch": 0.29, + "learning_rate": 0.000359197592778335, + "loss": 2.8601, + "theoretical_loss": 3.6658940208665047, + "tokens_seen": 953352192 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003591875626880642, + "loss": 2.8385, + "theoretical_loss": 3.6658698273361665, + "tokens_seen": 953417728 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003591775325977934, + "loss": 2.8584, + "theoretical_loss": 3.6658456359343927, + "tokens_seen": 953483264 + }, + { + "epoch": 0.29, + "objective/train/docs_used": 547886, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.527340888977051, + "objective/train/theoretical_loss": 3.66582144666085, + "objective/train/tokens_used": 974008800, + "theoretical_loss": 3.66582144666085, + "tokens_seen": 953548800 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035916750250752254, + "loss": 3.0438, + "theoretical_loss": 3.66582144666085, + "tokens_seen": 953548800 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003591574724172518, + "loss": 2.7089, + "theoretical_loss": 3.6657972595152053, + "tokens_seen": 953614336 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035914744232698096, + "loss": 2.4332, + "theoretical_loss": 3.6657730744971238, + "tokens_seen": 953679872 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035913741223671014, + "loss": 2.7254, + "theoretical_loss": 3.6657488916062735, + "tokens_seen": 953745408 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003591273821464393, + "loss": 2.7095, + "theoretical_loss": 3.665724710842321, + "tokens_seen": 953810944 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003591173520561685, + "loss": 2.8739, + "theoretical_loss": 3.665700532204932, + "tokens_seen": 953876480 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003591073219658977, + "loss": 2.6891, + "theoretical_loss": 3.6656763556937744, + "tokens_seen": 953942016 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003590972918756269, + "loss": 2.9942, + "theoretical_loss": 3.6656521813085154, + "tokens_seen": 954007552 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035908726178535605, + "loss": 2.2991, + "theoretical_loss": 3.6656280090488216, + "tokens_seen": 954073088 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003590772316950853, + "loss": 2.8385, + "theoretical_loss": 3.66560383891436, + "tokens_seen": 954138624 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003590672016048144, + "loss": 2.727, + "theoretical_loss": 3.6655796709047976, + "tokens_seen": 954204160 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035905717151454364, + "loss": 2.7121, + "theoretical_loss": 3.6655555050198023, + "tokens_seen": 954269696 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003590471414242728, + "loss": 2.7133, + "theoretical_loss": 3.665531341259041, + "tokens_seen": 954335232 + }, + { + "epoch": 0.29, + "learning_rate": 0.000359037111334002, + "loss": 2.8199, + "theoretical_loss": 3.665507179622182, + "tokens_seen": 954400768 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003590270812437312, + "loss": 2.6538, + "theoretical_loss": 3.6654830201088906, + "tokens_seen": 954466304 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035901705115346037, + "loss": 2.5035, + "theoretical_loss": 3.665458862718837, + "tokens_seen": 954531840 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035900702106318955, + "loss": 2.799, + "theoretical_loss": 3.6654347074516873, + "tokens_seen": 954597376 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003589969909729188, + "loss": 2.774, + "theoretical_loss": 3.6654105543071096, + "tokens_seen": 954662912 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003589869608826479, + "loss": 2.9062, + "theoretical_loss": 3.6653864032847716, + "tokens_seen": 954728448 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035897693079237715, + "loss": 2.7603, + "theoretical_loss": 3.665362254384341, + "tokens_seen": 954793984 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035896690070210633, + "loss": 2.5827, + "theoretical_loss": 3.6653381076054865, + "tokens_seen": 954859520 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003589568706118355, + "loss": 2.8857, + "theoretical_loss": 3.6653139629478755, + "tokens_seen": 954925056 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003589468405215647, + "loss": 2.7797, + "theoretical_loss": 3.6652898204111763, + "tokens_seen": 954990592 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035893681043129387, + "loss": 2.6953, + "theoretical_loss": 3.6652656799950565, + "tokens_seen": 955056128 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035892678034102305, + "loss": 2.3569, + "theoretical_loss": 3.665241541699185, + "tokens_seen": 955121664 + }, + { + "epoch": 0.29, + "objective/train/docs_used": 548540, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.2415659427642822, + "objective/train/theoretical_loss": 3.6652174055232303, + "objective/train/tokens_used": 975647200, + "theoretical_loss": 3.6652174055232303, + "tokens_seen": 955187200 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003589167502507523, + "loss": 2.7397, + "theoretical_loss": 3.6652174055232303, + "tokens_seen": 955187200 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003589067201604814, + "loss": 2.7946, + "theoretical_loss": 3.66519327146686, + "tokens_seen": 955252736 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035889669007021065, + "loss": 2.906, + "theoretical_loss": 3.665169139529743, + "tokens_seen": 955318272 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003588866599799398, + "loss": 2.5353, + "theoretical_loss": 3.665145009711548, + "tokens_seen": 955383808 + }, + { + "epoch": 0.29, + "learning_rate": 0.000358876629889669, + "loss": 2.8824, + "theoretical_loss": 3.6651208820119434, + "tokens_seen": 955449344 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003588665997993982, + "loss": 2.6533, + "theoretical_loss": 3.6650967564305983, + "tokens_seen": 955514880 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003588565697091274, + "loss": 2.956, + "theoretical_loss": 3.6650726329671803, + "tokens_seen": 955580416 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003588465396188566, + "loss": 2.9396, + "theoretical_loss": 3.6650485116213596, + "tokens_seen": 955645952 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035883650952858574, + "loss": 2.6032, + "theoretical_loss": 3.6650243923928043, + "tokens_seen": 955711488 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035882647943831497, + "loss": 2.9369, + "theoretical_loss": 3.6650002752811837, + "tokens_seen": 955777024 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035881644934804415, + "loss": 2.6835, + "theoretical_loss": 3.664976160286167, + "tokens_seen": 955842560 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035880641925777333, + "loss": 2.5322, + "theoretical_loss": 3.664952047407423, + "tokens_seen": 955908096 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003587963891675025, + "loss": 2.8914, + "theoretical_loss": 3.664927936644621, + "tokens_seen": 955973632 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035878635907723175, + "loss": 2.6669, + "theoretical_loss": 3.66490382799743, + "tokens_seen": 956039168 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003587763289869609, + "loss": 2.8197, + "theoretical_loss": 3.66487972146552, + "tokens_seen": 956104704 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003587662988966901, + "loss": 2.9293, + "theoretical_loss": 3.66485561704856, + "tokens_seen": 956170240 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035875626880641924, + "loss": 2.6933, + "theoretical_loss": 3.66483151474622, + "tokens_seen": 956235776 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003587462387161485, + "loss": 2.5356, + "theoretical_loss": 3.664807414558169, + "tokens_seen": 956301312 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035873620862587766, + "loss": 2.7276, + "theoretical_loss": 3.664783316484076, + "tokens_seen": 956366848 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035872617853560684, + "loss": 2.7561, + "theoretical_loss": 3.664759220523613, + "tokens_seen": 956432384 + }, + { + "epoch": 0.29, + "learning_rate": 0.000358716148445336, + "loss": 2.8387, + "theoretical_loss": 3.664735126676448, + "tokens_seen": 956497920 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003587061183550652, + "loss": 2.9331, + "theoretical_loss": 3.6647110349422505, + "tokens_seen": 956563456 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003586960882647944, + "loss": 2.7455, + "theoretical_loss": 3.6646869453206925, + "tokens_seen": 956628992 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003586860581745236, + "loss": 2.6588, + "theoretical_loss": 3.6646628578114417, + "tokens_seen": 956694528 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035867602808425274, + "loss": 2.7832, + "theoretical_loss": 3.66463877241417, + "tokens_seen": 956760064 + }, + { + "epoch": 0.29, + "objective/train/docs_used": 549880, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.8033275604248047, + "objective/train/theoretical_loss": 3.664614689128546, + "objective/train/tokens_used": 977285600, + "theoretical_loss": 3.664614689128546, + "tokens_seen": 956825600 + }, + { + "epoch": 0.29, + "learning_rate": 0.000358665997993982, + "loss": 2.7416, + "theoretical_loss": 3.664614689128546, + "tokens_seen": 956825600 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035865596790371116, + "loss": 2.7025, + "theoretical_loss": 3.6645906079542416, + "tokens_seen": 956891136 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035864593781344034, + "loss": 2.7759, + "theoretical_loss": 3.6645665288909255, + "tokens_seen": 956956672 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003586359077231695, + "loss": 2.687, + "theoretical_loss": 3.66454245193827, + "tokens_seen": 957022208 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003586258776328987, + "loss": 2.8093, + "theoretical_loss": 3.664518377095944, + "tokens_seen": 957087744 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003586158475426279, + "loss": 2.7905, + "theoretical_loss": 3.6644943043636182, + "tokens_seen": 957153280 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003586058174523571, + "loss": 2.8615, + "theoretical_loss": 3.6644702337409636, + "tokens_seen": 957218816 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035859578736208625, + "loss": 2.4014, + "theoretical_loss": 3.664446165227651, + "tokens_seen": 957284352 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003585857572718155, + "loss": 2.7414, + "theoretical_loss": 3.6644220988233513, + "tokens_seen": 957349888 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003585757271815446, + "loss": 2.6167, + "theoretical_loss": 3.6643980345277347, + "tokens_seen": 957415424 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035856569709127384, + "loss": 2.5804, + "theoretical_loss": 3.6643739723404725, + "tokens_seen": 957480960 + }, + { + "epoch": 0.29, + "learning_rate": 0.000358555667001003, + "loss": 2.6009, + "theoretical_loss": 3.664349912261236, + "tokens_seen": 957546496 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003585456369107322, + "loss": 2.7922, + "theoretical_loss": 3.6643258542896957, + "tokens_seen": 957612032 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003585356068204614, + "loss": 2.6896, + "theoretical_loss": 3.664301798425523, + "tokens_seen": 957677568 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035852557673019057, + "loss": 2.9789, + "theoretical_loss": 3.664277744668389, + "tokens_seen": 957743104 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035851554663991975, + "loss": 2.7048, + "theoretical_loss": 3.664253693017965, + "tokens_seen": 957808640 + }, + { + "epoch": 0.29, + "learning_rate": 0.000358505516549649, + "loss": 2.7744, + "theoretical_loss": 3.6642296434739228, + "tokens_seen": 957874176 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003584954864593781, + "loss": 2.8612, + "theoretical_loss": 3.664205596035933, + "tokens_seen": 957939712 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035848545636910735, + "loss": 2.8399, + "theoretical_loss": 3.6641815507036677, + "tokens_seen": 958005248 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035847542627883653, + "loss": 2.6487, + "theoretical_loss": 3.6641575074767982, + "tokens_seen": 958070784 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003584653961885657, + "loss": 2.874, + "theoretical_loss": 3.664133466354997, + "tokens_seen": 958136320 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003584553660982949, + "loss": 2.8552, + "theoretical_loss": 3.664109427337934, + "tokens_seen": 958201856 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035844533600802407, + "loss": 2.879, + "theoretical_loss": 3.664085390425283, + "tokens_seen": 958267392 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035843530591775325, + "loss": 2.7663, + "theoretical_loss": 3.6640613556167145, + "tokens_seen": 958332928 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003584252758274825, + "loss": 2.49, + "theoretical_loss": 3.664037322911901, + "tokens_seen": 958398464 + }, + { + "epoch": 0.29, + "objective/train/docs_used": 550706, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.51804518699646, + "objective/train/theoretical_loss": 3.664013292310515, + "objective/train/tokens_used": 978924000, + "theoretical_loss": 3.664013292310515, + "tokens_seen": 958464000 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003584152457372116, + "loss": 2.8182, + "theoretical_loss": 3.664013292310515, + "tokens_seen": 958464000 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035840521564694085, + "loss": 2.7688, + "theoretical_loss": 3.6639892638122276, + "tokens_seen": 958529536 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035839518555667, + "loss": 2.8112, + "theoretical_loss": 3.6639652374167113, + "tokens_seen": 958595072 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003583851554663992, + "loss": 3.0002, + "theoretical_loss": 3.663941213123639, + "tokens_seen": 958660608 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003583751253761284, + "loss": 2.5756, + "theoretical_loss": 3.6639171909326818, + "tokens_seen": 958726144 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003583650952858576, + "loss": 2.5717, + "theoretical_loss": 3.6638931708435134, + "tokens_seen": 958791680 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035835506519558676, + "loss": 2.6102, + "theoretical_loss": 3.6638691528558054, + "tokens_seen": 958857216 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035834503510531594, + "loss": 2.6326, + "theoretical_loss": 3.663845136969231, + "tokens_seen": 958922752 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003583350050150451, + "loss": 2.7544, + "theoretical_loss": 3.6638211231834616, + "tokens_seen": 958988288 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035832497492477435, + "loss": 2.6685, + "theoretical_loss": 3.6637971114981713, + "tokens_seen": 959053824 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003583149448345035, + "loss": 3.0154, + "theoretical_loss": 3.6637731019130317, + "tokens_seen": 959119360 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003583049147442327, + "loss": 2.9862, + "theoretical_loss": 3.6637490944277165, + "tokens_seen": 959184896 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003582948846539619, + "loss": 2.6679, + "theoretical_loss": 3.663725089041898, + "tokens_seen": 959250432 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003582848545636911, + "loss": 2.7592, + "theoretical_loss": 3.6637010857552497, + "tokens_seen": 959315968 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035827482447342026, + "loss": 2.5916, + "theoretical_loss": 3.6636770845674445, + "tokens_seen": 959381504 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035826479438314944, + "loss": 2.7301, + "theoretical_loss": 3.6636530854781553, + "tokens_seen": 959447040 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003582547642928786, + "loss": 2.8008, + "theoretical_loss": 3.6636290884870553, + "tokens_seen": 959512576 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035824473420260786, + "loss": 2.5103, + "theoretical_loss": 3.663605093593818, + "tokens_seen": 959578112 + }, + { + "epoch": 0.29, + "learning_rate": 0.000358234704112337, + "loss": 2.7472, + "theoretical_loss": 3.6635811007981163, + "tokens_seen": 959643648 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003582246740220662, + "loss": 2.5813, + "theoretical_loss": 3.6635571100996245, + "tokens_seen": 959709184 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035821464393179535, + "loss": 2.8559, + "theoretical_loss": 3.6635331214980154, + "tokens_seen": 959774720 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003582046138415246, + "loss": 2.8879, + "theoretical_loss": 3.663509134992962, + "tokens_seen": 959840256 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035819458375125376, + "loss": 2.7108, + "theoretical_loss": 3.6634851505841395, + "tokens_seen": 959905792 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035818455366098294, + "loss": 2.9047, + "theoretical_loss": 3.66346116827122, + "tokens_seen": 959971328 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003581745235707121, + "loss": 2.6694, + "theoretical_loss": 3.6634371880538783, + "tokens_seen": 960036864 + }, + { + "epoch": 0.29, + "objective/train/docs_used": 551132, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.232994794845581, + "objective/train/theoretical_loss": 3.6634132099317886, + "objective/train/tokens_used": 980562400, + "theoretical_loss": 3.6634132099317886, + "tokens_seen": 960102400 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035816449348044136, + "loss": 2.9548, + "theoretical_loss": 3.6634132099317886, + "tokens_seen": 960102400 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003581544633901705, + "loss": 2.6868, + "theoretical_loss": 3.6633892339046232, + "tokens_seen": 960167936 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003581444332998997, + "loss": 2.8028, + "theoretical_loss": 3.663365259972058, + "tokens_seen": 960233472 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035813440320962885, + "loss": 2.9587, + "theoretical_loss": 3.6633412881337653, + "tokens_seen": 960299008 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003581243731193581, + "loss": 2.789, + "theoretical_loss": 3.6633173183894208, + "tokens_seen": 960364544 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035811434302908727, + "loss": 2.9447, + "theoretical_loss": 3.6632933507386976, + "tokens_seen": 960430080 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035810431293881645, + "loss": 2.8157, + "theoretical_loss": 3.6632693851812705, + "tokens_seen": 960495616 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003580942828485457, + "loss": 2.6414, + "theoretical_loss": 3.663245421716814, + "tokens_seen": 960561152 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003580842527582748, + "loss": 2.8053, + "theoretical_loss": 3.663221460345002, + "tokens_seen": 960626688 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035807422266800404, + "loss": 2.9551, + "theoretical_loss": 3.6631975010655093, + "tokens_seen": 960692224 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003580641925777332, + "loss": 2.8168, + "theoretical_loss": 3.6631735438780106, + "tokens_seen": 960757760 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003580541624874624, + "loss": 2.6796, + "theoretical_loss": 3.663149588782181, + "tokens_seen": 960823296 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003580441323971916, + "loss": 2.7748, + "theoretical_loss": 3.6631256357776945, + "tokens_seen": 960888832 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035803410230692077, + "loss": 2.8139, + "theoretical_loss": 3.6631016848642255, + "tokens_seen": 960954368 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035802407221664995, + "loss": 2.7635, + "theoretical_loss": 3.6630777360414504, + "tokens_seen": 961019904 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003580140421263792, + "loss": 2.9129, + "theoretical_loss": 3.6630537893090422, + "tokens_seen": 961085440 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003580040120361083, + "loss": 2.7519, + "theoretical_loss": 3.663029844666678, + "tokens_seen": 961150976 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035799398194583755, + "loss": 2.5768, + "theoretical_loss": 3.663005902114031, + "tokens_seen": 961216512 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035798395185556673, + "loss": 2.4687, + "theoretical_loss": 3.6629819616507775, + "tokens_seen": 961282048 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003579739217652959, + "loss": 2.8777, + "theoretical_loss": 3.662958023276593, + "tokens_seen": 961347584 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003579638916750251, + "loss": 2.577, + "theoretical_loss": 3.662934086991151, + "tokens_seen": 961413120 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035795386158475427, + "loss": 2.9246, + "theoretical_loss": 3.6629101527941295, + "tokens_seen": 961478656 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035794383149448345, + "loss": 2.6441, + "theoretical_loss": 3.662886220685202, + "tokens_seen": 961544192 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003579338014042127, + "loss": 2.651, + "theoretical_loss": 3.6628622906640445, + "tokens_seen": 961609728 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003579237713139418, + "loss": 2.7711, + "theoretical_loss": 3.6628383627303327, + "tokens_seen": 961675264 + }, + { + "epoch": 0.29, + "objective/train/docs_used": 552317, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.286529064178467, + "objective/train/theoretical_loss": 3.6628144368837425, + "objective/train/tokens_used": 982200800, + "theoretical_loss": 3.6628144368837425, + "tokens_seen": 961740800 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035791374122367105, + "loss": 2.4988, + "theoretical_loss": 3.6628144368837425, + "tokens_seen": 961740800 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003579037111334002, + "loss": 2.5515, + "theoretical_loss": 3.662790513123949, + "tokens_seen": 961806336 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003578936810431294, + "loss": 2.6757, + "theoretical_loss": 3.662766591450629, + "tokens_seen": 961871872 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003578836509528586, + "loss": 2.6864, + "theoretical_loss": 3.662742671863458, + "tokens_seen": 961937408 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003578736208625878, + "loss": 2.7919, + "theoretical_loss": 3.6627187543621114, + "tokens_seen": 962002944 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035786359077231696, + "loss": 2.7932, + "theoretical_loss": 3.6626948389462655, + "tokens_seen": 962068480 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035785356068204614, + "loss": 2.7697, + "theoretical_loss": 3.662670925615597, + "tokens_seen": 962134016 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003578435305917753, + "loss": 2.7508, + "theoretical_loss": 3.6626470143697816, + "tokens_seen": 962199552 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035783350050150455, + "loss": 2.7527, + "theoretical_loss": 3.662623105208495, + "tokens_seen": 962265088 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003578234704112337, + "loss": 2.6663, + "theoretical_loss": 3.6625991981314145, + "tokens_seen": 962330624 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003578134403209629, + "loss": 2.7155, + "theoretical_loss": 3.6625752931382163, + "tokens_seen": 962396160 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003578034102306921, + "loss": 2.8594, + "theoretical_loss": 3.6625513902285767, + "tokens_seen": 962461696 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003577933801404213, + "loss": 2.8142, + "theoretical_loss": 3.6625274894021724, + "tokens_seen": 962527232 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035778335005015046, + "loss": 2.7512, + "theoretical_loss": 3.6625035906586794, + "tokens_seen": 962592768 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035777331995987964, + "loss": 2.838, + "theoretical_loss": 3.662479693997775, + "tokens_seen": 962658304 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003577632898696088, + "loss": 2.8718, + "theoretical_loss": 3.662455799419136, + "tokens_seen": 962723840 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035775325977933806, + "loss": 2.8056, + "theoretical_loss": 3.6624319069224387, + "tokens_seen": 962789376 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003577432296890672, + "loss": 2.8349, + "theoretical_loss": 3.66240801650736, + "tokens_seen": 962854912 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003577331995987964, + "loss": 2.8339, + "theoretical_loss": 3.662384128173578, + "tokens_seen": 962920448 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035772316950852555, + "loss": 2.8871, + "theoretical_loss": 3.6623602419207684, + "tokens_seen": 962985984 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003577131394182548, + "loss": 3.0937, + "theoretical_loss": 3.662336357748609, + "tokens_seen": 963051520 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035770310932798396, + "loss": 2.8346, + "theoretical_loss": 3.662312475656777, + "tokens_seen": 963117056 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035769307923771314, + "loss": 2.7321, + "theoretical_loss": 3.6622885956449496, + "tokens_seen": 963182592 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003576830491474423, + "loss": 2.4373, + "theoretical_loss": 3.6622647177128034, + "tokens_seen": 963248128 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035767301905717156, + "loss": 2.7596, + "theoretical_loss": 3.662240841860017, + "tokens_seen": 963313664 + }, + { + "epoch": 0.29, + "objective/train/docs_used": 552873, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.8712260723114014, + "objective/train/theoretical_loss": 3.662216968086267, + "objective/train/tokens_used": 983839200, + "theoretical_loss": 3.662216968086267, + "tokens_seen": 963379200 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003576629889669007, + "loss": 2.7574, + "theoretical_loss": 3.662216968086267, + "tokens_seen": 963379200 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003576529588766299, + "loss": 2.7536, + "theoretical_loss": 3.6621930963912317, + "tokens_seen": 963444736 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035764292878635905, + "loss": 2.8841, + "theoretical_loss": 3.662169226774588, + "tokens_seen": 963510272 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003576328986960883, + "loss": 2.7623, + "theoretical_loss": 3.662145359236014, + "tokens_seen": 963575808 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035762286860581747, + "loss": 2.6078, + "theoretical_loss": 3.6621214937751874, + "tokens_seen": 963641344 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035761283851554665, + "loss": 2.6939, + "theoretical_loss": 3.662097630391786, + "tokens_seen": 963706880 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035760280842527583, + "loss": 2.6624, + "theoretical_loss": 3.662073769085488, + "tokens_seen": 963772416 + }, + { + "epoch": 0.29, + "learning_rate": 0.000357592778335005, + "loss": 2.8644, + "theoretical_loss": 3.6620499098559707, + "tokens_seen": 963837952 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003575827482447342, + "loss": 2.8209, + "theoretical_loss": 3.6620260527029127, + "tokens_seen": 963903488 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003575727181544634, + "loss": 2.7228, + "theoretical_loss": 3.662002197625992, + "tokens_seen": 963969024 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035756268806419255, + "loss": 2.7868, + "theoretical_loss": 3.661978344624887, + "tokens_seen": 964034560 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003575526579739218, + "loss": 3.0674, + "theoretical_loss": 3.6619544936992754, + "tokens_seen": 964100096 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003575426278836509, + "loss": 2.806, + "theoretical_loss": 3.661930644848836, + "tokens_seen": 964165632 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035753259779338015, + "loss": 2.771, + "theoretical_loss": 3.661906798073247, + "tokens_seen": 964231168 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035752256770310933, + "loss": 2.8867, + "theoretical_loss": 3.6618829533721873, + "tokens_seen": 964296704 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003575125376128385, + "loss": 2.7098, + "theoretical_loss": 3.661859110745335, + "tokens_seen": 964362240 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003575025075225677, + "loss": 2.8184, + "theoretical_loss": 3.661835270192369, + "tokens_seen": 964427776 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035749247743229693, + "loss": 2.8266, + "theoretical_loss": 3.6618114317129677, + "tokens_seen": 964493312 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035748244734202606, + "loss": 2.7383, + "theoretical_loss": 3.6617875953068095, + "tokens_seen": 964558848 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003574724172517553, + "loss": 2.8184, + "theoretical_loss": 3.6617637609735745, + "tokens_seen": 964624384 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003574623871614844, + "loss": 2.7602, + "theoretical_loss": 3.661739928712941, + "tokens_seen": 964689920 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035745235707121365, + "loss": 2.7176, + "theoretical_loss": 3.661716098524588, + "tokens_seen": 964755456 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035744232698094283, + "loss": 2.9312, + "theoretical_loss": 3.661692270408194, + "tokens_seen": 964820992 + }, + { + "epoch": 0.29, + "learning_rate": 0.000357432296890672, + "loss": 2.5621, + "theoretical_loss": 3.661668444363438, + "tokens_seen": 964886528 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003574222668004012, + "loss": 2.722, + "theoretical_loss": 3.6616446203900006, + "tokens_seen": 964952064 + }, + { + "epoch": 0.29, + "objective/train/docs_used": 554313, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.084320545196533, + "objective/train/theoretical_loss": 3.6616207984875606, + "objective/train/tokens_used": 985477600, + "theoretical_loss": 3.6616207984875606, + "tokens_seen": 965017600 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003574122367101304, + "loss": 2.8319, + "theoretical_loss": 3.6616207984875606, + "tokens_seen": 965017600 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035740220661985956, + "loss": 2.8794, + "theoretical_loss": 3.6615969786557967, + "tokens_seen": 965083136 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003573921765295888, + "loss": 2.7008, + "theoretical_loss": 3.6615731608943882, + "tokens_seen": 965148672 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003573821464393179, + "loss": 2.4688, + "theoretical_loss": 3.6615493452030154, + "tokens_seen": 965214208 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035737211634904716, + "loss": 2.6848, + "theoretical_loss": 3.661525531581357, + "tokens_seen": 965279744 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003573620862587763, + "loss": 2.7393, + "theoretical_loss": 3.6615017200290936, + "tokens_seen": 965345280 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003573520561685055, + "loss": 2.7158, + "theoretical_loss": 3.661477910545904, + "tokens_seen": 965410816 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035734202607823475, + "loss": 2.7789, + "theoretical_loss": 3.6614541031314687, + "tokens_seen": 965476352 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003573319959879639, + "loss": 2.6427, + "theoretical_loss": 3.6614302977854667, + "tokens_seen": 965541888 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003573219658976931, + "loss": 2.7522, + "theoretical_loss": 3.6614064945075784, + "tokens_seen": 965607424 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003573119358074223, + "loss": 2.7014, + "theoretical_loss": 3.661382693297484, + "tokens_seen": 965672960 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003573019057171515, + "loss": 2.9907, + "theoretical_loss": 3.661358894154864, + "tokens_seen": 965738496 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035729187562688066, + "loss": 2.9057, + "theoretical_loss": 3.661335097079397, + "tokens_seen": 965804032 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035728184553660984, + "loss": 2.6417, + "theoretical_loss": 3.6613113020707644, + "tokens_seen": 965869568 + }, + { + "epoch": 0.29, + "learning_rate": 0.000357271815446339, + "loss": 2.5615, + "theoretical_loss": 3.6612875091286465, + "tokens_seen": 965935104 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035726178535606826, + "loss": 2.5768, + "theoretical_loss": 3.661263718252723, + "tokens_seen": 966000640 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003572517552657974, + "loss": 2.6855, + "theoretical_loss": 3.661239929442674, + "tokens_seen": 966066176 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003572417251755266, + "loss": 2.7967, + "theoretical_loss": 3.6612161426981817, + "tokens_seen": 966131712 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035723169508525575, + "loss": 2.6001, + "theoretical_loss": 3.6611923580189245, + "tokens_seen": 966197248 + }, + { + "epoch": 0.29, + "learning_rate": 0.000357221664994985, + "loss": 2.6888, + "theoretical_loss": 3.661168575404585, + "tokens_seen": 966262784 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035721163490471416, + "loss": 2.5276, + "theoretical_loss": 3.6611447948548426, + "tokens_seen": 966328320 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035720160481444334, + "loss": 2.8886, + "theoretical_loss": 3.6611210163693784, + "tokens_seen": 966393856 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003571915747241725, + "loss": 2.7423, + "theoretical_loss": 3.661097239947873, + "tokens_seen": 966459392 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035718154463390176, + "loss": 2.58, + "theoretical_loss": 3.661073465590008, + "tokens_seen": 966524928 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003571715145436309, + "loss": 2.9186, + "theoretical_loss": 3.6610496932954635, + "tokens_seen": 966590464 + }, + { + "epoch": 0.29, + "objective/train/docs_used": 554946, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.256721258163452, + "objective/train/theoretical_loss": 3.6610259230639217, + "objective/train/tokens_used": 987116000, + "theoretical_loss": 3.6610259230639217, + "tokens_seen": 966656000 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003571614844533601, + "loss": 2.7384, + "theoretical_loss": 3.6610259230639217, + "tokens_seen": 966656000 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035715145436308925, + "loss": 2.4726, + "theoretical_loss": 3.661002154895062, + "tokens_seen": 966721536 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003571414242728185, + "loss": 2.7803, + "theoretical_loss": 3.6609783887885676, + "tokens_seen": 966787072 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035713139418254767, + "loss": 2.6379, + "theoretical_loss": 3.6609546247441185, + "tokens_seen": 966852608 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035712136409227685, + "loss": 2.6972, + "theoretical_loss": 3.660930862761396, + "tokens_seen": 966918144 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035711133400200603, + "loss": 2.9712, + "theoretical_loss": 3.660907102840082, + "tokens_seen": 966983680 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003571013039117352, + "loss": 2.6973, + "theoretical_loss": 3.660883344979858, + "tokens_seen": 967049216 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003570912738214644, + "loss": 2.6, + "theoretical_loss": 3.6608595891804057, + "tokens_seen": 967114752 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003570812437311936, + "loss": 2.6018, + "theoretical_loss": 3.660835835441406, + "tokens_seen": 967180288 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035707121364092275, + "loss": 2.777, + "theoretical_loss": 3.660812083762541, + "tokens_seen": 967245824 + }, + { + "epoch": 0.29, + "learning_rate": 0.000357061183550652, + "loss": 2.6091, + "theoretical_loss": 3.660788334143492, + "tokens_seen": 967311360 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003570511534603811, + "loss": 2.8529, + "theoretical_loss": 3.6607645865839418, + "tokens_seen": 967376896 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035704112337011035, + "loss": 2.7116, + "theoretical_loss": 3.6607408410835713, + "tokens_seen": 967442432 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035703109327983953, + "loss": 2.5883, + "theoretical_loss": 3.6607170976420638, + "tokens_seen": 967507968 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003570210631895687, + "loss": 2.5894, + "theoretical_loss": 3.6606933562590998, + "tokens_seen": 967573504 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003570110330992979, + "loss": 2.7361, + "theoretical_loss": 3.660669616934362, + "tokens_seen": 967639040 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035700100300902713, + "loss": 2.4768, + "theoretical_loss": 3.660645879667533, + "tokens_seen": 967704576 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035699097291875626, + "loss": 2.7544, + "theoretical_loss": 3.6606221444582943, + "tokens_seen": 967770112 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003569809428284855, + "loss": 2.7132, + "theoretical_loss": 3.6605984113063292, + "tokens_seen": 967835648 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003569709127382146, + "loss": 2.6445, + "theoretical_loss": 3.6605746802113197, + "tokens_seen": 967901184 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035696088264794385, + "loss": 2.6094, + "theoretical_loss": 3.6605509511729473, + "tokens_seen": 967966720 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035695085255767303, + "loss": 2.6364, + "theoretical_loss": 3.660527224190896, + "tokens_seen": 968032256 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003569408224674022, + "loss": 2.6788, + "theoretical_loss": 3.6605034992648475, + "tokens_seen": 968097792 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003569307923771314, + "loss": 2.6751, + "theoretical_loss": 3.6604797763944843, + "tokens_seen": 968163328 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003569207622868606, + "loss": 2.7723, + "theoretical_loss": 3.6604560555794903, + "tokens_seen": 968228864 + }, + { + "epoch": 0.29, + "objective/train/docs_used": 556500, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.626742124557495, + "objective/train/theoretical_loss": 3.6604323368195475, + "objective/train/tokens_used": 988754400, + "theoretical_loss": 3.6604323368195475, + "tokens_seen": 968294400 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035691073219658976, + "loss": 2.5781, + "theoretical_loss": 3.6604323368195475, + "tokens_seen": 968294400 + }, + { + "epoch": 0.29, + "learning_rate": 0.000356900702106319, + "loss": 2.5801, + "theoretical_loss": 3.660408620114339, + "tokens_seen": 968359936 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003568906720160481, + "loss": 2.572, + "theoretical_loss": 3.6603849054635473, + "tokens_seen": 968425472 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035688064192577736, + "loss": 2.7129, + "theoretical_loss": 3.660361192866856, + "tokens_seen": 968491008 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003568706118355065, + "loss": 2.7842, + "theoretical_loss": 3.660337482323948, + "tokens_seen": 968556544 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003568605817452357, + "loss": 2.9533, + "theoretical_loss": 3.660313773834506, + "tokens_seen": 968622080 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003568505516549649, + "loss": 2.4242, + "theoretical_loss": 3.660290067398214, + "tokens_seen": 968687616 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003568405215646941, + "loss": 2.8214, + "theoretical_loss": 3.660266363014755, + "tokens_seen": 968753152 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035683049147442326, + "loss": 2.7149, + "theoretical_loss": 3.6602426606838128, + "tokens_seen": 968818688 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003568204613841525, + "loss": 2.8176, + "theoretical_loss": 3.6602189604050697, + "tokens_seen": 968884224 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003568104312938816, + "loss": 2.8382, + "theoretical_loss": 3.6601952621782106, + "tokens_seen": 968949760 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035680040120361086, + "loss": 2.6264, + "theoretical_loss": 3.660171566002918, + "tokens_seen": 969015296 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035679037111334, + "loss": 2.623, + "theoretical_loss": 3.660147871878876, + "tokens_seen": 969080832 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003567803410230692, + "loss": 2.8021, + "theoretical_loss": 3.660124179805769, + "tokens_seen": 969146368 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003567703109327984, + "loss": 2.7569, + "theoretical_loss": 3.6601004897832796, + "tokens_seen": 969211904 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003567602808425276, + "loss": 2.802, + "theoretical_loss": 3.6600768018110923, + "tokens_seen": 969277440 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035675025075225677, + "loss": 2.6978, + "theoretical_loss": 3.660053115888891, + "tokens_seen": 969342976 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035674022066198595, + "loss": 2.7472, + "theoretical_loss": 3.6600294320163593, + "tokens_seen": 969408512 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035673019057171513, + "loss": 2.753, + "theoretical_loss": 3.660005750193182, + "tokens_seen": 969474048 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035672016048144436, + "loss": 2.6382, + "theoretical_loss": 3.6599820704190433, + "tokens_seen": 969539584 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003567101303911735, + "loss": 2.5347, + "theoretical_loss": 3.6599583926936265, + "tokens_seen": 969605120 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003567001003009027, + "loss": 2.9088, + "theoretical_loss": 3.6599347170166165, + "tokens_seen": 969670656 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035669007021063185, + "loss": 2.6573, + "theoretical_loss": 3.659911043387698, + "tokens_seen": 969736192 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003566800401203611, + "loss": 2.6658, + "theoretical_loss": 3.6598873718065548, + "tokens_seen": 969801728 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035667001003009027, + "loss": 2.6196, + "theoretical_loss": 3.6598637022728715, + "tokens_seen": 969867264 + }, + { + "epoch": 0.29, + "objective/train/docs_used": 557201, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.023595094680786, + "objective/train/theoretical_loss": 3.659840034786333, + "objective/train/tokens_used": 990392800, + "theoretical_loss": 3.659840034786333, + "tokens_seen": 969932800 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035665997993981945, + "loss": 2.9494, + "theoretical_loss": 3.659840034786333, + "tokens_seen": 969932800 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035664994984954863, + "loss": 2.9918, + "theoretical_loss": 3.6598163693466232, + "tokens_seen": 969998336 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035663991975927787, + "loss": 2.7891, + "theoretical_loss": 3.6597927059534276, + "tokens_seen": 970063872 + }, + { + "epoch": 0.29, + "learning_rate": 0.000356629889669007, + "loss": 2.7887, + "theoretical_loss": 3.6597690446064313, + "tokens_seen": 970129408 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035661985957873623, + "loss": 2.6437, + "theoretical_loss": 3.659745385305318, + "tokens_seen": 970194944 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035660982948846536, + "loss": 2.7156, + "theoretical_loss": 3.6597217280497736, + "tokens_seen": 970260480 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003565997993981946, + "loss": 2.6954, + "theoretical_loss": 3.659698072839482, + "tokens_seen": 970326016 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003565897693079238, + "loss": 2.6848, + "theoretical_loss": 3.6596744196741295, + "tokens_seen": 970391552 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035657973921765295, + "loss": 2.7468, + "theoretical_loss": 3.6596507685534005, + "tokens_seen": 970457088 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003565697091273822, + "loss": 3.0558, + "theoretical_loss": 3.6596271194769807, + "tokens_seen": 970522624 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003565596790371113, + "loss": 2.6871, + "theoretical_loss": 3.659603472444555, + "tokens_seen": 970588160 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035654964894684055, + "loss": 2.7311, + "theoretical_loss": 3.6595798274558087, + "tokens_seen": 970653696 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035653961885656973, + "loss": 2.611, + "theoretical_loss": 3.6595561845104276, + "tokens_seen": 970719232 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003565295887662989, + "loss": 2.5287, + "theoretical_loss": 3.6595325436080968, + "tokens_seen": 970784768 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003565195586760281, + "loss": 2.919, + "theoretical_loss": 3.6595089047485017, + "tokens_seen": 970850304 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035650952858575733, + "loss": 2.6268, + "theoretical_loss": 3.6594852679313283, + "tokens_seen": 970915840 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035649949849548646, + "loss": 2.6663, + "theoretical_loss": 3.659461633156262, + "tokens_seen": 970981376 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003564894684052157, + "loss": 2.983, + "theoretical_loss": 3.6594380004229894, + "tokens_seen": 971046912 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003564794383149448, + "loss": 2.8703, + "theoretical_loss": 3.659414369731195, + "tokens_seen": 971112448 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035646940822467405, + "loss": 2.508, + "theoretical_loss": 3.659390741080565, + "tokens_seen": 971177984 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035645937813440323, + "loss": 2.6218, + "theoretical_loss": 3.659367114470786, + "tokens_seen": 971243520 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003564493480441324, + "loss": 2.6956, + "theoretical_loss": 3.659343489901544, + "tokens_seen": 971309056 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003564393179538616, + "loss": 2.8824, + "theoretical_loss": 3.659319867372525, + "tokens_seen": 971374592 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003564292878635908, + "loss": 2.7904, + "theoretical_loss": 3.6592962468834145, + "tokens_seen": 971440128 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035641925777331996, + "loss": 2.801, + "theoretical_loss": 3.6592726284339, + "tokens_seen": 971505664 + }, + { + "epoch": 0.29, + "objective/train/docs_used": 558372, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.128817319869995, + "objective/train/theoretical_loss": 3.6592490120236665, + "objective/train/tokens_used": 992031200, + "theoretical_loss": 3.6592490120236665, + "tokens_seen": 971571200 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003564092276830492, + "loss": 2.4792, + "theoretical_loss": 3.6592490120236665, + "tokens_seen": 971571200 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003563991975927783, + "loss": 2.8256, + "theoretical_loss": 3.6592253976524014, + "tokens_seen": 971636736 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035638916750250756, + "loss": 2.9438, + "theoretical_loss": 3.65920178531979, + "tokens_seen": 971702272 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003563791374122367, + "loss": 2.855, + "theoretical_loss": 3.65917817502552, + "tokens_seen": 971767808 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003563691073219659, + "loss": 2.9583, + "theoretical_loss": 3.6591545667692777, + "tokens_seen": 971833344 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003563590772316951, + "loss": 2.9139, + "theoretical_loss": 3.6591309605507494, + "tokens_seen": 971898880 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003563490471414243, + "loss": 2.8613, + "theoretical_loss": 3.6591073563696224, + "tokens_seen": 971964416 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035633901705115346, + "loss": 2.7056, + "theoretical_loss": 3.6590837542255827, + "tokens_seen": 972029952 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003563289869608827, + "loss": 2.8496, + "theoretical_loss": 3.6590601541183174, + "tokens_seen": 972095488 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003563189568706118, + "loss": 2.7058, + "theoretical_loss": 3.659036556047514, + "tokens_seen": 972161024 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035630892678034106, + "loss": 2.8395, + "theoretical_loss": 3.659012960012859, + "tokens_seen": 972226560 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003562988966900702, + "loss": 2.7317, + "theoretical_loss": 3.65898936601404, + "tokens_seen": 972292096 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003562888665997994, + "loss": 2.6731, + "theoretical_loss": 3.6589657740507437, + "tokens_seen": 972357632 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003562788365095286, + "loss": 2.6966, + "theoretical_loss": 3.658942184122657, + "tokens_seen": 972423168 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003562688064192578, + "loss": 2.6394, + "theoretical_loss": 3.6589185962294675, + "tokens_seen": 972488704 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035625877632898697, + "loss": 2.833, + "theoretical_loss": 3.658895010370863, + "tokens_seen": 972554240 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035624874623871615, + "loss": 2.5957, + "theoretical_loss": 3.65887142654653, + "tokens_seen": 972619776 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035623871614844533, + "loss": 2.7634, + "theoretical_loss": 3.658847844756157, + "tokens_seen": 972685312 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035622868605817456, + "loss": 2.8195, + "theoretical_loss": 3.6588242649994314, + "tokens_seen": 972750848 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003562186559679037, + "loss": 2.3889, + "theoretical_loss": 3.65880068727604, + "tokens_seen": 972816384 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003562086258776329, + "loss": 2.7761, + "theoretical_loss": 3.658777111585671, + "tokens_seen": 972881920 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035619859578736205, + "loss": 2.7698, + "theoretical_loss": 3.6587535379280123, + "tokens_seen": 972947456 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003561885656970913, + "loss": 2.869, + "theoretical_loss": 3.6587299663027517, + "tokens_seen": 973012992 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035617853560682047, + "loss": 2.7295, + "theoretical_loss": 3.6587063967095776, + "tokens_seen": 973078528 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035616850551654965, + "loss": 2.85, + "theoretical_loss": 3.6586828291481766, + "tokens_seen": 973144064 + }, + { + "epoch": 0.29, + "objective/train/docs_used": 558840, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.2617077827453613, + "objective/train/theoretical_loss": 3.6586592636182376, + "objective/train/tokens_used": 993669600, + "theoretical_loss": 3.6586592636182376, + "tokens_seen": 973209600 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035615847542627883, + "loss": 2.8221, + "theoretical_loss": 3.6586592636182376, + "tokens_seen": 973209600 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035614844533600807, + "loss": 2.5159, + "theoretical_loss": 3.658635700119449, + "tokens_seen": 973275136 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003561384152457372, + "loss": 2.8879, + "theoretical_loss": 3.658612138651498, + "tokens_seen": 973340672 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035612838515546643, + "loss": 2.6842, + "theoretical_loss": 3.6585885792140744, + "tokens_seen": 973406208 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035611835506519556, + "loss": 2.5066, + "theoretical_loss": 3.658565021806865, + "tokens_seen": 973471744 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003561083249749248, + "loss": 2.7656, + "theoretical_loss": 3.658541466429559, + "tokens_seen": 973537280 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035609829488465397, + "loss": 2.8484, + "theoretical_loss": 3.6585179130818446, + "tokens_seen": 973602816 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035608826479438315, + "loss": 2.6418, + "theoretical_loss": 3.6584943617634105, + "tokens_seen": 973668352 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035607823470411233, + "loss": 2.7734, + "theoretical_loss": 3.6584708124739453, + "tokens_seen": 973733888 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003560682046138415, + "loss": 2.8207, + "theoretical_loss": 3.658447265213137, + "tokens_seen": 973799424 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003560581745235707, + "loss": 2.738, + "theoretical_loss": 3.658423719980675, + "tokens_seen": 973864960 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035604814443329993, + "loss": 2.7117, + "theoretical_loss": 3.658400176776248, + "tokens_seen": 973930496 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035603811434302906, + "loss": 2.5512, + "theoretical_loss": 3.658376635599545, + "tokens_seen": 973996032 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003560280842527583, + "loss": 2.8137, + "theoretical_loss": 3.658353096450255, + "tokens_seen": 974061568 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003560180541624874, + "loss": 2.6937, + "theoretical_loss": 3.6583295593280667, + "tokens_seen": 974127104 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035600802407221666, + "loss": 2.6947, + "theoretical_loss": 3.658306024232669, + "tokens_seen": 974192640 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035599799398194584, + "loss": 2.881, + "theoretical_loss": 3.6582824911637513, + "tokens_seen": 974258176 + }, + { + "epoch": 0.3, + "learning_rate": 0.000355987963891675, + "loss": 2.6584, + "theoretical_loss": 3.658258960121003, + "tokens_seen": 974323712 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003559779338014042, + "loss": 2.7467, + "theoretical_loss": 3.6582354311041128, + "tokens_seen": 974389248 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035596790371113344, + "loss": 2.6844, + "theoretical_loss": 3.658211904112771, + "tokens_seen": 974454784 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035595787362086256, + "loss": 2.6847, + "theoretical_loss": 3.658188379146666, + "tokens_seen": 974520320 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003559478435305918, + "loss": 2.7858, + "theoretical_loss": 3.658164856205488, + "tokens_seen": 974585856 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003559378134403209, + "loss": 2.7402, + "theoretical_loss": 3.658141335288926, + "tokens_seen": 974651392 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035592778335005016, + "loss": 2.9206, + "theoretical_loss": 3.6581178163966706, + "tokens_seen": 974716928 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035591775325977934, + "loss": 2.7612, + "theoretical_loss": 3.6580942995284103, + "tokens_seen": 974782464 + }, + { + "epoch": 0.3, + "objective/train/docs_used": 560118, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.166886329650879, + "objective/train/theoretical_loss": 3.6580707846838356, + "objective/train/tokens_used": 995308000, + "theoretical_loss": 3.6580707846838356, + "tokens_seen": 974848000 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003559077231695085, + "loss": 2.8559, + "theoretical_loss": 3.6580707846838356, + "tokens_seen": 974848000 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003558976930792377, + "loss": 2.5144, + "theoretical_loss": 3.6580472718626353, + "tokens_seen": 974913536 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003558876629889669, + "loss": 2.7286, + "theoretical_loss": 3.6580237610645012, + "tokens_seen": 974979072 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035587763289869607, + "loss": 2.7843, + "theoretical_loss": 3.6580002522891215, + "tokens_seen": 975044608 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003558676028084253, + "loss": 2.5168, + "theoretical_loss": 3.657976745536187, + "tokens_seen": 975110144 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035585757271815443, + "loss": 2.8479, + "theoretical_loss": 3.657953240805388, + "tokens_seen": 975175680 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035584754262788366, + "loss": 2.5759, + "theoretical_loss": 3.6579297380964144, + "tokens_seen": 975241216 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003558375125376129, + "loss": 2.8865, + "theoretical_loss": 3.657906237408956, + "tokens_seen": 975306752 + }, + { + "epoch": 0.3, + "learning_rate": 0.000355827482447342, + "loss": 2.8445, + "theoretical_loss": 3.6578827387427038, + "tokens_seen": 975372288 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035581745235707126, + "loss": 2.5659, + "theoretical_loss": 3.6578592420973477, + "tokens_seen": 975437824 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003558074222668004, + "loss": 2.9561, + "theoretical_loss": 3.6578357474725784, + "tokens_seen": 975503360 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003557973921765296, + "loss": 2.73, + "theoretical_loss": 3.6578122548680865, + "tokens_seen": 975568896 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003557873620862588, + "loss": 2.7855, + "theoretical_loss": 3.657788764283562, + "tokens_seen": 975634432 + }, + { + "epoch": 0.3, + "learning_rate": 0.000355777331995988, + "loss": 2.8425, + "theoretical_loss": 3.6577652757186963, + "tokens_seen": 975699968 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035576730190571717, + "loss": 2.5808, + "theoretical_loss": 3.65774178917318, + "tokens_seen": 975765504 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035575727181544635, + "loss": 2.7233, + "theoretical_loss": 3.657718304646703, + "tokens_seen": 975831040 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035574724172517553, + "loss": 2.9573, + "theoretical_loss": 3.657694822138957, + "tokens_seen": 975896576 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035573721163490476, + "loss": 2.7861, + "theoretical_loss": 3.657671341649633, + "tokens_seen": 975962112 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003557271815446339, + "loss": 2.8094, + "theoretical_loss": 3.657647863178422, + "tokens_seen": 976027648 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003557171514543631, + "loss": 2.7859, + "theoretical_loss": 3.657624386725014, + "tokens_seen": 976093184 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035570712136409225, + "loss": 2.7932, + "theoretical_loss": 3.6576009122891016, + "tokens_seen": 976158720 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003556970912738215, + "loss": 2.8429, + "theoretical_loss": 3.657577439870375, + "tokens_seen": 976224256 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035568706118355067, + "loss": 2.8676, + "theoretical_loss": 3.657553969468526, + "tokens_seen": 976289792 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035567703109327985, + "loss": 2.7583, + "theoretical_loss": 3.6575305010832455, + "tokens_seen": 976355328 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035566700100300903, + "loss": 2.8168, + "theoretical_loss": 3.657507034714225, + "tokens_seen": 976420864 + }, + { + "epoch": 0.3, + "objective/train/docs_used": 560789, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.2082810401916504, + "objective/train/theoretical_loss": 3.6574835703611566, + "objective/train/tokens_used": 996946400, + "theoretical_loss": 3.6574835703611566, + "tokens_seen": 976486400 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035565697091273827, + "loss": 2.9093, + "theoretical_loss": 3.6574835703611566, + "tokens_seen": 976486400 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003556469408224674, + "loss": 2.7714, + "theoretical_loss": 3.657460108023731, + "tokens_seen": 976551936 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035563691073219663, + "loss": 2.8283, + "theoretical_loss": 3.6574366477016396, + "tokens_seen": 976617472 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035562688064192576, + "loss": 2.6195, + "theoretical_loss": 3.6574131893945747, + "tokens_seen": 976683008 + }, + { + "epoch": 0.3, + "learning_rate": 0.000355616850551655, + "loss": 2.5341, + "theoretical_loss": 3.657389733102228, + "tokens_seen": 976748544 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035560682046138417, + "loss": 2.688, + "theoretical_loss": 3.6573662788242913, + "tokens_seen": 976814080 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035559679037111335, + "loss": 2.8499, + "theoretical_loss": 3.6573428265604564, + "tokens_seen": 976879616 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035558676028084253, + "loss": 2.5356, + "theoretical_loss": 3.657319376310415, + "tokens_seen": 976945152 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003555767301905717, + "loss": 2.9294, + "theoretical_loss": 3.6572959280738595, + "tokens_seen": 977010688 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003555667001003009, + "loss": 2.7618, + "theoretical_loss": 3.6572724818504816, + "tokens_seen": 977076224 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035555667001003013, + "loss": 2.4052, + "theoretical_loss": 3.657249037639974, + "tokens_seen": 977141760 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035554663991975926, + "loss": 2.8651, + "theoretical_loss": 3.6572255954420276, + "tokens_seen": 977207296 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003555366098294885, + "loss": 2.7305, + "theoretical_loss": 3.6572021552563365, + "tokens_seen": 977272832 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003555265797392176, + "loss": 2.8875, + "theoretical_loss": 3.6571787170825925, + "tokens_seen": 977338368 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035551654964894686, + "loss": 2.6938, + "theoretical_loss": 3.657155280920487, + "tokens_seen": 977403904 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035550651955867604, + "loss": 2.8872, + "theoretical_loss": 3.657131846769713, + "tokens_seen": 977469440 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003554964894684052, + "loss": 2.955, + "theoretical_loss": 3.6571084146299633, + "tokens_seen": 977534976 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003554864593781344, + "loss": 2.5741, + "theoretical_loss": 3.657084984500931, + "tokens_seen": 977600512 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035547642928786364, + "loss": 2.8312, + "theoretical_loss": 3.6570615563823075, + "tokens_seen": 977666048 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035546639919759276, + "loss": 2.9623, + "theoretical_loss": 3.6570381302737864, + "tokens_seen": 977731584 + }, + { + "epoch": 0.3, + "learning_rate": 0.000355456369107322, + "loss": 2.7589, + "theoretical_loss": 3.657014706175061, + "tokens_seen": 977797120 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003554463390170511, + "loss": 2.9092, + "theoretical_loss": 3.656991284085823, + "tokens_seen": 977862656 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035543630892678036, + "loss": 2.7134, + "theoretical_loss": 3.6569678640057655, + "tokens_seen": 977928192 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035542627883650954, + "loss": 2.9054, + "theoretical_loss": 3.656944445934583, + "tokens_seen": 977993728 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003554162487462387, + "loss": 2.695, + "theoretical_loss": 3.6569210298719668, + "tokens_seen": 978059264 + }, + { + "epoch": 0.3, + "objective/train/docs_used": 561934, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6768414974212646, + "objective/train/theoretical_loss": 3.656897615817611, + "objective/train/tokens_used": 998584800, + "theoretical_loss": 3.656897615817611, + "tokens_seen": 978124800 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003554062186559679, + "loss": 2.7269, + "theoretical_loss": 3.656897615817611, + "tokens_seen": 978124800 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003553961885656971, + "loss": 2.5327, + "theoretical_loss": 3.656874203771208, + "tokens_seen": 978190336 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035538615847542627, + "loss": 2.6867, + "theoretical_loss": 3.6568507937324526, + "tokens_seen": 978255872 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003553761283851555, + "loss": 2.5836, + "theoretical_loss": 3.6568273857010363, + "tokens_seen": 978321408 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035536609829488463, + "loss": 2.7274, + "theoretical_loss": 3.6568039796766545, + "tokens_seen": 978386944 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035535606820461386, + "loss": 3.1734, + "theoretical_loss": 3.656780575658999, + "tokens_seen": 978452480 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035534603811434304, + "loss": 2.6707, + "theoretical_loss": 3.6567571736477644, + "tokens_seen": 978518016 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003553360080240722, + "loss": 2.5921, + "theoretical_loss": 3.6567337736426433, + "tokens_seen": 978583552 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003553259779338014, + "loss": 2.862, + "theoretical_loss": 3.656710375643331, + "tokens_seen": 978649088 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003553159478435306, + "loss": 2.87, + "theoretical_loss": 3.6566869796495194, + "tokens_seen": 978714624 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035530591775325977, + "loss": 2.5582, + "theoretical_loss": 3.6566635856609038, + "tokens_seen": 978780160 + }, + { + "epoch": 0.3, + "learning_rate": 0.000355295887662989, + "loss": 2.6182, + "theoretical_loss": 3.6566401936771773, + "tokens_seen": 978845696 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035528585757271813, + "loss": 2.6367, + "theoretical_loss": 3.6566168036980344, + "tokens_seen": 978911232 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035527582748244737, + "loss": 2.7342, + "theoretical_loss": 3.6565934157231688, + "tokens_seen": 978976768 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003552657973921765, + "loss": 2.7121, + "theoretical_loss": 3.6565700297522743, + "tokens_seen": 979042304 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035525576730190573, + "loss": 2.8119, + "theoretical_loss": 3.6565466457850455, + "tokens_seen": 979107840 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003552457372116349, + "loss": 2.8272, + "theoretical_loss": 3.656523263821177, + "tokens_seen": 979173376 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003552357071213641, + "loss": 2.7944, + "theoretical_loss": 3.656499883860362, + "tokens_seen": 979238912 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035522567703109327, + "loss": 2.5409, + "theoretical_loss": 3.6564765059022957, + "tokens_seen": 979304448 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035521564694082245, + "loss": 2.9313, + "theoretical_loss": 3.6564531299466725, + "tokens_seen": 979369984 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035520561685055163, + "loss": 2.6796, + "theoretical_loss": 3.6564297559931864, + "tokens_seen": 979435520 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035519558676028087, + "loss": 2.5426, + "theoretical_loss": 3.656406384041533, + "tokens_seen": 979501056 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035518555667001, + "loss": 2.4051, + "theoretical_loss": 3.6563830140914058, + "tokens_seen": 979566592 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035517552657973923, + "loss": 2.5261, + "theoretical_loss": 3.6563596461424996, + "tokens_seen": 979632128 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003551654964894684, + "loss": 2.8724, + "theoretical_loss": 3.65633628019451, + "tokens_seen": 979697664 + }, + { + "epoch": 0.3, + "objective/train/docs_used": 562605, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.710829257965088, + "objective/train/theoretical_loss": 3.6563129162471313, + "objective/train/tokens_used": 1000223200, + "theoretical_loss": 3.6563129162471313, + "tokens_seen": 979763200 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003551554663991976, + "loss": 2.8141, + "theoretical_loss": 3.6563129162471313, + "tokens_seen": 979763200 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003551454363089268, + "loss": 2.8598, + "theoretical_loss": 3.6562895543000584, + "tokens_seen": 979828736 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035513540621865596, + "loss": 2.5554, + "theoretical_loss": 3.656266194352986, + "tokens_seen": 979894272 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035512537612838514, + "loss": 2.7283, + "theoretical_loss": 3.65624283640561, + "tokens_seen": 979959808 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035511534603811437, + "loss": 2.8113, + "theoretical_loss": 3.6562194804576245, + "tokens_seen": 980025344 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003551053159478435, + "loss": 2.6216, + "theoretical_loss": 3.6561961265087253, + "tokens_seen": 980090880 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035509528585757273, + "loss": 2.6461, + "theoretical_loss": 3.6561727745586072, + "tokens_seen": 980156416 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003550852557673019, + "loss": 2.8014, + "theoretical_loss": 3.6561494246069666, + "tokens_seen": 980221952 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003550752256770311, + "loss": 2.5712, + "theoretical_loss": 3.6561260766534973, + "tokens_seen": 980287488 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035506519558676033, + "loss": 2.7309, + "theoretical_loss": 3.6561027306978957, + "tokens_seen": 980353024 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035505516549648946, + "loss": 2.5511, + "theoretical_loss": 3.656079386739857, + "tokens_seen": 980418560 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003550451354062187, + "loss": 2.7243, + "theoretical_loss": 3.656056044779077, + "tokens_seen": 980484096 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003550351053159478, + "loss": 2.7379, + "theoretical_loss": 3.656032704815251, + "tokens_seen": 980549632 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035502507522567706, + "loss": 2.6263, + "theoretical_loss": 3.656009366848075, + "tokens_seen": 980615168 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035501504513540624, + "loss": 2.7708, + "theoretical_loss": 3.6559860308772447, + "tokens_seen": 980680704 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003550050150451354, + "loss": 3.0059, + "theoretical_loss": 3.6559626969024563, + "tokens_seen": 980746240 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003549949849548646, + "loss": 2.6496, + "theoretical_loss": 3.655939364923405, + "tokens_seen": 980811776 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035498495486459384, + "loss": 2.5842, + "theoretical_loss": 3.6559160349397866, + "tokens_seen": 980877312 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035497492477432296, + "loss": 2.6637, + "theoretical_loss": 3.655892706951298, + "tokens_seen": 980942848 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003549648946840522, + "loss": 2.7586, + "theoretical_loss": 3.6558693809576353, + "tokens_seen": 981008384 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003549548645937813, + "loss": 2.544, + "theoretical_loss": 3.6558460569584943, + "tokens_seen": 981073920 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035494483450351056, + "loss": 2.749, + "theoretical_loss": 3.6558227349535706, + "tokens_seen": 981139456 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035493480441323974, + "loss": 2.727, + "theoretical_loss": 3.655799414942561, + "tokens_seen": 981204992 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003549247743229689, + "loss": 2.6803, + "theoretical_loss": 3.655776096925163, + "tokens_seen": 981270528 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003549147442326981, + "loss": 2.7434, + "theoretical_loss": 3.6557527809010715, + "tokens_seen": 981336064 + }, + { + "epoch": 0.3, + "objective/train/docs_used": 563033, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.9957189559936523, + "objective/train/theoretical_loss": 3.6557294668699836, + "objective/train/tokens_used": 1001861600, + "theoretical_loss": 3.6557294668699836, + "tokens_seen": 981401600 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003549047141424273, + "loss": 2.8148, + "theoretical_loss": 3.6557294668699836, + "tokens_seen": 981401600 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035489468405215647, + "loss": 2.9752, + "theoretical_loss": 3.655706154831596, + "tokens_seen": 981467136 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003548846539618857, + "loss": 2.6753, + "theoretical_loss": 3.6556828447856047, + "tokens_seen": 981532672 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035487462387161483, + "loss": 2.8501, + "theoretical_loss": 3.6556595367317075, + "tokens_seen": 981598208 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035486459378134406, + "loss": 2.6081, + "theoretical_loss": 3.6556362306696, + "tokens_seen": 981663744 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035485456369107324, + "loss": 2.6583, + "theoretical_loss": 3.6556129265989803, + "tokens_seen": 981729280 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003548445336008024, + "loss": 2.5497, + "theoretical_loss": 3.6555896245195436, + "tokens_seen": 981794816 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003548345035105316, + "loss": 2.7173, + "theoretical_loss": 3.655566324430989, + "tokens_seen": 981860352 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003548244734202608, + "loss": 2.8474, + "theoretical_loss": 3.6555430263330115, + "tokens_seen": 981925888 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035481444332998997, + "loss": 2.6676, + "theoretical_loss": 3.6555197302253095, + "tokens_seen": 981991424 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003548044132397192, + "loss": 2.9049, + "theoretical_loss": 3.6554964361075797, + "tokens_seen": 982056960 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035479438314944833, + "loss": 2.802, + "theoretical_loss": 3.6554731439795187, + "tokens_seen": 982122496 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035478435305917757, + "loss": 2.9521, + "theoretical_loss": 3.6554498538408255, + "tokens_seen": 982188032 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003547743229689067, + "loss": 2.7673, + "theoretical_loss": 3.655426565691196, + "tokens_seen": 982253568 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035476429287863593, + "loss": 2.7598, + "theoretical_loss": 3.6554032795303284, + "tokens_seen": 982319104 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003547542627883651, + "loss": 2.9587, + "theoretical_loss": 3.6553799953579196, + "tokens_seen": 982384640 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003547442326980943, + "loss": 2.8557, + "theoretical_loss": 3.655356713173667, + "tokens_seen": 982450176 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035473420260782347, + "loss": 2.6727, + "theoretical_loss": 3.6553334329772693, + "tokens_seen": 982515712 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035472417251755265, + "loss": 2.8693, + "theoretical_loss": 3.6553101547684235, + "tokens_seen": 982581248 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035471414242728183, + "loss": 2.7953, + "theoretical_loss": 3.6552868785468275, + "tokens_seen": 982646784 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035470411233701107, + "loss": 2.9148, + "theoretical_loss": 3.6552636043121787, + "tokens_seen": 982712320 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003546940822467402, + "loss": 2.7403, + "theoretical_loss": 3.655240332064176, + "tokens_seen": 982777856 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035468405215646943, + "loss": 2.8038, + "theoretical_loss": 3.6552170618025164, + "tokens_seen": 982843392 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003546740220661986, + "loss": 2.8159, + "theoretical_loss": 3.655193793526898, + "tokens_seen": 982908928 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003546639919759278, + "loss": 2.8576, + "theoretical_loss": 3.6551705272370194, + "tokens_seen": 982974464 + }, + { + "debugging/Self-BLEU-5": 0.41551008116403304, + "debugging/distinct-1-grams": 0.7823372661903831, + "debugging/distinct-2-grams": 0.9718703596543709, + "debugging/entropy-1-grams": 5.809135027053489, + "debugging/entropy-2-grams": 6.632451692658867, + "debugging/length": 482.0, + "debugging/num_segments": 11, + "debugging/score": 0.006270705493490795, + "debugging/score_std": 0.0057026996121831725, + "epoch": 0.3, + "objective/train/docs_used": 564216, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6628544330596924, + "objective/train/theoretical_loss": 3.6551472629325787, + "objective/train/tokens_used": 1003500000, + "theoretical_loss": 3.6551472629325787, + "tokens_seen": 983040000 + }, + { + "epoch": 0.3, + "learning_rate": 0.000354653961885657, + "loss": 2.8368, + "theoretical_loss": 3.6551472629325787, + "tokens_seen": 983040000 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035464393179538616, + "loss": 2.9336, + "theoretical_loss": 3.6551240006132737, + "tokens_seen": 983105536 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035463390170511534, + "loss": 2.7743, + "theoretical_loss": 3.655100740278803, + "tokens_seen": 983171072 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035462387161484457, + "loss": 2.7711, + "theoretical_loss": 3.655077481928865, + "tokens_seen": 983236608 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003546138415245737, + "loss": 2.8696, + "theoretical_loss": 3.655054225563158, + "tokens_seen": 983302144 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035460381143430293, + "loss": 2.6718, + "theoretical_loss": 3.6550309711813807, + "tokens_seen": 983367680 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035459378134403206, + "loss": 2.9678, + "theoretical_loss": 3.655007718783231, + "tokens_seen": 983433216 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003545837512537613, + "loss": 2.7624, + "theoretical_loss": 3.6549844683684083, + "tokens_seen": 983498752 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003545737211634905, + "loss": 2.8318, + "theoretical_loss": 3.654961219936611, + "tokens_seen": 983564288 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035456369107321966, + "loss": 2.6588, + "theoretical_loss": 3.6549379734875376, + "tokens_seen": 983629824 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035455366098294884, + "loss": 2.6437, + "theoretical_loss": 3.654914729020887, + "tokens_seen": 983695360 + }, + { + "epoch": 0.3, + "learning_rate": 0.000354543630892678, + "loss": 2.7436, + "theoretical_loss": 3.654891486536359, + "tokens_seen": 983760896 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003545336008024072, + "loss": 2.6911, + "theoretical_loss": 3.6548682460336512, + "tokens_seen": 983826432 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035452357071213644, + "loss": 2.8682, + "theoretical_loss": 3.6548450075124634, + "tokens_seen": 983891968 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035451354062186556, + "loss": 2.8968, + "theoretical_loss": 3.6548217709724944, + "tokens_seen": 983957504 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003545035105315948, + "loss": 2.9396, + "theoretical_loss": 3.6547985364134434, + "tokens_seen": 984023040 + }, + { + "epoch": 0.3, + "learning_rate": 0.000354493480441324, + "loss": 2.7011, + "theoretical_loss": 3.6547753038350095, + "tokens_seen": 984088576 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035448345035105316, + "loss": 2.9223, + "theoretical_loss": 3.6547520732368923, + "tokens_seen": 984154112 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035447342026078234, + "loss": 2.6631, + "theoretical_loss": 3.6547288446187913, + "tokens_seen": 984219648 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003544633901705115, + "loss": 2.6413, + "theoretical_loss": 3.6547056179804054, + "tokens_seen": 984285184 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003544533600802407, + "loss": 2.9834, + "theoretical_loss": 3.654682393321434, + "tokens_seen": 984350720 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035444332998996994, + "loss": 2.7545, + "theoretical_loss": 3.654659170641577, + "tokens_seen": 984416256 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035443329989969907, + "loss": 2.7285, + "theoretical_loss": 3.654635949940534, + "tokens_seen": 984481792 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003544232698094283, + "loss": 2.6874, + "theoretical_loss": 3.6546127312180046, + "tokens_seen": 984547328 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035441323971915743, + "loss": 2.83, + "theoretical_loss": 3.654589514473688, + "tokens_seen": 984612864 + }, + { + "epoch": 0.3, + "objective/train/docs_used": 564602, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.461876153945923, + "objective/train/theoretical_loss": 3.654566299707285, + "objective/train/tokens_used": 1005138400, + "theoretical_loss": 3.654566299707285, + "tokens_seen": 984678400 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035440320962888667, + "loss": 2.7541, + "theoretical_loss": 3.654566299707285, + "tokens_seen": 984678400 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035439317953861585, + "loss": 2.6857, + "theoretical_loss": 3.6545430869184954, + "tokens_seen": 984743936 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035438314944834503, + "loss": 2.6151, + "theoretical_loss": 3.6545198761070186, + "tokens_seen": 984809472 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003543731193580742, + "loss": 2.5855, + "theoretical_loss": 3.6544966672725545, + "tokens_seen": 984875008 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035436308926780344, + "loss": 2.7933, + "theoretical_loss": 3.654473460414803, + "tokens_seen": 984940544 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003543530591775326, + "loss": 2.7179, + "theoretical_loss": 3.6544502555334653, + "tokens_seen": 985006080 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003543430290872618, + "loss": 2.6704, + "theoretical_loss": 3.6544270526282405, + "tokens_seen": 985071616 + }, + { + "epoch": 0.3, + "learning_rate": 0.000354332998996991, + "loss": 2.5722, + "theoretical_loss": 3.6544038516988295, + "tokens_seen": 985137152 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035432296890672017, + "loss": 2.7282, + "theoretical_loss": 3.6543806527449325, + "tokens_seen": 985202688 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003543129388164494, + "loss": 2.8112, + "theoretical_loss": 3.6543574557662497, + "tokens_seen": 985268224 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035430290872617853, + "loss": 2.7309, + "theoretical_loss": 3.6543342607624814, + "tokens_seen": 985333760 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035429287863590777, + "loss": 2.801, + "theoretical_loss": 3.654311067733329, + "tokens_seen": 985399296 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003542828485456369, + "loss": 2.5378, + "theoretical_loss": 3.654287876678492, + "tokens_seen": 985464832 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035427281845536613, + "loss": 2.8312, + "theoretical_loss": 3.6542646875976716, + "tokens_seen": 985530368 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003542627883650953, + "loss": 2.5313, + "theoretical_loss": 3.654241500490569, + "tokens_seen": 985595904 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003542527582748245, + "loss": 2.8178, + "theoretical_loss": 3.654218315356884, + "tokens_seen": 985661440 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035424272818455367, + "loss": 2.7816, + "theoretical_loss": 3.6541951321963175, + "tokens_seen": 985726976 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035423269809428285, + "loss": 2.726, + "theoretical_loss": 3.6541719510085713, + "tokens_seen": 985792512 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035422266800401203, + "loss": 2.7122, + "theoretical_loss": 3.6541487717933463, + "tokens_seen": 985858048 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035421263791374127, + "loss": 2.6887, + "theoretical_loss": 3.654125594550343, + "tokens_seen": 985923584 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003542026078234704, + "loss": 2.846, + "theoretical_loss": 3.654102419279262, + "tokens_seen": 985989120 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035419257773319963, + "loss": 2.6776, + "theoretical_loss": 3.6540792459798057, + "tokens_seen": 986054656 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003541825476429288, + "loss": 2.8547, + "theoretical_loss": 3.6540560746516744, + "tokens_seen": 986120192 + }, + { + "epoch": 0.3, + "learning_rate": 0.000354172517552658, + "loss": 2.6325, + "theoretical_loss": 3.65403290529457, + "tokens_seen": 986185728 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003541624874623872, + "loss": 2.8815, + "theoretical_loss": 3.6540097379081935, + "tokens_seen": 986251264 + }, + { + "epoch": 0.3, + "objective/train/docs_used": 565772, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.818545341491699, + "objective/train/theoretical_loss": 3.653986572492247, + "objective/train/tokens_used": 1006776800, + "theoretical_loss": 3.653986572492247, + "tokens_seen": 986316800 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035415245737211636, + "loss": 2.7547, + "theoretical_loss": 3.653986572492247, + "tokens_seen": 986316800 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035414242728184554, + "loss": 2.7356, + "theoretical_loss": 3.653963409046431, + "tokens_seen": 986382336 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035413239719157477, + "loss": 2.4322, + "theoretical_loss": 3.653940247570448, + "tokens_seen": 986447872 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003541223671013039, + "loss": 2.929, + "theoretical_loss": 3.6539170880639986, + "tokens_seen": 986513408 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035411233701103314, + "loss": 2.6747, + "theoretical_loss": 3.6538939305267855, + "tokens_seen": 986578944 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035410230692076226, + "loss": 2.8283, + "theoretical_loss": 3.65387077495851, + "tokens_seen": 986644480 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003540922768304915, + "loss": 2.7726, + "theoretical_loss": 3.653847621358874, + "tokens_seen": 986710016 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003540822467402207, + "loss": 2.7862, + "theoretical_loss": 3.6538244697275792, + "tokens_seen": 986775552 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035407221664994986, + "loss": 2.8911, + "theoretical_loss": 3.653801320064328, + "tokens_seen": 986841088 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035406218655967904, + "loss": 2.6855, + "theoretical_loss": 3.6537781723688223, + "tokens_seen": 986906624 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003540521564694082, + "loss": 2.7844, + "theoretical_loss": 3.6537550266407637, + "tokens_seen": 986972160 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003540421263791374, + "loss": 2.7353, + "theoretical_loss": 3.6537318828798546, + "tokens_seen": 987037696 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035403209628886664, + "loss": 2.5104, + "theoretical_loss": 3.6537087410857976, + "tokens_seen": 987103232 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035402206619859577, + "loss": 2.9576, + "theoretical_loss": 3.6536856012582946, + "tokens_seen": 987168768 + }, + { + "epoch": 0.3, + "learning_rate": 0.000354012036108325, + "loss": 2.6131, + "theoretical_loss": 3.653662463397048, + "tokens_seen": 987234304 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003540020060180542, + "loss": 2.9106, + "theoretical_loss": 3.65363932750176, + "tokens_seen": 987299840 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035399197592778336, + "loss": 2.7959, + "theoretical_loss": 3.653616193572134, + "tokens_seen": 987365376 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035398194583751254, + "loss": 2.7339, + "theoretical_loss": 3.6535930616078716, + "tokens_seen": 987430912 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003539719157472417, + "loss": 2.7392, + "theoretical_loss": 3.6535699316086756, + "tokens_seen": 987496448 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003539618856569709, + "loss": 2.808, + "theoretical_loss": 3.6535468035742484, + "tokens_seen": 987561984 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035395185556670014, + "loss": 2.5625, + "theoretical_loss": 3.6535236775042934, + "tokens_seen": 987627520 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035394182547642927, + "loss": 2.2271, + "theoretical_loss": 3.6535005533985134, + "tokens_seen": 987693056 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003539317953861585, + "loss": 2.7749, + "theoretical_loss": 3.6534774312566105, + "tokens_seen": 987758592 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035392176529588763, + "loss": 2.9564, + "theoretical_loss": 3.6534543110782884, + "tokens_seen": 987824128 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035391173520561687, + "loss": 2.9576, + "theoretical_loss": 3.6534311928632497, + "tokens_seen": 987889664 + }, + { + "epoch": 0.3, + "objective/train/docs_used": 566398, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.868284225463867, + "objective/train/theoretical_loss": 3.653408076611197, + "objective/train/tokens_used": 1008415200, + "theoretical_loss": 3.653408076611197, + "tokens_seen": 987955200 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035390170511534605, + "loss": 2.862, + "theoretical_loss": 3.653408076611197, + "tokens_seen": 987955200 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035389167502507523, + "loss": 2.9369, + "theoretical_loss": 3.6533849623218346, + "tokens_seen": 988020736 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003538816449348044, + "loss": 2.9527, + "theoretical_loss": 3.653361849994865, + "tokens_seen": 988086272 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035387161484453364, + "loss": 2.7064, + "theoretical_loss": 3.6533387396299912, + "tokens_seen": 988151808 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035386158475426277, + "loss": 2.6127, + "theoretical_loss": 3.653315631226917, + "tokens_seen": 988217344 + }, + { + "epoch": 0.3, + "learning_rate": 0.000353851554663992, + "loss": 2.8811, + "theoretical_loss": 3.6532925247853454, + "tokens_seen": 988282880 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035384152457372113, + "loss": 2.8048, + "theoretical_loss": 3.65326942030498, + "tokens_seen": 988348416 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035383149448345037, + "loss": 2.8914, + "theoretical_loss": 3.6532463177855248, + "tokens_seen": 988413952 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035382146439317955, + "loss": 2.4781, + "theoretical_loss": 3.6532232172266825, + "tokens_seen": 988479488 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035381143430290873, + "loss": 2.7941, + "theoretical_loss": 3.653200118628157, + "tokens_seen": 988545024 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003538014042126379, + "loss": 2.6949, + "theoretical_loss": 3.653177021989653, + "tokens_seen": 988610560 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003537913741223671, + "loss": 2.5338, + "theoretical_loss": 3.653153927310873, + "tokens_seen": 988676096 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003537813440320963, + "loss": 2.732, + "theoretical_loss": 3.6531308345915208, + "tokens_seen": 988741632 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003537713139418255, + "loss": 2.76, + "theoretical_loss": 3.653107743831301, + "tokens_seen": 988807168 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035376128385155464, + "loss": 2.6865, + "theoretical_loss": 3.6530846550299176, + "tokens_seen": 988872704 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035375125376128387, + "loss": 2.7989, + "theoretical_loss": 3.6530615681870744, + "tokens_seen": 988938240 + }, + { + "epoch": 0.3, + "learning_rate": 0.000353741223671013, + "loss": 2.9433, + "theoretical_loss": 3.653038483302475, + "tokens_seen": 989003776 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035373119358074223, + "loss": 2.7141, + "theoretical_loss": 3.653015400375824, + "tokens_seen": 989069312 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003537211634904714, + "loss": 2.807, + "theoretical_loss": 3.652992319406826, + "tokens_seen": 989134848 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003537111334002006, + "loss": 2.565, + "theoretical_loss": 3.652969240395185, + "tokens_seen": 989200384 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003537011033099298, + "loss": 2.8931, + "theoretical_loss": 3.652946163340605, + "tokens_seen": 989265920 + }, + { + "epoch": 0.3, + "learning_rate": 0.000353691073219659, + "loss": 2.777, + "theoretical_loss": 3.6529230882427903, + "tokens_seen": 989331456 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035368104312938814, + "loss": 2.9949, + "theoretical_loss": 3.652900015101446, + "tokens_seen": 989396992 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003536710130391174, + "loss": 2.8874, + "theoretical_loss": 3.6528769439162767, + "tokens_seen": 989462528 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003536609829488465, + "loss": 2.768, + "theoretical_loss": 3.6528538746869863, + "tokens_seen": 989528064 + }, + { + "epoch": 0.3, + "objective/train/docs_used": 567574, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6882741451263428, + "objective/train/theoretical_loss": 3.65283080741328, + "objective/train/tokens_used": 1010053600, + "theoretical_loss": 3.65283080741328, + "tokens_seen": 989593600 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035365095285857574, + "loss": 2.8957, + "theoretical_loss": 3.65283080741328, + "tokens_seen": 989593600 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003536409227683049, + "loss": 2.5723, + "theoretical_loss": 3.652807742094862, + "tokens_seen": 989659136 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003536308926780341, + "loss": 2.7227, + "theoretical_loss": 3.652784678731438, + "tokens_seen": 989724672 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003536208625877633, + "loss": 2.7565, + "theoretical_loss": 3.652761617322712, + "tokens_seen": 989790208 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035361083249749246, + "loss": 2.7983, + "theoretical_loss": 3.65273855786839, + "tokens_seen": 989855744 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003536008024072217, + "loss": 2.604, + "theoretical_loss": 3.652715500368175, + "tokens_seen": 989921280 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003535907723169509, + "loss": 2.9243, + "theoretical_loss": 3.652692444821774, + "tokens_seen": 989986816 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035358074222668006, + "loss": 2.7679, + "theoretical_loss": 3.6526693912288914, + "tokens_seen": 990052352 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035357071213640924, + "loss": 2.6811, + "theoretical_loss": 3.652646339589232, + "tokens_seen": 990117888 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003535606820461384, + "loss": 2.6098, + "theoretical_loss": 3.652623289902502, + "tokens_seen": 990183424 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003535506519558676, + "loss": 2.8982, + "theoretical_loss": 3.6526002421684063, + "tokens_seen": 990248960 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035354062186559684, + "loss": 2.4424, + "theoretical_loss": 3.6525771963866496, + "tokens_seen": 990314496 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035353059177532597, + "loss": 2.8981, + "theoretical_loss": 3.652554152556938, + "tokens_seen": 990380032 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003535205616850552, + "loss": 2.614, + "theoretical_loss": 3.652531110678977, + "tokens_seen": 990445568 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003535105315947844, + "loss": 2.9645, + "theoretical_loss": 3.6525080707524715, + "tokens_seen": 990511104 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035350050150451356, + "loss": 2.8983, + "theoretical_loss": 3.652485032777128, + "tokens_seen": 990576640 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035349047141424274, + "loss": 2.8127, + "theoretical_loss": 3.652461996752652, + "tokens_seen": 990642176 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003534804413239719, + "loss": 2.9463, + "theoretical_loss": 3.6524389626787492, + "tokens_seen": 990707712 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003534704112337011, + "loss": 2.7451, + "theoretical_loss": 3.652415930555125, + "tokens_seen": 990773248 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035346038114343034, + "loss": 2.7748, + "theoretical_loss": 3.6523929003814857, + "tokens_seen": 990838784 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035345035105315947, + "loss": 2.8646, + "theoretical_loss": 3.652369872157537, + "tokens_seen": 990904320 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003534403209628887, + "loss": 2.6992, + "theoretical_loss": 3.6523468458829855, + "tokens_seen": 990969856 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035343029087261783, + "loss": 2.8358, + "theoretical_loss": 3.6523238215575367, + "tokens_seen": 991035392 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035342026078234707, + "loss": 2.9841, + "theoretical_loss": 3.6523007991808965, + "tokens_seen": 991100928 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035341023069207625, + "loss": 2.9709, + "theoretical_loss": 3.6522777787527714, + "tokens_seen": 991166464 + }, + { + "epoch": 0.3, + "objective/train/docs_used": 568180, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.3455541133880615, + "objective/train/theoretical_loss": 3.6522547602728683, + "objective/train/tokens_used": 1011692000, + "theoretical_loss": 3.6522547602728683, + "tokens_seen": 991232000 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035340020060180543, + "loss": 2.7301, + "theoretical_loss": 3.6522547602728683, + "tokens_seen": 991232000 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003533901705115346, + "loss": 2.5853, + "theoretical_loss": 3.652231743740893, + "tokens_seen": 991297536 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035338014042126384, + "loss": 2.6064, + "theoretical_loss": 3.652208729156551, + "tokens_seen": 991363072 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035337011033099297, + "loss": 2.8025, + "theoretical_loss": 3.6521857165195506, + "tokens_seen": 991428608 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003533600802407222, + "loss": 2.7678, + "theoretical_loss": 3.652162705829597, + "tokens_seen": 991494144 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035335005015045133, + "loss": 2.9635, + "theoretical_loss": 3.652139697086397, + "tokens_seen": 991559680 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035334002006018057, + "loss": 2.8122, + "theoretical_loss": 3.652116690289658, + "tokens_seen": 991625216 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035332998996990975, + "loss": 2.8804, + "theoretical_loss": 3.652093685439085, + "tokens_seen": 991690752 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035331995987963893, + "loss": 2.5648, + "theoretical_loss": 3.6520706825343865, + "tokens_seen": 991756288 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003533099297893681, + "loss": 2.786, + "theoretical_loss": 3.6520476815752687, + "tokens_seen": 991821824 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003532998996990973, + "loss": 2.685, + "theoretical_loss": 3.6520246825614384, + "tokens_seen": 991887360 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003532898696088265, + "loss": 2.715, + "theoretical_loss": 3.652001685492603, + "tokens_seen": 991952896 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003532798395185557, + "loss": 2.6665, + "theoretical_loss": 3.651978690368469, + "tokens_seen": 992018432 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035326980942828484, + "loss": 2.6679, + "theoretical_loss": 3.6519556971887437, + "tokens_seen": 992083968 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035325977933801407, + "loss": 2.682, + "theoretical_loss": 3.651932705953134, + "tokens_seen": 992149504 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003532497492477432, + "loss": 2.6389, + "theoretical_loss": 3.651909716661348, + "tokens_seen": 992215040 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035323971915747243, + "loss": 2.6615, + "theoretical_loss": 3.651886729313092, + "tokens_seen": 992280576 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003532296890672016, + "loss": 2.7523, + "theoretical_loss": 3.6518637439080734, + "tokens_seen": 992346112 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003532196589769308, + "loss": 2.6658, + "theoretical_loss": 3.651840760446, + "tokens_seen": 992411648 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035320962888666, + "loss": 2.9121, + "theoretical_loss": 3.6518177789265795, + "tokens_seen": 992477184 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003531995987963892, + "loss": 3.054, + "theoretical_loss": 3.651794799349519, + "tokens_seen": 992542720 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035318956870611834, + "loss": 2.8271, + "theoretical_loss": 3.651771821714526, + "tokens_seen": 992608256 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003531795386158476, + "loss": 2.9655, + "theoretical_loss": 3.6517488460213086, + "tokens_seen": 992673792 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003531695085255767, + "loss": 2.7269, + "theoretical_loss": 3.6517258722695747, + "tokens_seen": 992739328 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035315947843530594, + "loss": 2.964, + "theoretical_loss": 3.651702900459031, + "tokens_seen": 992804864 + }, + { + "epoch": 0.3, + "objective/train/docs_used": 569645, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.649104118347168, + "objective/train/theoretical_loss": 3.6516799305893866, + "objective/train/tokens_used": 1013330400, + "theoretical_loss": 3.6516799305893866, + "tokens_seen": 992870400 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003531494483450351, + "loss": 2.8156, + "theoretical_loss": 3.6516799305893866, + "tokens_seen": 992870400 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003531394182547643, + "loss": 2.7677, + "theoretical_loss": 3.651656962660349, + "tokens_seen": 992935936 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003531293881644935, + "loss": 2.7245, + "theoretical_loss": 3.6516339966716256, + "tokens_seen": 993001472 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035311935807422266, + "loss": 2.8212, + "theoretical_loss": 3.6516110326229247, + "tokens_seen": 993067008 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035310932798395184, + "loss": 2.8697, + "theoretical_loss": 3.651588070513955, + "tokens_seen": 993132544 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003530992978936811, + "loss": 2.7505, + "theoretical_loss": 3.651565110344424, + "tokens_seen": 993198080 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003530892678034102, + "loss": 2.7253, + "theoretical_loss": 3.6515421521140405, + "tokens_seen": 993263616 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035307923771313944, + "loss": 2.8219, + "theoretical_loss": 3.6515191958225124, + "tokens_seen": 993329152 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035306920762286857, + "loss": 2.6554, + "theoretical_loss": 3.651496241469548, + "tokens_seen": 993394688 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003530591775325978, + "loss": 2.6265, + "theoretical_loss": 3.6514732890548567, + "tokens_seen": 993460224 + }, + { + "epoch": 0.3, + "learning_rate": 0.000353049147442327, + "loss": 2.854, + "theoretical_loss": 3.6514503385781456, + "tokens_seen": 993525760 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035303911735205617, + "loss": 2.7416, + "theoretical_loss": 3.6514273900391236, + "tokens_seen": 993591296 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035302908726178535, + "loss": 2.6635, + "theoretical_loss": 3.6514044434374995, + "tokens_seen": 993656832 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003530190571715146, + "loss": 2.7221, + "theoretical_loss": 3.6513814987729822, + "tokens_seen": 993722368 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003530090270812437, + "loss": 2.5713, + "theoretical_loss": 3.65135855604528, + "tokens_seen": 993787904 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035299899699097294, + "loss": 2.7223, + "theoretical_loss": 3.6513356152541023, + "tokens_seen": 993853440 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035298896690070207, + "loss": 2.8612, + "theoretical_loss": 3.651312676399158, + "tokens_seen": 993918976 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003529789368104313, + "loss": 2.8658, + "theoretical_loss": 3.6512897394801547, + "tokens_seen": 993984512 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003529689067201605, + "loss": 2.8514, + "theoretical_loss": 3.651266804496803, + "tokens_seen": 994050048 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035295887662988967, + "loss": 2.6491, + "theoretical_loss": 3.651243871448811, + "tokens_seen": 994115584 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035294884653961885, + "loss": 2.8216, + "theoretical_loss": 3.651220940335888, + "tokens_seen": 994181120 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035293881644934803, + "loss": 2.8118, + "theoretical_loss": 3.651198011157743, + "tokens_seen": 994246656 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003529287863590772, + "loss": 2.947, + "theoretical_loss": 3.6511750839140866, + "tokens_seen": 994312192 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035291875626880645, + "loss": 2.9168, + "theoretical_loss": 3.651152158604626, + "tokens_seen": 994377728 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003529087261785356, + "loss": 2.9427, + "theoretical_loss": 3.651129235229072, + "tokens_seen": 994443264 + }, + { + "epoch": 0.3, + "objective/train/docs_used": 570064, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7230284214019775, + "objective/train/theoretical_loss": 3.6511063137871336, + "objective/train/tokens_used": 1014968800, + "theoretical_loss": 3.6511063137871336, + "tokens_seen": 994508800 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003528986960882648, + "loss": 2.8699, + "theoretical_loss": 3.6511063137871336, + "tokens_seen": 994508800 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035288866599799394, + "loss": 3.1177, + "theoretical_loss": 3.65108339427852, + "tokens_seen": 994574336 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035287863590772317, + "loss": 2.8565, + "theoretical_loss": 3.6510604767029413, + "tokens_seen": 994639872 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035286860581745235, + "loss": 2.9145, + "theoretical_loss": 3.6510375610601065, + "tokens_seen": 994705408 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035285857572718153, + "loss": 2.8725, + "theoretical_loss": 3.6510146473497262, + "tokens_seen": 994770944 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035284854563691077, + "loss": 2.6387, + "theoretical_loss": 3.6509917355715094, + "tokens_seen": 994836480 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035283851554663995, + "loss": 2.6792, + "theoretical_loss": 3.6509688257251662, + "tokens_seen": 994902016 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035282848545636913, + "loss": 2.8448, + "theoretical_loss": 3.650945917810406, + "tokens_seen": 994967552 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003528184553660983, + "loss": 2.7388, + "theoretical_loss": 3.650923011826939, + "tokens_seen": 995033088 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003528084252758275, + "loss": 2.7228, + "theoretical_loss": 3.6509001077744756, + "tokens_seen": 995098624 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003527983951855567, + "loss": 2.7621, + "theoretical_loss": 3.6508772056527263, + "tokens_seen": 995164160 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003527883650952859, + "loss": 2.7531, + "theoretical_loss": 3.6508543054613996, + "tokens_seen": 995229696 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035277833500501504, + "loss": 2.8787, + "theoretical_loss": 3.650831407200206, + "tokens_seen": 995295232 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035276830491474427, + "loss": 2.8108, + "theoretical_loss": 3.650808510868858, + "tokens_seen": 995360768 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003527582748244734, + "loss": 2.7316, + "theoretical_loss": 3.650785616467063, + "tokens_seen": 995426304 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035274824473420263, + "loss": 2.8154, + "theoretical_loss": 3.6507627239945326, + "tokens_seen": 995491840 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003527382146439318, + "loss": 2.8825, + "theoretical_loss": 3.6507398334509773, + "tokens_seen": 995557376 + }, + { + "epoch": 0.3, + "learning_rate": 0.000352728184553661, + "loss": 2.9597, + "theoretical_loss": 3.650716944836107, + "tokens_seen": 995622912 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003527181544633902, + "loss": 3.011, + "theoretical_loss": 3.650694058149634, + "tokens_seen": 995688448 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003527081243731194, + "loss": 2.6709, + "theoretical_loss": 3.6506711733912667, + "tokens_seen": 995753984 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035269809428284854, + "loss": 2.9617, + "theoretical_loss": 3.650648290560717, + "tokens_seen": 995819520 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003526880641925778, + "loss": 2.976, + "theoretical_loss": 3.6506254096576956, + "tokens_seen": 995885056 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003526780341023069, + "loss": 2.9255, + "theoretical_loss": 3.6506025306819128, + "tokens_seen": 995950592 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035266800401203614, + "loss": 2.8566, + "theoretical_loss": 3.65057965363308, + "tokens_seen": 996016128 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003526579739217653, + "loss": 3.1169, + "theoretical_loss": 3.650556778510908, + "tokens_seen": 996081664 + }, + { + "epoch": 0.3, + "objective/train/docs_used": 571551, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.333362102508545, + "objective/train/theoretical_loss": 3.6505339053151076, + "objective/train/tokens_used": 1016607200, + "theoretical_loss": 3.6505339053151076, + "tokens_seen": 996147200 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003526479438314945, + "loss": 2.9734, + "theoretical_loss": 3.6505339053151076, + "tokens_seen": 996147200 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003526379137412237, + "loss": 2.8977, + "theoretical_loss": 3.65051103404539, + "tokens_seen": 996212736 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035262788365095286, + "loss": 2.6582, + "theoretical_loss": 3.650488164701466, + "tokens_seen": 996278272 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035261785356068204, + "loss": 2.8289, + "theoretical_loss": 3.6504652972830476, + "tokens_seen": 996343808 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003526078234704113, + "loss": 2.6944, + "theoretical_loss": 3.6504424317898447, + "tokens_seen": 996409344 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003525977933801404, + "loss": 2.7801, + "theoretical_loss": 3.6504195682215697, + "tokens_seen": 996474880 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035258776328986964, + "loss": 2.8807, + "theoretical_loss": 3.650396706577934, + "tokens_seen": 996540416 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035257773319959877, + "loss": 2.4202, + "theoretical_loss": 3.6503738468586486, + "tokens_seen": 996605952 + }, + { + "epoch": 0.3, + "learning_rate": 0.000352567703109328, + "loss": 2.4863, + "theoretical_loss": 3.6503509890634254, + "tokens_seen": 996671488 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003525576730190572, + "loss": 2.8406, + "theoretical_loss": 3.650328133191975, + "tokens_seen": 996737024 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035254764292878637, + "loss": 2.8054, + "theoretical_loss": 3.65030527924401, + "tokens_seen": 996802560 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035253761283851555, + "loss": 3.0078, + "theoretical_loss": 3.650282427219242, + "tokens_seen": 996868096 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003525275827482448, + "loss": 2.7922, + "theoretical_loss": 3.650259577117382, + "tokens_seen": 996933632 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003525175526579739, + "loss": 2.8068, + "theoretical_loss": 3.6502367289381423, + "tokens_seen": 996999168 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035250752256770314, + "loss": 2.6318, + "theoretical_loss": 3.650213882681235, + "tokens_seen": 997064704 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035249749247743227, + "loss": 2.7184, + "theoretical_loss": 3.650191038346372, + "tokens_seen": 997130240 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003524874623871615, + "loss": 2.8327, + "theoretical_loss": 3.6501681959332646, + "tokens_seen": 997195776 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003524774322968907, + "loss": 2.9303, + "theoretical_loss": 3.6501453554416257, + "tokens_seen": 997261312 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035246740220661987, + "loss": 2.781, + "theoretical_loss": 3.650122516871167, + "tokens_seen": 997326848 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035245737211634905, + "loss": 2.5285, + "theoretical_loss": 3.650099680221601, + "tokens_seen": 997392384 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035244734202607823, + "loss": 2.8837, + "theoretical_loss": 3.650076845492639, + "tokens_seen": 997457920 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003524373119358074, + "loss": 2.5407, + "theoretical_loss": 3.650054012683994, + "tokens_seen": 997523456 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035242728184553665, + "loss": 2.6506, + "theoretical_loss": 3.6500311817953786, + "tokens_seen": 997588992 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003524172517552658, + "loss": 2.6817, + "theoretical_loss": 3.6500083528265046, + "tokens_seen": 997654528 + }, + { + "epoch": 0.3, + "learning_rate": 0.000352407221664995, + "loss": 2.8539, + "theoretical_loss": 3.6499855257770855, + "tokens_seen": 997720064 + }, + { + "epoch": 0.3, + "objective/train/docs_used": 572250, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.9232091903686523, + "objective/train/theoretical_loss": 3.6499627006468325, + "objective/train/tokens_used": 1018245600, + "theoretical_loss": 3.6499627006468325, + "tokens_seen": 997785600 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035239719157472414, + "loss": 2.8161, + "theoretical_loss": 3.6499627006468325, + "tokens_seen": 997785600 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035238716148445337, + "loss": 3.0906, + "theoretical_loss": 3.6499398774354592, + "tokens_seen": 997851136 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035237713139418255, + "loss": 2.9953, + "theoretical_loss": 3.6499170561426784, + "tokens_seen": 997916672 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035236710130391173, + "loss": 2.7025, + "theoretical_loss": 3.6498942367682017, + "tokens_seen": 997982208 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003523570712136409, + "loss": 2.6802, + "theoretical_loss": 3.6498714193117427, + "tokens_seen": 998047744 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035234704112337015, + "loss": 2.698, + "theoretical_loss": 3.6498486037730142, + "tokens_seen": 998113280 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003523370110330993, + "loss": 2.9316, + "theoretical_loss": 3.6498257901517293, + "tokens_seen": 998178816 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003523269809428285, + "loss": 2.8804, + "theoretical_loss": 3.6498029784476005, + "tokens_seen": 998244352 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035231695085255764, + "loss": 2.7959, + "theoretical_loss": 3.6497801686603415, + "tokens_seen": 998309888 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003523069207622869, + "loss": 2.8759, + "theoretical_loss": 3.649757360789664, + "tokens_seen": 998375424 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035229689067201606, + "loss": 2.9551, + "theoretical_loss": 3.649734554835283, + "tokens_seen": 998440960 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035228686058174524, + "loss": 2.9078, + "theoretical_loss": 3.649711750796911, + "tokens_seen": 998506496 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003522768304914744, + "loss": 2.7866, + "theoretical_loss": 3.6496889486742607, + "tokens_seen": 998572032 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003522668004012036, + "loss": 2.8472, + "theoretical_loss": 3.6496661484670465, + "tokens_seen": 998637568 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003522567703109328, + "loss": 2.7142, + "theoretical_loss": 3.6496433501749808, + "tokens_seen": 998703104 + }, + { + "epoch": 0.3, + "learning_rate": 0.000352246740220662, + "loss": 2.7087, + "theoretical_loss": 3.649620553797777, + "tokens_seen": 998768640 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035223671013039114, + "loss": 2.8337, + "theoretical_loss": 3.6495977593351503, + "tokens_seen": 998834176 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003522266800401204, + "loss": 2.844, + "theoretical_loss": 3.6495749667868123, + "tokens_seen": 998899712 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035221664994984956, + "loss": 2.7285, + "theoretical_loss": 3.649552176152478, + "tokens_seen": 998965248 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035220661985957874, + "loss": 2.7266, + "theoretical_loss": 3.64952938743186, + "tokens_seen": 999030784 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003521965897693079, + "loss": 2.7699, + "theoretical_loss": 3.6495066006246732, + "tokens_seen": 999096320 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003521865596790371, + "loss": 2.7324, + "theoretical_loss": 3.6494838157306306, + "tokens_seen": 999161856 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003521765295887663, + "loss": 2.9057, + "theoretical_loss": 3.649461032749447, + "tokens_seen": 999227392 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003521664994984955, + "loss": 2.6274, + "theoretical_loss": 3.6494382516808357, + "tokens_seen": 999292928 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035215646940822465, + "loss": 2.8404, + "theoretical_loss": 3.6494154725245105, + "tokens_seen": 999358464 + }, + { + "epoch": 0.3, + "objective/train/docs_used": 573370, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.806417942047119, + "objective/train/theoretical_loss": 3.649392695280186, + "objective/train/tokens_used": 1019884000, + "theoretical_loss": 3.649392695280186, + "tokens_seen": 999424000 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003521464393179539, + "loss": 2.9672, + "theoretical_loss": 3.649392695280186, + "tokens_seen": 999424000 + }, + { + "epoch": 0.3, + "learning_rate": 0.000352136409227683, + "loss": 2.6774, + "theoretical_loss": 3.649369919947577, + "tokens_seen": 999489536 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035212637913741224, + "loss": 2.7706, + "theoretical_loss": 3.649347146526396, + "tokens_seen": 999555072 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003521163490471414, + "loss": 2.9799, + "theoretical_loss": 3.6493243750163584, + "tokens_seen": 999620608 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003521063189568706, + "loss": 2.8104, + "theoretical_loss": 3.649301605417178, + "tokens_seen": 999686144 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035209628886659984, + "loss": 2.594, + "theoretical_loss": 3.64927883772857, + "tokens_seen": 999751680 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035208625877632897, + "loss": 2.8925, + "theoretical_loss": 3.6492560719502483, + "tokens_seen": 999817216 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003520762286860582, + "loss": 2.7776, + "theoretical_loss": 3.6492333080819277, + "tokens_seen": 999882752 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003520661985957874, + "loss": 2.6188, + "theoretical_loss": 3.649210546123322, + "tokens_seen": 999948288 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035205616850551657, + "loss": 2.8714, + "theoretical_loss": 3.649187786074147, + "tokens_seen": 1000013824 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035204613841524575, + "loss": 2.7148, + "theoretical_loss": 3.6491650279341163, + "tokens_seen": 1000079360 + }, + { + "epoch": 0.3, + "learning_rate": 0.000352036108324975, + "loss": 2.8301, + "theoretical_loss": 3.649142271702946, + "tokens_seen": 1000144896 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003520260782347041, + "loss": 2.9736, + "theoretical_loss": 3.6491195173803495, + "tokens_seen": 1000210432 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035201604814443334, + "loss": 2.8222, + "theoretical_loss": 3.6490967649660426, + "tokens_seen": 1000275968 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035200601805416247, + "loss": 2.8509, + "theoretical_loss": 3.64907401445974, + "tokens_seen": 1000341504 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003519959879638917, + "loss": 3.024, + "theoretical_loss": 3.6490512658611562, + "tokens_seen": 1000407040 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003519859578736209, + "loss": 2.7419, + "theoretical_loss": 3.6490285191700074, + "tokens_seen": 1000472576 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035197592778335007, + "loss": 2.5506, + "theoretical_loss": 3.6490057743860076, + "tokens_seen": 1000538112 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035196589769307925, + "loss": 2.9733, + "theoretical_loss": 3.6489830315088727, + "tokens_seen": 1000603648 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035195586760280843, + "loss": 2.762, + "theoretical_loss": 3.6489602905383176, + "tokens_seen": 1000669184 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003519458375125376, + "loss": 2.7699, + "theoretical_loss": 3.6489375514740576, + "tokens_seen": 1000734720 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035193580742226685, + "loss": 2.8435, + "theoretical_loss": 3.648914814315809, + "tokens_seen": 1000800256 + }, + { + "epoch": 0.3, + "learning_rate": 0.000351925777331996, + "loss": 2.7319, + "theoretical_loss": 3.6488920790632857, + "tokens_seen": 1000865792 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003519157472417252, + "loss": 2.8933, + "theoretical_loss": 3.6488693457162045, + "tokens_seen": 1000931328 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035190571715145434, + "loss": 2.9106, + "theoretical_loss": 3.64884661427428, + "tokens_seen": 1000996864 + }, + { + "epoch": 0.3, + "objective/train/docs_used": 574030, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.889488458633423, + "objective/train/theoretical_loss": 3.6488238847372285, + "objective/train/tokens_used": 1021522400, + "theoretical_loss": 3.6488238847372285, + "tokens_seen": 1001062400 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035189568706118357, + "loss": 2.7585, + "theoretical_loss": 3.6488238847372285, + "tokens_seen": 1001062400 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035188565697091275, + "loss": 2.6913, + "theoretical_loss": 3.6488011571047654, + "tokens_seen": 1001127936 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035187562688064193, + "loss": 2.3341, + "theoretical_loss": 3.6487784313766065, + "tokens_seen": 1001193472 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003518655967903711, + "loss": 2.4424, + "theoretical_loss": 3.6487557075524677, + "tokens_seen": 1001259008 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035185556670010035, + "loss": 2.808, + "theoretical_loss": 3.6487329856320647, + "tokens_seen": 1001324544 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003518455366098295, + "loss": 2.7049, + "theoretical_loss": 3.6487102656151134, + "tokens_seen": 1001390080 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003518355065195587, + "loss": 2.7219, + "theoretical_loss": 3.6486875475013303, + "tokens_seen": 1001455616 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035182547642928784, + "loss": 2.7721, + "theoretical_loss": 3.6486648312904304, + "tokens_seen": 1001521152 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003518154463390171, + "loss": 2.6588, + "theoretical_loss": 3.648642116982131, + "tokens_seen": 1001586688 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035180541624874626, + "loss": 2.5789, + "theoretical_loss": 3.6486194045761473, + "tokens_seen": 1001652224 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035179538615847544, + "loss": 2.7727, + "theoretical_loss": 3.6485966940721957, + "tokens_seen": 1001717760 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003517853560682046, + "loss": 2.7398, + "theoretical_loss": 3.6485739854699935, + "tokens_seen": 1001783296 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003517753259779338, + "loss": 2.8169, + "theoretical_loss": 3.648551278769256, + "tokens_seen": 1001848832 + }, + { + "epoch": 0.3, + "learning_rate": 0.000351765295887663, + "loss": 2.8466, + "theoretical_loss": 3.6485285739696995, + "tokens_seen": 1001914368 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003517552657973922, + "loss": 2.8887, + "theoretical_loss": 3.648505871071041, + "tokens_seen": 1001979904 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035174523570712134, + "loss": 2.7629, + "theoretical_loss": 3.648483170072997, + "tokens_seen": 1002045440 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003517352056168506, + "loss": 2.6051, + "theoretical_loss": 3.6484604709752846, + "tokens_seen": 1002110976 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035172517552657976, + "loss": 2.7626, + "theoretical_loss": 3.6484377737776192, + "tokens_seen": 1002176512 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035171514543630894, + "loss": 2.798, + "theoretical_loss": 3.6484150784797187, + "tokens_seen": 1002242048 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003517051153460381, + "loss": 2.5994, + "theoretical_loss": 3.6483923850812987, + "tokens_seen": 1002307584 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003516950852557673, + "loss": 2.7399, + "theoretical_loss": 3.6483696935820777, + "tokens_seen": 1002373120 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003516850551654965, + "loss": 2.4331, + "theoretical_loss": 3.6483470039817707, + "tokens_seen": 1002438656 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003516750250752257, + "loss": 2.8169, + "theoretical_loss": 3.648324316280096, + "tokens_seen": 1002504192 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035166499498495485, + "loss": 2.5726, + "theoretical_loss": 3.6483016304767704, + "tokens_seen": 1002569728 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003516549648946841, + "loss": 2.6555, + "theoretical_loss": 3.6482789465715104, + "tokens_seen": 1002635264 + }, + { + "epoch": 0.3, + "objective/train/docs_used": 574442, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.802563190460205, + "objective/train/theoretical_loss": 3.6482562645640337, + "objective/train/tokens_used": 1023160800, + "theoretical_loss": 3.6482562645640337, + "tokens_seen": 1002700800 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003516449348044132, + "loss": 2.8908, + "theoretical_loss": 3.6482562645640337, + "tokens_seen": 1002700800 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035163490471414244, + "loss": 2.4215, + "theoretical_loss": 3.648233584454057, + "tokens_seen": 1002766336 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003516248746238716, + "loss": 2.6889, + "theoretical_loss": 3.6482109062412986, + "tokens_seen": 1002831872 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003516148445336008, + "loss": 2.9218, + "theoretical_loss": 3.6481882299254744, + "tokens_seen": 1002897408 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035160481444333, + "loss": 3.061, + "theoretical_loss": 3.648165555506303, + "tokens_seen": 1002962944 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035159478435305917, + "loss": 2.7356, + "theoretical_loss": 3.648142882983501, + "tokens_seen": 1003028480 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035158475426278835, + "loss": 3.0071, + "theoretical_loss": 3.6481202123567864, + "tokens_seen": 1003094016 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003515747241725176, + "loss": 2.5451, + "theoretical_loss": 3.6480975436258767, + "tokens_seen": 1003159552 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003515646940822467, + "loss": 2.7872, + "theoretical_loss": 3.6480748767904894, + "tokens_seen": 1003225088 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035155466399197595, + "loss": 2.8908, + "theoretical_loss": 3.6480522118503425, + "tokens_seen": 1003290624 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035154463390170513, + "loss": 2.8634, + "theoretical_loss": 3.6480295488051535, + "tokens_seen": 1003356160 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003515346038114343, + "loss": 2.8649, + "theoretical_loss": 3.64800688765464, + "tokens_seen": 1003421696 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003515245737211635, + "loss": 2.8774, + "theoretical_loss": 3.64798422839852, + "tokens_seen": 1003487232 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035151454363089267, + "loss": 2.7771, + "theoretical_loss": 3.647961571036512, + "tokens_seen": 1003552768 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035150451354062185, + "loss": 2.7988, + "theoretical_loss": 3.647938915568333, + "tokens_seen": 1003618304 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003514944834503511, + "loss": 3.0265, + "theoretical_loss": 3.647916261993702, + "tokens_seen": 1003683840 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003514844533600802, + "loss": 2.497, + "theoretical_loss": 3.6478936103123365, + "tokens_seen": 1003749376 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035147442326980945, + "loss": 2.8102, + "theoretical_loss": 3.6478709605239548, + "tokens_seen": 1003814912 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003514643931795386, + "loss": 2.8625, + "theoretical_loss": 3.647848312628275, + "tokens_seen": 1003880448 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003514543630892678, + "loss": 2.7732, + "theoretical_loss": 3.6478256666250157, + "tokens_seen": 1003945984 + }, + { + "epoch": 0.3, + "learning_rate": 0.000351444332998997, + "loss": 2.7814, + "theoretical_loss": 3.6478030225138953, + "tokens_seen": 1004011520 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003514343029087262, + "loss": 2.6363, + "theoretical_loss": 3.647780380294632, + "tokens_seen": 1004077056 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035142427281845536, + "loss": 2.9135, + "theoretical_loss": 3.647757739966944, + "tokens_seen": 1004142592 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035141424272818454, + "loss": 2.7157, + "theoretical_loss": 3.6477351015305506, + "tokens_seen": 1004208128 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003514042126379137, + "loss": 2.7695, + "theoretical_loss": 3.6477124649851698, + "tokens_seen": 1004273664 + }, + { + "epoch": 0.3, + "objective/train/docs_used": 575683, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7143306732177734, + "objective/train/theoretical_loss": 3.64768983033052, + "objective/train/tokens_used": 1024799200, + "theoretical_loss": 3.64768983033052, + "tokens_seen": 1004339200 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035139418254764295, + "loss": 2.7203, + "theoretical_loss": 3.64768983033052, + "tokens_seen": 1004339200 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003513841524573721, + "loss": 2.7271, + "theoretical_loss": 3.647667197566321, + "tokens_seen": 1004404736 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003513741223671013, + "loss": 2.7022, + "theoretical_loss": 3.6476445666922905, + "tokens_seen": 1004470272 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003513640922768305, + "loss": 2.6145, + "theoretical_loss": 3.6476219377081476, + "tokens_seen": 1004535808 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003513540621865597, + "loss": 2.9513, + "theoretical_loss": 3.6475993106136118, + "tokens_seen": 1004601344 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003513440320962889, + "loss": 2.7464, + "theoretical_loss": 3.647576685408401, + "tokens_seen": 1004666880 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035133400200601804, + "loss": 2.7284, + "theoretical_loss": 3.6475540620922353, + "tokens_seen": 1004732416 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003513239719157473, + "loss": 2.737, + "theoretical_loss": 3.6475314406648334, + "tokens_seen": 1004797952 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035131394182547646, + "loss": 2.9295, + "theoretical_loss": 3.647508821125914, + "tokens_seen": 1004863488 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035130391173520564, + "loss": 2.7276, + "theoretical_loss": 3.6474862034751965, + "tokens_seen": 1004929024 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003512938816449348, + "loss": 2.6849, + "theoretical_loss": 3.6474635877124, + "tokens_seen": 1004994560 + }, + { + "epoch": 0.3, + "learning_rate": 0.000351283851554664, + "loss": 2.8561, + "theoretical_loss": 3.6474409738372446, + "tokens_seen": 1005060096 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003512738214643932, + "loss": 2.6324, + "theoretical_loss": 3.647418361849449, + "tokens_seen": 1005125632 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003512637913741224, + "loss": 3.0691, + "theoretical_loss": 3.647395751748733, + "tokens_seen": 1005191168 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035125376128385154, + "loss": 2.7431, + "theoretical_loss": 3.6473731435348156, + "tokens_seen": 1005256704 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003512437311935808, + "loss": 2.7852, + "theoretical_loss": 3.6473505372074166, + "tokens_seen": 1005322240 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035123370110330996, + "loss": 2.9097, + "theoretical_loss": 3.647327932766256, + "tokens_seen": 1005387776 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035122367101303914, + "loss": 2.768, + "theoretical_loss": 3.647305330211053, + "tokens_seen": 1005453312 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003512136409227683, + "loss": 2.7207, + "theoretical_loss": 3.6472827295415273, + "tokens_seen": 1005518848 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003512036108324975, + "loss": 2.7768, + "theoretical_loss": 3.647260130757399, + "tokens_seen": 1005584384 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003511935807422267, + "loss": 2.5983, + "theoretical_loss": 3.647237533858388, + "tokens_seen": 1005649920 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003511835506519559, + "loss": 2.8383, + "theoretical_loss": 3.647214938844214, + "tokens_seen": 1005715456 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035117352056168505, + "loss": 2.6759, + "theoretical_loss": 3.647192345714597, + "tokens_seen": 1005780992 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003511634904714143, + "loss": 2.8947, + "theoretical_loss": 3.6471697544692567, + "tokens_seen": 1005846528 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003511534603811434, + "loss": 2.6597, + "theoretical_loss": 3.6471471651079135, + "tokens_seen": 1005912064 + }, + { + "epoch": 0.3, + "objective/train/docs_used": 576356, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.5811333656311035, + "objective/train/theoretical_loss": 3.6471245776302883, + "objective/train/tokens_used": 1026437600, + "theoretical_loss": 3.6471245776302883, + "tokens_seen": 1005977600 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035114343029087264, + "loss": 2.8224, + "theoretical_loss": 3.6471245776302883, + "tokens_seen": 1005977600 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003511334002006018, + "loss": 2.8391, + "theoretical_loss": 3.6471019920361, + "tokens_seen": 1006043136 + }, + { + "epoch": 0.3, + "learning_rate": 0.000351123370110331, + "loss": 2.8754, + "theoretical_loss": 3.6470794083250695, + "tokens_seen": 1006108672 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003511133400200602, + "loss": 2.8797, + "theoretical_loss": 3.6470568264969176, + "tokens_seen": 1006174208 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035110330992978937, + "loss": 2.7388, + "theoretical_loss": 3.647034246551364, + "tokens_seen": 1006239744 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035109327983951855, + "loss": 2.5076, + "theoretical_loss": 3.6470116684881297, + "tokens_seen": 1006305280 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003510832497492478, + "loss": 2.8662, + "theoretical_loss": 3.646989092306934, + "tokens_seen": 1006370816 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003510732196589769, + "loss": 2.6386, + "theoretical_loss": 3.646966518007499, + "tokens_seen": 1006436352 + }, + { + "epoch": 0.31, + "learning_rate": 0.00035106318956870615, + "loss": 2.5959, + "theoretical_loss": 3.6469439455895447, + "tokens_seen": 1006501888 + }, + { + "epoch": 0.31, + "learning_rate": 0.00035105315947843533, + "loss": 2.6767, + "theoretical_loss": 3.646921375052792, + "tokens_seen": 1006567424 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003510431293881645, + "loss": 2.6978, + "theoretical_loss": 3.646898806396961, + "tokens_seen": 1006632960 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003510330992978937, + "loss": 2.5124, + "theoretical_loss": 3.646876239621773, + "tokens_seen": 1006698496 + }, + { + "epoch": 0.31, + "learning_rate": 0.00035102306920762287, + "loss": 3.0031, + "theoretical_loss": 3.646853674726949, + "tokens_seen": 1006764032 + }, + { + "epoch": 0.31, + "learning_rate": 0.00035101303911735205, + "loss": 2.8924, + "theoretical_loss": 3.64683111171221, + "tokens_seen": 1006829568 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003510030090270813, + "loss": 2.7971, + "theoretical_loss": 3.6468085505772763, + "tokens_seen": 1006895104 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003509929789368104, + "loss": 2.6092, + "theoretical_loss": 3.6467859913218694, + "tokens_seen": 1006960640 + }, + { + "epoch": 0.31, + "learning_rate": 0.00035098294884653965, + "loss": 2.5674, + "theoretical_loss": 3.646763433945711, + "tokens_seen": 1007026176 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003509729187562688, + "loss": 2.7497, + "theoretical_loss": 3.6467408784485214, + "tokens_seen": 1007091712 + }, + { + "epoch": 0.31, + "learning_rate": 0.000350962888665998, + "loss": 2.7896, + "theoretical_loss": 3.6467183248300215, + "tokens_seen": 1007157248 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003509528585757272, + "loss": 2.6715, + "theoretical_loss": 3.6466957730899336, + "tokens_seen": 1007222784 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003509428284854564, + "loss": 2.7412, + "theoretical_loss": 3.646673223227979, + "tokens_seen": 1007288320 + }, + { + "epoch": 0.31, + "learning_rate": 0.00035093279839518556, + "loss": 2.6717, + "theoretical_loss": 3.6466506752438788, + "tokens_seen": 1007353856 + }, + { + "epoch": 0.31, + "learning_rate": 0.00035092276830491474, + "loss": 2.8398, + "theoretical_loss": 3.646628129137354, + "tokens_seen": 1007419392 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003509127382146439, + "loss": 2.8167, + "theoretical_loss": 3.6466055849081265, + "tokens_seen": 1007484928 + }, + { + "epoch": 0.31, + "learning_rate": 0.00035090270812437315, + "loss": 2.785, + "theoretical_loss": 3.6465830425559185, + "tokens_seen": 1007550464 + }, + { + "epoch": 0.31, + "objective/train/docs_used": 577861, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.47003436088562, + "objective/train/theoretical_loss": 3.6465605020804506, + "objective/train/tokens_used": 1028076000, + "theoretical_loss": 3.6465605020804506, + "tokens_seen": 1007616000 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003508926780341023, + "loss": 2.5462, + "theoretical_loss": 3.6465605020804506, + "tokens_seen": 1007616000 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003508826479438315, + "loss": 2.4548, + "theoretical_loss": 3.646537963481445, + "tokens_seen": 1007681536 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003508726178535607, + "loss": 2.7762, + "theoretical_loss": 3.646515426758624, + "tokens_seen": 1007747072 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003508625877632899, + "loss": 2.7092, + "theoretical_loss": 3.646492891911708, + "tokens_seen": 1007812608 + }, + { + "epoch": 0.31, + "learning_rate": 0.00035085255767301906, + "loss": 2.772, + "theoretical_loss": 3.6464703589404204, + "tokens_seen": 1007878144 + }, + { + "epoch": 0.31, + "learning_rate": 0.00035084252758274824, + "loss": 2.6412, + "theoretical_loss": 3.646447827844482, + "tokens_seen": 1007943680 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003508324974924774, + "loss": 2.8119, + "theoretical_loss": 3.646425298623616, + "tokens_seen": 1008009216 + }, + { + "epoch": 0.31, + "learning_rate": 0.00035082246740220666, + "loss": 2.3924, + "theoretical_loss": 3.6464027712775433, + "tokens_seen": 1008074752 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003508124373119358, + "loss": 2.6974, + "theoretical_loss": 3.646380245805986, + "tokens_seen": 1008140288 + }, + { + "epoch": 0.31, + "learning_rate": 0.000350802407221665, + "loss": 2.8924, + "theoretical_loss": 3.6463577222086676, + "tokens_seen": 1008205824 + }, + { + "epoch": 0.31, + "learning_rate": 0.00035079237713139415, + "loss": 2.8118, + "theoretical_loss": 3.6463352004853093, + "tokens_seen": 1008271360 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003507823470411234, + "loss": 2.6372, + "theoretical_loss": 3.646312680635633, + "tokens_seen": 1008336896 + }, + { + "epoch": 0.31, + "learning_rate": 0.00035077231695085256, + "loss": 2.5175, + "theoretical_loss": 3.646290162659363, + "tokens_seen": 1008402432 + }, + { + "epoch": 0.31, + "learning_rate": 0.00035076228686058174, + "loss": 2.9351, + "theoretical_loss": 3.646267646556219, + "tokens_seen": 1008467968 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003507522567703109, + "loss": 2.6388, + "theoretical_loss": 3.6462451323259257, + "tokens_seen": 1008533504 + }, + { + "epoch": 0.31, + "learning_rate": 0.00035074222668004016, + "loss": 2.8205, + "theoretical_loss": 3.6462226199682046, + "tokens_seen": 1008599040 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003507321965897693, + "loss": 2.5539, + "theoretical_loss": 3.646200109482778, + "tokens_seen": 1008664576 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003507221664994985, + "loss": 2.7278, + "theoretical_loss": 3.646177600869369, + "tokens_seen": 1008730112 + }, + { + "epoch": 0.31, + "learning_rate": 0.00035071213640922765, + "loss": 2.7777, + "theoretical_loss": 3.646155094127701, + "tokens_seen": 1008795648 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003507021063189569, + "loss": 2.5319, + "theoretical_loss": 3.646132589257496, + "tokens_seen": 1008861184 + }, + { + "epoch": 0.31, + "learning_rate": 0.00035069207622868607, + "loss": 2.7216, + "theoretical_loss": 3.6461100862584765, + "tokens_seen": 1008926720 + }, + { + "epoch": 0.31, + "learning_rate": 0.00035068204613841525, + "loss": 2.6221, + "theoretical_loss": 3.646087585130366, + "tokens_seen": 1008992256 + }, + { + "epoch": 0.31, + "learning_rate": 0.00035067201604814443, + "loss": 2.5606, + "theoretical_loss": 3.6460650858728876, + "tokens_seen": 1009057792 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003506619859578736, + "loss": 2.5822, + "theoretical_loss": 3.6460425884857637, + "tokens_seen": 1009123328 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003506519558676028, + "loss": 2.6976, + "theoretical_loss": 3.6460200929687177, + "tokens_seen": 1009188864 + }, + { + "epoch": 0.31, + "objective/train/docs_used": 578450, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.908143997192383, + "objective/train/theoretical_loss": 3.6459975993214724, + "objective/train/tokens_used": 1029714400, + "theoretical_loss": 3.6459975993214724, + "tokens_seen": 1009254400 + }, + { + "epoch": 0.31, + "learning_rate": 0.000350641925777332, + "loss": 2.7492, + "theoretical_loss": 3.6459975993214724, + "tokens_seen": 1009254400 + }, + { + "epoch": 0.31, + "learning_rate": 0.00035063189568706115, + "loss": 2.77, + "theoretical_loss": 3.6459751075437516, + "tokens_seen": 1009319936 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003506218655967904, + "loss": 2.6826, + "theoretical_loss": 3.645952617635278, + "tokens_seen": 1009385472 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003506118355065195, + "loss": 2.7402, + "theoretical_loss": 3.645930129595775, + "tokens_seen": 1009451008 + }, + { + "epoch": 0.31, + "learning_rate": 0.00035060180541624875, + "loss": 2.7609, + "theoretical_loss": 3.645907643424966, + "tokens_seen": 1009516544 + }, + { + "epoch": 0.31, + "learning_rate": 0.000350591775325978, + "loss": 2.8985, + "theoretical_loss": 3.6458851591225754, + "tokens_seen": 1009582080 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003505817452357071, + "loss": 2.8505, + "theoretical_loss": 3.645862676688325, + "tokens_seen": 1009647616 + }, + { + "epoch": 0.31, + "learning_rate": 0.00035057171514543635, + "loss": 2.5282, + "theoretical_loss": 3.645840196121939, + "tokens_seen": 1009713152 + }, + { + "epoch": 0.31, + "learning_rate": 0.00035056168505516553, + "loss": 2.7067, + "theoretical_loss": 3.6458177174231414, + "tokens_seen": 1009778688 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003505516549648947, + "loss": 2.5714, + "theoretical_loss": 3.6457952405916556, + "tokens_seen": 1009844224 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003505416248746239, + "loss": 2.7524, + "theoretical_loss": 3.6457727656272043, + "tokens_seen": 1009909760 + }, + { + "epoch": 0.31, + "learning_rate": 0.00035053159478435307, + "loss": 2.4596, + "theoretical_loss": 3.645750292529513, + "tokens_seen": 1009975296 + }, + { + "epoch": 0.31, + "learning_rate": 0.00035052156469408225, + "loss": 2.7061, + "theoretical_loss": 3.6457278212983053, + "tokens_seen": 1010040832 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003505115346038115, + "loss": 2.4885, + "theoretical_loss": 3.645705351933304, + "tokens_seen": 1010106368 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003505015045135406, + "loss": 2.7954, + "theoretical_loss": 3.6456828844342337, + "tokens_seen": 1010171904 + }, + { + "epoch": 0.31, + "learning_rate": 0.00035049147442326985, + "loss": 2.6005, + "theoretical_loss": 3.6456604188008184, + "tokens_seen": 1010237440 + }, + { + "epoch": 0.31, + "learning_rate": 0.000350481444332999, + "loss": 2.7985, + "theoretical_loss": 3.6456379550327815, + "tokens_seen": 1010302976 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003504714142427282, + "loss": 2.7055, + "theoretical_loss": 3.6456154931298483, + "tokens_seen": 1010368512 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003504613841524574, + "loss": 2.7006, + "theoretical_loss": 3.6455930330917425, + "tokens_seen": 1010434048 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003504513540621866, + "loss": 2.7961, + "theoretical_loss": 3.645570574918188, + "tokens_seen": 1010499584 + }, + { + "epoch": 0.31, + "learning_rate": 0.00035044132397191576, + "loss": 2.6502, + "theoretical_loss": 3.645548118608909, + "tokens_seen": 1010565120 + }, + { + "epoch": 0.31, + "learning_rate": 0.00035043129388164494, + "loss": 2.682, + "theoretical_loss": 3.6455256641636304, + "tokens_seen": 1010630656 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003504212637913741, + "loss": 2.7945, + "theoretical_loss": 3.6455032115820765, + "tokens_seen": 1010696192 + }, + { + "epoch": 0.31, + "learning_rate": 0.00035041123370110335, + "loss": 2.5379, + "theoretical_loss": 3.645480760863972, + "tokens_seen": 1010761728 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003504012036108325, + "loss": 2.6981, + "theoretical_loss": 3.6454583120090405, + "tokens_seen": 1010827264 + }, + { + "epoch": 0.31, + "objective/train/docs_used": 579792, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.9241912364959717, + "objective/train/theoretical_loss": 3.6454358650170073, + "objective/train/tokens_used": 1031352800, + "theoretical_loss": 3.6454358650170073, + "tokens_seen": 1010892800 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003503911735205617, + "loss": 2.9682, + "theoretical_loss": 3.6454358650170073, + "tokens_seen": 1010892800 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003503811434302909, + "loss": 2.583, + "theoretical_loss": 3.645413419887597, + "tokens_seen": 1010958336 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003503711133400201, + "loss": 2.6421, + "theoretical_loss": 3.6453909766205346, + "tokens_seen": 1011023872 + }, + { + "epoch": 0.31, + "learning_rate": 0.00035036108324974926, + "loss": 2.6315, + "theoretical_loss": 3.6453685352155443, + "tokens_seen": 1011089408 + }, + { + "epoch": 0.31, + "learning_rate": 0.00035035105315947844, + "loss": 2.6428, + "theoretical_loss": 3.645346095672352, + "tokens_seen": 1011154944 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003503410230692076, + "loss": 2.7126, + "theoretical_loss": 3.6453236579906805, + "tokens_seen": 1011220480 + }, + { + "epoch": 0.31, + "learning_rate": 0.00035033099297893686, + "loss": 2.5518, + "theoretical_loss": 3.6453012221702563, + "tokens_seen": 1011286016 + }, + { + "epoch": 0.31, + "learning_rate": 0.000350320962888666, + "loss": 3.1047, + "theoretical_loss": 3.6452787882108044, + "tokens_seen": 1011351552 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003503109327983952, + "loss": 2.578, + "theoretical_loss": 3.64525635611205, + "tokens_seen": 1011417088 + }, + { + "epoch": 0.31, + "learning_rate": 0.00035030090270812435, + "loss": 2.479, + "theoretical_loss": 3.645233925873717, + "tokens_seen": 1011482624 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003502908726178536, + "loss": 2.4378, + "theoretical_loss": 3.6452114974955316, + "tokens_seen": 1011548160 + }, + { + "epoch": 0.31, + "learning_rate": 0.00035028084252758276, + "loss": 2.7474, + "theoretical_loss": 3.645189070977219, + "tokens_seen": 1011613696 + }, + { + "epoch": 0.31, + "learning_rate": 0.00035027081243731194, + "loss": 2.5031, + "theoretical_loss": 3.645166646318504, + "tokens_seen": 1011679232 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003502607823470411, + "loss": 2.8781, + "theoretical_loss": 3.6451442235191127, + "tokens_seen": 1011744768 + }, + { + "epoch": 0.31, + "learning_rate": 0.00035025075225677036, + "loss": 2.8065, + "theoretical_loss": 3.64512180257877, + "tokens_seen": 1011810304 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003502407221664995, + "loss": 2.8696, + "theoretical_loss": 3.645099383497201, + "tokens_seen": 1011875840 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003502306920762287, + "loss": 2.8428, + "theoretical_loss": 3.6450769662741322, + "tokens_seen": 1011941376 + }, + { + "epoch": 0.31, + "learning_rate": 0.00035022066198595785, + "loss": 2.7668, + "theoretical_loss": 3.6450545509092884, + "tokens_seen": 1012006912 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003502106318956871, + "loss": 2.6901, + "theoretical_loss": 3.6450321374023957, + "tokens_seen": 1012072448 + }, + { + "epoch": 0.31, + "learning_rate": 0.00035020060180541627, + "loss": 2.7531, + "theoretical_loss": 3.6450097257531793, + "tokens_seen": 1012137984 + }, + { + "epoch": 0.31, + "learning_rate": 0.00035019057171514545, + "loss": 2.7556, + "theoretical_loss": 3.6449873159613655, + "tokens_seen": 1012203520 + }, + { + "epoch": 0.31, + "learning_rate": 0.00035018054162487463, + "loss": 2.6658, + "theoretical_loss": 3.64496490802668, + "tokens_seen": 1012269056 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003501705115346038, + "loss": 2.8573, + "theoretical_loss": 3.644942501948848, + "tokens_seen": 1012334592 + }, + { + "epoch": 0.31, + "learning_rate": 0.000350160481444333, + "loss": 2.8175, + "theoretical_loss": 3.6449200977275966, + "tokens_seen": 1012400128 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003501504513540622, + "loss": 2.8505, + "theoretical_loss": 3.6448976953626513, + "tokens_seen": 1012465664 + }, + { + "epoch": 0.31, + "objective/train/docs_used": 580179, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.3930437564849854, + "objective/train/theoretical_loss": 3.6448752948537377, + "objective/train/tokens_used": 1032991200, + "theoretical_loss": 3.6448752948537377, + "tokens_seen": 1012531200 + }, + { + "epoch": 0.31, + "learning_rate": 0.00035014042126379135, + "loss": 3.029, + "theoretical_loss": 3.6448752948537377, + "tokens_seen": 1012531200 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003501303911735206, + "loss": 3.0943, + "theoretical_loss": 3.6448528962005824, + "tokens_seen": 1012596736 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003501203610832497, + "loss": 3.0421, + "theoretical_loss": 3.6448304994029113, + "tokens_seen": 1012662272 + }, + { + "epoch": 0.31, + "learning_rate": 0.00035011033099297895, + "loss": 2.8042, + "theoretical_loss": 3.644808104460451, + "tokens_seen": 1012727808 + }, + { + "epoch": 0.31, + "learning_rate": 0.00035010030090270813, + "loss": 2.9248, + "theoretical_loss": 3.6447857113729274, + "tokens_seen": 1012793344 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003500902708124373, + "loss": 2.6852, + "theoretical_loss": 3.6447633201400667, + "tokens_seen": 1012858880 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003500802407221665, + "loss": 2.843, + "theoretical_loss": 3.6447409307615963, + "tokens_seen": 1012924416 + }, + { + "epoch": 0.31, + "learning_rate": 0.00035007021063189573, + "loss": 2.7913, + "theoretical_loss": 3.644718543237241, + "tokens_seen": 1012989952 + }, + { + "epoch": 0.31, + "learning_rate": 0.00035006018054162486, + "loss": 2.8401, + "theoretical_loss": 3.644696157566729, + "tokens_seen": 1013055488 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003500501504513541, + "loss": 2.9712, + "theoretical_loss": 3.644673773749786, + "tokens_seen": 1013121024 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003500401203610832, + "loss": 2.6187, + "theoretical_loss": 3.644651391786139, + "tokens_seen": 1013186560 + }, + { + "epoch": 0.31, + "learning_rate": 0.00035003009027081245, + "loss": 2.6597, + "theoretical_loss": 3.6446290116755145, + "tokens_seen": 1013252096 + }, + { + "epoch": 0.31, + "learning_rate": 0.00035002006018054163, + "loss": 2.6615, + "theoretical_loss": 3.644606633417639, + "tokens_seen": 1013317632 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003500100300902708, + "loss": 2.8412, + "theoretical_loss": 3.6445842570122395, + "tokens_seen": 1013383168 + }, + { + "epoch": 0.31, + "learning_rate": 0.00035, + "loss": 2.8491, + "theoretical_loss": 3.644561882459043, + "tokens_seen": 1013448704 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003499899699097292, + "loss": 2.7624, + "theoretical_loss": 3.6445395097577764, + "tokens_seen": 1013514240 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034997993981945836, + "loss": 2.92, + "theoretical_loss": 3.6445171389081663, + "tokens_seen": 1013579776 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003499699097291876, + "loss": 2.8834, + "theoretical_loss": 3.64449476990994, + "tokens_seen": 1013645312 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003499598796389167, + "loss": 2.7117, + "theoretical_loss": 3.6444724027628252, + "tokens_seen": 1013710848 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034994984954864596, + "loss": 2.849, + "theoretical_loss": 3.644450037466548, + "tokens_seen": 1013776384 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003499398194583751, + "loss": 2.6253, + "theoretical_loss": 3.6444276740208363, + "tokens_seen": 1013841920 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003499297893681043, + "loss": 2.88, + "theoretical_loss": 3.6444053124254174, + "tokens_seen": 1013907456 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003499197592778335, + "loss": 2.5481, + "theoretical_loss": 3.644382952680018, + "tokens_seen": 1013972992 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003499097291875627, + "loss": 2.7435, + "theoretical_loss": 3.6443605947843665, + "tokens_seen": 1014038528 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034989969909729186, + "loss": 2.5695, + "theoretical_loss": 3.644338238738189, + "tokens_seen": 1014104064 + }, + { + "epoch": 0.31, + "objective/train/docs_used": 581599, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.9870293140411377, + "objective/train/theoretical_loss": 3.6443158845412142, + "objective/train/tokens_used": 1034629600, + "theoretical_loss": 3.6443158845412142, + "tokens_seen": 1014169600 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003498896690070211, + "loss": 2.7469, + "theoretical_loss": 3.6443158845412142, + "tokens_seen": 1014169600 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003498796389167502, + "loss": 2.5881, + "theoretical_loss": 3.6442935321931693, + "tokens_seen": 1014235136 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034986960882647946, + "loss": 2.8075, + "theoretical_loss": 3.6442711816937816, + "tokens_seen": 1014300672 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003498595787362086, + "loss": 2.8776, + "theoretical_loss": 3.644248833042779, + "tokens_seen": 1014366208 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003498495486459378, + "loss": 2.8352, + "theoretical_loss": 3.644226486239889, + "tokens_seen": 1014431744 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034983951855566706, + "loss": 2.844, + "theoretical_loss": 3.64420414128484, + "tokens_seen": 1014497280 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003498294884653962, + "loss": 2.9222, + "theoretical_loss": 3.6441817981773594, + "tokens_seen": 1014562816 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003498194583751254, + "loss": 2.5364, + "theoretical_loss": 3.6441594569171754, + "tokens_seen": 1014628352 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034980942828485455, + "loss": 2.6019, + "theoretical_loss": 3.6441371175040156, + "tokens_seen": 1014693888 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003497993981945838, + "loss": 2.6437, + "theoretical_loss": 3.644114779937608, + "tokens_seen": 1014759424 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034978936810431296, + "loss": 2.7834, + "theoretical_loss": 3.644092444217681, + "tokens_seen": 1014824960 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034977933801404214, + "loss": 2.9565, + "theoretical_loss": 3.6440701103439626, + "tokens_seen": 1014890496 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003497693079237713, + "loss": 2.9657, + "theoretical_loss": 3.644047778316181, + "tokens_seen": 1014956032 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034975927783350056, + "loss": 2.8679, + "theoretical_loss": 3.644025448134064, + "tokens_seen": 1015021568 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003497492477432297, + "loss": 2.9711, + "theoretical_loss": 3.644003119797341, + "tokens_seen": 1015087104 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003497392176529589, + "loss": 2.6752, + "theoretical_loss": 3.6439807933057393, + "tokens_seen": 1015152640 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034972918756268805, + "loss": 2.7547, + "theoretical_loss": 3.643958468658988, + "tokens_seen": 1015218176 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003497191574724173, + "loss": 2.5455, + "theoretical_loss": 3.6439361458568147, + "tokens_seen": 1015283712 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034970912738214647, + "loss": 2.7375, + "theoretical_loss": 3.6439138248989495, + "tokens_seen": 1015349248 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034969909729187565, + "loss": 2.8744, + "theoretical_loss": 3.6438915057851196, + "tokens_seen": 1015414784 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034968906720160483, + "loss": 2.7847, + "theoretical_loss": 3.643869188515054, + "tokens_seen": 1015480320 + }, + { + "epoch": 0.31, + "learning_rate": 0.000349679037111334, + "loss": 2.8876, + "theoretical_loss": 3.6438468730884814, + "tokens_seen": 1015545856 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003496690070210632, + "loss": 2.7832, + "theoretical_loss": 3.643824559505131, + "tokens_seen": 1015611392 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003496589769307924, + "loss": 2.808, + "theoretical_loss": 3.6438022477647314, + "tokens_seen": 1015676928 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034964894684052155, + "loss": 2.5508, + "theoretical_loss": 3.643779937867011, + "tokens_seen": 1015742464 + }, + { + "epoch": 0.31, + "objective/train/docs_used": 582330, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.8800933361053467, + "objective/train/theoretical_loss": 3.6437576298116996, + "objective/train/tokens_used": 1036268000, + "theoretical_loss": 3.6437576298116996, + "tokens_seen": 1015808000 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003496389167502508, + "loss": 2.741, + "theoretical_loss": 3.6437576298116996, + "tokens_seen": 1015808000 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003496288866599799, + "loss": 2.5943, + "theoretical_loss": 3.6437353235985253, + "tokens_seen": 1015873536 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034961885656970915, + "loss": 2.7954, + "theoretical_loss": 3.6437130192272176, + "tokens_seen": 1015939072 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034960882647943833, + "loss": 2.7477, + "theoretical_loss": 3.6436907166975057, + "tokens_seen": 1016004608 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003495987963891675, + "loss": 2.6713, + "theoretical_loss": 3.6436684160091186, + "tokens_seen": 1016070144 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003495887662988967, + "loss": 2.6791, + "theoretical_loss": 3.6436461171617864, + "tokens_seen": 1016135680 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034957873620862593, + "loss": 2.7401, + "theoretical_loss": 3.643623820155237, + "tokens_seen": 1016201216 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034956870611835506, + "loss": 2.6888, + "theoretical_loss": 3.6436015249892, + "tokens_seen": 1016266752 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003495586760280843, + "loss": 2.8085, + "theoretical_loss": 3.6435792316634057, + "tokens_seen": 1016332288 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003495486459378134, + "loss": 2.5099, + "theoretical_loss": 3.6435569401775822, + "tokens_seen": 1016397824 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034953861584754265, + "loss": 2.6178, + "theoretical_loss": 3.6435346505314605, + "tokens_seen": 1016463360 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034952858575727183, + "loss": 2.6781, + "theoretical_loss": 3.643512362724769, + "tokens_seen": 1016528896 + }, + { + "epoch": 0.31, + "learning_rate": 0.000349518555667001, + "loss": 2.6614, + "theoretical_loss": 3.6434900767572382, + "tokens_seen": 1016594432 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003495085255767302, + "loss": 2.5157, + "theoretical_loss": 3.6434677926285977, + "tokens_seen": 1016659968 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003494984954864594, + "loss": 2.9407, + "theoretical_loss": 3.6434455103385766, + "tokens_seen": 1016725504 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034948846539618856, + "loss": 2.5569, + "theoretical_loss": 3.6434232298869045, + "tokens_seen": 1016791040 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003494784353059178, + "loss": 2.7666, + "theoretical_loss": 3.6434009512733123, + "tokens_seen": 1016856576 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003494684052156469, + "loss": 2.3313, + "theoretical_loss": 3.643378674497529, + "tokens_seen": 1016922112 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034945837512537616, + "loss": 2.9162, + "theoretical_loss": 3.643356399559286, + "tokens_seen": 1016987648 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003494483450351053, + "loss": 2.5875, + "theoretical_loss": 3.6433341264583117, + "tokens_seen": 1017053184 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003494383149448345, + "loss": 2.6881, + "theoretical_loss": 3.6433118551943364, + "tokens_seen": 1017118720 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003494282848545637, + "loss": 2.637, + "theoretical_loss": 3.643289585767091, + "tokens_seen": 1017184256 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003494182547642929, + "loss": 2.5439, + "theoretical_loss": 3.6432673181763056, + "tokens_seen": 1017249792 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034940822467402206, + "loss": 2.5029, + "theoretical_loss": 3.64324505242171, + "tokens_seen": 1017315328 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003493981945837513, + "loss": 2.8158, + "theoretical_loss": 3.643222788503034, + "tokens_seen": 1017380864 + }, + { + "epoch": 0.31, + "objective/train/docs_used": 582938, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7228047847747803, + "objective/train/theoretical_loss": 3.64320052642001, + "objective/train/tokens_used": 1037906400, + "theoretical_loss": 3.64320052642001, + "tokens_seen": 1017446400 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003493881644934804, + "loss": 2.6733, + "theoretical_loss": 3.64320052642001, + "tokens_seen": 1017446400 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034937813440320966, + "loss": 2.8868, + "theoretical_loss": 3.643178266172366, + "tokens_seen": 1017511936 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003493681043129388, + "loss": 2.7095, + "theoretical_loss": 3.643156007759834, + "tokens_seen": 1017577472 + }, + { + "epoch": 0.31, + "learning_rate": 0.000349358074222668, + "loss": 2.8682, + "theoretical_loss": 3.6431337511821438, + "tokens_seen": 1017643008 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003493480441323972, + "loss": 2.5822, + "theoretical_loss": 3.643111496439027, + "tokens_seen": 1017708544 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003493380140421264, + "loss": 2.889, + "theoretical_loss": 3.6430892435302136, + "tokens_seen": 1017774080 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034932798395185557, + "loss": 2.7182, + "theoretical_loss": 3.6430669924554335, + "tokens_seen": 1017839616 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034931795386158475, + "loss": 2.5725, + "theoretical_loss": 3.6430447432144186, + "tokens_seen": 1017905152 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034930792377131393, + "loss": 2.7366, + "theoretical_loss": 3.6430224958068997, + "tokens_seen": 1017970688 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034929789368104316, + "loss": 2.6078, + "theoretical_loss": 3.643000250232607, + "tokens_seen": 1018036224 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003492878635907723, + "loss": 2.8245, + "theoretical_loss": 3.642978006491272, + "tokens_seen": 1018101760 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003492778335005015, + "loss": 2.6025, + "theoretical_loss": 3.6429557645826254, + "tokens_seen": 1018167296 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034926780341023065, + "loss": 2.5496, + "theoretical_loss": 3.6429335245063985, + "tokens_seen": 1018232832 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003492577733199599, + "loss": 2.6617, + "theoretical_loss": 3.642911286262322, + "tokens_seen": 1018298368 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034924774322968907, + "loss": 2.438, + "theoretical_loss": 3.642889049850128, + "tokens_seen": 1018363904 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034923771313941825, + "loss": 2.7004, + "theoretical_loss": 3.6428668152695467, + "tokens_seen": 1018429440 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034922768304914743, + "loss": 2.8968, + "theoretical_loss": 3.6428445825203095, + "tokens_seen": 1018494976 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034921765295887667, + "loss": 2.7715, + "theoretical_loss": 3.642822351602148, + "tokens_seen": 1018560512 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003492076228686058, + "loss": 2.8807, + "theoretical_loss": 3.6428001225147937, + "tokens_seen": 1018626048 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034919759277833503, + "loss": 2.5589, + "theoretical_loss": 3.642777895257978, + "tokens_seen": 1018691584 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034918756268806416, + "loss": 2.7364, + "theoretical_loss": 3.6427556698314323, + "tokens_seen": 1018757120 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003491775325977934, + "loss": 2.8622, + "theoretical_loss": 3.6427334462348884, + "tokens_seen": 1018822656 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034916750250752257, + "loss": 2.7157, + "theoretical_loss": 3.6427112244680773, + "tokens_seen": 1018888192 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034915747241725175, + "loss": 2.8452, + "theoretical_loss": 3.642689004530731, + "tokens_seen": 1018953728 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034914744232698093, + "loss": 2.6818, + "theoretical_loss": 3.642666786422582, + "tokens_seen": 1019019264 + }, + { + "epoch": 0.31, + "objective/train/docs_used": 584109, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.996743679046631, + "objective/train/theoretical_loss": 3.6426445701433607, + "objective/train/tokens_used": 1039544800, + "theoretical_loss": 3.6426445701433607, + "tokens_seen": 1019084800 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003491374122367101, + "loss": 2.6503, + "theoretical_loss": 3.6426445701433607, + "tokens_seen": 1019084800 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003491273821464393, + "loss": 2.5936, + "theoretical_loss": 3.6426223556928, + "tokens_seen": 1019150336 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034911735205616853, + "loss": 2.6019, + "theoretical_loss": 3.6426001430706316, + "tokens_seen": 1019215872 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034910732196589766, + "loss": 2.7685, + "theoretical_loss": 3.642577932276587, + "tokens_seen": 1019281408 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003490972918756269, + "loss": 2.5364, + "theoretical_loss": 3.642555723310399, + "tokens_seen": 1019346944 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034908726178535613, + "loss": 2.5531, + "theoretical_loss": 3.642533516171799, + "tokens_seen": 1019412480 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034907723169508526, + "loss": 2.8211, + "theoretical_loss": 3.6425113108605194, + "tokens_seen": 1019478016 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003490672016048145, + "loss": 2.6898, + "theoretical_loss": 3.642489107376292, + "tokens_seen": 1019543552 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003490571715145436, + "loss": 2.8711, + "theoretical_loss": 3.6424669057188495, + "tokens_seen": 1019609088 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034904714142427285, + "loss": 2.8004, + "theoretical_loss": 3.6424447058879243, + "tokens_seen": 1019674624 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034903711133400203, + "loss": 2.6168, + "theoretical_loss": 3.642422507883248, + "tokens_seen": 1019740160 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003490270812437312, + "loss": 2.657, + "theoretical_loss": 3.642400311704554, + "tokens_seen": 1019805696 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003490170511534604, + "loss": 2.8466, + "theoretical_loss": 3.6423781173515746, + "tokens_seen": 1019871232 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003490070210631896, + "loss": 2.7977, + "theoretical_loss": 3.6423559248240416, + "tokens_seen": 1019936768 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034899699097291876, + "loss": 2.612, + "theoretical_loss": 3.6423337341216877, + "tokens_seen": 1020002304 + }, + { + "epoch": 0.31, + "learning_rate": 0.000348986960882648, + "loss": 2.7051, + "theoretical_loss": 3.6423115452442465, + "tokens_seen": 1020067840 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003489769307923771, + "loss": 2.8261, + "theoretical_loss": 3.6422893581914497, + "tokens_seen": 1020133376 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034896690070210636, + "loss": 2.6246, + "theoretical_loss": 3.64226717296303, + "tokens_seen": 1020198912 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003489568706118355, + "loss": 2.7986, + "theoretical_loss": 3.642244989558721, + "tokens_seen": 1020264448 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003489468405215647, + "loss": 2.7107, + "theoretical_loss": 3.6422228079782553, + "tokens_seen": 1020329984 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003489368104312939, + "loss": 2.576, + "theoretical_loss": 3.6422006282213655, + "tokens_seen": 1020395520 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003489267803410231, + "loss": 2.6551, + "theoretical_loss": 3.642178450287785, + "tokens_seen": 1020461056 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034891675025075226, + "loss": 3.014, + "theoretical_loss": 3.642156274177246, + "tokens_seen": 1020526592 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003489067201604815, + "loss": 2.7514, + "theoretical_loss": 3.642134099889483, + "tokens_seen": 1020592128 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003488966900702106, + "loss": 2.6577, + "theoretical_loss": 3.642111927424228, + "tokens_seen": 1020657664 + }, + { + "epoch": 0.31, + "objective/train/docs_used": 584789, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.098395586013794, + "objective/train/theoretical_loss": 3.642089756781214, + "objective/train/tokens_used": 1041183200, + "theoretical_loss": 3.642089756781214, + "tokens_seen": 1020723200 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034888665997993986, + "loss": 2.8457, + "theoretical_loss": 3.642089756781214, + "tokens_seen": 1020723200 + }, + { + "epoch": 0.31, + "learning_rate": 0.000348876629889669, + "loss": 2.8021, + "theoretical_loss": 3.642067587960175, + "tokens_seen": 1020788736 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003488665997993982, + "loss": 2.8013, + "theoretical_loss": 3.6420454209608444, + "tokens_seen": 1020854272 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003488565697091274, + "loss": 2.7521, + "theoretical_loss": 3.6420232557829553, + "tokens_seen": 1020919808 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003488465396188566, + "loss": 2.8693, + "theoretical_loss": 3.6420010924262405, + "tokens_seen": 1020985344 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034883650952858577, + "loss": 2.8671, + "theoretical_loss": 3.6419789308904345, + "tokens_seen": 1021050880 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034882647943831495, + "loss": 2.8414, + "theoretical_loss": 3.6419567711752703, + "tokens_seen": 1021116416 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034881644934804413, + "loss": 2.7493, + "theoretical_loss": 3.6419346132804815, + "tokens_seen": 1021181952 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034880641925777336, + "loss": 2.8599, + "theoretical_loss": 3.641912457205802, + "tokens_seen": 1021247488 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003487963891675025, + "loss": 2.6261, + "theoretical_loss": 3.6418903029509653, + "tokens_seen": 1021313024 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003487863590772317, + "loss": 2.6819, + "theoretical_loss": 3.6418681505157053, + "tokens_seen": 1021378560 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034877632898696085, + "loss": 2.7968, + "theoretical_loss": 3.641845999899755, + "tokens_seen": 1021444096 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003487662988966901, + "loss": 2.5944, + "theoretical_loss": 3.64182385110285, + "tokens_seen": 1021509632 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034875626880641927, + "loss": 2.6248, + "theoretical_loss": 3.6418017041247226, + "tokens_seen": 1021575168 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034874623871614845, + "loss": 2.7073, + "theoretical_loss": 3.6417795589651076, + "tokens_seen": 1021640704 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034873620862587763, + "loss": 2.8114, + "theoretical_loss": 3.6417574156237387, + "tokens_seen": 1021706240 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034872617853560687, + "loss": 2.8429, + "theoretical_loss": 3.64173527410035, + "tokens_seen": 1021771776 + }, + { + "epoch": 0.31, + "learning_rate": 0.000348716148445336, + "loss": 2.4108, + "theoretical_loss": 3.641713134394676, + "tokens_seen": 1021837312 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034870611835506523, + "loss": 2.7061, + "theoretical_loss": 3.6416909965064512, + "tokens_seen": 1021902848 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034869608826479436, + "loss": 2.9169, + "theoretical_loss": 3.6416688604354084, + "tokens_seen": 1021968384 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003486860581745236, + "loss": 2.5726, + "theoretical_loss": 3.6416467261812837, + "tokens_seen": 1022033920 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034867602808425277, + "loss": 2.6644, + "theoretical_loss": 3.64162459374381, + "tokens_seen": 1022099456 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034866599799398195, + "loss": 2.4884, + "theoretical_loss": 3.641602463122723, + "tokens_seen": 1022164992 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034865596790371113, + "loss": 2.9141, + "theoretical_loss": 3.641580334317756, + "tokens_seen": 1022230528 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003486459378134403, + "loss": 2.8229, + "theoretical_loss": 3.6415582073286448, + "tokens_seen": 1022296064 + }, + { + "epoch": 0.31, + "objective/train/docs_used": 586155, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.2513537406921387, + "objective/train/theoretical_loss": 3.6415360821551226, + "objective/train/tokens_used": 1042821600, + "theoretical_loss": 3.6415360821551226, + "tokens_seen": 1022361600 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003486359077231695, + "loss": 2.6438, + "theoretical_loss": 3.6415360821551226, + "tokens_seen": 1022361600 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034862587763289873, + "loss": 2.418, + "theoretical_loss": 3.641513958796925, + "tokens_seen": 1022427136 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034861584754262786, + "loss": 2.7724, + "theoretical_loss": 3.6414918372537866, + "tokens_seen": 1022492672 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003486058174523571, + "loss": 2.6699, + "theoretical_loss": 3.6414697175254425, + "tokens_seen": 1022558208 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003485957873620862, + "loss": 2.7834, + "theoretical_loss": 3.6414475996116265, + "tokens_seen": 1022623744 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034858575727181546, + "loss": 2.8153, + "theoretical_loss": 3.641425483512074, + "tokens_seen": 1022689280 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034857572718154464, + "loss": 2.7645, + "theoretical_loss": 3.6414033692265204, + "tokens_seen": 1022754816 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003485656970912738, + "loss": 2.8039, + "theoretical_loss": 3.6413812567547, + "tokens_seen": 1022820352 + }, + { + "epoch": 0.31, + "learning_rate": 0.000348555667001003, + "loss": 2.5724, + "theoretical_loss": 3.641359146096348, + "tokens_seen": 1022885888 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034854563691073223, + "loss": 2.5706, + "theoretical_loss": 3.6413370372512, + "tokens_seen": 1022951424 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034853560682046136, + "loss": 2.7568, + "theoretical_loss": 3.6413149302189907, + "tokens_seen": 1023016960 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003485255767301906, + "loss": 2.7516, + "theoretical_loss": 3.6412928249994554, + "tokens_seen": 1023082496 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003485155466399197, + "loss": 2.5073, + "theoretical_loss": 3.6412707215923295, + "tokens_seen": 1023148032 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034850551654964896, + "loss": 2.5328, + "theoretical_loss": 3.6412486199973477, + "tokens_seen": 1023213568 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034849548645937814, + "loss": 2.4615, + "theoretical_loss": 3.641226520214246, + "tokens_seen": 1023279104 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003484854563691073, + "loss": 2.6278, + "theoretical_loss": 3.64120442224276, + "tokens_seen": 1023344640 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003484754262788365, + "loss": 2.4062, + "theoretical_loss": 3.6411823260826246, + "tokens_seen": 1023410176 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003484653961885657, + "loss": 2.7154, + "theoretical_loss": 3.6411602317335756, + "tokens_seen": 1023475712 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034845536609829486, + "loss": 2.6828, + "theoretical_loss": 3.6411381391953492, + "tokens_seen": 1023541248 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003484453360080241, + "loss": 2.7837, + "theoretical_loss": 3.64111604846768, + "tokens_seen": 1023606784 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003484353059177532, + "loss": 2.8578, + "theoretical_loss": 3.6410939595503047, + "tokens_seen": 1023672320 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034842527582748246, + "loss": 2.9192, + "theoretical_loss": 3.641071872442958, + "tokens_seen": 1023737856 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034841524573721164, + "loss": 2.6856, + "theoretical_loss": 3.641049787145376, + "tokens_seen": 1023803392 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003484052156469408, + "loss": 2.577, + "theoretical_loss": 3.641027703657296, + "tokens_seen": 1023868928 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034839518555667, + "loss": 2.586, + "theoretical_loss": 3.6410056219784526, + "tokens_seen": 1023934464 + }, + { + "epoch": 0.31, + "objective/train/docs_used": 586672, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.3583452701568604, + "objective/train/theoretical_loss": 3.6409835421085814, + "objective/train/tokens_used": 1044460000, + "theoretical_loss": 3.6409835421085814, + "tokens_seen": 1024000000 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003483851554663992, + "loss": 2.6136, + "theoretical_loss": 3.6409835421085814, + "tokens_seen": 1024000000 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034837512537612837, + "loss": 2.7439, + "theoretical_loss": 3.640961464047419, + "tokens_seen": 1024065536 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003483650952858576, + "loss": 2.7051, + "theoretical_loss": 3.640939387794702, + "tokens_seen": 1024131072 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034835506519558673, + "loss": 2.6484, + "theoretical_loss": 3.640917313350166, + "tokens_seen": 1024196608 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034834503510531597, + "loss": 2.5977, + "theoretical_loss": 3.6408952407135473, + "tokens_seen": 1024262144 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034833500501504515, + "loss": 2.527, + "theoretical_loss": 3.640873169884582, + "tokens_seen": 1024327680 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034832497492477433, + "loss": 2.5782, + "theoretical_loss": 3.6408511008630073, + "tokens_seen": 1024393216 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034831494483450356, + "loss": 2.6198, + "theoretical_loss": 3.6408290336485583, + "tokens_seen": 1024458752 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003483049147442327, + "loss": 2.6609, + "theoretical_loss": 3.6408069682409723, + "tokens_seen": 1024524288 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003482948846539619, + "loss": 2.6009, + "theoretical_loss": 3.640784904639985, + "tokens_seen": 1024589824 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034828485456369105, + "loss": 2.7204, + "theoretical_loss": 3.640762842845334, + "tokens_seen": 1024655360 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003482748244734203, + "loss": 2.735, + "theoretical_loss": 3.6407407828567555, + "tokens_seen": 1024720896 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034826479438314947, + "loss": 2.7663, + "theoretical_loss": 3.6407187246739854, + "tokens_seen": 1024786432 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034825476429287865, + "loss": 2.938, + "theoretical_loss": 3.640696668296761, + "tokens_seen": 1024851968 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034824473420260783, + "loss": 2.6089, + "theoretical_loss": 3.6406746137248196, + "tokens_seen": 1024917504 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034823470411233707, + "loss": 2.4941, + "theoretical_loss": 3.6406525609578972, + "tokens_seen": 1024983040 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003482246740220662, + "loss": 2.7548, + "theoretical_loss": 3.6406305099957312, + "tokens_seen": 1025048576 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034821464393179543, + "loss": 2.729, + "theoretical_loss": 3.6406084608380582, + "tokens_seen": 1025114112 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034820461384152456, + "loss": 2.9915, + "theoretical_loss": 3.6405864134846153, + "tokens_seen": 1025179648 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003481945837512538, + "loss": 2.6314, + "theoretical_loss": 3.6405643679351396, + "tokens_seen": 1025245184 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034818455366098297, + "loss": 2.71, + "theoretical_loss": 3.6405423241893677, + "tokens_seen": 1025310720 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034817452357071215, + "loss": 2.8017, + "theoretical_loss": 3.6405202822470377, + "tokens_seen": 1025376256 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034816449348044133, + "loss": 2.7367, + "theoretical_loss": 3.6404982421078858, + "tokens_seen": 1025441792 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003481544633901705, + "loss": 2.506, + "theoretical_loss": 3.6404762037716503, + "tokens_seen": 1025507328 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003481444332998997, + "loss": 2.765, + "theoretical_loss": 3.640454167238068, + "tokens_seen": 1025572864 + }, + { + "epoch": 0.31, + "objective/train/docs_used": 587795, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7853541374206543, + "objective/train/theoretical_loss": 3.6404321325068754, + "objective/train/tokens_used": 1046098400, + "theoretical_loss": 3.6404321325068754, + "tokens_seen": 1025638400 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034813440320962893, + "loss": 2.5706, + "theoretical_loss": 3.6404321325068754, + "tokens_seen": 1025638400 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034812437311935806, + "loss": 2.7371, + "theoretical_loss": 3.6404100995778115, + "tokens_seen": 1025703936 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003481143430290873, + "loss": 2.795, + "theoretical_loss": 3.640388068450613, + "tokens_seen": 1025769472 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003481043129388164, + "loss": 2.7781, + "theoretical_loss": 3.640366039125017, + "tokens_seen": 1025835008 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034809428284854566, + "loss": 2.6449, + "theoretical_loss": 3.640344011600762, + "tokens_seen": 1025900544 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034808425275827484, + "loss": 2.663, + "theoretical_loss": 3.640321985877585, + "tokens_seen": 1025966080 + }, + { + "epoch": 0.31, + "learning_rate": 0.000348074222668004, + "loss": 2.5169, + "theoretical_loss": 3.6402999619552245, + "tokens_seen": 1026031616 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003480641925777332, + "loss": 2.6988, + "theoretical_loss": 3.640277939833417, + "tokens_seen": 1026097152 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034805416248746243, + "loss": 2.8381, + "theoretical_loss": 3.640255919511901, + "tokens_seen": 1026162688 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034804413239719156, + "loss": 2.6385, + "theoretical_loss": 3.640233900990415, + "tokens_seen": 1026228224 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003480341023069208, + "loss": 2.5172, + "theoretical_loss": 3.6402118842686955, + "tokens_seen": 1026293760 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003480240722166499, + "loss": 2.8238, + "theoretical_loss": 3.6401898693464814, + "tokens_seen": 1026359296 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034801404212637916, + "loss": 2.5891, + "theoretical_loss": 3.6401678562235107, + "tokens_seen": 1026424832 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034800401203610834, + "loss": 2.7217, + "theoretical_loss": 3.6401458448995214, + "tokens_seen": 1026490368 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003479939819458375, + "loss": 2.6334, + "theoretical_loss": 3.6401238353742515, + "tokens_seen": 1026555904 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003479839518555667, + "loss": 2.8416, + "theoretical_loss": 3.6401018276474395, + "tokens_seen": 1026621440 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003479739217652959, + "loss": 2.7436, + "theoretical_loss": 3.6400798217188233, + "tokens_seen": 1026686976 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034796389167502506, + "loss": 2.7709, + "theoretical_loss": 3.6400578175881417, + "tokens_seen": 1026752512 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003479538615847543, + "loss": 2.8797, + "theoretical_loss": 3.640035815255132, + "tokens_seen": 1026818048 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034794383149448343, + "loss": 2.6561, + "theoretical_loss": 3.640013814719534, + "tokens_seen": 1026883584 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034793380140421266, + "loss": 2.6316, + "theoretical_loss": 3.6399918159810847, + "tokens_seen": 1026949120 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034792377131394184, + "loss": 2.5455, + "theoretical_loss": 3.639969819039524, + "tokens_seen": 1027014656 + }, + { + "epoch": 0.31, + "learning_rate": 0.000347913741223671, + "loss": 2.6521, + "theoretical_loss": 3.63994782389459, + "tokens_seen": 1027080192 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003479037111334002, + "loss": 2.5803, + "theoretical_loss": 3.639925830546021, + "tokens_seen": 1027145728 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003478936810431294, + "loss": 2.7146, + "theoretical_loss": 3.639903838993556, + "tokens_seen": 1027211264 + }, + { + "epoch": 0.31, + "objective/train/docs_used": 588595, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.0338401794433594, + "objective/train/theoretical_loss": 3.6398818492369336, + "objective/train/tokens_used": 1047736800, + "theoretical_loss": 3.6398818492369336, + "tokens_seen": 1027276800 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034788365095285857, + "loss": 2.9332, + "theoretical_loss": 3.6398818492369336, + "tokens_seen": 1027276800 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003478736208625878, + "loss": 2.6021, + "theoretical_loss": 3.639859861275893, + "tokens_seen": 1027342336 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034786359077231693, + "loss": 2.5805, + "theoretical_loss": 3.639837875110172, + "tokens_seen": 1027407872 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034785356068204617, + "loss": 2.6794, + "theoretical_loss": 3.6398158907395106, + "tokens_seen": 1027473408 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003478435305917753, + "loss": 2.8133, + "theoretical_loss": 3.6397939081636475, + "tokens_seen": 1027538944 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034783350050150453, + "loss": 2.7606, + "theoretical_loss": 3.6397719273823212, + "tokens_seen": 1027604480 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003478234704112337, + "loss": 2.805, + "theoretical_loss": 3.6397499483952718, + "tokens_seen": 1027670016 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003478134403209629, + "loss": 2.7671, + "theoretical_loss": 3.639727971202237, + "tokens_seen": 1027735552 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034780341023069207, + "loss": 2.6075, + "theoretical_loss": 3.6397059958029576, + "tokens_seen": 1027801088 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034779338014042125, + "loss": 2.6638, + "theoretical_loss": 3.639684022197171, + "tokens_seen": 1027866624 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034778335005015043, + "loss": 2.7425, + "theoretical_loss": 3.639662050384618, + "tokens_seen": 1027932160 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034777331995987967, + "loss": 2.9488, + "theoretical_loss": 3.6396400803650377, + "tokens_seen": 1027997696 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003477632898696088, + "loss": 2.8707, + "theoretical_loss": 3.639618112138169, + "tokens_seen": 1028063232 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034775325977933803, + "loss": 2.9239, + "theoretical_loss": 3.639596145703752, + "tokens_seen": 1028128768 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003477432296890672, + "loss": 2.7521, + "theoretical_loss": 3.6395741810615254, + "tokens_seen": 1028194304 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003477331995987964, + "loss": 2.6803, + "theoretical_loss": 3.6395522182112288, + "tokens_seen": 1028259840 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003477231695085256, + "loss": 2.6393, + "theoretical_loss": 3.6395302571526025, + "tokens_seen": 1028325376 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034771313941825476, + "loss": 2.5703, + "theoretical_loss": 3.639508297885386, + "tokens_seen": 1028390912 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034770310932798394, + "loss": 2.5074, + "theoretical_loss": 3.6394863404093183, + "tokens_seen": 1028456448 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034769307923771317, + "loss": 2.8029, + "theoretical_loss": 3.6394643847241404, + "tokens_seen": 1028521984 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003476830491474423, + "loss": 2.8572, + "theoretical_loss": 3.6394424308295914, + "tokens_seen": 1028587520 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034767301905717153, + "loss": 2.986, + "theoretical_loss": 3.6394204787254107, + "tokens_seen": 1028653056 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034766298896690066, + "loss": 2.8997, + "theoretical_loss": 3.6393985284113395, + "tokens_seen": 1028718592 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003476529588766299, + "loss": 2.6233, + "theoretical_loss": 3.6393765798871165, + "tokens_seen": 1028784128 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003476429287863591, + "loss": 2.683, + "theoretical_loss": 3.6393546331524824, + "tokens_seen": 1028849664 + }, + { + "epoch": 0.31, + "objective/train/docs_used": 589927, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.343308448791504, + "objective/train/theoretical_loss": 3.639332688207178, + "objective/train/tokens_used": 1049375200, + "theoretical_loss": 3.639332688207178, + "tokens_seen": 1028915200 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034763289869608826, + "loss": 2.6073, + "theoretical_loss": 3.639332688207178, + "tokens_seen": 1028915200 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034762286860581744, + "loss": 2.6045, + "theoretical_loss": 3.639310745050942, + "tokens_seen": 1028980736 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003476128385155466, + "loss": 2.8443, + "theoretical_loss": 3.639288803683515, + "tokens_seen": 1029046272 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003476028084252758, + "loss": 2.6795, + "theoretical_loss": 3.639266864104638, + "tokens_seen": 1029111808 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034759277833500504, + "loss": 2.7822, + "theoretical_loss": 3.639244926314051, + "tokens_seen": 1029177344 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003475827482447342, + "loss": 2.6223, + "theoretical_loss": 3.6392229903114943, + "tokens_seen": 1029242880 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003475727181544634, + "loss": 2.5421, + "theoretical_loss": 3.6392010560967085, + "tokens_seen": 1029308416 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034756268806419263, + "loss": 2.7771, + "theoretical_loss": 3.6391791236694333, + "tokens_seen": 1029373952 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034755265797392176, + "loss": 2.75, + "theoretical_loss": 3.63915719302941, + "tokens_seen": 1029439488 + }, + { + "epoch": 0.31, + "learning_rate": 0.000347542627883651, + "loss": 2.7818, + "theoretical_loss": 3.6391352641763794, + "tokens_seen": 1029505024 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003475325977933801, + "loss": 2.5195, + "theoretical_loss": 3.6391133371100817, + "tokens_seen": 1029570560 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034752256770310936, + "loss": 2.7429, + "theoretical_loss": 3.6390914118302575, + "tokens_seen": 1029636096 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034751253761283854, + "loss": 2.6975, + "theoretical_loss": 3.639069488336648, + "tokens_seen": 1029701632 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003475025075225677, + "loss": 2.7511, + "theoretical_loss": 3.6390475666289936, + "tokens_seen": 1029767168 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003474924774322969, + "loss": 2.6733, + "theoretical_loss": 3.639025646707035, + "tokens_seen": 1029832704 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003474824473420261, + "loss": 2.9669, + "theoretical_loss": 3.639003728570514, + "tokens_seen": 1029898240 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034747241725175527, + "loss": 2.5577, + "theoretical_loss": 3.638981812219171, + "tokens_seen": 1029963776 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003474623871614845, + "loss": 2.5564, + "theoretical_loss": 3.6389598976527466, + "tokens_seen": 1030029312 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034745235707121363, + "loss": 2.5409, + "theoretical_loss": 3.6389379848709824, + "tokens_seen": 1030094848 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034744232698094286, + "loss": 2.6629, + "theoretical_loss": 3.6389160738736193, + "tokens_seen": 1030160384 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034743229689067204, + "loss": 2.6851, + "theoretical_loss": 3.638894164660399, + "tokens_seen": 1030225920 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003474222668004012, + "loss": 2.8912, + "theoretical_loss": 3.6388722572310623, + "tokens_seen": 1030291456 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003474122367101304, + "loss": 2.8359, + "theoretical_loss": 3.6388503515853503, + "tokens_seen": 1030356992 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003474022066198596, + "loss": 2.6322, + "theoretical_loss": 3.6388284477230046, + "tokens_seen": 1030422528 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034739217652958877, + "loss": 2.8248, + "theoretical_loss": 3.6388065456437673, + "tokens_seen": 1030488064 + }, + { + "epoch": 0.31, + "objective/train/docs_used": 590720, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.665179967880249, + "objective/train/theoretical_loss": 3.6387846453473784, + "objective/train/tokens_used": 1051013600, + "theoretical_loss": 3.6387846453473784, + "tokens_seen": 1030553600 + }, + { + "epoch": 0.31, + "learning_rate": 0.000347382146439318, + "loss": 2.6953, + "theoretical_loss": 3.6387846453473784, + "tokens_seen": 1030553600 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034737211634904713, + "loss": 2.8616, + "theoretical_loss": 3.6387627468335806, + "tokens_seen": 1030619136 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034736208625877637, + "loss": 2.7247, + "theoretical_loss": 3.638740850102115, + "tokens_seen": 1030684672 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003473520561685055, + "loss": 2.6063, + "theoretical_loss": 3.6387189551527226, + "tokens_seen": 1030750208 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034734202607823473, + "loss": 2.8228, + "theoretical_loss": 3.6386970619851464, + "tokens_seen": 1030815744 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003473319959879639, + "loss": 2.6144, + "theoretical_loss": 3.6386751705991274, + "tokens_seen": 1030881280 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003473219658976931, + "loss": 2.5845, + "theoretical_loss": 3.6386532809944074, + "tokens_seen": 1030946816 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034731193580742227, + "loss": 2.6959, + "theoretical_loss": 3.638631393170728, + "tokens_seen": 1031012352 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034730190571715145, + "loss": 2.6184, + "theoretical_loss": 3.638609507127832, + "tokens_seen": 1031077888 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034729187562688063, + "loss": 2.6805, + "theoretical_loss": 3.6385876228654603, + "tokens_seen": 1031143424 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034728184553660987, + "loss": 2.6608, + "theoretical_loss": 3.6385657403833553, + "tokens_seen": 1031208960 + }, + { + "epoch": 0.31, + "learning_rate": 0.000347271815446339, + "loss": 2.7898, + "theoretical_loss": 3.6385438596812594, + "tokens_seen": 1031274496 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034726178535606823, + "loss": 2.444, + "theoretical_loss": 3.638521980758914, + "tokens_seen": 1031340032 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003472517552657974, + "loss": 2.6381, + "theoretical_loss": 3.638500103616062, + "tokens_seen": 1031405568 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003472417251755266, + "loss": 2.7054, + "theoretical_loss": 3.638478228252445, + "tokens_seen": 1031471104 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003472316950852558, + "loss": 2.8486, + "theoretical_loss": 3.6384563546678055, + "tokens_seen": 1031536640 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034722166499498496, + "loss": 2.7188, + "theoretical_loss": 3.6384344828618858, + "tokens_seen": 1031602176 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034721163490471414, + "loss": 2.6648, + "theoretical_loss": 3.6384126128344283, + "tokens_seen": 1031667712 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034720160481444337, + "loss": 2.7066, + "theoretical_loss": 3.6383907445851755, + "tokens_seen": 1031733248 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003471915747241725, + "loss": 2.8733, + "theoretical_loss": 3.6383688781138694, + "tokens_seen": 1031798784 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034718154463390173, + "loss": 2.6089, + "theoretical_loss": 3.638347013420254, + "tokens_seen": 1031864320 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034717151454363086, + "loss": 2.6149, + "theoretical_loss": 3.6383251505040697, + "tokens_seen": 1031929856 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003471614844533601, + "loss": 2.8473, + "theoretical_loss": 3.6383032893650604, + "tokens_seen": 1031995392 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003471514543630893, + "loss": 2.9024, + "theoretical_loss": 3.6382814300029693, + "tokens_seen": 1032060928 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034714142427281846, + "loss": 2.3506, + "theoretical_loss": 3.638259572417538, + "tokens_seen": 1032126464 + }, + { + "epoch": 0.31, + "objective/train/docs_used": 591991, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.758350372314453, + "objective/train/theoretical_loss": 3.6382377166085096, + "objective/train/tokens_used": 1052652000, + "theoretical_loss": 3.6382377166085096, + "tokens_seen": 1032192000 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034713139418254764, + "loss": 2.5972, + "theoretical_loss": 3.6382377166085096, + "tokens_seen": 1032192000 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003471213640922768, + "loss": 2.7929, + "theoretical_loss": 3.6382158625756276, + "tokens_seen": 1032257536 + }, + { + "epoch": 0.31, + "learning_rate": 0.000347111334002006, + "loss": 2.5425, + "theoretical_loss": 3.6381940103186343, + "tokens_seen": 1032323072 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034710130391173524, + "loss": 2.795, + "theoretical_loss": 3.638172159837273, + "tokens_seen": 1032388608 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034709127382146436, + "loss": 2.7402, + "theoretical_loss": 3.6381503111312865, + "tokens_seen": 1032454144 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003470812437311936, + "loss": 2.625, + "theoretical_loss": 3.6381284642004177, + "tokens_seen": 1032519680 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003470712136409228, + "loss": 2.7265, + "theoretical_loss": 3.63810661904441, + "tokens_seen": 1032585216 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034706118355065196, + "loss": 2.7078, + "theoretical_loss": 3.638084775663007, + "tokens_seen": 1032650752 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034705115346038114, + "loss": 2.5945, + "theoretical_loss": 3.6380629340559505, + "tokens_seen": 1032716288 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003470411233701103, + "loss": 2.8322, + "theoretical_loss": 3.6380410942229853, + "tokens_seen": 1032781824 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003470310932798395, + "loss": 2.8284, + "theoretical_loss": 3.6380192561638545, + "tokens_seen": 1032847360 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034702106318956874, + "loss": 2.6081, + "theoretical_loss": 3.6379974198783005, + "tokens_seen": 1032912896 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034701103309929787, + "loss": 2.6289, + "theoretical_loss": 3.6379755853660676, + "tokens_seen": 1032978432 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003470010030090271, + "loss": 2.7921, + "theoretical_loss": 3.637953752626899, + "tokens_seen": 1033043968 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034699097291875623, + "loss": 2.5529, + "theoretical_loss": 3.6379319216605386, + "tokens_seen": 1033109504 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034698094282848547, + "loss": 2.8529, + "theoretical_loss": 3.6379100924667296, + "tokens_seen": 1033175040 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034697091273821465, + "loss": 2.7692, + "theoretical_loss": 3.637888265045216, + "tokens_seen": 1033240576 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034696088264794383, + "loss": 2.7461, + "theoretical_loss": 3.637866439395741, + "tokens_seen": 1033306112 + }, + { + "epoch": 0.31, + "learning_rate": 0.000346950852557673, + "loss": 2.7241, + "theoretical_loss": 3.6378446155180484, + "tokens_seen": 1033371648 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034694082246740224, + "loss": 2.4661, + "theoretical_loss": 3.6378227934118827, + "tokens_seen": 1033437184 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034693079237713137, + "loss": 2.85, + "theoretical_loss": 3.6378009730769874, + "tokens_seen": 1033502720 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003469207622868606, + "loss": 2.8995, + "theoretical_loss": 3.6377791545131064, + "tokens_seen": 1033568256 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034691073219658973, + "loss": 2.9056, + "theoretical_loss": 3.6377573377199837, + "tokens_seen": 1033633792 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034690070210631897, + "loss": 2.7773, + "theoretical_loss": 3.6377355226973633, + "tokens_seen": 1033699328 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034689067201604815, + "loss": 2.4875, + "theoretical_loss": 3.6377137094449887, + "tokens_seen": 1033764864 + }, + { + "epoch": 0.31, + "objective/train/docs_used": 592462, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.698251962661743, + "objective/train/theoretical_loss": 3.6376918979626054, + "objective/train/tokens_used": 1054290400, + "theoretical_loss": 3.6376918979626054, + "tokens_seen": 1033830400 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034688064192577733, + "loss": 2.8148, + "theoretical_loss": 3.6376918979626054, + "tokens_seen": 1033830400 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003468706118355065, + "loss": 3.0153, + "theoretical_loss": 3.6376700882499566, + "tokens_seen": 1033895936 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003468605817452357, + "loss": 2.6659, + "theoretical_loss": 3.637648280306786, + "tokens_seen": 1033961472 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003468505516549649, + "loss": 2.6632, + "theoretical_loss": 3.6376264741328397, + "tokens_seen": 1034027008 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003468405215646941, + "loss": 2.6456, + "theoretical_loss": 3.6376046697278603, + "tokens_seen": 1034092544 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003468304914744233, + "loss": 2.6186, + "theoretical_loss": 3.6375828670915933, + "tokens_seen": 1034158080 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034682046138415247, + "loss": 2.6635, + "theoretical_loss": 3.6375610662237827, + "tokens_seen": 1034223616 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034681043129388165, + "loss": 2.8968, + "theoretical_loss": 3.637539267124173, + "tokens_seen": 1034289152 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034680040120361083, + "loss": 2.9101, + "theoretical_loss": 3.637517469792508, + "tokens_seen": 1034354688 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034679037111334007, + "loss": 2.6541, + "theoretical_loss": 3.6374956742285343, + "tokens_seen": 1034420224 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003467803410230692, + "loss": 2.7401, + "theoretical_loss": 3.637473880431995, + "tokens_seen": 1034485760 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034677031093279843, + "loss": 2.6351, + "theoretical_loss": 3.6374520884026347, + "tokens_seen": 1034551296 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003467602808425276, + "loss": 2.8485, + "theoretical_loss": 3.637430298140199, + "tokens_seen": 1034616832 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003467502507522568, + "loss": 2.6275, + "theoretical_loss": 3.6374085096444326, + "tokens_seen": 1034682368 + }, + { + "epoch": 0.31, + "learning_rate": 0.000346740220661986, + "loss": 2.8068, + "theoretical_loss": 3.63738672291508, + "tokens_seen": 1034747904 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034673019057171516, + "loss": 2.763, + "theoretical_loss": 3.6373649379518866, + "tokens_seen": 1034813440 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034672016048144434, + "loss": 2.7394, + "theoretical_loss": 3.637343154754597, + "tokens_seen": 1034878976 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034671013039117357, + "loss": 2.5258, + "theoretical_loss": 3.637321373322956, + "tokens_seen": 1034944512 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003467001003009027, + "loss": 2.603, + "theoretical_loss": 3.6372995936567096, + "tokens_seen": 1035010048 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034669007021063193, + "loss": 2.7538, + "theoretical_loss": 3.637277815755602, + "tokens_seen": 1035075584 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034668004012036106, + "loss": 2.9486, + "theoretical_loss": 3.6372560396193787, + "tokens_seen": 1035141120 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003466700100300903, + "loss": 2.7035, + "theoretical_loss": 3.6372342652477854, + "tokens_seen": 1035206656 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003466599799398195, + "loss": 2.7222, + "theoretical_loss": 3.637212492640567, + "tokens_seen": 1035272192 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034664994984954866, + "loss": 2.6645, + "theoretical_loss": 3.6371907217974684, + "tokens_seen": 1035337728 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034663991975927784, + "loss": 2.5969, + "theoretical_loss": 3.637168952718236, + "tokens_seen": 1035403264 + }, + { + "epoch": 0.31, + "objective/train/docs_used": 593828, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.932084083557129, + "objective/train/theoretical_loss": 3.6371471854026147, + "objective/train/tokens_used": 1055928800, + "theoretical_loss": 3.6371471854026147, + "tokens_seen": 1035468800 + }, + { + "epoch": 0.31, + "learning_rate": 0.000346629889669007, + "loss": 2.7516, + "theoretical_loss": 3.6371471854026147, + "tokens_seen": 1035468800 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003466198595787362, + "loss": 2.6325, + "theoretical_loss": 3.63712541985035, + "tokens_seen": 1035534336 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034660982948846544, + "loss": 2.8506, + "theoretical_loss": 3.6371036560611874, + "tokens_seen": 1035599872 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034659979939819456, + "loss": 2.596, + "theoretical_loss": 3.637081894034873, + "tokens_seen": 1035665408 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003465897693079238, + "loss": 2.8458, + "theoretical_loss": 3.6370601337711514, + "tokens_seen": 1035730944 + }, + { + "epoch": 0.31, + "learning_rate": 0.000346579739217653, + "loss": 2.4655, + "theoretical_loss": 3.63703837526977, + "tokens_seen": 1035796480 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034656970912738216, + "loss": 2.8439, + "theoretical_loss": 3.637016618530473, + "tokens_seen": 1035862016 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034655967903711134, + "loss": 2.8886, + "theoretical_loss": 3.6369948635530074, + "tokens_seen": 1035927552 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003465496489468405, + "loss": 3.0587, + "theoretical_loss": 3.6369731103371183, + "tokens_seen": 1035993088 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003465396188565697, + "loss": 2.6988, + "theoretical_loss": 3.636951358882552, + "tokens_seen": 1036058624 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034652958876629894, + "loss": 2.8252, + "theoretical_loss": 3.6369296091890537, + "tokens_seen": 1036124160 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034651955867602807, + "loss": 2.5462, + "theoretical_loss": 3.636907861256371, + "tokens_seen": 1036189696 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003465095285857573, + "loss": 2.6812, + "theoretical_loss": 3.6368861150842484, + "tokens_seen": 1036255232 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034649949849548643, + "loss": 2.7817, + "theoretical_loss": 3.6368643706724333, + "tokens_seen": 1036320768 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034648946840521567, + "loss": 2.5474, + "theoretical_loss": 3.636842628020671, + "tokens_seen": 1036386304 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034647943831494485, + "loss": 2.7485, + "theoretical_loss": 3.6368208871287084, + "tokens_seen": 1036451840 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034646940822467403, + "loss": 2.6356, + "theoretical_loss": 3.6367991479962916, + "tokens_seen": 1036517376 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003464593781344032, + "loss": 2.8492, + "theoretical_loss": 3.6367774106231665, + "tokens_seen": 1036582912 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034644934804413244, + "loss": 2.6997, + "theoretical_loss": 3.6367556750090797, + "tokens_seen": 1036648448 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034643931795386157, + "loss": 2.7899, + "theoretical_loss": 3.6367339411537785, + "tokens_seen": 1036713984 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003464292878635908, + "loss": 2.4874, + "theoretical_loss": 3.6367122090570088, + "tokens_seen": 1036779520 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034641925777331993, + "loss": 2.8197, + "theoretical_loss": 3.6366904787185166, + "tokens_seen": 1036845056 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034640922768304917, + "loss": 2.8723, + "theoretical_loss": 3.6366687501380492, + "tokens_seen": 1036910592 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034639919759277835, + "loss": 2.8535, + "theoretical_loss": 3.636647023315353, + "tokens_seen": 1036976128 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034638916750250753, + "loss": 2.6143, + "theoretical_loss": 3.6366252982501757, + "tokens_seen": 1037041664 + }, + { + "epoch": 0.31, + "objective/train/docs_used": 594513, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.8957436084747314, + "objective/train/theoretical_loss": 3.6366035749422627, + "objective/train/tokens_used": 1057567200, + "theoretical_loss": 3.6366035749422627, + "tokens_seen": 1037107200 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003463791374122367, + "loss": 2.6859, + "theoretical_loss": 3.6366035749422627, + "tokens_seen": 1037107200 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003463691073219659, + "loss": 2.6229, + "theoretical_loss": 3.6365818533913608, + "tokens_seen": 1037172736 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003463590772316951, + "loss": 2.8337, + "theoretical_loss": 3.6365601335972184, + "tokens_seen": 1037238272 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003463490471414243, + "loss": 2.5818, + "theoretical_loss": 3.636538415559581, + "tokens_seen": 1037303808 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034633901705115344, + "loss": 2.6391, + "theoretical_loss": 3.636516699278196, + "tokens_seen": 1037369344 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034632898696088267, + "loss": 2.9942, + "theoretical_loss": 3.6364949847528107, + "tokens_seen": 1037434880 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003463189568706118, + "loss": 2.5854, + "theoretical_loss": 3.636473271983172, + "tokens_seen": 1037500416 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034630892678034103, + "loss": 2.7216, + "theoretical_loss": 3.636451560969027, + "tokens_seen": 1037565952 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003462988966900702, + "loss": 2.6403, + "theoretical_loss": 3.636429851710123, + "tokens_seen": 1037631488 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003462888665997994, + "loss": 2.5831, + "theoretical_loss": 3.6364081442062073, + "tokens_seen": 1037697024 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003462788365095286, + "loss": 2.5644, + "theoretical_loss": 3.636386438457027, + "tokens_seen": 1037762560 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003462688064192578, + "loss": 2.5274, + "theoretical_loss": 3.6363647344623296, + "tokens_seen": 1037828096 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034625877632898694, + "loss": 2.7663, + "theoretical_loss": 3.636343032221862, + "tokens_seen": 1037893632 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003462487462387162, + "loss": 2.8208, + "theoretical_loss": 3.6363213317353726, + "tokens_seen": 1037959168 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003462387161484453, + "loss": 2.5846, + "theoretical_loss": 3.6362996330026087, + "tokens_seen": 1038024704 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034622868605817454, + "loss": 2.6524, + "theoretical_loss": 3.6362779360233173, + "tokens_seen": 1038090240 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003462186559679037, + "loss": 2.5714, + "theoretical_loss": 3.636256240797246, + "tokens_seen": 1038155776 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003462086258776329, + "loss": 2.524, + "theoretical_loss": 3.6362345473241433, + "tokens_seen": 1038221312 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003461985957873621, + "loss": 2.6278, + "theoretical_loss": 3.636212855603756, + "tokens_seen": 1038286848 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034618856569709126, + "loss": 2.6327, + "theoretical_loss": 3.6361911656358323, + "tokens_seen": 1038352384 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034617853560682044, + "loss": 2.8812, + "theoretical_loss": 3.63616947742012, + "tokens_seen": 1038417920 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003461685055165497, + "loss": 2.634, + "theoretical_loss": 3.6361477909563673, + "tokens_seen": 1038483456 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003461584754262788, + "loss": 2.8152, + "theoretical_loss": 3.6361261062443218, + "tokens_seen": 1038548992 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034614844533600804, + "loss": 2.7671, + "theoretical_loss": 3.636104423283731, + "tokens_seen": 1038614528 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034613841524573717, + "loss": 2.6844, + "theoretical_loss": 3.6360827420743442, + "tokens_seen": 1038680064 + }, + { + "epoch": 0.31, + "objective/train/docs_used": 595196, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.387634515762329, + "objective/train/theoretical_loss": 3.6360610626159087, + "objective/train/tokens_used": 1059205600, + "theoretical_loss": 3.6360610626159087, + "tokens_seen": 1038745600 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003461283851554664, + "loss": 2.8943, + "theoretical_loss": 3.6360610626159087, + "tokens_seen": 1038745600 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003461183550651956, + "loss": 2.7764, + "theoretical_loss": 3.6360393849081722, + "tokens_seen": 1038811136 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034610832497492476, + "loss": 2.583, + "theoretical_loss": 3.636017708950883, + "tokens_seen": 1038876672 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034609829488465395, + "loss": 2.6657, + "theoretical_loss": 3.6359960347437905, + "tokens_seen": 1038942208 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003460882647943832, + "loss": 2.6773, + "theoretical_loss": 3.6359743622866416, + "tokens_seen": 1039007744 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034607823470411236, + "loss": 2.5882, + "theoretical_loss": 3.6359526915791855, + "tokens_seen": 1039073280 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034606820461384154, + "loss": 2.7421, + "theoretical_loss": 3.63593102262117, + "tokens_seen": 1039138816 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003460581745235707, + "loss": 2.7581, + "theoretical_loss": 3.6359093554123447, + "tokens_seen": 1039204352 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003460481444332999, + "loss": 2.7209, + "theoretical_loss": 3.6358876899524564, + "tokens_seen": 1039269888 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034603811434302914, + "loss": 2.6575, + "theoretical_loss": 3.635866026241255, + "tokens_seen": 1039335424 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034602808425275827, + "loss": 2.7294, + "theoretical_loss": 3.6358443642784883, + "tokens_seen": 1039400960 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003460180541624875, + "loss": 2.71, + "theoretical_loss": 3.6358227040639055, + "tokens_seen": 1039466496 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034600802407221663, + "loss": 2.5774, + "theoretical_loss": 3.635801045597255, + "tokens_seen": 1039532032 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034599799398194587, + "loss": 2.6288, + "theoretical_loss": 3.6357793888782863, + "tokens_seen": 1039597568 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034598796389167505, + "loss": 2.7648, + "theoretical_loss": 3.635757733906747, + "tokens_seen": 1039663104 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034597793380140423, + "loss": 2.9598, + "theoretical_loss": 3.6357360806823866, + "tokens_seen": 1039728640 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003459679037111334, + "loss": 2.573, + "theoretical_loss": 3.635714429204954, + "tokens_seen": 1039794176 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034595787362086264, + "loss": 2.7907, + "theoretical_loss": 3.6356927794741978, + "tokens_seen": 1039859712 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034594784353059177, + "loss": 2.4407, + "theoretical_loss": 3.6356711314898678, + "tokens_seen": 1039925248 + }, + { + "epoch": 0.32, + "learning_rate": 0.000345937813440321, + "loss": 2.8387, + "theoretical_loss": 3.6356494852517125, + "tokens_seen": 1039990784 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034592778335005013, + "loss": 2.7633, + "theoretical_loss": 3.635627840759481, + "tokens_seen": 1040056320 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034591775325977937, + "loss": 2.5899, + "theoretical_loss": 3.6356061980129235, + "tokens_seen": 1040121856 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034590772316950855, + "loss": 2.7339, + "theoretical_loss": 3.6355845570117875, + "tokens_seen": 1040187392 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034589769307923773, + "loss": 2.7634, + "theoretical_loss": 3.6355629177558235, + "tokens_seen": 1040252928 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003458876629889669, + "loss": 2.6106, + "theoretical_loss": 3.63554128024478, + "tokens_seen": 1040318464 + }, + { + "epoch": 0.32, + "objective/train/docs_used": 596146, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.691300630569458, + "objective/train/theoretical_loss": 3.635519644478408, + "objective/train/tokens_used": 1060844000, + "theoretical_loss": 3.635519644478408, + "tokens_seen": 1040384000 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003458776328986961, + "loss": 2.6513, + "theoretical_loss": 3.635519644478408, + "tokens_seen": 1040384000 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003458676028084253, + "loss": 2.8975, + "theoretical_loss": 3.6354980104564545, + "tokens_seen": 1040449536 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003458575727181545, + "loss": 2.6274, + "theoretical_loss": 3.635476378178671, + "tokens_seen": 1040515072 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034584754262788364, + "loss": 2.8437, + "theoretical_loss": 3.6354547476448067, + "tokens_seen": 1040580608 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034583751253761287, + "loss": 2.8093, + "theoretical_loss": 3.63543311885461, + "tokens_seen": 1040646144 + }, + { + "epoch": 0.32, + "learning_rate": 0.000345827482447342, + "loss": 3.0119, + "theoretical_loss": 3.6354114918078317, + "tokens_seen": 1040711680 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034581745235707123, + "loss": 2.5918, + "theoretical_loss": 3.6353898665042212, + "tokens_seen": 1040777216 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003458074222668004, + "loss": 2.8139, + "theoretical_loss": 3.6353682429435286, + "tokens_seen": 1040842752 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003457973921765296, + "loss": 2.7758, + "theoretical_loss": 3.635346621125503, + "tokens_seen": 1040908288 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003457873620862588, + "loss": 2.6856, + "theoretical_loss": 3.6353250010498943, + "tokens_seen": 1040973824 + }, + { + "epoch": 0.32, + "learning_rate": 0.000345777331995988, + "loss": 2.8113, + "theoretical_loss": 3.635303382716453, + "tokens_seen": 1041039360 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034576730190571714, + "loss": 2.8213, + "theoretical_loss": 3.635281766124929, + "tokens_seen": 1041104896 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003457572718154464, + "loss": 2.9416, + "theoretical_loss": 3.6352601512750713, + "tokens_seen": 1041170432 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003457472417251755, + "loss": 2.5308, + "theoretical_loss": 3.6352385381666315, + "tokens_seen": 1041235968 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034573721163490474, + "loss": 2.7986, + "theoretical_loss": 3.635216926799358, + "tokens_seen": 1041301504 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003457271815446339, + "loss": 2.8505, + "theoretical_loss": 3.6351953171730025, + "tokens_seen": 1041367040 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003457171514543631, + "loss": 2.7238, + "theoretical_loss": 3.635173709287314, + "tokens_seen": 1041432576 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003457071213640923, + "loss": 2.8888, + "theoretical_loss": 3.635152103142044, + "tokens_seen": 1041498112 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034569709127382146, + "loss": 2.8266, + "theoretical_loss": 3.6351304987369417, + "tokens_seen": 1041563648 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034568706118355064, + "loss": 2.9319, + "theoretical_loss": 3.635108896071759, + "tokens_seen": 1041629184 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003456770310932799, + "loss": 2.8362, + "theoretical_loss": 3.635087295146244, + "tokens_seen": 1041694720 + }, + { + "epoch": 0.32, + "learning_rate": 0.000345667001003009, + "loss": 2.807, + "theoretical_loss": 3.6350656959601486, + "tokens_seen": 1041760256 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034565697091273824, + "loss": 2.2996, + "theoretical_loss": 3.635044098513223, + "tokens_seen": 1041825792 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034564694082246737, + "loss": 2.7532, + "theoretical_loss": 3.6350225028052185, + "tokens_seen": 1041891328 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003456369107321966, + "loss": 2.5693, + "theoretical_loss": 3.6350009088358846, + "tokens_seen": 1041956864 + }, + { + "epoch": 0.32, + "objective/train/docs_used": 596810, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7167646884918213, + "objective/train/theoretical_loss": 3.634979316604973, + "objective/train/tokens_used": 1062482400, + "theoretical_loss": 3.634979316604973, + "tokens_seen": 1042022400 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003456268806419258, + "loss": 2.5767, + "theoretical_loss": 3.634979316604973, + "tokens_seen": 1042022400 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034561685055165496, + "loss": 2.6354, + "theoretical_loss": 3.6349577261122334, + "tokens_seen": 1042087936 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034560682046138415, + "loss": 2.8368, + "theoretical_loss": 3.6349361373574176, + "tokens_seen": 1042153472 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003455967903711134, + "loss": 2.8294, + "theoretical_loss": 3.6349145503402758, + "tokens_seen": 1042219008 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003455867602808425, + "loss": 2.646, + "theoretical_loss": 3.6348929650605584, + "tokens_seen": 1042284544 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034557673019057174, + "loss": 2.6774, + "theoretical_loss": 3.6348713815180176, + "tokens_seen": 1042350080 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034556670010030087, + "loss": 2.7323, + "theoretical_loss": 3.634849799712404, + "tokens_seen": 1042415616 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003455566700100301, + "loss": 2.8602, + "theoretical_loss": 3.6348282196434676, + "tokens_seen": 1042481152 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003455466399197593, + "loss": 2.613, + "theoretical_loss": 3.63480664131096, + "tokens_seen": 1042546688 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034553660982948847, + "loss": 2.6413, + "theoretical_loss": 3.6347850647146336, + "tokens_seen": 1042612224 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034552657973921765, + "loss": 2.646, + "theoretical_loss": 3.6347634898542385, + "tokens_seen": 1042677760 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034551654964894683, + "loss": 2.8543, + "theoretical_loss": 3.634741916729525, + "tokens_seen": 1042743296 + }, + { + "epoch": 0.32, + "learning_rate": 0.000345506519558676, + "loss": 2.8361, + "theoretical_loss": 3.6347203453402464, + "tokens_seen": 1042808832 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034549648946840525, + "loss": 2.6943, + "theoretical_loss": 3.634698775686153, + "tokens_seen": 1042874368 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003454864593781344, + "loss": 2.7488, + "theoretical_loss": 3.6346772077669955, + "tokens_seen": 1042939904 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003454764292878636, + "loss": 2.586, + "theoretical_loss": 3.634655641582527, + "tokens_seen": 1043005440 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034546639919759274, + "loss": 2.8642, + "theoretical_loss": 3.6346340771324974, + "tokens_seen": 1043070976 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034545636910732197, + "loss": 2.6401, + "theoretical_loss": 3.6346125144166592, + "tokens_seen": 1043136512 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034544633901705115, + "loss": 2.4307, + "theoretical_loss": 3.634590953434764, + "tokens_seen": 1043202048 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034543630892678033, + "loss": 2.6713, + "theoretical_loss": 3.6345693941865624, + "tokens_seen": 1043267584 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003454262788365095, + "loss": 2.7338, + "theoretical_loss": 3.634547836671808, + "tokens_seen": 1043333120 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034541624874623875, + "loss": 2.7467, + "theoretical_loss": 3.6345262808902508, + "tokens_seen": 1043398656 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003454062186559679, + "loss": 2.7464, + "theoretical_loss": 3.6345047268416435, + "tokens_seen": 1043464192 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003453961885656971, + "loss": 2.5193, + "theoretical_loss": 3.6344831745257373, + "tokens_seen": 1043529728 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034538615847542624, + "loss": 2.7831, + "theoretical_loss": 3.6344616239422844, + "tokens_seen": 1043595264 + }, + { + "epoch": 0.32, + "objective/train/docs_used": 598197, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.5348751544952393, + "objective/train/theoretical_loss": 3.634440075091037, + "objective/train/tokens_used": 1064120800, + "theoretical_loss": 3.634440075091037, + "tokens_seen": 1043660800 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003453761283851555, + "loss": 2.5298, + "theoretical_loss": 3.634440075091037, + "tokens_seen": 1043660800 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034536609829488466, + "loss": 2.6719, + "theoretical_loss": 3.6344185279717474, + "tokens_seen": 1043726336 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034535606820461384, + "loss": 2.6142, + "theoretical_loss": 3.634396982584167, + "tokens_seen": 1043791872 + }, + { + "epoch": 0.32, + "learning_rate": 0.000345346038114343, + "loss": 2.8735, + "theoretical_loss": 3.634375438928048, + "tokens_seen": 1043857408 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003453360080240722, + "loss": 2.7239, + "theoretical_loss": 3.6343538970031433, + "tokens_seen": 1043922944 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034532597793380143, + "loss": 2.5252, + "theoretical_loss": 3.634332356809204, + "tokens_seen": 1043988480 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003453159478435306, + "loss": 2.9766, + "theoretical_loss": 3.6343108183459827, + "tokens_seen": 1044054016 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003453059177532598, + "loss": 2.5889, + "theoretical_loss": 3.634289281613232, + "tokens_seen": 1044119552 + }, + { + "epoch": 0.32, + "learning_rate": 0.000345295887662989, + "loss": 2.8217, + "theoretical_loss": 3.6342677466107043, + "tokens_seen": 1044185088 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003452858575727182, + "loss": 2.8705, + "theoretical_loss": 3.634246213338152, + "tokens_seen": 1044250624 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034527582748244734, + "loss": 2.608, + "theoretical_loss": 3.6342246817953274, + "tokens_seen": 1044316160 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003452657973921766, + "loss": 2.58, + "theoretical_loss": 3.634203151981983, + "tokens_seen": 1044381696 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003452557673019057, + "loss": 2.7392, + "theoretical_loss": 3.6341816238978715, + "tokens_seen": 1044447232 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034524573721163494, + "loss": 2.6955, + "theoretical_loss": 3.634160097542745, + "tokens_seen": 1044512768 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003452357071213641, + "loss": 2.5541, + "theoretical_loss": 3.634138572916357, + "tokens_seen": 1044578304 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003452256770310933, + "loss": 2.6838, + "theoretical_loss": 3.6341170500184603, + "tokens_seen": 1044643840 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003452156469408225, + "loss": 2.7417, + "theoretical_loss": 3.6340955288488064, + "tokens_seen": 1044709376 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034520561685055166, + "loss": 2.5506, + "theoretical_loss": 3.6340740094071493, + "tokens_seen": 1044774912 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034519558676028084, + "loss": 2.5669, + "theoretical_loss": 3.6340524916932413, + "tokens_seen": 1044840448 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003451855566700101, + "loss": 2.4199, + "theoretical_loss": 3.6340309757068354, + "tokens_seen": 1044905984 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003451755265797392, + "loss": 2.6435, + "theoretical_loss": 3.634009461447685, + "tokens_seen": 1044971520 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034516549648946844, + "loss": 2.6944, + "theoretical_loss": 3.6339879489155433, + "tokens_seen": 1045037056 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034515546639919757, + "loss": 2.5304, + "theoretical_loss": 3.6339664381101624, + "tokens_seen": 1045102592 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003451454363089268, + "loss": 2.6684, + "theoretical_loss": 3.633944929031296, + "tokens_seen": 1045168128 + }, + { + "epoch": 0.32, + "learning_rate": 0.000345135406218656, + "loss": 2.6685, + "theoretical_loss": 3.6339234216786975, + "tokens_seen": 1045233664 + }, + { + "epoch": 0.32, + "objective/train/docs_used": 598880, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.0017547607421875, + "objective/train/theoretical_loss": 3.6339019160521198, + "objective/train/tokens_used": 1065759200, + "theoretical_loss": 3.6339019160521198, + "tokens_seen": 1045299200 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034512537612838517, + "loss": 2.6466, + "theoretical_loss": 3.6339019160521198, + "tokens_seen": 1045299200 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034511534603811435, + "loss": 2.4864, + "theoretical_loss": 3.633880412151316, + "tokens_seen": 1045364736 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003451053159478436, + "loss": 2.817, + "theoretical_loss": 3.6338589099760394, + "tokens_seen": 1045430272 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003450952858575727, + "loss": 2.8222, + "theoretical_loss": 3.633837409526044, + "tokens_seen": 1045495808 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034508525576730194, + "loss": 2.8463, + "theoretical_loss": 3.6338159108010832, + "tokens_seen": 1045561344 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034507522567703107, + "loss": 2.722, + "theoretical_loss": 3.63379441380091, + "tokens_seen": 1045626880 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003450651955867603, + "loss": 2.6809, + "theoretical_loss": 3.6337729185252776, + "tokens_seen": 1045692416 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003450551654964895, + "loss": 2.6905, + "theoretical_loss": 3.6337514249739407, + "tokens_seen": 1045757952 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034504513540621867, + "loss": 2.585, + "theoretical_loss": 3.633729933146652, + "tokens_seen": 1045823488 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034503510531594785, + "loss": 2.5853, + "theoretical_loss": 3.6337084430431656, + "tokens_seen": 1045889024 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034502507522567703, + "loss": 2.849, + "theoretical_loss": 3.633686954663235, + "tokens_seen": 1045954560 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003450150451354062, + "loss": 2.8259, + "theoretical_loss": 3.6336654680066145, + "tokens_seen": 1046020096 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034500501504513545, + "loss": 2.8492, + "theoretical_loss": 3.6336439830730574, + "tokens_seen": 1046085632 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003449949849548646, + "loss": 2.7853, + "theoretical_loss": 3.6336224998623177, + "tokens_seen": 1046151168 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003449849548645938, + "loss": 2.6331, + "theoretical_loss": 3.6336010183741494, + "tokens_seen": 1046216704 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034497492477432294, + "loss": 2.5808, + "theoretical_loss": 3.6335795386083065, + "tokens_seen": 1046282240 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034496489468405217, + "loss": 2.8715, + "theoretical_loss": 3.6335580605645434, + "tokens_seen": 1046347776 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034495486459378135, + "loss": 2.7676, + "theoretical_loss": 3.633536584242613, + "tokens_seen": 1046413312 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034494483450351053, + "loss": 2.8036, + "theoretical_loss": 3.6335151096422713, + "tokens_seen": 1046478848 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003449348044132397, + "loss": 2.7083, + "theoretical_loss": 3.6334936367632706, + "tokens_seen": 1046544384 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034492477432296895, + "loss": 2.8613, + "theoretical_loss": 3.633472165605366, + "tokens_seen": 1046609920 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003449147442326981, + "loss": 2.578, + "theoretical_loss": 3.633450696168312, + "tokens_seen": 1046675456 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003449047141424273, + "loss": 2.9851, + "theoretical_loss": 3.6334292284518632, + "tokens_seen": 1046740992 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034489468405215644, + "loss": 2.8185, + "theoretical_loss": 3.6334077624557732, + "tokens_seen": 1046806528 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003448846539618857, + "loss": 2.7573, + "theoretical_loss": 3.633386298179797, + "tokens_seen": 1046872064 + }, + { + "epoch": 0.32, + "objective/train/docs_used": 599993, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.5509517192840576, + "objective/train/theoretical_loss": 3.6333648356236883, + "objective/train/tokens_used": 1067397600, + "theoretical_loss": 3.6333648356236883, + "tokens_seen": 1046937600 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034487462387161486, + "loss": 2.6535, + "theoretical_loss": 3.6333648356236883, + "tokens_seen": 1046937600 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034486459378134404, + "loss": 2.8468, + "theoretical_loss": 3.6333433747872026, + "tokens_seen": 1047003136 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003448545636910732, + "loss": 2.5095, + "theoretical_loss": 3.633321915670094, + "tokens_seen": 1047068672 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003448445336008024, + "loss": 2.5423, + "theoretical_loss": 3.6333004582721173, + "tokens_seen": 1047134208 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003448345035105316, + "loss": 2.6071, + "theoretical_loss": 3.633279002593027, + "tokens_seen": 1047199744 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003448244734202608, + "loss": 2.6695, + "theoretical_loss": 3.6332575486325784, + "tokens_seen": 1047265280 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034481444332998994, + "loss": 2.4582, + "theoretical_loss": 3.6332360963905255, + "tokens_seen": 1047330816 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003448044132397192, + "loss": 2.8089, + "theoretical_loss": 3.633214645866624, + "tokens_seen": 1047396352 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034479438314944836, + "loss": 2.6347, + "theoretical_loss": 3.6331931970606277, + "tokens_seen": 1047461888 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034478435305917754, + "loss": 2.6302, + "theoretical_loss": 3.6331717499722926, + "tokens_seen": 1047527424 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003447743229689067, + "loss": 2.5898, + "theoretical_loss": 3.633150304601373, + "tokens_seen": 1047592960 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003447642928786359, + "loss": 2.7539, + "theoretical_loss": 3.6331288609476244, + "tokens_seen": 1047658496 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003447542627883651, + "loss": 2.4385, + "theoretical_loss": 3.633107419010802, + "tokens_seen": 1047724032 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003447442326980943, + "loss": 2.8129, + "theoretical_loss": 3.6330859787906604, + "tokens_seen": 1047789568 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034473420260782345, + "loss": 2.6833, + "theoretical_loss": 3.6330645402869552, + "tokens_seen": 1047855104 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003447241725175527, + "loss": 2.6778, + "theoretical_loss": 3.6330431034994417, + "tokens_seen": 1047920640 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003447141424272818, + "loss": 2.667, + "theoretical_loss": 3.6330216684278747, + "tokens_seen": 1047986176 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034470411233701104, + "loss": 2.8517, + "theoretical_loss": 3.63300023507201, + "tokens_seen": 1048051712 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003446940822467402, + "loss": 2.6234, + "theoretical_loss": 3.6329788034316026, + "tokens_seen": 1048117248 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003446840521564694, + "loss": 2.6275, + "theoretical_loss": 3.6329573735064087, + "tokens_seen": 1048182784 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003446740220661986, + "loss": 2.8087, + "theoretical_loss": 3.6329359452961834, + "tokens_seen": 1048248320 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034466399197592777, + "loss": 2.7613, + "theoretical_loss": 3.6329145188006815, + "tokens_seen": 1048313856 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034465396188565695, + "loss": 2.783, + "theoretical_loss": 3.6328930940196598, + "tokens_seen": 1048379392 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003446439317953862, + "loss": 2.7231, + "theoretical_loss": 3.6328716709528734, + "tokens_seen": 1048444928 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003446339017051153, + "loss": 2.9136, + "theoretical_loss": 3.632850249600078, + "tokens_seen": 1048510464 + }, + { + "epoch": 0.32, + "objective/train/docs_used": 601088, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7792372703552246, + "objective/train/theoretical_loss": 3.632828829961029, + "objective/train/tokens_used": 1069036000, + "theoretical_loss": 3.632828829961029, + "tokens_seen": 1048576000 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034462387161484455, + "loss": 2.7244, + "theoretical_loss": 3.632828829961029, + "tokens_seen": 1048576000 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034461384152457373, + "loss": 2.6533, + "theoretical_loss": 3.632807412035483, + "tokens_seen": 1048641536 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003446038114343029, + "loss": 2.9389, + "theoretical_loss": 3.6327859958231956, + "tokens_seen": 1048707072 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034459378134403214, + "loss": 2.5312, + "theoretical_loss": 3.632764581323922, + "tokens_seen": 1048772608 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034458375125376127, + "loss": 2.7008, + "theoretical_loss": 3.632743168537419, + "tokens_seen": 1048838144 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003445737211634905, + "loss": 2.9017, + "theoretical_loss": 3.6327217574634423, + "tokens_seen": 1048903680 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003445636910732197, + "loss": 2.781, + "theoretical_loss": 3.632700348101748, + "tokens_seen": 1048969216 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034455366098294887, + "loss": 2.5106, + "theoretical_loss": 3.632678940452092, + "tokens_seen": 1049034752 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034454363089267805, + "loss": 2.818, + "theoretical_loss": 3.6326575345142307, + "tokens_seen": 1049100288 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034453360080240723, + "loss": 2.6281, + "theoretical_loss": 3.63263613028792, + "tokens_seen": 1049165824 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003445235707121364, + "loss": 2.9862, + "theoretical_loss": 3.632614727772917, + "tokens_seen": 1049231360 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034451354062186565, + "loss": 2.9908, + "theoretical_loss": 3.6325933269689763, + "tokens_seen": 1049296896 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003445035105315948, + "loss": 2.6828, + "theoretical_loss": 3.6325719278758557, + "tokens_seen": 1049362432 + }, + { + "epoch": 0.32, + "learning_rate": 0.000344493480441324, + "loss": 2.7238, + "theoretical_loss": 3.6325505304933117, + "tokens_seen": 1049427968 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034448345035105314, + "loss": 2.9632, + "theoretical_loss": 3.6325291348211, + "tokens_seen": 1049493504 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034447342026078237, + "loss": 2.9243, + "theoretical_loss": 3.632507740858977, + "tokens_seen": 1049559040 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034446339017051155, + "loss": 2.8214, + "theoretical_loss": 3.6324863486067, + "tokens_seen": 1049624576 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034445336008024073, + "loss": 2.98, + "theoretical_loss": 3.632464958064025, + "tokens_seen": 1049690112 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003444433299899699, + "loss": 2.8977, + "theoretical_loss": 3.6324435692307087, + "tokens_seen": 1049755648 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034443329989969915, + "loss": 2.827, + "theoretical_loss": 3.6324221821065077, + "tokens_seen": 1049821184 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003444232698094283, + "loss": 2.6642, + "theoretical_loss": 3.6324007966911798, + "tokens_seen": 1049886720 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003444132397191575, + "loss": 2.8095, + "theoretical_loss": 3.6323794129844806, + "tokens_seen": 1049952256 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034440320962888664, + "loss": 2.7009, + "theoretical_loss": 3.6323580309861674, + "tokens_seen": 1050017792 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003443931795386159, + "loss": 2.6426, + "theoretical_loss": 3.632336650695997, + "tokens_seen": 1050083328 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034438314944834506, + "loss": 2.6632, + "theoretical_loss": 3.6323152721137264, + "tokens_seen": 1050148864 + }, + { + "epoch": 0.32, + "objective/train/docs_used": 601638, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6265807151794434, + "objective/train/theoretical_loss": 3.6322938952391124, + "objective/train/tokens_used": 1070674400, + "theoretical_loss": 3.6322938952391124, + "tokens_seen": 1050214400 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034437311935807424, + "loss": 2.9085, + "theoretical_loss": 3.6322938952391124, + "tokens_seen": 1050214400 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003443630892678034, + "loss": 2.7781, + "theoretical_loss": 3.632272520071912, + "tokens_seen": 1050279936 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003443530591775326, + "loss": 2.7129, + "theoretical_loss": 3.632251146611883, + "tokens_seen": 1050345472 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003443430290872618, + "loss": 2.8101, + "theoretical_loss": 3.632229774858782, + "tokens_seen": 1050411008 + }, + { + "epoch": 0.32, + "learning_rate": 0.000344332998996991, + "loss": 2.9306, + "theoretical_loss": 3.6322084048123666, + "tokens_seen": 1050476544 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034432296890672014, + "loss": 2.6188, + "theoretical_loss": 3.632187036472393, + "tokens_seen": 1050542080 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003443129388164494, + "loss": 2.8198, + "theoretical_loss": 3.63216566983862, + "tokens_seen": 1050607616 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034430290872617856, + "loss": 2.5511, + "theoretical_loss": 3.632144304910804, + "tokens_seen": 1050673152 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034429287863590774, + "loss": 2.6806, + "theoretical_loss": 3.6321229416887024, + "tokens_seen": 1050738688 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003442828485456369, + "loss": 2.4646, + "theoretical_loss": 3.6321015801720726, + "tokens_seen": 1050804224 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003442728184553661, + "loss": 2.8023, + "theoretical_loss": 3.632080220360673, + "tokens_seen": 1050869760 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003442627883650953, + "loss": 2.7111, + "theoretical_loss": 3.63205886225426, + "tokens_seen": 1050935296 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003442527582748245, + "loss": 2.7092, + "theoretical_loss": 3.6320375058525913, + "tokens_seen": 1051000832 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034424272818455365, + "loss": 2.8757, + "theoretical_loss": 3.632016151155425, + "tokens_seen": 1051066368 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003442326980942829, + "loss": 2.5614, + "theoretical_loss": 3.631994798162519, + "tokens_seen": 1051131904 + }, + { + "epoch": 0.32, + "learning_rate": 0.000344222668004012, + "loss": 2.7355, + "theoretical_loss": 3.631973446873631, + "tokens_seen": 1051197440 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034421263791374124, + "loss": 3.0354, + "theoretical_loss": 3.6319520972885178, + "tokens_seen": 1051262976 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003442026078234704, + "loss": 2.6231, + "theoretical_loss": 3.6319307494069384, + "tokens_seen": 1051328512 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003441925777331996, + "loss": 2.7072, + "theoretical_loss": 3.63190940322865, + "tokens_seen": 1051394048 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003441825476429288, + "loss": 2.7603, + "theoretical_loss": 3.631888058753411, + "tokens_seen": 1051459584 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034417251755265797, + "loss": 2.8314, + "theoretical_loss": 3.6318667159809794, + "tokens_seen": 1051525120 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034416248746238715, + "loss": 2.7417, + "theoretical_loss": 3.631845374911113, + "tokens_seen": 1051590656 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003441524573721164, + "loss": 2.8893, + "theoretical_loss": 3.63182403554357, + "tokens_seen": 1051656192 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003441424272818455, + "loss": 2.6317, + "theoretical_loss": 3.6318026978781077, + "tokens_seen": 1051721728 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034413239719157475, + "loss": 2.6096, + "theoretical_loss": 3.631781361914485, + "tokens_seen": 1051787264 + }, + { + "epoch": 0.32, + "objective/train/docs_used": 602388, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.636488676071167, + "objective/train/theoretical_loss": 3.631760027652461, + "objective/train/tokens_used": 1072312800, + "theoretical_loss": 3.631760027652461, + "tokens_seen": 1051852800 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034412236710130393, + "loss": 2.6459, + "theoretical_loss": 3.631760027652461, + "tokens_seen": 1051852800 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003441123370110331, + "loss": 2.8544, + "theoretical_loss": 3.631738695091793, + "tokens_seen": 1051918336 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003441023069207623, + "loss": 2.8765, + "theoretical_loss": 3.631717364232239, + "tokens_seen": 1051983872 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034409227683049147, + "loss": 2.9941, + "theoretical_loss": 3.6316960350735585, + "tokens_seen": 1052049408 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034408224674022065, + "loss": 2.6593, + "theoretical_loss": 3.631674707615509, + "tokens_seen": 1052114944 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003440722166499499, + "loss": 2.6419, + "theoretical_loss": 3.6316533818578494, + "tokens_seen": 1052180480 + }, + { + "epoch": 0.32, + "learning_rate": 0.000344062186559679, + "loss": 2.7217, + "theoretical_loss": 3.6316320578003385, + "tokens_seen": 1052246016 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034405215646940825, + "loss": 2.779, + "theoretical_loss": 3.631610735442734, + "tokens_seen": 1052311552 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003440421263791374, + "loss": 2.7338, + "theoretical_loss": 3.631589414784795, + "tokens_seen": 1052377088 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003440320962888666, + "loss": 2.7192, + "theoretical_loss": 3.631568095826281, + "tokens_seen": 1052442624 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003440220661985958, + "loss": 2.8274, + "theoretical_loss": 3.631546778566949, + "tokens_seen": 1052508160 + }, + { + "epoch": 0.32, + "learning_rate": 0.000344012036108325, + "loss": 2.6441, + "theoretical_loss": 3.631525463006559, + "tokens_seen": 1052573696 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034400200601805416, + "loss": 2.887, + "theoretical_loss": 3.6315041491448703, + "tokens_seen": 1052639232 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034399197592778334, + "loss": 2.6618, + "theoretical_loss": 3.6314828369816405, + "tokens_seen": 1052704768 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003439819458375125, + "loss": 2.8792, + "theoretical_loss": 3.6314615265166292, + "tokens_seen": 1052770304 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034397191574724175, + "loss": 2.7309, + "theoretical_loss": 3.6314402177495957, + "tokens_seen": 1052835840 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003439618856569709, + "loss": 2.4914, + "theoretical_loss": 3.6314189106802974, + "tokens_seen": 1052901376 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003439518555667001, + "loss": 2.7305, + "theoretical_loss": 3.6313976053084955, + "tokens_seen": 1052966912 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003439418254764293, + "loss": 2.6785, + "theoretical_loss": 3.6313763016339484, + "tokens_seen": 1053032448 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003439317953861585, + "loss": 2.7708, + "theoretical_loss": 3.6313549996564145, + "tokens_seen": 1053097984 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034392176529588766, + "loss": 2.7598, + "theoretical_loss": 3.631333699375654, + "tokens_seen": 1053163520 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034391173520561684, + "loss": 2.6934, + "theoretical_loss": 3.6313124007914253, + "tokens_seen": 1053229056 + }, + { + "epoch": 0.32, + "learning_rate": 0.000343901705115346, + "loss": 2.7674, + "theoretical_loss": 3.6312911039034885, + "tokens_seen": 1053294592 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034389167502507526, + "loss": 2.6415, + "theoretical_loss": 3.6312698087116027, + "tokens_seen": 1053360128 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003438816449348044, + "loss": 2.8464, + "theoretical_loss": 3.6312485152155274, + "tokens_seen": 1053425664 + }, + { + "epoch": 0.32, + "objective/train/docs_used": 603451, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.558467388153076, + "objective/train/theoretical_loss": 3.6312272234150216, + "objective/train/tokens_used": 1073951200, + "theoretical_loss": 3.6312272234150216, + "tokens_seen": 1053491200 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003438716148445336, + "loss": 2.5967, + "theoretical_loss": 3.6312272234150216, + "tokens_seen": 1053491200 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034386158475426275, + "loss": 2.7545, + "theoretical_loss": 3.6312059333098454, + "tokens_seen": 1053556736 + }, + { + "epoch": 0.32, + "learning_rate": 0.000343851554663992, + "loss": 2.6856, + "theoretical_loss": 3.631184644899758, + "tokens_seen": 1053622272 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003438415245737212, + "loss": 2.7763, + "theoretical_loss": 3.6311633581845193, + "tokens_seen": 1053687808 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034383149448345034, + "loss": 2.7686, + "theoretical_loss": 3.631142073163889, + "tokens_seen": 1053753344 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003438214643931796, + "loss": 2.8325, + "theoretical_loss": 3.631120789837626, + "tokens_seen": 1053818880 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034381143430290876, + "loss": 2.7579, + "theoretical_loss": 3.6310995082054913, + "tokens_seen": 1053884416 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034380140421263794, + "loss": 2.736, + "theoretical_loss": 3.631078228267244, + "tokens_seen": 1053949952 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003437913741223671, + "loss": 2.622, + "theoretical_loss": 3.6310569500226437, + "tokens_seen": 1054015488 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003437813440320963, + "loss": 2.7355, + "theoretical_loss": 3.631035673471451, + "tokens_seen": 1054081024 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003437713139418255, + "loss": 2.7603, + "theoretical_loss": 3.6310143986134253, + "tokens_seen": 1054146560 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003437612838515547, + "loss": 2.8809, + "theoretical_loss": 3.6309931254483274, + "tokens_seen": 1054212096 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034375125376128385, + "loss": 2.6522, + "theoretical_loss": 3.6309718539759164, + "tokens_seen": 1054277632 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003437412236710131, + "loss": 2.8721, + "theoretical_loss": 3.630950584195953, + "tokens_seen": 1054343168 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003437311935807422, + "loss": 2.8551, + "theoretical_loss": 3.6309293161081966, + "tokens_seen": 1054408704 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034372116349047144, + "loss": 2.6339, + "theoretical_loss": 3.6309080497124087, + "tokens_seen": 1054474240 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003437111334002006, + "loss": 2.7648, + "theoretical_loss": 3.6308867850083484, + "tokens_seen": 1054539776 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003437011033099298, + "loss": 2.6046, + "theoretical_loss": 3.6308655219957764, + "tokens_seen": 1054605312 + }, + { + "epoch": 0.32, + "learning_rate": 0.000343691073219659, + "loss": 2.94, + "theoretical_loss": 3.6308442606744533, + "tokens_seen": 1054670848 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034368104312938817, + "loss": 3.0762, + "theoretical_loss": 3.6308230010441394, + "tokens_seen": 1054736384 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034367101303911735, + "loss": 2.7697, + "theoretical_loss": 3.6308017431045947, + "tokens_seen": 1054801920 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003436609829488466, + "loss": 2.8754, + "theoretical_loss": 3.6307804868555804, + "tokens_seen": 1054867456 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003436509528585757, + "loss": 2.8024, + "theoretical_loss": 3.6307592322968563, + "tokens_seen": 1054932992 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034364092276830495, + "loss": 2.8058, + "theoretical_loss": 3.6307379794281838, + "tokens_seen": 1054998528 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034363089267803413, + "loss": 2.8597, + "theoretical_loss": 3.6307167282493227, + "tokens_seen": 1055064064 + }, + { + "epoch": 0.32, + "objective/train/docs_used": 603451, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7778093814849854, + "objective/train/theoretical_loss": 3.630695478760034, + "objective/train/tokens_used": 1075589600, + "theoretical_loss": 3.630695478760034, + "tokens_seen": 1055129600 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003436208625877633, + "loss": 2.757, + "theoretical_loss": 3.630695478760034, + "tokens_seen": 1055129600 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003436108324974925, + "loss": 2.7407, + "theoretical_loss": 3.6306742309600786, + "tokens_seen": 1055195136 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034360080240722167, + "loss": 2.7055, + "theoretical_loss": 3.6306529848492177, + "tokens_seen": 1055260672 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034359077231695085, + "loss": 3.0244, + "theoretical_loss": 3.6306317404272113, + "tokens_seen": 1055326208 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003435807422266801, + "loss": 2.997, + "theoretical_loss": 3.6306104976938203, + "tokens_seen": 1055391744 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003435707121364092, + "loss": 2.8282, + "theoretical_loss": 3.6305892566488067, + "tokens_seen": 1055457280 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034356068204613845, + "loss": 2.8036, + "theoretical_loss": 3.6305680172919303, + "tokens_seen": 1055522816 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003435506519558676, + "loss": 2.7039, + "theoretical_loss": 3.6305467796229527, + "tokens_seen": 1055588352 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003435406218655968, + "loss": 2.7812, + "theoretical_loss": 3.6305255436416353, + "tokens_seen": 1055653888 + }, + { + "epoch": 0.32, + "learning_rate": 0.000343530591775326, + "loss": 2.7483, + "theoretical_loss": 3.6305043093477383, + "tokens_seen": 1055719424 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003435205616850552, + "loss": 2.8625, + "theoretical_loss": 3.630483076741023, + "tokens_seen": 1055784960 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034351053159478436, + "loss": 2.865, + "theoretical_loss": 3.6304618458212516, + "tokens_seen": 1055850496 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034350050150451354, + "loss": 2.7542, + "theoretical_loss": 3.630440616588185, + "tokens_seen": 1055916032 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003434904714142427, + "loss": 2.879, + "theoretical_loss": 3.630419389041584, + "tokens_seen": 1055981568 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034348044132397195, + "loss": 2.6634, + "theoretical_loss": 3.6303981631812103, + "tokens_seen": 1056047104 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003434704112337011, + "loss": 2.7825, + "theoretical_loss": 3.6303769390068257, + "tokens_seen": 1056112640 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003434603811434303, + "loss": 3.1067, + "theoretical_loss": 3.6303557165181912, + "tokens_seen": 1056178176 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003434503510531595, + "loss": 3.0015, + "theoretical_loss": 3.630334495715068, + "tokens_seen": 1056243712 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003434403209628887, + "loss": 2.8421, + "theoretical_loss": 3.630313276597218, + "tokens_seen": 1056309248 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034343029087261786, + "loss": 2.9641, + "theoretical_loss": 3.630292059164404, + "tokens_seen": 1056374784 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034342026078234704, + "loss": 2.7949, + "theoretical_loss": 3.630270843416385, + "tokens_seen": 1056440320 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003434102306920762, + "loss": 2.8654, + "theoretical_loss": 3.6302496293529254, + "tokens_seen": 1056505856 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034340020060180546, + "loss": 2.7665, + "theoretical_loss": 3.6302284169737855, + "tokens_seen": 1056571392 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003433901705115346, + "loss": 2.603, + "theoretical_loss": 3.630207206278727, + "tokens_seen": 1056636928 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003433801404212638, + "loss": 2.9841, + "theoretical_loss": 3.630185997267513, + "tokens_seen": 1056702464 + }, + { + "epoch": 0.32, + "objective/train/docs_used": 603451, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.389645576477051, + "objective/train/theoretical_loss": 3.6301647899399043, + "objective/train/tokens_used": 1077228000, + "theoretical_loss": 3.6301647899399043, + "tokens_seen": 1056768000 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034337011033099295, + "loss": 2.6396, + "theoretical_loss": 3.6301647899399043, + "tokens_seen": 1056768000 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003433600802407222, + "loss": 2.9659, + "theoretical_loss": 3.630143584295663, + "tokens_seen": 1056833536 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034335005015045136, + "loss": 2.9358, + "theoretical_loss": 3.6301223803345515, + "tokens_seen": 1056899072 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034334002006018054, + "loss": 2.8693, + "theoretical_loss": 3.6301011780563313, + "tokens_seen": 1056964608 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003433299899699097, + "loss": 2.8913, + "theoretical_loss": 3.630079977460765, + "tokens_seen": 1057030144 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003433199598796389, + "loss": 2.7994, + "theoretical_loss": 3.630058778547615, + "tokens_seen": 1057095680 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003433099297893681, + "loss": 2.718, + "theoretical_loss": 3.630037581316642, + "tokens_seen": 1057161216 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003432998996990973, + "loss": 2.8121, + "theoretical_loss": 3.63001638576761, + "tokens_seen": 1057226752 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034328986960882645, + "loss": 2.7863, + "theoretical_loss": 3.6299951919002806, + "tokens_seen": 1057292288 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003432798395185557, + "loss": 2.9746, + "theoretical_loss": 3.6299739997144163, + "tokens_seen": 1057357824 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034326980942828487, + "loss": 2.9017, + "theoretical_loss": 3.6299528092097795, + "tokens_seen": 1057423360 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034325977933801405, + "loss": 2.8035, + "theoretical_loss": 3.629931620386132, + "tokens_seen": 1057488896 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034324974924774323, + "loss": 2.6389, + "theoretical_loss": 3.6299104332432366, + "tokens_seen": 1057554432 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003432397191574724, + "loss": 2.8848, + "theoretical_loss": 3.6298892477808566, + "tokens_seen": 1057619968 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003432296890672016, + "loss": 2.7983, + "theoretical_loss": 3.6298680639987535, + "tokens_seen": 1057685504 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003432196589769308, + "loss": 2.7758, + "theoretical_loss": 3.6298468818966905, + "tokens_seen": 1057751040 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034320962888665995, + "loss": 2.8584, + "theoretical_loss": 3.6298257014744304, + "tokens_seen": 1057816576 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003431995987963892, + "loss": 2.8647, + "theoretical_loss": 3.6298045227317357, + "tokens_seen": 1057882112 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003431895687061183, + "loss": 2.7919, + "theoretical_loss": 3.6297833456683692, + "tokens_seen": 1057947648 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034317953861584755, + "loss": 2.7487, + "theoretical_loss": 3.629762170284094, + "tokens_seen": 1058013184 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034316950852557673, + "loss": 2.8929, + "theoretical_loss": 3.629740996578672, + "tokens_seen": 1058078720 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003431594784353059, + "loss": 2.7257, + "theoretical_loss": 3.6297198245518674, + "tokens_seen": 1058144256 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003431494483450351, + "loss": 2.8459, + "theoretical_loss": 3.629698654203442, + "tokens_seen": 1058209792 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034313941825476433, + "loss": 2.7694, + "theoretical_loss": 3.6296774855331604, + "tokens_seen": 1058275328 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034312938816449345, + "loss": 2.6542, + "theoretical_loss": 3.6296563185407837, + "tokens_seen": 1058340864 + }, + { + "epoch": 0.32, + "objective/train/docs_used": 604503, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.553166151046753, + "objective/train/theoretical_loss": 3.6296351532260767, + "objective/train/tokens_used": 1078866400, + "theoretical_loss": 3.6296351532260767, + "tokens_seen": 1058406400 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003431193580742227, + "loss": 2.5634, + "theoretical_loss": 3.6296351532260767, + "tokens_seen": 1058406400 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003431093279839518, + "loss": 2.8122, + "theoretical_loss": 3.629613989588801, + "tokens_seen": 1058471936 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034309929789368105, + "loss": 2.9786, + "theoretical_loss": 3.6295928276287217, + "tokens_seen": 1058537472 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003430892678034103, + "loss": 2.734, + "theoretical_loss": 3.6295716673456004, + "tokens_seen": 1058603008 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003430792377131394, + "loss": 2.8131, + "theoretical_loss": 3.629550508739201, + "tokens_seen": 1058668544 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034306920762286865, + "loss": 2.873, + "theoretical_loss": 3.6295293518092873, + "tokens_seen": 1058734080 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003430591775325978, + "loss": 2.8453, + "theoretical_loss": 3.629508196555622, + "tokens_seen": 1058799616 + }, + { + "epoch": 0.32, + "learning_rate": 0.000343049147442327, + "loss": 2.8413, + "theoretical_loss": 3.629487042977969, + "tokens_seen": 1058865152 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003430391173520562, + "loss": 2.9058, + "theoretical_loss": 3.6294658910760913, + "tokens_seen": 1058930688 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003430290872617854, + "loss": 2.6502, + "theoretical_loss": 3.6294447408497525, + "tokens_seen": 1058996224 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034301905717151456, + "loss": 2.6363, + "theoretical_loss": 3.6294235922987177, + "tokens_seen": 1059061760 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034300902708124374, + "loss": 2.9184, + "theoretical_loss": 3.6294024454227483, + "tokens_seen": 1059127296 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003429989969909729, + "loss": 2.5751, + "theoretical_loss": 3.6293813002216093, + "tokens_seen": 1059192832 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034298896690070215, + "loss": 2.6484, + "theoretical_loss": 3.629360156695064, + "tokens_seen": 1059258368 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003429789368104313, + "loss": 2.8876, + "theoretical_loss": 3.629339014842876, + "tokens_seen": 1059323904 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003429689067201605, + "loss": 2.7748, + "theoretical_loss": 3.6293178746648103, + "tokens_seen": 1059389440 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003429588766298897, + "loss": 2.7669, + "theoretical_loss": 3.6292967361606294, + "tokens_seen": 1059454976 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003429488465396189, + "loss": 2.9142, + "theoretical_loss": 3.6292755993300982, + "tokens_seen": 1059520512 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034293881644934806, + "loss": 2.8876, + "theoretical_loss": 3.62925446417298, + "tokens_seen": 1059586048 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034292878635907724, + "loss": 2.9082, + "theoretical_loss": 3.629233330689039, + "tokens_seen": 1059651584 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003429187562688064, + "loss": 2.8625, + "theoretical_loss": 3.6292121988780393, + "tokens_seen": 1059717120 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034290872617853566, + "loss": 3.0508, + "theoretical_loss": 3.6291910687397455, + "tokens_seen": 1059782656 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003428986960882648, + "loss": 2.6365, + "theoretical_loss": 3.629169940273921, + "tokens_seen": 1059848192 + }, + { + "epoch": 0.32, + "learning_rate": 0.000342888665997994, + "loss": 2.6082, + "theoretical_loss": 3.6291488134803305, + "tokens_seen": 1059913728 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034287863590772315, + "loss": 2.8241, + "theoretical_loss": 3.629127688358738, + "tokens_seen": 1059979264 + }, + { + "epoch": 0.32, + "objective/train/docs_used": 605021, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.5084993839263916, + "objective/train/theoretical_loss": 3.6291065649089083, + "objective/train/tokens_used": 1080504800, + "theoretical_loss": 3.6291065649089083, + "tokens_seen": 1060044800 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003428686058174524, + "loss": 2.7752, + "theoretical_loss": 3.6291065649089083, + "tokens_seen": 1060044800 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034285857572718156, + "loss": 3.0004, + "theoretical_loss": 3.629085443130605, + "tokens_seen": 1060110336 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034284854563691074, + "loss": 2.6841, + "theoretical_loss": 3.6290643230235924, + "tokens_seen": 1060175872 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003428385155466399, + "loss": 2.8473, + "theoretical_loss": 3.6290432045876364, + "tokens_seen": 1060241408 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003428284854563691, + "loss": 2.9134, + "theoretical_loss": 3.6290220878225004, + "tokens_seen": 1060306944 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003428184553660983, + "loss": 2.8376, + "theoretical_loss": 3.629000972727949, + "tokens_seen": 1060372480 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003428084252758275, + "loss": 2.8398, + "theoretical_loss": 3.6289798593037466, + "tokens_seen": 1060438016 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034279839518555665, + "loss": 2.9166, + "theoretical_loss": 3.6289587475496585, + "tokens_seen": 1060503552 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003427883650952859, + "loss": 2.9059, + "theoretical_loss": 3.628937637465449, + "tokens_seen": 1060569088 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034277833500501507, + "loss": 2.7971, + "theoretical_loss": 3.628916529050883, + "tokens_seen": 1060634624 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034276830491474425, + "loss": 2.9997, + "theoretical_loss": 3.628895422305725, + "tokens_seen": 1060700160 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034275827482447343, + "loss": 2.9778, + "theoretical_loss": 3.6288743172297404, + "tokens_seen": 1060765696 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003427482447342026, + "loss": 2.7168, + "theoretical_loss": 3.628853213822693, + "tokens_seen": 1060831232 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003427382146439318, + "loss": 2.7087, + "theoretical_loss": 3.6288321120843494, + "tokens_seen": 1060896768 + }, + { + "epoch": 0.32, + "learning_rate": 0.000342728184553661, + "loss": 2.8008, + "theoretical_loss": 3.6288110120144728, + "tokens_seen": 1060962304 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034271815446339015, + "loss": 2.8665, + "theoretical_loss": 3.6287899136128297, + "tokens_seen": 1061027840 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003427081243731194, + "loss": 2.9721, + "theoretical_loss": 3.6287688168791843, + "tokens_seen": 1061093376 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003426980942828485, + "loss": 2.7802, + "theoretical_loss": 3.6287477218133017, + "tokens_seen": 1061158912 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034268806419257775, + "loss": 2.7762, + "theoretical_loss": 3.6287266284149475, + "tokens_seen": 1061224448 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034267803410230693, + "loss": 3.0388, + "theoretical_loss": 3.6287055366838867, + "tokens_seen": 1061289984 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003426680040120361, + "loss": 2.8567, + "theoretical_loss": 3.628684446619885, + "tokens_seen": 1061355520 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003426579739217653, + "loss": 2.9571, + "theoretical_loss": 3.628663358222707, + "tokens_seen": 1061421056 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034264794383149453, + "loss": 2.8665, + "theoretical_loss": 3.6286422714921187, + "tokens_seen": 1061486592 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034263791374122366, + "loss": 2.9273, + "theoretical_loss": 3.6286211864278846, + "tokens_seen": 1061552128 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003426278836509529, + "loss": 3.0076, + "theoretical_loss": 3.628600103029771, + "tokens_seen": 1061617664 + }, + { + "epoch": 0.32, + "objective/train/docs_used": 606310, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6500792503356934, + "objective/train/theoretical_loss": 3.6285790212975435, + "objective/train/tokens_used": 1082143200, + "theoretical_loss": 3.6285790212975435, + "tokens_seen": 1061683200 + }, + { + "epoch": 0.32, + "learning_rate": 0.000342617853560682, + "loss": 2.8789, + "theoretical_loss": 3.6285790212975435, + "tokens_seen": 1061683200 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034260782347041125, + "loss": 2.6082, + "theoretical_loss": 3.6285579412309668, + "tokens_seen": 1061748736 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034259779338014043, + "loss": 2.9098, + "theoretical_loss": 3.628536862829807, + "tokens_seen": 1061814272 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003425877632898696, + "loss": 2.7178, + "theoretical_loss": 3.62851578609383, + "tokens_seen": 1061879808 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003425777331995988, + "loss": 2.5725, + "theoretical_loss": 3.6284947110228014, + "tokens_seen": 1061945344 + }, + { + "epoch": 0.32, + "learning_rate": 0.000342567703109328, + "loss": 3.0116, + "theoretical_loss": 3.6284736376164872, + "tokens_seen": 1062010880 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034255767301905716, + "loss": 2.8026, + "theoretical_loss": 3.6284525658746523, + "tokens_seen": 1062076416 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003425476429287864, + "loss": 2.7983, + "theoretical_loss": 3.628431495797063, + "tokens_seen": 1062141952 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003425376128385155, + "loss": 2.8089, + "theoretical_loss": 3.6284104273834847, + "tokens_seen": 1062207488 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034252758274824476, + "loss": 2.7864, + "theoretical_loss": 3.628389360633684, + "tokens_seen": 1062273024 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003425175526579739, + "loss": 2.8937, + "theoretical_loss": 3.6283682955474275, + "tokens_seen": 1062338560 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003425075225677031, + "loss": 2.972, + "theoretical_loss": 3.62834723212448, + "tokens_seen": 1062404096 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003424974924774323, + "loss": 2.8654, + "theoretical_loss": 3.628326170364608, + "tokens_seen": 1062469632 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003424874623871615, + "loss": 2.7632, + "theoretical_loss": 3.6283051102675774, + "tokens_seen": 1062535168 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034247743229689066, + "loss": 2.9251, + "theoretical_loss": 3.6282840518331545, + "tokens_seen": 1062600704 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003424674022066199, + "loss": 2.7058, + "theoretical_loss": 3.628262995061106, + "tokens_seen": 1062666240 + }, + { + "epoch": 0.32, + "learning_rate": 0.000342457372116349, + "loss": 2.9026, + "theoretical_loss": 3.628241939951198, + "tokens_seen": 1062731776 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034244734202607826, + "loss": 2.7201, + "theoretical_loss": 3.628220886503196, + "tokens_seen": 1062797312 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003424373119358074, + "loss": 2.9517, + "theoretical_loss": 3.628199834716867, + "tokens_seen": 1062862848 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003424272818455366, + "loss": 2.8719, + "theoretical_loss": 3.6281787845919777, + "tokens_seen": 1062928384 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003424172517552658, + "loss": 2.9413, + "theoretical_loss": 3.628157736128294, + "tokens_seen": 1062993920 + }, + { + "epoch": 0.32, + "learning_rate": 0.000342407221664995, + "loss": 2.9108, + "theoretical_loss": 3.6281366893255824, + "tokens_seen": 1063059456 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034239719157472416, + "loss": 2.8455, + "theoretical_loss": 3.6281156441836098, + "tokens_seen": 1063124992 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034238716148445335, + "loss": 3.0117, + "theoretical_loss": 3.628094600702143, + "tokens_seen": 1063190528 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003423771313941825, + "loss": 2.8063, + "theoretical_loss": 3.6280735588809474, + "tokens_seen": 1063256064 + }, + { + "epoch": 0.32, + "objective/train/docs_used": 606853, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.1918704509735107, + "objective/train/theoretical_loss": 3.628052518719791, + "objective/train/tokens_used": 1083781600, + "theoretical_loss": 3.628052518719791, + "tokens_seen": 1063321600 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034236710130391176, + "loss": 2.8517, + "theoretical_loss": 3.628052518719791, + "tokens_seen": 1063321600 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003423570712136409, + "loss": 2.862, + "theoretical_loss": 3.62803148021844, + "tokens_seen": 1063387136 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003423470411233701, + "loss": 2.9037, + "theoretical_loss": 3.628010443376662, + "tokens_seen": 1063452672 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003423370110330993, + "loss": 2.9186, + "theoretical_loss": 3.6279894081942223, + "tokens_seen": 1063518208 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003423269809428285, + "loss": 2.9507, + "theoretical_loss": 3.6279683746708886, + "tokens_seen": 1063583744 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003423169508525577, + "loss": 2.9116, + "theoretical_loss": 3.627947342806428, + "tokens_seen": 1063649280 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034230692076228685, + "loss": 3.1611, + "theoretical_loss": 3.627926312600607, + "tokens_seen": 1063714816 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003422968906720161, + "loss": 2.8771, + "theoretical_loss": 3.6279052840531936, + "tokens_seen": 1063780352 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034228686058174527, + "loss": 3.0187, + "theoretical_loss": 3.627884257163953, + "tokens_seen": 1063845888 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034227683049147445, + "loss": 2.8274, + "theoretical_loss": 3.6278632319326545, + "tokens_seen": 1063911424 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034226680040120363, + "loss": 2.8214, + "theoretical_loss": 3.6278422083590636, + "tokens_seen": 1063976960 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003422567703109328, + "loss": 2.8985, + "theoretical_loss": 3.627821186442948, + "tokens_seen": 1064042496 + }, + { + "epoch": 0.32, + "learning_rate": 0.000342246740220662, + "loss": 2.6914, + "theoretical_loss": 3.627800166184076, + "tokens_seen": 1064108032 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003422367101303912, + "loss": 2.8644, + "theoretical_loss": 3.6277791475822134, + "tokens_seen": 1064173568 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034222668004012035, + "loss": 2.6989, + "theoretical_loss": 3.627758130637128, + "tokens_seen": 1064239104 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003422166499498496, + "loss": 2.81, + "theoretical_loss": 3.6277371153485873, + "tokens_seen": 1064304640 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003422066198595787, + "loss": 3.0286, + "theoretical_loss": 3.6277161017163593, + "tokens_seen": 1064370176 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034219658976930795, + "loss": 3.0704, + "theoretical_loss": 3.62769508974021, + "tokens_seen": 1064435712 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034218655967903713, + "loss": 2.8213, + "theoretical_loss": 3.6276740794199087, + "tokens_seen": 1064501248 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003421765295887663, + "loss": 3.0809, + "theoretical_loss": 3.6276530707552217, + "tokens_seen": 1064566784 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003421664994984955, + "loss": 2.6297, + "theoretical_loss": 3.627632063745917, + "tokens_seen": 1064632320 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034215646940822473, + "loss": 2.9563, + "theoretical_loss": 3.6276110583917625, + "tokens_seen": 1064697856 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034214643931795386, + "loss": 2.826, + "theoretical_loss": 3.6275900546925257, + "tokens_seen": 1064763392 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003421364092276831, + "loss": 2.9191, + "theoretical_loss": 3.6275690526479747, + "tokens_seen": 1064828928 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003421263791374122, + "loss": 2.9951, + "theoretical_loss": 3.627548052257877, + "tokens_seen": 1064894464 + }, + { + "epoch": 0.32, + "objective/train/docs_used": 607789, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.4026103019714355, + "objective/train/theoretical_loss": 3.6275270535220008, + "objective/train/tokens_used": 1085420000, + "theoretical_loss": 3.6275270535220008, + "tokens_seen": 1064960000 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034211634904714145, + "loss": 2.7203, + "theoretical_loss": 3.6275270535220008, + "tokens_seen": 1064960000 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034210631895687063, + "loss": 2.8995, + "theoretical_loss": 3.627506056440113, + "tokens_seen": 1065025536 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003420962888665998, + "loss": 2.8442, + "theoretical_loss": 3.627485061011983, + "tokens_seen": 1065091072 + }, + { + "epoch": 0.32, + "learning_rate": 0.000342086258776329, + "loss": 2.8242, + "theoretical_loss": 3.6274640672373772, + "tokens_seen": 1065156608 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003420762286860582, + "loss": 2.8854, + "theoretical_loss": 3.6274430751160653, + "tokens_seen": 1065222144 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034206619859578736, + "loss": 2.7236, + "theoretical_loss": 3.6274220846478142, + "tokens_seen": 1065287680 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003420561685055166, + "loss": 2.4308, + "theoretical_loss": 3.627401095832393, + "tokens_seen": 1065353216 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003420461384152457, + "loss": 2.7998, + "theoretical_loss": 3.627380108669569, + "tokens_seen": 1065418752 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034203610832497496, + "loss": 2.7385, + "theoretical_loss": 3.6273591231591107, + "tokens_seen": 1065484288 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003420260782347041, + "loss": 2.9335, + "theoretical_loss": 3.627338139300787, + "tokens_seen": 1065549824 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003420160481444333, + "loss": 2.8329, + "theoretical_loss": 3.627317157094365, + "tokens_seen": 1065615360 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003420060180541625, + "loss": 3.1238, + "theoretical_loss": 3.6272961765396143, + "tokens_seen": 1065680896 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003419959879638917, + "loss": 2.9674, + "theoretical_loss": 3.627275197636303, + "tokens_seen": 1065746432 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034198595787362086, + "loss": 2.8419, + "theoretical_loss": 3.627254220384199, + "tokens_seen": 1065811968 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003419759277833501, + "loss": 2.9711, + "theoretical_loss": 3.6272332447830715, + "tokens_seen": 1065877504 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003419658976930792, + "loss": 2.8748, + "theoretical_loss": 3.6272122708326884, + "tokens_seen": 1065943040 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034195586760280846, + "loss": 2.4948, + "theoretical_loss": 3.6271912985328187, + "tokens_seen": 1066008576 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003419458375125376, + "loss": 2.7201, + "theoretical_loss": 3.6271703278832312, + "tokens_seen": 1066074112 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003419358074222668, + "loss": 2.7851, + "theoretical_loss": 3.6271493588836945, + "tokens_seen": 1066139648 + }, + { + "epoch": 0.32, + "learning_rate": 0.000341925777331996, + "loss": 2.8361, + "theoretical_loss": 3.6271283915339776, + "tokens_seen": 1066205184 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003419157472417252, + "loss": 2.752, + "theoretical_loss": 3.6271074258338487, + "tokens_seen": 1066270720 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034190571715145436, + "loss": 2.8748, + "theoretical_loss": 3.6270864617830765, + "tokens_seen": 1066336256 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034189568706118355, + "loss": 2.8798, + "theoretical_loss": 3.627065499381431, + "tokens_seen": 1066401792 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003418856569709127, + "loss": 2.7507, + "theoretical_loss": 3.62704453862868, + "tokens_seen": 1066467328 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034187562688064196, + "loss": 3.0367, + "theoretical_loss": 3.627023579524593, + "tokens_seen": 1066532864 + }, + { + "epoch": 0.32, + "objective/train/docs_used": 608475, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.9811666011810303, + "objective/train/theoretical_loss": 3.627002622068939, + "objective/train/tokens_used": 1087058400, + "theoretical_loss": 3.627002622068939, + "tokens_seen": 1066598400 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003418655967903711, + "loss": 2.8227, + "theoretical_loss": 3.627002622068939, + "tokens_seen": 1066598400 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003418555667001003, + "loss": 2.7874, + "theoretical_loss": 3.6269816662614875, + "tokens_seen": 1066663936 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034184553660982945, + "loss": 3.0939, + "theoretical_loss": 3.626960712102006, + "tokens_seen": 1066729472 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003418355065195587, + "loss": 2.7039, + "theoretical_loss": 3.6269397595902664, + "tokens_seen": 1066795008 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034182547642928787, + "loss": 2.8489, + "theoretical_loss": 3.6269188087260353, + "tokens_seen": 1066860544 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034181544633901705, + "loss": 2.8331, + "theoretical_loss": 3.6268978595090835, + "tokens_seen": 1066926080 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034180541624874623, + "loss": 2.999, + "theoretical_loss": 3.6268769119391795, + "tokens_seen": 1066991616 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034179538615847547, + "loss": 2.8222, + "theoretical_loss": 3.6268559660160933, + "tokens_seen": 1067057152 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003417853560682046, + "loss": 2.9422, + "theoretical_loss": 3.626835021739594, + "tokens_seen": 1067122688 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034177532597793383, + "loss": 2.8443, + "theoretical_loss": 3.626814079109451, + "tokens_seen": 1067188224 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034176529588766295, + "loss": 2.8355, + "theoretical_loss": 3.6267931381254335, + "tokens_seen": 1067253760 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003417552657973922, + "loss": 2.8643, + "theoretical_loss": 3.626772198787312, + "tokens_seen": 1067319296 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034174523570712137, + "loss": 3.1466, + "theoretical_loss": 3.6267512610948547, + "tokens_seen": 1067384832 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034173520561685055, + "loss": 2.929, + "theoretical_loss": 3.6267303250478324, + "tokens_seen": 1067450368 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034172517552657973, + "loss": 2.8143, + "theoretical_loss": 3.626709390646014, + "tokens_seen": 1067515904 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003417151454363089, + "loss": 3.0625, + "theoretical_loss": 3.6266884578891703, + "tokens_seen": 1067581440 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003417051153460381, + "loss": 2.9147, + "theoretical_loss": 3.6266675267770703, + "tokens_seen": 1067646976 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034169508525576733, + "loss": 2.7833, + "theoretical_loss": 3.6266465973094832, + "tokens_seen": 1067712512 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034168505516549646, + "loss": 2.833, + "theoretical_loss": 3.62662566948618, + "tokens_seen": 1067778048 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003416750250752257, + "loss": 2.922, + "theoretical_loss": 3.62660474330693, + "tokens_seen": 1067843584 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003416649949849548, + "loss": 2.7559, + "theoretical_loss": 3.626583818771504, + "tokens_seen": 1067909120 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034165496489468406, + "loss": 2.7823, + "theoretical_loss": 3.6265628958796707, + "tokens_seen": 1067974656 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034164493480441324, + "loss": 2.8859, + "theoretical_loss": 3.626541974631201, + "tokens_seen": 1068040192 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003416349047141424, + "loss": 2.6829, + "theoretical_loss": 3.626521055025864, + "tokens_seen": 1068105728 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003416248746238716, + "loss": 2.9408, + "theoretical_loss": 3.6265001370634318, + "tokens_seen": 1068171264 + }, + { + "epoch": 0.32, + "objective/train/docs_used": 609657, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.649430274963379, + "objective/train/theoretical_loss": 3.626479220743673, + "objective/train/tokens_used": 1088696800, + "theoretical_loss": 3.626479220743673, + "tokens_seen": 1068236800 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034161484453360083, + "loss": 2.7688, + "theoretical_loss": 3.626479220743673, + "tokens_seen": 1068236800 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034160481444332996, + "loss": 2.6576, + "theoretical_loss": 3.6264583060663584, + "tokens_seen": 1068302336 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003415947843530592, + "loss": 2.8885, + "theoretical_loss": 3.6264373930312575, + "tokens_seen": 1068367872 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003415847542627884, + "loss": 2.9877, + "theoretical_loss": 3.6264164816381417, + "tokens_seen": 1068433408 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034157472417251756, + "loss": 2.769, + "theoretical_loss": 3.626395571886781, + "tokens_seen": 1068498944 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003415646940822468, + "loss": 3.0801, + "theoretical_loss": 3.626374663776946, + "tokens_seen": 1068564480 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003415546639919759, + "loss": 3.1225, + "theoretical_loss": 3.6263537573084066, + "tokens_seen": 1068630016 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034154463390170516, + "loss": 2.9672, + "theoretical_loss": 3.6263328524809335, + "tokens_seen": 1068695552 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003415346038114343, + "loss": 2.9757, + "theoretical_loss": 3.6263119492942977, + "tokens_seen": 1068761088 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003415245737211635, + "loss": 2.5134, + "theoretical_loss": 3.6262910477482695, + "tokens_seen": 1068826624 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003415145436308927, + "loss": 2.9517, + "theoretical_loss": 3.62627014784262, + "tokens_seen": 1068892160 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003415045135406219, + "loss": 2.952, + "theoretical_loss": 3.6262492495771186, + "tokens_seen": 1068957696 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034149448345035106, + "loss": 2.5716, + "theoretical_loss": 3.6262283529515376, + "tokens_seen": 1069023232 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003414844533600803, + "loss": 2.5247, + "theoretical_loss": 3.6262074579656467, + "tokens_seen": 1069088768 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003414744232698094, + "loss": 2.7415, + "theoretical_loss": 3.626186564619217, + "tokens_seen": 1069154304 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034146439317953866, + "loss": 2.881, + "theoretical_loss": 3.6261656729120197, + "tokens_seen": 1069219840 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003414543630892678, + "loss": 2.8606, + "theoretical_loss": 3.6261447828438254, + "tokens_seen": 1069285376 + }, + { + "epoch": 0.32, + "learning_rate": 0.000341444332998997, + "loss": 2.955, + "theoretical_loss": 3.6261238944144054, + "tokens_seen": 1069350912 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003414343029087262, + "loss": 2.8541, + "theoretical_loss": 3.6261030076235303, + "tokens_seen": 1069416448 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003414242728184554, + "loss": 2.7894, + "theoretical_loss": 3.6260821224709714, + "tokens_seen": 1069481984 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034141424272818456, + "loss": 2.8315, + "theoretical_loss": 3.6260612389565, + "tokens_seen": 1069547520 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034140421263791375, + "loss": 2.7903, + "theoretical_loss": 3.626040357079887, + "tokens_seen": 1069613056 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034139418254764293, + "loss": 2.6672, + "theoretical_loss": 3.626019476840904, + "tokens_seen": 1069678592 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034138415245737216, + "loss": 2.8537, + "theoretical_loss": 3.6259985982393217, + "tokens_seen": 1069744128 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003413741223671013, + "loss": 2.9128, + "theoretical_loss": 3.6259777212749116, + "tokens_seen": 1069809664 + }, + { + "epoch": 0.32, + "objective/train/docs_used": 610408, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.762554168701172, + "objective/train/theoretical_loss": 3.625956845947445, + "objective/train/tokens_used": 1090335200, + "theoretical_loss": 3.625956845947445, + "tokens_seen": 1069875200 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003413640922768305, + "loss": 2.7203, + "theoretical_loss": 3.625956845947445, + "tokens_seen": 1069875200 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034135406218655965, + "loss": 2.6732, + "theoretical_loss": 3.625935972256694, + "tokens_seen": 1069940736 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003413440320962889, + "loss": 2.8757, + "theoretical_loss": 3.6259151002024286, + "tokens_seen": 1070006272 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034133400200601807, + "loss": 2.709, + "theoretical_loss": 3.6258942297844214, + "tokens_seen": 1070071808 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034132397191574725, + "loss": 3.0307, + "theoretical_loss": 3.6258733610024434, + "tokens_seen": 1070137344 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034131394182547643, + "loss": 3.0535, + "theoretical_loss": 3.6258524938562666, + "tokens_seen": 1070202880 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034130391173520567, + "loss": 2.8827, + "theoretical_loss": 3.6258316283456624, + "tokens_seen": 1070268416 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003412938816449348, + "loss": 2.8905, + "theoretical_loss": 3.625810764470402, + "tokens_seen": 1070333952 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034128385155466403, + "loss": 2.9564, + "theoretical_loss": 3.6257899022302578, + "tokens_seen": 1070399488 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034127382146439315, + "loss": 3.0119, + "theoretical_loss": 3.6257690416250012, + "tokens_seen": 1070465024 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003412637913741224, + "loss": 2.7948, + "theoretical_loss": 3.6257481826544047, + "tokens_seen": 1070530560 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034125376128385157, + "loss": 3.0444, + "theoretical_loss": 3.6257273253182394, + "tokens_seen": 1070596096 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034124373119358075, + "loss": 2.8809, + "theoretical_loss": 3.6257064696162766, + "tokens_seen": 1070661632 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034123370110330993, + "loss": 2.9107, + "theoretical_loss": 3.62568561554829, + "tokens_seen": 1070727168 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003412236710130391, + "loss": 2.8435, + "theoretical_loss": 3.6256647631140497, + "tokens_seen": 1070792704 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003412136409227683, + "loss": 2.6885, + "theoretical_loss": 3.625643912313329, + "tokens_seen": 1070858240 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034120361083249753, + "loss": 2.8571, + "theoretical_loss": 3.6256230631458997, + "tokens_seen": 1070923776 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034119358074222666, + "loss": 2.6341, + "theoretical_loss": 3.6256022156115337, + "tokens_seen": 1070989312 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003411835506519559, + "loss": 2.9415, + "theoretical_loss": 3.6255813697100034, + "tokens_seen": 1071054848 + }, + { + "epoch": 0.32, + "learning_rate": 0.000341173520561685, + "loss": 2.8487, + "theoretical_loss": 3.625560525441081, + "tokens_seen": 1071120384 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034116349047141426, + "loss": 2.7576, + "theoretical_loss": 3.625539682804538, + "tokens_seen": 1071185920 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034115346038114344, + "loss": 3.1281, + "theoretical_loss": 3.6255188418001483, + "tokens_seen": 1071251456 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003411434302908726, + "loss": 2.9333, + "theoretical_loss": 3.6254980024276824, + "tokens_seen": 1071316992 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003411334002006018, + "loss": 2.8272, + "theoretical_loss": 3.625477164686914, + "tokens_seen": 1071382528 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034112337011033103, + "loss": 2.5998, + "theoretical_loss": 3.625456328577615, + "tokens_seen": 1071448064 + }, + { + "epoch": 0.32, + "objective/train/docs_used": 611059, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.9438235759735107, + "objective/train/theoretical_loss": 3.6254354940995586, + "objective/train/tokens_used": 1091973600, + "theoretical_loss": 3.6254354940995586, + "tokens_seen": 1071513600 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034111334002006016, + "loss": 2.8028, + "theoretical_loss": 3.6254354940995586, + "tokens_seen": 1071513600 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003411033099297894, + "loss": 2.5908, + "theoretical_loss": 3.625414661252516, + "tokens_seen": 1071579136 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003410932798395185, + "loss": 2.8883, + "theoretical_loss": 3.625393830036261, + "tokens_seen": 1071644672 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034108324974924776, + "loss": 2.8411, + "theoretical_loss": 3.6253730004505655, + "tokens_seen": 1071710208 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034107321965897694, + "loss": 2.9231, + "theoretical_loss": 3.6253521724952025, + "tokens_seen": 1071775744 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003410631895687061, + "loss": 2.6598, + "theoretical_loss": 3.6253313461699452, + "tokens_seen": 1071841280 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003410531594784353, + "loss": 2.8306, + "theoretical_loss": 3.6253105214745647, + "tokens_seen": 1071906816 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003410431293881645, + "loss": 2.7032, + "theoretical_loss": 3.625289698408836, + "tokens_seen": 1071972352 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034103309929789366, + "loss": 3.0398, + "theoretical_loss": 3.6252688769725303, + "tokens_seen": 1072037888 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003410230692076229, + "loss": 3.0151, + "theoretical_loss": 3.625248057165421, + "tokens_seen": 1072103424 + }, + { + "epoch": 0.32, + "learning_rate": 0.000341013039117352, + "loss": 2.6389, + "theoretical_loss": 3.6252272389872813, + "tokens_seen": 1072168960 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034100300902708126, + "loss": 2.8668, + "theoretical_loss": 3.625206422437884, + "tokens_seen": 1072234496 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034099297893681044, + "loss": 2.9389, + "theoretical_loss": 3.625185607517002, + "tokens_seen": 1072300032 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003409829488465396, + "loss": 2.7522, + "theoretical_loss": 3.6251647942244087, + "tokens_seen": 1072365568 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003409729187562688, + "loss": 3.0151, + "theoretical_loss": 3.625143982559877, + "tokens_seen": 1072431104 + }, + { + "epoch": 0.32, + "learning_rate": 0.000340962888665998, + "loss": 2.6466, + "theoretical_loss": 3.62512317252318, + "tokens_seen": 1072496640 + }, + { + "epoch": 0.33, + "learning_rate": 0.00034095285857572717, + "loss": 2.844, + "theoretical_loss": 3.625102364114091, + "tokens_seen": 1072562176 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003409428284854564, + "loss": 2.9951, + "theoretical_loss": 3.6250815573323836, + "tokens_seen": 1072627712 + }, + { + "epoch": 0.33, + "learning_rate": 0.00034093279839518553, + "loss": 2.8237, + "theoretical_loss": 3.6250607521778306, + "tokens_seen": 1072693248 + }, + { + "epoch": 0.33, + "learning_rate": 0.00034092276830491477, + "loss": 2.7825, + "theoretical_loss": 3.6250399486502056, + "tokens_seen": 1072758784 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003409127382146439, + "loss": 2.7819, + "theoretical_loss": 3.6250191467492816, + "tokens_seen": 1072824320 + }, + { + "epoch": 0.33, + "learning_rate": 0.00034090270812437313, + "loss": 2.995, + "theoretical_loss": 3.6249983464748325, + "tokens_seen": 1072889856 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003408926780341023, + "loss": 2.7502, + "theoretical_loss": 3.624977547826632, + "tokens_seen": 1072955392 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003408826479438315, + "loss": 2.7813, + "theoretical_loss": 3.6249567508044533, + "tokens_seen": 1073020928 + }, + { + "epoch": 0.33, + "learning_rate": 0.00034087261785356067, + "loss": 2.8362, + "theoretical_loss": 3.62493595540807, + "tokens_seen": 1073086464 + }, + { + "epoch": 0.33, + "objective/train/docs_used": 612309, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.1632614135742188, + "objective/train/theoretical_loss": 3.6249151616372552, + "objective/train/tokens_used": 1093612000, + "theoretical_loss": 3.6249151616372552, + "tokens_seen": 1073152000 + }, + { + "epoch": 0.33, + "learning_rate": 0.00034086258776328985, + "loss": 2.9813, + "theoretical_loss": 3.6249151616372552, + "tokens_seen": 1073152000 + }, + { + "epoch": 0.33, + "learning_rate": 0.00034085255767301903, + "loss": 2.9443, + "theoretical_loss": 3.6248943694917832, + "tokens_seen": 1073217536 + }, + { + "epoch": 0.33, + "learning_rate": 0.00034084252758274827, + "loss": 2.8603, + "theoretical_loss": 3.624873578971428, + "tokens_seen": 1073283072 + }, + { + "epoch": 0.33, + "learning_rate": 0.00034083249749247745, + "loss": 2.981, + "theoretical_loss": 3.6248527900759626, + "tokens_seen": 1073348608 + }, + { + "epoch": 0.33, + "learning_rate": 0.00034082246740220663, + "loss": 2.7924, + "theoretical_loss": 3.6248320028051615, + "tokens_seen": 1073414144 + }, + { + "epoch": 0.33, + "learning_rate": 0.00034081243731193587, + "loss": 2.6268, + "theoretical_loss": 3.624811217158798, + "tokens_seen": 1073479680 + }, + { + "epoch": 0.33, + "learning_rate": 0.000340802407221665, + "loss": 2.9026, + "theoretical_loss": 3.6247904331366465, + "tokens_seen": 1073545216 + }, + { + "epoch": 0.33, + "learning_rate": 0.00034079237713139423, + "loss": 3.0145, + "theoretical_loss": 3.6247696507384806, + "tokens_seen": 1073610752 + }, + { + "epoch": 0.33, + "learning_rate": 0.00034078234704112335, + "loss": 3.1453, + "theoretical_loss": 3.6247488699640735, + "tokens_seen": 1073676288 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003407723169508526, + "loss": 3.005, + "theoretical_loss": 3.6247280908132007, + "tokens_seen": 1073741824 + }, + { + "epoch": 0.33, + "learning_rate": 0.00034076228686058177, + "loss": 2.9818, + "theoretical_loss": 3.6247073132856364, + "tokens_seen": 1073807360 + }, + { + "epoch": 0.33, + "learning_rate": 0.00034075225677031095, + "loss": 2.9317, + "theoretical_loss": 3.624686537381153, + "tokens_seen": 1073872896 + }, + { + "epoch": 0.33, + "learning_rate": 0.00034074222668004013, + "loss": 2.8471, + "theoretical_loss": 3.624665763099526, + "tokens_seen": 1073938432 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003407321965897693, + "loss": 2.7046, + "theoretical_loss": 3.6246449904405287, + "tokens_seen": 1074003968 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003407221664994985, + "loss": 2.8474, + "theoretical_loss": 3.6246242194039366, + "tokens_seen": 1074069504 + }, + { + "epoch": 0.33, + "learning_rate": 0.00034071213640922773, + "loss": 2.9473, + "theoretical_loss": 3.6246034499895234, + "tokens_seen": 1074135040 + }, + { + "epoch": 0.33, + "learning_rate": 0.00034070210631895686, + "loss": 2.8643, + "theoretical_loss": 3.6245826821970626, + "tokens_seen": 1074200576 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003406920762286861, + "loss": 2.6662, + "theoretical_loss": 3.6245619160263303, + "tokens_seen": 1074266112 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003406820461384152, + "loss": 2.8472, + "theoretical_loss": 3.624541151477099, + "tokens_seen": 1074331648 + }, + { + "epoch": 0.33, + "learning_rate": 0.00034067201604814446, + "loss": 2.8107, + "theoretical_loss": 3.6245203885491453, + "tokens_seen": 1074397184 + }, + { + "epoch": 0.33, + "learning_rate": 0.00034066198595787364, + "loss": 2.9788, + "theoretical_loss": 3.6244996272422414, + "tokens_seen": 1074462720 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003406519558676028, + "loss": 2.6493, + "theoretical_loss": 3.624478867556164, + "tokens_seen": 1074528256 + }, + { + "epoch": 0.33, + "learning_rate": 0.000340641925777332, + "loss": 2.9302, + "theoretical_loss": 3.6244581094906865, + "tokens_seen": 1074593792 + }, + { + "epoch": 0.33, + "learning_rate": 0.00034063189568706123, + "loss": 3.0455, + "theoretical_loss": 3.6244373530455842, + "tokens_seen": 1074659328 + }, + { + "epoch": 0.33, + "learning_rate": 0.00034062186559679036, + "loss": 2.6912, + "theoretical_loss": 3.624416598220631, + "tokens_seen": 1074724864 + }, + { + "epoch": 0.33, + "objective/train/docs_used": 613082, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.9263360500335693, + "objective/train/theoretical_loss": 3.624395845015602, + "objective/train/tokens_used": 1095250400, + "theoretical_loss": 3.624395845015602, + "tokens_seen": 1074790400 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003406118355065196, + "loss": 2.8121, + "theoretical_loss": 3.624395845015602, + "tokens_seen": 1074790400 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003406018054162487, + "loss": 2.6544, + "theoretical_loss": 3.624375093430273, + "tokens_seen": 1074855936 + }, + { + "epoch": 0.33, + "learning_rate": 0.00034059177532597796, + "loss": 2.7314, + "theoretical_loss": 3.624354343464417, + "tokens_seen": 1074921472 + }, + { + "epoch": 0.33, + "learning_rate": 0.00034058174523570714, + "loss": 2.9165, + "theoretical_loss": 3.62433359511781, + "tokens_seen": 1074987008 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003405717151454363, + "loss": 2.8441, + "theoretical_loss": 3.6243128483902276, + "tokens_seen": 1075052544 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003405616850551655, + "loss": 2.9276, + "theoretical_loss": 3.6242921032814435, + "tokens_seen": 1075118080 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003405516549648947, + "loss": 2.7259, + "theoretical_loss": 3.6242713597912335, + "tokens_seen": 1075183616 + }, + { + "epoch": 0.33, + "learning_rate": 0.00034054162487462386, + "loss": 2.6169, + "theoretical_loss": 3.6242506179193716, + "tokens_seen": 1075249152 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003405315947843531, + "loss": 3.0901, + "theoretical_loss": 3.624229877665634, + "tokens_seen": 1075314688 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003405215646940822, + "loss": 3.1089, + "theoretical_loss": 3.624209139029796, + "tokens_seen": 1075380224 + }, + { + "epoch": 0.33, + "learning_rate": 0.00034051153460381146, + "loss": 2.8348, + "theoretical_loss": 3.624188402011632, + "tokens_seen": 1075445760 + }, + { + "epoch": 0.33, + "learning_rate": 0.00034050150451354064, + "loss": 2.8731, + "theoretical_loss": 3.624167666610918, + "tokens_seen": 1075511296 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003404914744232698, + "loss": 2.6944, + "theoretical_loss": 3.6241469328274283, + "tokens_seen": 1075576832 + }, + { + "epoch": 0.33, + "learning_rate": 0.000340481444332999, + "loss": 2.504, + "theoretical_loss": 3.6241262006609394, + "tokens_seen": 1075642368 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003404714142427282, + "loss": 2.9181, + "theoretical_loss": 3.6241054701112256, + "tokens_seen": 1075707904 + }, + { + "epoch": 0.33, + "learning_rate": 0.00034046138415245737, + "loss": 2.816, + "theoretical_loss": 3.6240847411780637, + "tokens_seen": 1075773440 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003404513540621866, + "loss": 2.4249, + "theoretical_loss": 3.6240640138612275, + "tokens_seen": 1075838976 + }, + { + "epoch": 0.33, + "learning_rate": 0.00034044132397191573, + "loss": 2.783, + "theoretical_loss": 3.6240432881604936, + "tokens_seen": 1075904512 + }, + { + "epoch": 0.33, + "learning_rate": 0.00034043129388164497, + "loss": 3.0601, + "theoretical_loss": 3.6240225640756374, + "tokens_seen": 1075970048 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003404212637913741, + "loss": 2.7947, + "theoretical_loss": 3.624001841606434, + "tokens_seen": 1076035584 + }, + { + "epoch": 0.33, + "learning_rate": 0.00034041123370110333, + "loss": 2.6833, + "theoretical_loss": 3.62398112075266, + "tokens_seen": 1076101120 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003404012036108325, + "loss": 2.8068, + "theoretical_loss": 3.6239604015140907, + "tokens_seen": 1076166656 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003403911735205617, + "loss": 3.0181, + "theoretical_loss": 3.6239396838905016, + "tokens_seen": 1076232192 + }, + { + "epoch": 0.33, + "learning_rate": 0.00034038114343029087, + "loss": 2.5246, + "theoretical_loss": 3.623918967881668, + "tokens_seen": 1076297728 + }, + { + "epoch": 0.33, + "learning_rate": 0.00034037111334002005, + "loss": 2.5533, + "theoretical_loss": 3.6238982534873667, + "tokens_seen": 1076363264 + }, + { + "epoch": 0.33, + "objective/train/docs_used": 614240, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.0069870948791504, + "objective/train/theoretical_loss": 3.6238775407073733, + "objective/train/tokens_used": 1096888800, + "theoretical_loss": 3.6238775407073733, + "tokens_seen": 1076428800 + }, + { + "epoch": 0.33, + "learning_rate": 0.00034036108324974923, + "loss": 2.8369, + "theoretical_loss": 3.6238775407073733, + "tokens_seen": 1076428800 + }, + { + "epoch": 0.33, + "learning_rate": 0.00034035105315947847, + "loss": 2.8159, + "theoretical_loss": 3.623856829541464, + "tokens_seen": 1076494336 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003403410230692076, + "loss": 2.7912, + "theoretical_loss": 3.623836119989414, + "tokens_seen": 1076559872 + }, + { + "epoch": 0.33, + "learning_rate": 0.00034033099297893683, + "loss": 2.7323, + "theoretical_loss": 3.6238154120509996, + "tokens_seen": 1076625408 + }, + { + "epoch": 0.33, + "learning_rate": 0.000340320962888666, + "loss": 2.8018, + "theoretical_loss": 3.623794705725997, + "tokens_seen": 1076690944 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003403109327983952, + "loss": 2.986, + "theoretical_loss": 3.6237740010141826, + "tokens_seen": 1076756480 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003403009027081244, + "loss": 2.663, + "theoretical_loss": 3.6237532979153317, + "tokens_seen": 1076822016 + }, + { + "epoch": 0.33, + "learning_rate": 0.00034029087261785356, + "loss": 2.9006, + "theoretical_loss": 3.6237325964292215, + "tokens_seen": 1076887552 + }, + { + "epoch": 0.33, + "learning_rate": 0.00034028084252758274, + "loss": 2.7358, + "theoretical_loss": 3.6237118965556276, + "tokens_seen": 1076953088 + }, + { + "epoch": 0.33, + "learning_rate": 0.00034027081243731197, + "loss": 2.7473, + "theoretical_loss": 3.6236911982943267, + "tokens_seen": 1077018624 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003402607823470411, + "loss": 2.7207, + "theoretical_loss": 3.623670501645094, + "tokens_seen": 1077084160 + }, + { + "epoch": 0.33, + "learning_rate": 0.00034025075225677033, + "loss": 2.5474, + "theoretical_loss": 3.623649806607707, + "tokens_seen": 1077149696 + }, + { + "epoch": 0.33, + "learning_rate": 0.00034024072216649946, + "loss": 2.794, + "theoretical_loss": 3.6236291131819427, + "tokens_seen": 1077215232 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003402306920762287, + "loss": 2.6533, + "theoretical_loss": 3.623608421367576, + "tokens_seen": 1077280768 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003402206619859579, + "loss": 2.6205, + "theoretical_loss": 3.623587731164384, + "tokens_seen": 1077346304 + }, + { + "epoch": 0.33, + "learning_rate": 0.00034021063189568706, + "loss": 2.7893, + "theoretical_loss": 3.6235670425721436, + "tokens_seen": 1077411840 + }, + { + "epoch": 0.33, + "learning_rate": 0.00034020060180541624, + "loss": 2.7813, + "theoretical_loss": 3.623546355590631, + "tokens_seen": 1077477376 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003401905717151454, + "loss": 2.8552, + "theoretical_loss": 3.623525670219623, + "tokens_seen": 1077542912 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003401805416248746, + "loss": 2.8653, + "theoretical_loss": 3.6235049864588964, + "tokens_seen": 1077608448 + }, + { + "epoch": 0.33, + "learning_rate": 0.00034017051153460384, + "loss": 2.703, + "theoretical_loss": 3.623484304308228, + "tokens_seen": 1077673984 + }, + { + "epoch": 0.33, + "learning_rate": 0.00034016048144433296, + "loss": 2.6445, + "theoretical_loss": 3.6234636237673934, + "tokens_seen": 1077739520 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003401504513540622, + "loss": 2.9982, + "theoretical_loss": 3.623442944836171, + "tokens_seen": 1077805056 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003401404212637914, + "loss": 2.9628, + "theoretical_loss": 3.623422267514337, + "tokens_seen": 1077870592 + }, + { + "epoch": 0.33, + "learning_rate": 0.00034013039117352056, + "loss": 2.8934, + "theoretical_loss": 3.6234015918016684, + "tokens_seen": 1077936128 + }, + { + "epoch": 0.33, + "learning_rate": 0.00034012036108324974, + "loss": 2.7416, + "theoretical_loss": 3.6233809176979417, + "tokens_seen": 1078001664 + }, + { + "epoch": 0.33, + "objective/train/docs_used": 615029, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.2141013145446777, + "objective/train/theoretical_loss": 3.6233602452029348, + "objective/train/tokens_used": 1098527200, + "theoretical_loss": 3.6233602452029348, + "tokens_seen": 1078067200 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003401103309929789, + "loss": 2.7761, + "theoretical_loss": 3.6233602452029348, + "tokens_seen": 1078067200 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003401003009027081, + "loss": 2.7931, + "theoretical_loss": 3.6233395743164243, + "tokens_seen": 1078132736 + }, + { + "epoch": 0.33, + "learning_rate": 0.00034009027081243734, + "loss": 2.8483, + "theoretical_loss": 3.623318905038187, + "tokens_seen": 1078198272 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003400802407221665, + "loss": 2.6341, + "theoretical_loss": 3.6232982373679996, + "tokens_seen": 1078263808 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003400702106318957, + "loss": 2.7314, + "theoretical_loss": 3.6232775713056404, + "tokens_seen": 1078329344 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003400601805416249, + "loss": 2.9296, + "theoretical_loss": 3.623256906850886, + "tokens_seen": 1078394880 + }, + { + "epoch": 0.33, + "learning_rate": 0.00034005015045135406, + "loss": 2.8733, + "theoretical_loss": 3.6232362440035137, + "tokens_seen": 1078460416 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003400401203610833, + "loss": 2.7425, + "theoretical_loss": 3.623215582763301, + "tokens_seen": 1078525952 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003400300902708124, + "loss": 2.8198, + "theoretical_loss": 3.623194923130025, + "tokens_seen": 1078591488 + }, + { + "epoch": 0.33, + "learning_rate": 0.00034002006018054166, + "loss": 2.8348, + "theoretical_loss": 3.6231742651034633, + "tokens_seen": 1078657024 + }, + { + "epoch": 0.33, + "learning_rate": 0.00034001003009027084, + "loss": 2.7931, + "theoretical_loss": 3.623153608683393, + "tokens_seen": 1078722560 + }, + { + "epoch": 0.33, + "learning_rate": 0.00034, + "loss": 2.6992, + "theoretical_loss": 3.623132953869592, + "tokens_seen": 1078788096 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003399899699097292, + "loss": 2.7412, + "theoretical_loss": 3.6231123006618375, + "tokens_seen": 1078853632 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003399799398194584, + "loss": 3.0068, + "theoretical_loss": 3.6230916490599077, + "tokens_seen": 1078919168 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033996990972918757, + "loss": 2.9322, + "theoretical_loss": 3.623070999063579, + "tokens_seen": 1078984704 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003399598796389168, + "loss": 2.7205, + "theoretical_loss": 3.6230503506726306, + "tokens_seen": 1079050240 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033994984954864593, + "loss": 2.8571, + "theoretical_loss": 3.623029703886839, + "tokens_seen": 1079115776 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033993981945837517, + "loss": 2.9509, + "theoretical_loss": 3.623009058705982, + "tokens_seen": 1079181312 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003399297893681043, + "loss": 2.7831, + "theoretical_loss": 3.622988415129838, + "tokens_seen": 1079246848 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033991975927783353, + "loss": 2.647, + "theoretical_loss": 3.6229677731581846, + "tokens_seen": 1079312384 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003399097291875627, + "loss": 2.7705, + "theoretical_loss": 3.6229471327907996, + "tokens_seen": 1079377920 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003398996990972919, + "loss": 2.7651, + "theoretical_loss": 3.6229264940274613, + "tokens_seen": 1079443456 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033988966900702107, + "loss": 2.7049, + "theoretical_loss": 3.6229058568679466, + "tokens_seen": 1079508992 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033987963891675025, + "loss": 2.7955, + "theoretical_loss": 3.6228852213120346, + "tokens_seen": 1079574528 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033986960882647943, + "loss": 2.8825, + "theoretical_loss": 3.622864587359503, + "tokens_seen": 1079640064 + }, + { + "epoch": 0.33, + "objective/train/docs_used": 616160, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6415908336639404, + "objective/train/theoretical_loss": 3.62284395501013, + "objective/train/tokens_used": 1100165600, + "theoretical_loss": 3.62284395501013, + "tokens_seen": 1079705600 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033985957873620867, + "loss": 2.8522, + "theoretical_loss": 3.62284395501013, + "tokens_seen": 1079705600 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003398495486459378, + "loss": 2.6656, + "theoretical_loss": 3.622823324263694, + "tokens_seen": 1079771136 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033983951855566703, + "loss": 2.5958, + "theoretical_loss": 3.6228026951199723, + "tokens_seen": 1079836672 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003398294884653962, + "loss": 2.7387, + "theoretical_loss": 3.6227820675787443, + "tokens_seen": 1079902208 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003398194583751254, + "loss": 2.8057, + "theoretical_loss": 3.6227614416397866, + "tokens_seen": 1079967744 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003398094282848546, + "loss": 2.6636, + "theoretical_loss": 3.6227408173028794, + "tokens_seen": 1080033280 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033979939819458376, + "loss": 2.8359, + "theoretical_loss": 3.6227201945678003, + "tokens_seen": 1080098816 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033978936810431294, + "loss": 2.864, + "theoretical_loss": 3.6226995734343275, + "tokens_seen": 1080164352 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033977933801404217, + "loss": 2.6828, + "theoretical_loss": 3.6226789539022395, + "tokens_seen": 1080229888 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003397693079237713, + "loss": 2.9217, + "theoretical_loss": 3.6226583359713147, + "tokens_seen": 1080295424 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033975927783350053, + "loss": 2.7289, + "theoretical_loss": 3.622637719641332, + "tokens_seen": 1080360960 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033974924774322966, + "loss": 3.0248, + "theoretical_loss": 3.6226171049120697, + "tokens_seen": 1080426496 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003397392176529589, + "loss": 2.8641, + "theoretical_loss": 3.6225964917833067, + "tokens_seen": 1080492032 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003397291875626881, + "loss": 2.8577, + "theoretical_loss": 3.6225758802548214, + "tokens_seen": 1080557568 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033971915747241726, + "loss": 2.9925, + "theoretical_loss": 3.622555270326393, + "tokens_seen": 1080623104 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033970912738214644, + "loss": 2.7394, + "theoretical_loss": 3.6225346619977996, + "tokens_seen": 1080688640 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003396990972918756, + "loss": 2.779, + "theoretical_loss": 3.6225140552688204, + "tokens_seen": 1080754176 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003396890672016048, + "loss": 2.5084, + "theoretical_loss": 3.6224934501392343, + "tokens_seen": 1080819712 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033967903711133404, + "loss": 2.745, + "theoretical_loss": 3.62247284660882, + "tokens_seen": 1080885248 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033966900702106316, + "loss": 3.0291, + "theoretical_loss": 3.6224522446773566, + "tokens_seen": 1080950784 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003396589769307924, + "loss": 2.8396, + "theoretical_loss": 3.6224316443446227, + "tokens_seen": 1081016320 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003396489468405216, + "loss": 2.6911, + "theoretical_loss": 3.6224110456103977, + "tokens_seen": 1081081856 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033963891675025076, + "loss": 2.7142, + "theoretical_loss": 3.622390448474461, + "tokens_seen": 1081147392 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033962888665997994, + "loss": 2.8434, + "theoretical_loss": 3.6223698529365906, + "tokens_seen": 1081212928 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003396188565697091, + "loss": 2.8612, + "theoretical_loss": 3.622349258996567, + "tokens_seen": 1081278464 + }, + { + "epoch": 0.33, + "objective/train/docs_used": 616786, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.5360734462738037, + "objective/train/theoretical_loss": 3.6223286666541683, + "objective/train/tokens_used": 1101804000, + "theoretical_loss": 3.6223286666541683, + "tokens_seen": 1081344000 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003396088264794383, + "loss": 2.7399, + "theoretical_loss": 3.6223286666541683, + "tokens_seen": 1081344000 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033959879638916754, + "loss": 2.8223, + "theoretical_loss": 3.6223080759091744, + "tokens_seen": 1081409536 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033958876629889667, + "loss": 2.9926, + "theoretical_loss": 3.6222874867613646, + "tokens_seen": 1081475072 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003395787362086259, + "loss": 2.9275, + "theoretical_loss": 3.6222668992105174, + "tokens_seen": 1081540608 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033956870611835503, + "loss": 2.8446, + "theoretical_loss": 3.6222463132564133, + "tokens_seen": 1081606144 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033955867602808426, + "loss": 2.855, + "theoretical_loss": 3.6222257288988313, + "tokens_seen": 1081671680 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033954864593781345, + "loss": 2.6914, + "theoretical_loss": 3.62220514613755, + "tokens_seen": 1081737216 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003395386158475426, + "loss": 2.9186, + "theoretical_loss": 3.6221845649723505, + "tokens_seen": 1081802752 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003395285857572718, + "loss": 2.8569, + "theoretical_loss": 3.6221639854030117, + "tokens_seen": 1081868288 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033951855566700104, + "loss": 2.9483, + "theoretical_loss": 3.6221434074293124, + "tokens_seen": 1081933824 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033950852557673017, + "loss": 2.8563, + "theoretical_loss": 3.6221228310510334, + "tokens_seen": 1081999360 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003394984954864594, + "loss": 2.7528, + "theoretical_loss": 3.6221022562679535, + "tokens_seen": 1082064896 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033948846539618853, + "loss": 3.0155, + "theoretical_loss": 3.6220816830798532, + "tokens_seen": 1082130432 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033947843530591777, + "loss": 2.8725, + "theoretical_loss": 3.6220611114865116, + "tokens_seen": 1082195968 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033946840521564695, + "loss": 2.9052, + "theoretical_loss": 3.622040541487709, + "tokens_seen": 1082261504 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033945837512537613, + "loss": 2.9108, + "theoretical_loss": 3.6220199730832245, + "tokens_seen": 1082327040 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003394483450351053, + "loss": 2.766, + "theoretical_loss": 3.621999406272839, + "tokens_seen": 1082392576 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003394383149448345, + "loss": 2.8041, + "theoretical_loss": 3.6219788410563316, + "tokens_seen": 1082458112 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003394282848545637, + "loss": 2.8635, + "theoretical_loss": 3.6219582774334835, + "tokens_seen": 1082523648 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003394182547642929, + "loss": 2.7891, + "theoretical_loss": 3.6219377154040733, + "tokens_seen": 1082589184 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033940822467402204, + "loss": 2.7864, + "theoretical_loss": 3.621917154967882, + "tokens_seen": 1082654720 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033939819458375127, + "loss": 2.9599, + "theoretical_loss": 3.6218965961246887, + "tokens_seen": 1082720256 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003393881644934804, + "loss": 2.9539, + "theoretical_loss": 3.621876038874275, + "tokens_seen": 1082785792 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033937813440320963, + "loss": 2.9477, + "theoretical_loss": 3.6218554832164203, + "tokens_seen": 1082851328 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003393681043129388, + "loss": 2.7347, + "theoretical_loss": 3.621834929150905, + "tokens_seen": 1082916864 + }, + { + "epoch": 0.33, + "objective/train/docs_used": 617451, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.633185386657715, + "objective/train/theoretical_loss": 3.6218143766775093, + "objective/train/tokens_used": 1103442400, + "theoretical_loss": 3.6218143766775093, + "tokens_seen": 1082982400 + }, + { + "epoch": 0.33, + "learning_rate": 0.000339358074222668, + "loss": 2.4618, + "theoretical_loss": 3.6218143766775093, + "tokens_seen": 1082982400 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003393480441323972, + "loss": 2.6262, + "theoretical_loss": 3.621793825796013, + "tokens_seen": 1083047936 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003393380140421264, + "loss": 3.1751, + "theoretical_loss": 3.6217732765061976, + "tokens_seen": 1083113472 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003393279839518556, + "loss": 3.0513, + "theoretical_loss": 3.621752728807843, + "tokens_seen": 1083179008 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003393179538615848, + "loss": 2.8251, + "theoretical_loss": 3.62173218270073, + "tokens_seen": 1083244544 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033930792377131396, + "loss": 2.9146, + "theoretical_loss": 3.621711638184638, + "tokens_seen": 1083310080 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033929789368104314, + "loss": 2.8361, + "theoretical_loss": 3.6216910952593486, + "tokens_seen": 1083375616 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033928786359077237, + "loss": 2.8677, + "theoretical_loss": 3.621670553924642, + "tokens_seen": 1083441152 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003392778335005015, + "loss": 2.9648, + "theoretical_loss": 3.6216500141802994, + "tokens_seen": 1083506688 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033926780341023073, + "loss": 2.7732, + "theoretical_loss": 3.6216294760261007, + "tokens_seen": 1083572224 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033925777331995986, + "loss": 2.8369, + "theoretical_loss": 3.621608939461827, + "tokens_seen": 1083637760 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003392477432296891, + "loss": 2.7674, + "theoretical_loss": 3.6215884044872593, + "tokens_seen": 1083703296 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003392377131394183, + "loss": 2.8257, + "theoretical_loss": 3.621567871102178, + "tokens_seen": 1083768832 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033922768304914746, + "loss": 2.8896, + "theoretical_loss": 3.6215473393063644, + "tokens_seen": 1083834368 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033921765295887664, + "loss": 2.8236, + "theoretical_loss": 3.621526809099599, + "tokens_seen": 1083899904 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003392076228686058, + "loss": 2.7537, + "theoretical_loss": 3.621506280481663, + "tokens_seen": 1083965440 + }, + { + "epoch": 0.33, + "learning_rate": 0.000339197592778335, + "loss": 2.6675, + "theoretical_loss": 3.621485753452337, + "tokens_seen": 1084030976 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033918756268806424, + "loss": 2.9946, + "theoretical_loss": 3.6214652280114024, + "tokens_seen": 1084096512 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033917753259779336, + "loss": 2.6778, + "theoretical_loss": 3.62144470415864, + "tokens_seen": 1084162048 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003391675025075226, + "loss": 2.8716, + "theoretical_loss": 3.6214241818938318, + "tokens_seen": 1084227584 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003391574724172518, + "loss": 2.9214, + "theoretical_loss": 3.621403661216758, + "tokens_seen": 1084293120 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033914744232698096, + "loss": 2.7397, + "theoretical_loss": 3.6213831421272005, + "tokens_seen": 1084358656 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033913741223671014, + "loss": 2.8811, + "theoretical_loss": 3.6213626246249393, + "tokens_seen": 1084424192 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003391273821464393, + "loss": 2.7686, + "theoretical_loss": 3.621342108709757, + "tokens_seen": 1084489728 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003391173520561685, + "loss": 2.9005, + "theoretical_loss": 3.6213215943814347, + "tokens_seen": 1084555264 + }, + { + "epoch": 0.33, + "objective/train/docs_used": 618647, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.9332821369171143, + "objective/train/theoretical_loss": 3.621301081639753, + "objective/train/tokens_used": 1105080800, + "theoretical_loss": 3.621301081639753, + "tokens_seen": 1084620800 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033910732196589774, + "loss": 2.836, + "theoretical_loss": 3.621301081639753, + "tokens_seen": 1084620800 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033909729187562687, + "loss": 2.7247, + "theoretical_loss": 3.621280570484494, + "tokens_seen": 1084686336 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003390872617853561, + "loss": 2.7581, + "theoretical_loss": 3.6212600609154393, + "tokens_seen": 1084751872 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033907723169508523, + "loss": 2.7286, + "theoretical_loss": 3.62123955293237, + "tokens_seen": 1084817408 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033906720160481446, + "loss": 2.6939, + "theoretical_loss": 3.621219046535068, + "tokens_seen": 1084882944 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033905717151454365, + "loss": 2.6277, + "theoretical_loss": 3.6211985417233157, + "tokens_seen": 1084948480 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033904714142427283, + "loss": 2.8194, + "theoretical_loss": 3.6211780384968923, + "tokens_seen": 1085014016 + }, + { + "epoch": 0.33, + "learning_rate": 0.000339037111334002, + "loss": 2.8098, + "theoretical_loss": 3.6211575368555815, + "tokens_seen": 1085079552 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033902708124373124, + "loss": 2.8801, + "theoretical_loss": 3.6211370367991647, + "tokens_seen": 1085145088 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033901705115346037, + "loss": 2.7294, + "theoretical_loss": 3.6211165383274233, + "tokens_seen": 1085210624 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003390070210631896, + "loss": 2.8037, + "theoretical_loss": 3.621096041440139, + "tokens_seen": 1085276160 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033899699097291873, + "loss": 2.8804, + "theoretical_loss": 3.621075546137094, + "tokens_seen": 1085341696 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033898696088264797, + "loss": 2.7322, + "theoretical_loss": 3.621055052418071, + "tokens_seen": 1085407232 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033897693079237715, + "loss": 2.8731, + "theoretical_loss": 3.62103456028285, + "tokens_seen": 1085472768 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033896690070210633, + "loss": 2.9282, + "theoretical_loss": 3.6210140697312143, + "tokens_seen": 1085538304 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003389568706118355, + "loss": 2.6806, + "theoretical_loss": 3.6209935807629465, + "tokens_seen": 1085603840 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003389468405215647, + "loss": 2.6054, + "theoretical_loss": 3.620973093377827, + "tokens_seen": 1085669376 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003389368104312939, + "loss": 2.8093, + "theoretical_loss": 3.620952607575639, + "tokens_seen": 1085734912 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003389267803410231, + "loss": 2.4797, + "theoretical_loss": 3.6209321233561647, + "tokens_seen": 1085800448 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033891675025075224, + "loss": 2.8042, + "theoretical_loss": 3.6209116407191857, + "tokens_seen": 1085865984 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033890672016048147, + "loss": 2.8362, + "theoretical_loss": 3.620891159664485, + "tokens_seen": 1085931520 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003388966900702106, + "loss": 2.6543, + "theoretical_loss": 3.620870680191844, + "tokens_seen": 1085997056 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033888665997993983, + "loss": 2.8688, + "theoretical_loss": 3.620850202301046, + "tokens_seen": 1086062592 + }, + { + "epoch": 0.33, + "learning_rate": 0.000338876629889669, + "loss": 2.8394, + "theoretical_loss": 3.620829725991873, + "tokens_seen": 1086128128 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003388665997993982, + "loss": 2.6662, + "theoretical_loss": 3.620809251264107, + "tokens_seen": 1086193664 + }, + { + "epoch": 0.33, + "objective/train/docs_used": 619458, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.568040132522583, + "objective/train/theoretical_loss": 3.6207887781175305, + "objective/train/tokens_used": 1106719200, + "theoretical_loss": 3.6207887781175305, + "tokens_seen": 1086259200 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003388565697091274, + "loss": 2.7664, + "theoretical_loss": 3.6207887781175305, + "tokens_seen": 1086259200 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003388465396188566, + "loss": 2.7643, + "theoretical_loss": 3.620768306551927, + "tokens_seen": 1086324736 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033883650952858574, + "loss": 2.7177, + "theoretical_loss": 3.620747836567078, + "tokens_seen": 1086390272 + }, + { + "epoch": 0.33, + "learning_rate": 0.000338826479438315, + "loss": 2.6571, + "theoretical_loss": 3.6207273681627665, + "tokens_seen": 1086455808 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003388164493480441, + "loss": 2.8876, + "theoretical_loss": 3.620706901338775, + "tokens_seen": 1086521344 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033880641925777334, + "loss": 3.1274, + "theoretical_loss": 3.6206864360948865, + "tokens_seen": 1086586880 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003387963891675025, + "loss": 2.7335, + "theoretical_loss": 3.620665972430883, + "tokens_seen": 1086652416 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003387863590772317, + "loss": 2.6078, + "theoretical_loss": 3.620645510346548, + "tokens_seen": 1086717952 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003387763289869609, + "loss": 2.9976, + "theoretical_loss": 3.620625049841664, + "tokens_seen": 1086783488 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033876629889669006, + "loss": 2.9847, + "theoretical_loss": 3.620604590916014, + "tokens_seen": 1086849024 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033875626880641924, + "loss": 3.001, + "theoretical_loss": 3.6205841335693805, + "tokens_seen": 1086914560 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003387462387161485, + "loss": 2.8169, + "theoretical_loss": 3.620563677801547, + "tokens_seen": 1086980096 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003387362086258776, + "loss": 2.6679, + "theoretical_loss": 3.6205432236122963, + "tokens_seen": 1087045632 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033872617853560684, + "loss": 2.9451, + "theoretical_loss": 3.620522771001411, + "tokens_seen": 1087111168 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033871614844533597, + "loss": 2.786, + "theoretical_loss": 3.6205023199686743, + "tokens_seen": 1087176704 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003387061183550652, + "loss": 2.9861, + "theoretical_loss": 3.6204818705138697, + "tokens_seen": 1087242240 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003386960882647944, + "loss": 2.7302, + "theoretical_loss": 3.6204614226367804, + "tokens_seen": 1087307776 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033868605817452356, + "loss": 2.9137, + "theoretical_loss": 3.620440976337189, + "tokens_seen": 1087373312 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033867602808425275, + "loss": 2.8122, + "theoretical_loss": 3.6204205316148794, + "tokens_seen": 1087438848 + }, + { + "epoch": 0.33, + "learning_rate": 0.000338665997993982, + "loss": 2.8908, + "theoretical_loss": 3.6204000884696343, + "tokens_seen": 1087504384 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003386559679037111, + "loss": 2.9411, + "theoretical_loss": 3.6203796469012373, + "tokens_seen": 1087569920 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033864593781344034, + "loss": 2.9194, + "theoretical_loss": 3.6203592069094714, + "tokens_seen": 1087635456 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033863590772316947, + "loss": 2.6298, + "theoretical_loss": 3.620338768494121, + "tokens_seen": 1087700992 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003386258776328987, + "loss": 2.7466, + "theoretical_loss": 3.6203183316549685, + "tokens_seen": 1087766528 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003386158475426279, + "loss": 2.6553, + "theoretical_loss": 3.620297896391798, + "tokens_seen": 1087832064 + }, + { + "epoch": 0.33, + "objective/train/docs_used": 620460, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.974976062774658, + "objective/train/theoretical_loss": 3.6202774627043923, + "objective/train/tokens_used": 1108357600, + "theoretical_loss": 3.6202774627043923, + "tokens_seen": 1087897600 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033860581745235707, + "loss": 2.8833, + "theoretical_loss": 3.6202774627043923, + "tokens_seen": 1087897600 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033859578736208625, + "loss": 2.824, + "theoretical_loss": 3.620257030592536, + "tokens_seen": 1087963136 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033858575727181543, + "loss": 2.8824, + "theoretical_loss": 3.6202366000560118, + "tokens_seen": 1088028672 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033857572718154467, + "loss": 3.1579, + "theoretical_loss": 3.6202161710946044, + "tokens_seen": 1088094208 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033856569709127385, + "loss": 2.7135, + "theoretical_loss": 3.620195743708096, + "tokens_seen": 1088159744 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033855566700100303, + "loss": 2.7726, + "theoretical_loss": 3.6201753178962717, + "tokens_seen": 1088225280 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003385456369107322, + "loss": 2.8389, + "theoretical_loss": 3.620154893658915, + "tokens_seen": 1088290816 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033853560682046144, + "loss": 2.7851, + "theoretical_loss": 3.620134470995809, + "tokens_seen": 1088356352 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033852557673019057, + "loss": 2.8981, + "theoretical_loss": 3.620114049906739, + "tokens_seen": 1088421888 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003385155466399198, + "loss": 2.7789, + "theoretical_loss": 3.6200936303914872, + "tokens_seen": 1088487424 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033850551654964893, + "loss": 2.608, + "theoretical_loss": 3.6200732124498387, + "tokens_seen": 1088552960 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033849548645937817, + "loss": 2.953, + "theoretical_loss": 3.6200527960815774, + "tokens_seen": 1088618496 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033848545636910735, + "loss": 2.9394, + "theoretical_loss": 3.620032381286487, + "tokens_seen": 1088684032 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033847542627883653, + "loss": 2.8162, + "theoretical_loss": 3.6200119680643517, + "tokens_seen": 1088749568 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003384653961885657, + "loss": 2.8363, + "theoretical_loss": 3.6199915564149556, + "tokens_seen": 1088815104 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003384553660982949, + "loss": 2.7672, + "theoretical_loss": 3.6199711463380835, + "tokens_seen": 1088880640 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003384453360080241, + "loss": 2.8507, + "theoretical_loss": 3.6199507378335185, + "tokens_seen": 1088946176 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003384353059177533, + "loss": 2.9751, + "theoretical_loss": 3.6199303309010458, + "tokens_seen": 1089011712 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033842527582748244, + "loss": 2.5372, + "theoretical_loss": 3.6199099255404494, + "tokens_seen": 1089077248 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033841524573721167, + "loss": 2.9739, + "theoretical_loss": 3.6198895217515132, + "tokens_seen": 1089142784 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003384052156469408, + "loss": 2.8163, + "theoretical_loss": 3.6198691195340222, + "tokens_seen": 1089208320 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033839518555667003, + "loss": 2.8807, + "theoretical_loss": 3.6198487188877606, + "tokens_seen": 1089273856 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003383851554663992, + "loss": 2.7718, + "theoretical_loss": 3.619828319812513, + "tokens_seen": 1089339392 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003383751253761284, + "loss": 2.697, + "theoretical_loss": 3.6198079223080635, + "tokens_seen": 1089404928 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003383650952858576, + "loss": 2.7297, + "theoretical_loss": 3.619787526374197, + "tokens_seen": 1089470464 + }, + { + "epoch": 0.33, + "objective/train/docs_used": 620976, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.815063953399658, + "objective/train/theoretical_loss": 3.6197671320106983, + "objective/train/tokens_used": 1109996000, + "theoretical_loss": 3.6197671320106983, + "tokens_seen": 1089536000 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003383550651955868, + "loss": 2.8682, + "theoretical_loss": 3.6197671320106983, + "tokens_seen": 1089536000 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033834503510531594, + "loss": 2.7622, + "theoretical_loss": 3.6197467392173515, + "tokens_seen": 1089601536 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003383350050150452, + "loss": 2.9224, + "theoretical_loss": 3.619726347993942, + "tokens_seen": 1089667072 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003383249749247743, + "loss": 2.8569, + "theoretical_loss": 3.619705958340254, + "tokens_seen": 1089732608 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033831494483450354, + "loss": 2.8069, + "theoretical_loss": 3.619685570256072, + "tokens_seen": 1089798144 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003383049147442327, + "loss": 3.0527, + "theoretical_loss": 3.6196651837411817, + "tokens_seen": 1089863680 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003382948846539619, + "loss": 2.5986, + "theoretical_loss": 3.6196447987953677, + "tokens_seen": 1089929216 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003382848545636911, + "loss": 2.7295, + "theoretical_loss": 3.619624415418414, + "tokens_seen": 1089994752 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033827482447342026, + "loss": 2.602, + "theoretical_loss": 3.6196040336101065, + "tokens_seen": 1090060288 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033826479438314944, + "loss": 2.8757, + "theoretical_loss": 3.6195836533702295, + "tokens_seen": 1090125824 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003382547642928787, + "loss": 2.8242, + "theoretical_loss": 3.619563274698569, + "tokens_seen": 1090191360 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003382447342026078, + "loss": 2.8484, + "theoretical_loss": 3.619542897594909, + "tokens_seen": 1090256896 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033823470411233704, + "loss": 2.7804, + "theoretical_loss": 3.619522522059035, + "tokens_seen": 1090322432 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033822467402206617, + "loss": 3.0528, + "theoretical_loss": 3.619502148090733, + "tokens_seen": 1090387968 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003382146439317954, + "loss": 2.8757, + "theoretical_loss": 3.6194817756897866, + "tokens_seen": 1090453504 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003382046138415246, + "loss": 2.7916, + "theoretical_loss": 3.6194614048559828, + "tokens_seen": 1090519040 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033819458375125376, + "loss": 2.8731, + "theoretical_loss": 3.619441035589105, + "tokens_seen": 1090584576 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033818455366098295, + "loss": 2.7836, + "theoretical_loss": 3.6194206678889396, + "tokens_seen": 1090650112 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003381745235707122, + "loss": 2.7583, + "theoretical_loss": 3.619400301755272, + "tokens_seen": 1090715648 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003381644934804413, + "loss": 2.7524, + "theoretical_loss": 3.6193799371878876, + "tokens_seen": 1090781184 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033815446339017054, + "loss": 2.8784, + "theoretical_loss": 3.619359574186572, + "tokens_seen": 1090846720 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033814443329989967, + "loss": 2.8183, + "theoretical_loss": 3.6193392127511097, + "tokens_seen": 1090912256 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003381344032096289, + "loss": 2.6517, + "theoretical_loss": 3.619318852881287, + "tokens_seen": 1090977792 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003381243731193581, + "loss": 2.7468, + "theoretical_loss": 3.61929849457689, + "tokens_seen": 1091043328 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033811434302908727, + "loss": 2.7003, + "theoretical_loss": 3.619278137837703, + "tokens_seen": 1091108864 + }, + { + "epoch": 0.33, + "objective/train/docs_used": 621707, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.5296897888183594, + "objective/train/theoretical_loss": 3.619257782663513, + "objective/train/tokens_used": 1111634400, + "theoretical_loss": 3.619257782663513, + "tokens_seen": 1091174400 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033810431293881645, + "loss": 2.7684, + "theoretical_loss": 3.619257782663513, + "tokens_seen": 1091174400 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033809428284854563, + "loss": 2.8331, + "theoretical_loss": 3.6192374290541047, + "tokens_seen": 1091239936 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003380842527582748, + "loss": 3.0038, + "theoretical_loss": 3.619217077009264, + "tokens_seen": 1091305472 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033807422266800405, + "loss": 2.7942, + "theoretical_loss": 3.619196726528777, + "tokens_seen": 1091371008 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003380641925777332, + "loss": 2.9958, + "theoretical_loss": 3.6191763776124297, + "tokens_seen": 1091436544 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003380541624874624, + "loss": 2.9046, + "theoretical_loss": 3.619156030260008, + "tokens_seen": 1091502080 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033804413239719154, + "loss": 2.8238, + "theoretical_loss": 3.619135684471297, + "tokens_seen": 1091567616 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033803410230692077, + "loss": 2.8689, + "theoretical_loss": 3.6191153402460836, + "tokens_seen": 1091633152 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033802407221664995, + "loss": 2.6574, + "theoretical_loss": 3.6190949975841527, + "tokens_seen": 1091698688 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033801404212637913, + "loss": 2.6205, + "theoretical_loss": 3.6190746564852914, + "tokens_seen": 1091764224 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003380040120361083, + "loss": 2.8344, + "theoretical_loss": 3.6190543169492857, + "tokens_seen": 1091829760 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033799398194583755, + "loss": 2.8608, + "theoretical_loss": 3.6190339789759207, + "tokens_seen": 1091895296 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003379839518555667, + "loss": 2.6476, + "theoretical_loss": 3.6190136425649837, + "tokens_seen": 1091960832 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003379739217652959, + "loss": 2.7307, + "theoretical_loss": 3.6189933077162606, + "tokens_seen": 1092026368 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033796389167502504, + "loss": 2.6518, + "theoretical_loss": 3.6189729744295374, + "tokens_seen": 1092091904 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003379538615847543, + "loss": 2.8566, + "theoretical_loss": 3.618952642704601, + "tokens_seen": 1092157440 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033794383149448346, + "loss": 3.009, + "theoretical_loss": 3.6189323125412365, + "tokens_seen": 1092222976 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033793380140421264, + "loss": 2.5639, + "theoretical_loss": 3.618911983939231, + "tokens_seen": 1092288512 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003379237713139418, + "loss": 2.8416, + "theoretical_loss": 3.618891656898372, + "tokens_seen": 1092354048 + }, + { + "epoch": 0.33, + "learning_rate": 0.000337913741223671, + "loss": 2.7869, + "theoretical_loss": 3.618871331418444, + "tokens_seen": 1092419584 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003379037111334002, + "loss": 2.9362, + "theoretical_loss": 3.6188510074992344, + "tokens_seen": 1092485120 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003378936810431294, + "loss": 2.7854, + "theoretical_loss": 3.61883068514053, + "tokens_seen": 1092550656 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033788365095285854, + "loss": 2.6472, + "theoretical_loss": 3.6188103643421172, + "tokens_seen": 1092616192 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003378736208625878, + "loss": 2.9506, + "theoretical_loss": 3.6187900451037827, + "tokens_seen": 1092681728 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003378635907723169, + "loss": 2.865, + "theoretical_loss": 3.618769727425313, + "tokens_seen": 1092747264 + }, + { + "epoch": 0.33, + "objective/train/docs_used": 622698, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7957966327667236, + "objective/train/theoretical_loss": 3.618749411306495, + "objective/train/tokens_used": 1113272800, + "theoretical_loss": 3.618749411306495, + "tokens_seen": 1092812800 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033785356068204614, + "loss": 2.9377, + "theoretical_loss": 3.618749411306495, + "tokens_seen": 1092812800 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003378435305917753, + "loss": 2.8781, + "theoretical_loss": 3.618729096747115, + "tokens_seen": 1092878336 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003378335005015045, + "loss": 2.8333, + "theoretical_loss": 3.6187087837469605, + "tokens_seen": 1092943872 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033782347041123374, + "loss": 2.8423, + "theoretical_loss": 3.6186884723058172, + "tokens_seen": 1093009408 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003378134403209629, + "loss": 2.6144, + "theoretical_loss": 3.6186681624234733, + "tokens_seen": 1093074944 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003378034102306921, + "loss": 2.9321, + "theoretical_loss": 3.6186478540997156, + "tokens_seen": 1093140480 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003377933801404213, + "loss": 2.7128, + "theoretical_loss": 3.61862754733433, + "tokens_seen": 1093206016 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033778335005015046, + "loss": 2.746, + "theoretical_loss": 3.618607242127104, + "tokens_seen": 1093271552 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033777331995987964, + "loss": 2.8417, + "theoretical_loss": 3.6185869384778253, + "tokens_seen": 1093337088 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003377632898696089, + "loss": 2.9675, + "theoretical_loss": 3.61856663638628, + "tokens_seen": 1093402624 + }, + { + "epoch": 0.33, + "learning_rate": 0.000337753259779338, + "loss": 2.8952, + "theoretical_loss": 3.6185463358522565, + "tokens_seen": 1093468160 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033774322968906724, + "loss": 2.9549, + "theoretical_loss": 3.61852603687554, + "tokens_seen": 1093533696 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033773319959879637, + "loss": 3.0052, + "theoretical_loss": 3.61850573945592, + "tokens_seen": 1093599232 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003377231695085256, + "loss": 2.5477, + "theoretical_loss": 3.618485443593182, + "tokens_seen": 1093664768 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003377131394182548, + "loss": 2.933, + "theoretical_loss": 3.618465149287114, + "tokens_seen": 1093730304 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033770310932798396, + "loss": 2.82, + "theoretical_loss": 3.6184448565375034, + "tokens_seen": 1093795840 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033769307923771315, + "loss": 2.8009, + "theoretical_loss": 3.6184245653441374, + "tokens_seen": 1093861376 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003376830491474424, + "loss": 2.7675, + "theoretical_loss": 3.6184042757068036, + "tokens_seen": 1093926912 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003376730190571715, + "loss": 2.8857, + "theoretical_loss": 3.6183839876252892, + "tokens_seen": 1093992448 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033766298896690074, + "loss": 2.6906, + "theoretical_loss": 3.618363701099382, + "tokens_seen": 1094057984 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033765295887662987, + "loss": 2.992, + "theoretical_loss": 3.6183434161288694, + "tokens_seen": 1094123520 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003376429287863591, + "loss": 2.7279, + "theoretical_loss": 3.618323132713539, + "tokens_seen": 1094189056 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003376328986960883, + "loss": 2.8448, + "theoretical_loss": 3.6183028508531785, + "tokens_seen": 1094254592 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033762286860581747, + "loss": 2.7623, + "theoretical_loss": 3.618282570547575, + "tokens_seen": 1094320128 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033761283851554665, + "loss": 2.6394, + "theoretical_loss": 3.6182622917965173, + "tokens_seen": 1094385664 + }, + { + "epoch": 0.33, + "objective/train/docs_used": 623416, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.533064365386963, + "objective/train/theoretical_loss": 3.618242014599793, + "objective/train/tokens_used": 1114911200, + "theoretical_loss": 3.618242014599793, + "tokens_seen": 1094451200 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033760280842527583, + "loss": 2.7738, + "theoretical_loss": 3.618242014599793, + "tokens_seen": 1094451200 + }, + { + "epoch": 0.33, + "learning_rate": 0.000337592778335005, + "loss": 2.8115, + "theoretical_loss": 3.618221738957189, + "tokens_seen": 1094516736 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033758274824473425, + "loss": 2.6558, + "theoretical_loss": 3.618201464868493, + "tokens_seen": 1094582272 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003375727181544634, + "loss": 2.702, + "theoretical_loss": 3.6181811923334943, + "tokens_seen": 1094647808 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003375626880641926, + "loss": 2.8236, + "theoretical_loss": 3.61816092135198, + "tokens_seen": 1094713344 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033755265797392174, + "loss": 2.9504, + "theoretical_loss": 3.6181406519237376, + "tokens_seen": 1094778880 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033754262788365097, + "loss": 2.7125, + "theoretical_loss": 3.618120384048556, + "tokens_seen": 1094844416 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033753259779338015, + "loss": 2.8418, + "theoretical_loss": 3.618100117726223, + "tokens_seen": 1094909952 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033752256770310933, + "loss": 2.7961, + "theoretical_loss": 3.6180798529565266, + "tokens_seen": 1094975488 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003375125376128385, + "loss": 2.978, + "theoretical_loss": 3.6180595897392545, + "tokens_seen": 1095041024 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033750250752256775, + "loss": 2.8671, + "theoretical_loss": 3.6180393280741954, + "tokens_seen": 1095106560 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003374924774322969, + "loss": 2.6766, + "theoretical_loss": 3.6180190679611375, + "tokens_seen": 1095172096 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003374824473420261, + "loss": 2.9238, + "theoretical_loss": 3.617998809399869, + "tokens_seen": 1095237632 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033747241725175524, + "loss": 2.9739, + "theoretical_loss": 3.6179785523901775, + "tokens_seen": 1095303168 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003374623871614845, + "loss": 2.7271, + "theoretical_loss": 3.6179582969318527, + "tokens_seen": 1095368704 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033745235707121366, + "loss": 2.768, + "theoretical_loss": 3.617938043024682, + "tokens_seen": 1095434240 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033744232698094284, + "loss": 2.9015, + "theoretical_loss": 3.6179177906684536, + "tokens_seen": 1095499776 + }, + { + "epoch": 0.33, + "learning_rate": 0.000337432296890672, + "loss": 2.6495, + "theoretical_loss": 3.617897539862957, + "tokens_seen": 1095565312 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003374222668004012, + "loss": 2.7574, + "theoretical_loss": 3.6178772906079804, + "tokens_seen": 1095630848 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003374122367101304, + "loss": 2.7673, + "theoretical_loss": 3.617857042903312, + "tokens_seen": 1095696384 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003374022066198596, + "loss": 2.8336, + "theoretical_loss": 3.6178367967487404, + "tokens_seen": 1095761920 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033739217652958874, + "loss": 2.7375, + "theoretical_loss": 3.617816552144054, + "tokens_seen": 1095827456 + }, + { + "epoch": 0.33, + "learning_rate": 0.000337382146439318, + "loss": 2.8566, + "theoretical_loss": 3.6177963090890417, + "tokens_seen": 1095892992 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003373721163490471, + "loss": 3.0084, + "theoretical_loss": 3.6177760675834927, + "tokens_seen": 1095958528 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033736208625877634, + "loss": 2.9808, + "theoretical_loss": 3.617755827627195, + "tokens_seen": 1096024064 + }, + { + "epoch": 0.33, + "objective/train/docs_used": 624708, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.919447422027588, + "objective/train/theoretical_loss": 3.6177355892199383, + "objective/train/tokens_used": 1116549600, + "theoretical_loss": 3.6177355892199383, + "tokens_seen": 1096089600 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003373520561685055, + "loss": 2.7467, + "theoretical_loss": 3.6177355892199383, + "tokens_seen": 1096089600 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003373420260782347, + "loss": 2.8654, + "theoretical_loss": 3.6177153523615107, + "tokens_seen": 1096155136 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003373319959879639, + "loss": 2.556, + "theoretical_loss": 3.617695117051701, + "tokens_seen": 1096220672 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003373219658976931, + "loss": 2.9015, + "theoretical_loss": 3.6176748832902987, + "tokens_seen": 1096286208 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033731193580742225, + "loss": 2.8972, + "theoretical_loss": 3.6176546510770926, + "tokens_seen": 1096351744 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003373019057171515, + "loss": 2.5119, + "theoretical_loss": 3.6176344204118713, + "tokens_seen": 1096417280 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003372918756268806, + "loss": 2.5876, + "theoretical_loss": 3.617614191294424, + "tokens_seen": 1096482816 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033728184553660984, + "loss": 2.9728, + "theoretical_loss": 3.6175939637245405, + "tokens_seen": 1096548352 + }, + { + "epoch": 0.33, + "learning_rate": 0.000337271815446339, + "loss": 2.8694, + "theoretical_loss": 3.6175737377020085, + "tokens_seen": 1096613888 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003372617853560682, + "loss": 2.8237, + "theoretical_loss": 3.6175535132266186, + "tokens_seen": 1096679424 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003372517552657974, + "loss": 2.7239, + "theoretical_loss": 3.6175332902981587, + "tokens_seen": 1096744960 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033724172517552657, + "loss": 2.66, + "theoretical_loss": 3.6175130689164194, + "tokens_seen": 1096810496 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033723169508525575, + "loss": 2.9751, + "theoretical_loss": 3.6174928490811893, + "tokens_seen": 1096876032 + }, + { + "epoch": 0.33, + "learning_rate": 0.000337221664994985, + "loss": 2.7232, + "theoretical_loss": 3.617472630792258, + "tokens_seen": 1096941568 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003372116349047141, + "loss": 2.6941, + "theoretical_loss": 3.617452414049414, + "tokens_seen": 1097007104 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033720160481444335, + "loss": 2.89, + "theoretical_loss": 3.617432198852448, + "tokens_seen": 1097072640 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003371915747241725, + "loss": 2.8757, + "theoretical_loss": 3.6174119852011484, + "tokens_seen": 1097138176 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003371815446339017, + "loss": 2.797, + "theoretical_loss": 3.6173917730953056, + "tokens_seen": 1097203712 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003371715145436309, + "loss": 2.8792, + "theoretical_loss": 3.617371562534708, + "tokens_seen": 1097269248 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033716148445336007, + "loss": 2.7201, + "theoretical_loss": 3.6173513535191466, + "tokens_seen": 1097334784 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033715145436308925, + "loss": 2.7757, + "theoretical_loss": 3.61733114604841, + "tokens_seen": 1097400320 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003371414242728185, + "loss": 2.8821, + "theoretical_loss": 3.6173109401222883, + "tokens_seen": 1097465856 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003371313941825476, + "loss": 2.6266, + "theoretical_loss": 3.6172907357405704, + "tokens_seen": 1097531392 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033712136409227685, + "loss": 2.614, + "theoretical_loss": 3.617270532903047, + "tokens_seen": 1097596928 + }, + { + "epoch": 0.33, + "learning_rate": 0.000337111334002006, + "loss": 2.856, + "theoretical_loss": 3.617250331609508, + "tokens_seen": 1097662464 + }, + { + "epoch": 0.33, + "objective/train/docs_used": 625328, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.957606792449951, + "objective/train/theoretical_loss": 3.617230131859743, + "objective/train/tokens_used": 1118188000, + "theoretical_loss": 3.617230131859743, + "tokens_seen": 1097728000 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003371013039117352, + "loss": 2.807, + "theoretical_loss": 3.617230131859743, + "tokens_seen": 1097728000 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003370912738214644, + "loss": 2.8049, + "theoretical_loss": 3.617209933653541, + "tokens_seen": 1097793536 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003370812437311936, + "loss": 2.6515, + "theoretical_loss": 3.6171897369906922, + "tokens_seen": 1097859072 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003370712136409228, + "loss": 2.8353, + "theoretical_loss": 3.6171695418709877, + "tokens_seen": 1097924608 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033706118355065194, + "loss": 2.7082, + "theoretical_loss": 3.617149348294217, + "tokens_seen": 1097990144 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033705115346038117, + "loss": 2.9072, + "theoretical_loss": 3.6171291562601695, + "tokens_seen": 1098055680 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033704112337011035, + "loss": 2.8727, + "theoretical_loss": 3.6171089657686357, + "tokens_seen": 1098121216 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033703109327983953, + "loss": 2.7816, + "theoretical_loss": 3.617088776819406, + "tokens_seen": 1098186752 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003370210631895687, + "loss": 2.6581, + "theoretical_loss": 3.61706858941227, + "tokens_seen": 1098252288 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033701103309929795, + "loss": 2.8799, + "theoretical_loss": 3.6170484035470176, + "tokens_seen": 1098317824 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003370010030090271, + "loss": 2.8505, + "theoretical_loss": 3.61702821922344, + "tokens_seen": 1098383360 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003369909729187563, + "loss": 2.8928, + "theoretical_loss": 3.6170080364413275, + "tokens_seen": 1098448896 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033698094282848544, + "loss": 2.7809, + "theoretical_loss": 3.6169878552004695, + "tokens_seen": 1098514432 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003369709127382147, + "loss": 2.5962, + "theoretical_loss": 3.616967675500657, + "tokens_seen": 1098579968 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033696088264794386, + "loss": 2.9829, + "theoretical_loss": 3.6169474973416804, + "tokens_seen": 1098645504 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033695085255767304, + "loss": 2.6946, + "theoretical_loss": 3.6169273207233292, + "tokens_seen": 1098711040 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003369408224674022, + "loss": 3.0566, + "theoretical_loss": 3.6169071456453956, + "tokens_seen": 1098776576 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003369307923771314, + "loss": 2.9306, + "theoretical_loss": 3.6168869721076686, + "tokens_seen": 1098842112 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003369207622868606, + "loss": 2.8365, + "theoretical_loss": 3.61686680010994, + "tokens_seen": 1098907648 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003369107321965898, + "loss": 2.6402, + "theoretical_loss": 3.616846629651999, + "tokens_seen": 1098973184 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033690070210631894, + "loss": 2.8273, + "theoretical_loss": 3.6168264607336376, + "tokens_seen": 1099038720 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003368906720160482, + "loss": 2.6659, + "theoretical_loss": 3.616806293354646, + "tokens_seen": 1099104256 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003368806419257773, + "loss": 2.7483, + "theoretical_loss": 3.6167861275148145, + "tokens_seen": 1099169792 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033687061183550654, + "loss": 2.769, + "theoretical_loss": 3.6167659632139344, + "tokens_seen": 1099235328 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003368605817452357, + "loss": 2.8283, + "theoretical_loss": 3.6167458004517963, + "tokens_seen": 1099300864 + }, + { + "epoch": 0.33, + "objective/train/docs_used": 626712, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.77760648727417, + "objective/train/theoretical_loss": 3.616725639228191, + "objective/train/tokens_used": 1119826400, + "theoretical_loss": 3.616725639228191, + "tokens_seen": 1099366400 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003368505516549649, + "loss": 2.9228, + "theoretical_loss": 3.616725639228191, + "tokens_seen": 1099366400 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003368405215646941, + "loss": 2.8445, + "theoretical_loss": 3.61670547954291, + "tokens_seen": 1099431936 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003368304914744233, + "loss": 2.9232, + "theoretical_loss": 3.6166853213957433, + "tokens_seen": 1099497472 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033682046138415245, + "loss": 2.9415, + "theoretical_loss": 3.616665164786482, + "tokens_seen": 1099563008 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003368104312938817, + "loss": 2.7541, + "theoretical_loss": 3.616645009714918, + "tokens_seen": 1099628544 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003368004012036108, + "loss": 2.7823, + "theoretical_loss": 3.6166248561808416, + "tokens_seen": 1099694080 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033679037111334004, + "loss": 2.868, + "theoretical_loss": 3.616604704184044, + "tokens_seen": 1099759616 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003367803410230692, + "loss": 3.1415, + "theoretical_loss": 3.616584553724316, + "tokens_seen": 1099825152 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003367703109327984, + "loss": 2.7907, + "theoretical_loss": 3.6165644048014505, + "tokens_seen": 1099890688 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003367602808425276, + "loss": 2.8857, + "theoretical_loss": 3.616544257415236, + "tokens_seen": 1099956224 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033675025075225677, + "loss": 2.7619, + "theoretical_loss": 3.6165241115654663, + "tokens_seen": 1100021760 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033674022066198595, + "loss": 2.704, + "theoretical_loss": 3.616503967251931, + "tokens_seen": 1100087296 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003367301905717152, + "loss": 2.7738, + "theoretical_loss": 3.616483824474422, + "tokens_seen": 1100152832 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003367201604814443, + "loss": 2.745, + "theoretical_loss": 3.6164636832327313, + "tokens_seen": 1100218368 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033671013039117355, + "loss": 2.658, + "theoretical_loss": 3.616443543526649, + "tokens_seen": 1100283904 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033670010030090273, + "loss": 2.9539, + "theoretical_loss": 3.6164234053559676, + "tokens_seen": 1100349440 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003366900702106319, + "loss": 2.8099, + "theoretical_loss": 3.616403268720479, + "tokens_seen": 1100414976 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003366800401203611, + "loss": 2.4884, + "theoretical_loss": 3.616383133619973, + "tokens_seen": 1100480512 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033667001003009027, + "loss": 2.6961, + "theoretical_loss": 3.6163630000542435, + "tokens_seen": 1100546048 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033665997993981945, + "loss": 2.9331, + "theoretical_loss": 3.6163428680230796, + "tokens_seen": 1100611584 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003366499498495487, + "loss": 2.7126, + "theoretical_loss": 3.616322737526275, + "tokens_seen": 1100677120 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003366399197592778, + "loss": 2.5203, + "theoretical_loss": 3.6163026085636205, + "tokens_seen": 1100742656 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033662988966900705, + "loss": 2.8215, + "theoretical_loss": 3.6162824811349084, + "tokens_seen": 1100808192 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003366198595787362, + "loss": 2.657, + "theoretical_loss": 3.61626235523993, + "tokens_seen": 1100873728 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003366098294884654, + "loss": 3.0526, + "theoretical_loss": 3.616242230878477, + "tokens_seen": 1100939264 + }, + { + "epoch": 0.33, + "objective/train/docs_used": 627263, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7997748851776123, + "objective/train/theoretical_loss": 3.6162221080503416, + "objective/train/tokens_used": 1121464800, + "theoretical_loss": 3.6162221080503416, + "tokens_seen": 1101004800 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003365997993981946, + "loss": 2.7248, + "theoretical_loss": 3.6162221080503416, + "tokens_seen": 1101004800 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003365897693079238, + "loss": 2.6942, + "theoretical_loss": 3.6162019867553155, + "tokens_seen": 1101070336 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033657973921765295, + "loss": 2.9403, + "theoretical_loss": 3.616181866993191, + "tokens_seen": 1101135872 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033656970912738214, + "loss": 2.8927, + "theoretical_loss": 3.6161617487637603, + "tokens_seen": 1101201408 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003365596790371113, + "loss": 2.7933, + "theoretical_loss": 3.6161416320668147, + "tokens_seen": 1101266944 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033654964894684055, + "loss": 2.8472, + "theoretical_loss": 3.6161215169021466, + "tokens_seen": 1101332480 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003365396188565697, + "loss": 2.9318, + "theoretical_loss": 3.616101403269548, + "tokens_seen": 1101398016 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003365295887662989, + "loss": 2.9445, + "theoretical_loss": 3.616081291168811, + "tokens_seen": 1101463552 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003365195586760281, + "loss": 2.8915, + "theoretical_loss": 3.6160611805997283, + "tokens_seen": 1101529088 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003365095285857573, + "loss": 3.003, + "theoretical_loss": 3.616041071562092, + "tokens_seen": 1101594624 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033649949849548646, + "loss": 2.7045, + "theoretical_loss": 3.616020964055694, + "tokens_seen": 1101660160 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033648946840521564, + "loss": 3.0042, + "theoretical_loss": 3.616000858080327, + "tokens_seen": 1101725696 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003364794383149448, + "loss": 3.0318, + "theoretical_loss": 3.615980753635783, + "tokens_seen": 1101791232 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033646940822467406, + "loss": 2.8623, + "theoretical_loss": 3.615960650721855, + "tokens_seen": 1101856768 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003364593781344032, + "loss": 2.6692, + "theoretical_loss": 3.6159405493383345, + "tokens_seen": 1101922304 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003364493480441324, + "loss": 3.0541, + "theoretical_loss": 3.6159204494850146, + "tokens_seen": 1101987840 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033643931795386154, + "loss": 3.001, + "theoretical_loss": 3.615900351161688, + "tokens_seen": 1102053376 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003364292878635908, + "loss": 2.7191, + "theoretical_loss": 3.6158802543681468, + "tokens_seen": 1102118912 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033641925777331996, + "loss": 2.7872, + "theoretical_loss": 3.615860159104184, + "tokens_seen": 1102184448 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033640922768304914, + "loss": 2.7161, + "theoretical_loss": 3.615840065369592, + "tokens_seen": 1102249984 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003363991975927783, + "loss": 2.8629, + "theoretical_loss": 3.6158199731641636, + "tokens_seen": 1102315520 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003363891675025075, + "loss": 2.7311, + "theoretical_loss": 3.6157998824876914, + "tokens_seen": 1102381056 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003363791374122367, + "loss": 2.7388, + "theoretical_loss": 3.6157797933399682, + "tokens_seen": 1102446592 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003363691073219659, + "loss": 2.873, + "theoretical_loss": 3.6157597057207873, + "tokens_seen": 1102512128 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033635907723169505, + "loss": 2.7788, + "theoretical_loss": 3.6157396196299403, + "tokens_seen": 1102577664 + }, + { + "epoch": 0.33, + "objective/train/docs_used": 628470, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.9640238285064697, + "objective/train/theoretical_loss": 3.615719535067221, + "objective/train/tokens_used": 1123103200, + "theoretical_loss": 3.615719535067221, + "tokens_seen": 1102643200 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003363490471414243, + "loss": 2.7905, + "theoretical_loss": 3.615719535067221, + "tokens_seen": 1102643200 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033633901705115346, + "loss": 2.8242, + "theoretical_loss": 3.6156994520324224, + "tokens_seen": 1102708736 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033632898696088265, + "loss": 2.8001, + "theoretical_loss": 3.6156793705253376, + "tokens_seen": 1102774272 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003363189568706119, + "loss": 2.9828, + "theoretical_loss": 3.615659290545759, + "tokens_seen": 1102839808 + }, + { + "epoch": 0.33, + "learning_rate": 0.000336308926780341, + "loss": 2.8555, + "theoretical_loss": 3.6156392120934795, + "tokens_seen": 1102905344 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033629889669007024, + "loss": 2.6872, + "theoretical_loss": 3.6156191351682936, + "tokens_seen": 1102970880 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003362888665997994, + "loss": 2.7234, + "theoretical_loss": 3.615599059769993, + "tokens_seen": 1103036416 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003362788365095286, + "loss": 2.7006, + "theoretical_loss": 3.615578985898371, + "tokens_seen": 1103101952 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003362688064192578, + "loss": 2.6149, + "theoretical_loss": 3.615558913553221, + "tokens_seen": 1103167488 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033625877632898697, + "loss": 2.8813, + "theoretical_loss": 3.615538842734337, + "tokens_seen": 1103233024 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033624874623871615, + "loss": 2.7927, + "theoretical_loss": 3.615518773441511, + "tokens_seen": 1103298560 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003362387161484454, + "loss": 2.8666, + "theoretical_loss": 3.615498705674537, + "tokens_seen": 1103364096 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003362286860581745, + "loss": 2.91, + "theoretical_loss": 3.615478639433209, + "tokens_seen": 1103429632 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033621865596790375, + "loss": 3.0292, + "theoretical_loss": 3.6154585747173194, + "tokens_seen": 1103495168 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033620862587763293, + "loss": 2.5587, + "theoretical_loss": 3.615438511526662, + "tokens_seen": 1103560704 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003361985957873621, + "loss": 2.7936, + "theoretical_loss": 3.6154184498610307, + "tokens_seen": 1103626240 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003361885656970913, + "loss": 2.9072, + "theoretical_loss": 3.6153983897202187, + "tokens_seen": 1103691776 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033617853560682047, + "loss": 2.5974, + "theoretical_loss": 3.6153783311040186, + "tokens_seen": 1103757312 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033616850551654965, + "loss": 2.5525, + "theoretical_loss": 3.615358274012226, + "tokens_seen": 1103822848 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003361584754262789, + "loss": 2.8614, + "theoretical_loss": 3.6153382184446325, + "tokens_seen": 1103888384 + }, + { + "epoch": 0.33, + "learning_rate": 0.000336148445336008, + "loss": 2.4938, + "theoretical_loss": 3.6153181644010335, + "tokens_seen": 1103953920 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033613841524573725, + "loss": 2.7168, + "theoretical_loss": 3.6152981118812217, + "tokens_seen": 1104019456 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003361283851554664, + "loss": 2.8884, + "theoretical_loss": 3.6152780608849913, + "tokens_seen": 1104084992 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003361183550651956, + "loss": 2.829, + "theoretical_loss": 3.615258011412136, + "tokens_seen": 1104150528 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003361083249749248, + "loss": 2.7764, + "theoretical_loss": 3.6152379634624494, + "tokens_seen": 1104216064 + }, + { + "epoch": 0.33, + "objective/train/docs_used": 629051, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.695175886154175, + "objective/train/theoretical_loss": 3.615217917035726, + "objective/train/tokens_used": 1124741600, + "theoretical_loss": 3.615217917035726, + "tokens_seen": 1104281600 + }, + { + "epoch": 0.33, + "learning_rate": 0.000336098294884654, + "loss": 2.8306, + "theoretical_loss": 3.615217917035726, + "tokens_seen": 1104281600 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033608826479438316, + "loss": 2.8245, + "theoretical_loss": 3.615197872131759, + "tokens_seen": 1104347136 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033607823470411234, + "loss": 2.6761, + "theoretical_loss": 3.615177828750343, + "tokens_seen": 1104412672 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003360682046138415, + "loss": 2.6768, + "theoretical_loss": 3.6151577868912717, + "tokens_seen": 1104478208 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033605817452357075, + "loss": 2.6808, + "theoretical_loss": 3.615137746554339, + "tokens_seen": 1104543744 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003360481444332999, + "loss": 2.9501, + "theoretical_loss": 3.6151177077393397, + "tokens_seen": 1104609280 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003360381143430291, + "loss": 2.8945, + "theoretical_loss": 3.615097670446067, + "tokens_seen": 1104674816 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003360280842527583, + "loss": 2.6155, + "theoretical_loss": 3.615077634674316, + "tokens_seen": 1104740352 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003360180541624875, + "loss": 2.8755, + "theoretical_loss": 3.6150576004238797, + "tokens_seen": 1104805888 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033600802407221666, + "loss": 2.8912, + "theoretical_loss": 3.6150375676945536, + "tokens_seen": 1104871424 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033599799398194584, + "loss": 2.832, + "theoretical_loss": 3.6150175364861314, + "tokens_seen": 1104936960 + }, + { + "epoch": 0.33, + "learning_rate": 0.000335987963891675, + "loss": 2.7801, + "theoretical_loss": 3.6149975067984075, + "tokens_seen": 1105002496 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033597793380140426, + "loss": 2.913, + "theoretical_loss": 3.614977478631176, + "tokens_seen": 1105068032 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003359679037111334, + "loss": 2.7242, + "theoretical_loss": 3.6149574519842322, + "tokens_seen": 1105133568 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003359578736208626, + "loss": 2.7592, + "theoretical_loss": 3.6149374268573697, + "tokens_seen": 1105199104 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033594784353059174, + "loss": 2.7311, + "theoretical_loss": 3.6149174032503835, + "tokens_seen": 1105264640 + }, + { + "epoch": 0.33, + "learning_rate": 0.000335937813440321, + "loss": 2.8126, + "theoretical_loss": 3.6148973811630674, + "tokens_seen": 1105330176 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033592778335005016, + "loss": 2.919, + "theoretical_loss": 3.6148773605952167, + "tokens_seen": 1105395712 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033591775325977934, + "loss": 2.9777, + "theoretical_loss": 3.6148573415466254, + "tokens_seen": 1105461248 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003359077231695085, + "loss": 2.72, + "theoretical_loss": 3.6148373240170892, + "tokens_seen": 1105526784 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003358976930792377, + "loss": 2.8915, + "theoretical_loss": 3.614817308006401, + "tokens_seen": 1105592320 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003358876629889669, + "loss": 2.8444, + "theoretical_loss": 3.614797293514358, + "tokens_seen": 1105657856 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003358776328986961, + "loss": 2.7208, + "theoretical_loss": 3.614777280540753, + "tokens_seen": 1105723392 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033586760280842525, + "loss": 2.6324, + "theoretical_loss": 3.6147572690853815, + "tokens_seen": 1105788928 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003358575727181545, + "loss": 2.9243, + "theoretical_loss": 3.614737259148038, + "tokens_seen": 1105854464 + }, + { + "epoch": 0.34, + "objective/train/docs_used": 630354, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.1543846130371094, + "objective/train/theoretical_loss": 3.614717250728518, + "objective/train/tokens_used": 1126380000, + "theoretical_loss": 3.614717250728518, + "tokens_seen": 1105920000 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033584754262788366, + "loss": 2.9578, + "theoretical_loss": 3.614717250728518, + "tokens_seen": 1105920000 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033583751253761285, + "loss": 2.8399, + "theoretical_loss": 3.6146972438266163, + "tokens_seen": 1105985536 + }, + { + "epoch": 0.34, + "learning_rate": 0.000335827482447342, + "loss": 2.7268, + "theoretical_loss": 3.6146772384421273, + "tokens_seen": 1106051072 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003358174523570712, + "loss": 2.5028, + "theoretical_loss": 3.6146572345748464, + "tokens_seen": 1106116608 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003358074222668004, + "loss": 2.8004, + "theoretical_loss": 3.614637232224569, + "tokens_seen": 1106182144 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003357973921765296, + "loss": 2.6896, + "theoretical_loss": 3.6146172313910894, + "tokens_seen": 1106247680 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033578736208625875, + "loss": 2.8121, + "theoretical_loss": 3.6145972320742037, + "tokens_seen": 1106313216 + }, + { + "epoch": 0.34, + "learning_rate": 0.000335777331995988, + "loss": 2.9344, + "theoretical_loss": 3.614577234273706, + "tokens_seen": 1106378752 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003357673019057171, + "loss": 2.9275, + "theoretical_loss": 3.6145572379893927, + "tokens_seen": 1106444288 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033575727181544635, + "loss": 2.9514, + "theoretical_loss": 3.614537243221058, + "tokens_seen": 1106509824 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033574724172517553, + "loss": 2.8776, + "theoretical_loss": 3.6145172499684977, + "tokens_seen": 1106575360 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003357372116349047, + "loss": 2.984, + "theoretical_loss": 3.6144972582315065, + "tokens_seen": 1106640896 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003357271815446339, + "loss": 2.7178, + "theoretical_loss": 3.614477268009881, + "tokens_seen": 1106706432 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033571715145436313, + "loss": 2.6776, + "theoretical_loss": 3.6144572793034158, + "tokens_seen": 1106771968 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033570712136409225, + "loss": 2.9554, + "theoretical_loss": 3.6144372921119063, + "tokens_seen": 1106837504 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003356970912738215, + "loss": 2.7329, + "theoretical_loss": 3.614417306435148, + "tokens_seen": 1106903040 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003356870611835506, + "loss": 2.5433, + "theoretical_loss": 3.614397322272937, + "tokens_seen": 1106968576 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033567703109327985, + "loss": 2.9015, + "theoretical_loss": 3.6143773396250687, + "tokens_seen": 1107034112 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033566700100300903, + "loss": 2.7667, + "theoretical_loss": 3.614357358491338, + "tokens_seen": 1107099648 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003356569709127382, + "loss": 2.9542, + "theoretical_loss": 3.614337378871541, + "tokens_seen": 1107165184 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003356469408224674, + "loss": 2.6445, + "theoretical_loss": 3.6143174007654735, + "tokens_seen": 1107230720 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003356369107321966, + "loss": 2.8545, + "theoretical_loss": 3.614297424172931, + "tokens_seen": 1107296256 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033562688064192576, + "loss": 3.1025, + "theoretical_loss": 3.6142774490937093, + "tokens_seen": 1107361792 + }, + { + "epoch": 0.34, + "learning_rate": 0.000335616850551655, + "loss": 2.8094, + "theoretical_loss": 3.6142574755276047, + "tokens_seen": 1107427328 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003356068204613841, + "loss": 2.5111, + "theoretical_loss": 3.6142375034744125, + "tokens_seen": 1107492864 + }, + { + "epoch": 0.34, + "objective/train/docs_used": 630918, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.0654361248016357, + "objective/train/theoretical_loss": 3.614217532933929, + "objective/train/tokens_used": 1128018400, + "theoretical_loss": 3.614217532933929, + "tokens_seen": 1107558400 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033559679037111336, + "loss": 2.872, + "theoretical_loss": 3.614217532933929, + "tokens_seen": 1107558400 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003355867602808425, + "loss": 2.7998, + "theoretical_loss": 3.6141975639059494, + "tokens_seen": 1107623936 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003355767301905717, + "loss": 2.9765, + "theoretical_loss": 3.6141775963902703, + "tokens_seen": 1107689472 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033556670010030095, + "loss": 2.9205, + "theoretical_loss": 3.6141576303866874, + "tokens_seen": 1107755008 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003355566700100301, + "loss": 2.7464, + "theoretical_loss": 3.6141376658949973, + "tokens_seen": 1107820544 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003355466399197593, + "loss": 3.0312, + "theoretical_loss": 3.614117702914995, + "tokens_seen": 1107886080 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003355366098294885, + "loss": 2.684, + "theoretical_loss": 3.614097741446478, + "tokens_seen": 1107951616 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003355265797392177, + "loss": 2.906, + "theoretical_loss": 3.6140777814892413, + "tokens_seen": 1108017152 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033551654964894686, + "loss": 2.7426, + "theoretical_loss": 3.6140578230430815, + "tokens_seen": 1108082688 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033550651955867604, + "loss": 2.7537, + "theoretical_loss": 3.6140378661077954, + "tokens_seen": 1108148224 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003354964894684052, + "loss": 2.8885, + "theoretical_loss": 3.6140179106831787, + "tokens_seen": 1108213760 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033548645937813446, + "loss": 2.7789, + "theoretical_loss": 3.6139979567690275, + "tokens_seen": 1108279296 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003354764292878636, + "loss": 2.8793, + "theoretical_loss": 3.613978004365139, + "tokens_seen": 1108344832 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003354663991975928, + "loss": 2.6812, + "theoretical_loss": 3.6139580534713085, + "tokens_seen": 1108410368 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033545636910732195, + "loss": 2.6601, + "theoretical_loss": 3.613938104087333, + "tokens_seen": 1108475904 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003354463390170512, + "loss": 2.8458, + "theoretical_loss": 3.6139181562130096, + "tokens_seen": 1108541440 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033543630892678036, + "loss": 2.9397, + "theoretical_loss": 3.6138982098481334, + "tokens_seen": 1108606976 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033542627883650954, + "loss": 2.8445, + "theoretical_loss": 3.6138782649925023, + "tokens_seen": 1108672512 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003354162487462387, + "loss": 2.9118, + "theoretical_loss": 3.613858321645912, + "tokens_seen": 1108738048 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003354062186559679, + "loss": 2.9033, + "theoretical_loss": 3.61383837980816, + "tokens_seen": 1108803584 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003353961885656971, + "loss": 2.706, + "theoretical_loss": 3.613818439479042, + "tokens_seen": 1108869120 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003353861584754263, + "loss": 2.9654, + "theoretical_loss": 3.6137985006583557, + "tokens_seen": 1108934656 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033537612838515545, + "loss": 3.005, + "theoretical_loss": 3.613778563345897, + "tokens_seen": 1109000192 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003353660982948847, + "loss": 2.6576, + "theoretical_loss": 3.613758627541463, + "tokens_seen": 1109065728 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033535606820461386, + "loss": 2.8184, + "theoretical_loss": 3.6137386932448505, + "tokens_seen": 1109131264 + }, + { + "epoch": 0.34, + "objective/train/docs_used": 632203, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.0943145751953125, + "objective/train/theoretical_loss": 3.613718760455857, + "objective/train/tokens_used": 1129656800, + "theoretical_loss": 3.613718760455857, + "tokens_seen": 1109196800 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033534603811434305, + "loss": 2.986, + "theoretical_loss": 3.613718760455857, + "tokens_seen": 1109196800 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003353360080240722, + "loss": 2.7588, + "theoretical_loss": 3.613698829174278, + "tokens_seen": 1109262336 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003353259779338014, + "loss": 3.0704, + "theoretical_loss": 3.613678899399912, + "tokens_seen": 1109327872 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003353159478435306, + "loss": 2.7734, + "theoretical_loss": 3.6136589711325557, + "tokens_seen": 1109393408 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003353059177532598, + "loss": 2.8086, + "theoretical_loss": 3.613639044372005, + "tokens_seen": 1109458944 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033529588766298895, + "loss": 2.6376, + "theoretical_loss": 3.613619119118058, + "tokens_seen": 1109524480 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003352858575727182, + "loss": 2.8826, + "theoretical_loss": 3.6135991953705116, + "tokens_seen": 1109590016 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003352758274824473, + "loss": 2.77, + "theoretical_loss": 3.6135792731291625, + "tokens_seen": 1109655552 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033526579739217655, + "loss": 2.7135, + "theoretical_loss": 3.6135593523938088, + "tokens_seen": 1109721088 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033525576730190573, + "loss": 2.8717, + "theoretical_loss": 3.613539433164247, + "tokens_seen": 1109786624 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003352457372116349, + "loss": 2.7744, + "theoretical_loss": 3.613519515440274, + "tokens_seen": 1109852160 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003352357071213641, + "loss": 2.6309, + "theoretical_loss": 3.6134995992216883, + "tokens_seen": 1109917696 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033522567703109333, + "loss": 2.827, + "theoretical_loss": 3.6134796845082864, + "tokens_seen": 1109983232 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033521564694082245, + "loss": 2.8097, + "theoretical_loss": 3.613459771299866, + "tokens_seen": 1110048768 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003352056168505517, + "loss": 2.6723, + "theoretical_loss": 3.613439859596224, + "tokens_seen": 1110114304 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003351955867602808, + "loss": 2.5586, + "theoretical_loss": 3.613419949397159, + "tokens_seen": 1110179840 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033518555667001005, + "loss": 2.7258, + "theoretical_loss": 3.6134000407024676, + "tokens_seen": 1110245376 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033517552657973923, + "loss": 2.6444, + "theoretical_loss": 3.6133801335119475, + "tokens_seen": 1110310912 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003351654964894684, + "loss": 2.9695, + "theoretical_loss": 3.613360227825396, + "tokens_seen": 1110376448 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003351554663991976, + "loss": 2.6985, + "theoretical_loss": 3.6133403236426114, + "tokens_seen": 1110441984 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003351454363089268, + "loss": 2.8705, + "theoretical_loss": 3.6133204209633907, + "tokens_seen": 1110507520 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033513540621865596, + "loss": 2.9051, + "theoretical_loss": 3.613300519787532, + "tokens_seen": 1110573056 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003351253761283852, + "loss": 2.7934, + "theoretical_loss": 3.613280620114833, + "tokens_seen": 1110638592 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003351153460381143, + "loss": 2.6601, + "theoretical_loss": 3.6132607219450916, + "tokens_seen": 1110704128 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033510531594784356, + "loss": 2.9004, + "theoretical_loss": 3.6132408252781048, + "tokens_seen": 1110769664 + }, + { + "epoch": 0.34, + "objective/train/docs_used": 632721, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.85196852684021, + "objective/train/theoretical_loss": 3.6132209301136715, + "objective/train/tokens_used": 1131295200, + "theoretical_loss": 3.6132209301136715, + "tokens_seen": 1110835200 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003350952858575727, + "loss": 2.9093, + "theoretical_loss": 3.6132209301136715, + "tokens_seen": 1110835200 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003350852557673019, + "loss": 2.8115, + "theoretical_loss": 3.6132010364515894, + "tokens_seen": 1110900736 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003350752256770311, + "loss": 2.8412, + "theoretical_loss": 3.613181144291656, + "tokens_seen": 1110966272 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003350651955867603, + "loss": 2.9338, + "theoretical_loss": 3.6131612536336695, + "tokens_seen": 1111031808 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033505516549648946, + "loss": 2.8114, + "theoretical_loss": 3.6131413644774275, + "tokens_seen": 1111097344 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003350451354062187, + "loss": 2.8687, + "theoretical_loss": 3.613121476822729, + "tokens_seen": 1111162880 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003350351053159478, + "loss": 2.9487, + "theoretical_loss": 3.613101590669371, + "tokens_seen": 1111228416 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033502507522567706, + "loss": 3.0188, + "theoretical_loss": 3.6130817060171525, + "tokens_seen": 1111293952 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003350150451354062, + "loss": 2.5555, + "theoretical_loss": 3.6130618228658715, + "tokens_seen": 1111359488 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003350050150451354, + "loss": 2.7118, + "theoretical_loss": 3.6130419412153256, + "tokens_seen": 1111425024 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003349949849548646, + "loss": 2.856, + "theoretical_loss": 3.6130220610653137, + "tokens_seen": 1111490560 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003349849548645938, + "loss": 2.6239, + "theoretical_loss": 3.613002182415634, + "tokens_seen": 1111556096 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033497492477432296, + "loss": 2.7339, + "theoretical_loss": 3.6129823052660845, + "tokens_seen": 1111621632 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033496489468405215, + "loss": 2.8535, + "theoretical_loss": 3.6129624296164637, + "tokens_seen": 1111687168 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003349548645937813, + "loss": 2.8399, + "theoretical_loss": 3.61294255546657, + "tokens_seen": 1111752704 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033494483450351056, + "loss": 2.5883, + "theoretical_loss": 3.612922682816202, + "tokens_seen": 1111818240 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003349348044132397, + "loss": 2.6914, + "theoretical_loss": 3.6129028116651583, + "tokens_seen": 1111883776 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003349247743229689, + "loss": 2.7452, + "theoretical_loss": 3.612882942013237, + "tokens_seen": 1111949312 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033491474423269805, + "loss": 2.6961, + "theoretical_loss": 3.612863073860237, + "tokens_seen": 1112014848 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003349047141424273, + "loss": 2.5755, + "theoretical_loss": 3.6128432072059566, + "tokens_seen": 1112080384 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033489468405215647, + "loss": 2.7884, + "theoretical_loss": 3.6128233420501945, + "tokens_seen": 1112145920 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033488465396188565, + "loss": 2.9253, + "theoretical_loss": 3.61280347839275, + "tokens_seen": 1112211456 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033487462387161483, + "loss": 2.9888, + "theoretical_loss": 3.6127836162334206, + "tokens_seen": 1112276992 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033486459378134406, + "loss": 2.7683, + "theoretical_loss": 3.612763755572006, + "tokens_seen": 1112342528 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003348545636910732, + "loss": 2.8703, + "theoretical_loss": 3.6127438964083045, + "tokens_seen": 1112408064 + }, + { + "epoch": 0.34, + "objective/train/docs_used": 634025, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.646425485610962, + "objective/train/theoretical_loss": 3.6127240387421153, + "objective/train/tokens_used": 1132933600, + "theoretical_loss": 3.6127240387421153, + "tokens_seen": 1112473600 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003348445336008024, + "loss": 2.7428, + "theoretical_loss": 3.6127240387421153, + "tokens_seen": 1112473600 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003348345035105316, + "loss": 2.7945, + "theoretical_loss": 3.6127041825732373, + "tokens_seen": 1112539136 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003348244734202608, + "loss": 2.6325, + "theoretical_loss": 3.612684327901469, + "tokens_seen": 1112604672 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033481444332999, + "loss": 2.8851, + "theoretical_loss": 3.6126644747266097, + "tokens_seen": 1112670208 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033480441323971915, + "loss": 2.8663, + "theoretical_loss": 3.6126446230484577, + "tokens_seen": 1112735744 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003347943831494484, + "loss": 2.7085, + "theoretical_loss": 3.612624772866813, + "tokens_seen": 1112801280 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003347843530591775, + "loss": 2.7654, + "theoretical_loss": 3.6126049241814746, + "tokens_seen": 1112866816 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033477432296890675, + "loss": 2.6204, + "theoretical_loss": 3.6125850769922407, + "tokens_seen": 1112932352 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033476429287863593, + "loss": 2.7107, + "theoretical_loss": 3.612565231298911, + "tokens_seen": 1112997888 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003347542627883651, + "loss": 2.8912, + "theoretical_loss": 3.612545387101285, + "tokens_seen": 1113063424 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003347442326980943, + "loss": 2.8544, + "theoretical_loss": 3.6125255443991615, + "tokens_seen": 1113128960 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033473420260782353, + "loss": 2.8361, + "theoretical_loss": 3.61250570319234, + "tokens_seen": 1113194496 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033472417251755265, + "loss": 2.7455, + "theoretical_loss": 3.6124858634806194, + "tokens_seen": 1113260032 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003347141424272819, + "loss": 2.8991, + "theoretical_loss": 3.612466025263799, + "tokens_seen": 1113325568 + }, + { + "epoch": 0.34, + "learning_rate": 0.000334704112337011, + "loss": 2.7228, + "theoretical_loss": 3.612446188541679, + "tokens_seen": 1113391104 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033469408224674025, + "loss": 2.9362, + "theoretical_loss": 3.612426353314058, + "tokens_seen": 1113456640 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033468405215646943, + "loss": 3.0716, + "theoretical_loss": 3.6124065195807358, + "tokens_seen": 1113522176 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003346740220661986, + "loss": 2.8211, + "theoretical_loss": 3.6123866873415116, + "tokens_seen": 1113587712 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003346639919759278, + "loss": 2.6047, + "theoretical_loss": 3.612366856596185, + "tokens_seen": 1113653248 + }, + { + "epoch": 0.34, + "learning_rate": 0.000334653961885657, + "loss": 2.8605, + "theoretical_loss": 3.612347027344556, + "tokens_seen": 1113718784 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033464393179538616, + "loss": 2.9608, + "theoretical_loss": 3.6123271995864235, + "tokens_seen": 1113784320 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003346339017051154, + "loss": 2.5576, + "theoretical_loss": 3.6123073733215882, + "tokens_seen": 1113849856 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003346238716148445, + "loss": 2.7201, + "theoretical_loss": 3.6122875485498485, + "tokens_seen": 1113915392 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033461384152457376, + "loss": 3.0712, + "theoretical_loss": 3.612267725271005, + "tokens_seen": 1113980928 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003346038114343029, + "loss": 2.5518, + "theoretical_loss": 3.6122479034848576, + "tokens_seen": 1114046464 + }, + { + "epoch": 0.34, + "objective/train/docs_used": 634689, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.0542807579040527, + "objective/train/theoretical_loss": 3.612228083191205, + "objective/train/tokens_used": 1134572000, + "theoretical_loss": 3.612228083191205, + "tokens_seen": 1114112000 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003345937813440321, + "loss": 2.7864, + "theoretical_loss": 3.612228083191205, + "tokens_seen": 1114112000 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003345837512537613, + "loss": 2.6419, + "theoretical_loss": 3.6122082643898485, + "tokens_seen": 1114177536 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003345737211634905, + "loss": 2.8113, + "theoretical_loss": 3.6121884470805865, + "tokens_seen": 1114243072 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033456369107321966, + "loss": 2.7092, + "theoretical_loss": 3.61216863126322, + "tokens_seen": 1114308608 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003345536609829489, + "loss": 2.8143, + "theoretical_loss": 3.612148816937548, + "tokens_seen": 1114374144 + }, + { + "epoch": 0.34, + "learning_rate": 0.000334543630892678, + "loss": 2.9402, + "theoretical_loss": 3.6121290041033722, + "tokens_seen": 1114439680 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033453360080240726, + "loss": 2.8054, + "theoretical_loss": 3.6121091927604905, + "tokens_seen": 1114505216 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003345235707121364, + "loss": 2.8467, + "theoretical_loss": 3.612089382908705, + "tokens_seen": 1114570752 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003345135406218656, + "loss": 2.7999, + "theoretical_loss": 3.6120695745478137, + "tokens_seen": 1114636288 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003345035105315948, + "loss": 2.5859, + "theoretical_loss": 3.612049767677618, + "tokens_seen": 1114701824 + }, + { + "epoch": 0.34, + "learning_rate": 0.000334493480441324, + "loss": 2.8155, + "theoretical_loss": 3.6120299622979184, + "tokens_seen": 1114767360 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033448345035105316, + "loss": 2.6153, + "theoretical_loss": 3.6120101584085145, + "tokens_seen": 1114832896 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033447342026078235, + "loss": 2.7435, + "theoretical_loss": 3.6119903560092066, + "tokens_seen": 1114898432 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003344633901705115, + "loss": 2.8516, + "theoretical_loss": 3.611970555099795, + "tokens_seen": 1114963968 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033445336008024076, + "loss": 2.6626, + "theoretical_loss": 3.6119507556800805, + "tokens_seen": 1115029504 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003344433299899699, + "loss": 2.9688, + "theoretical_loss": 3.611930957749863, + "tokens_seen": 1115095040 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003344332998996991, + "loss": 2.7055, + "theoretical_loss": 3.611911161308943, + "tokens_seen": 1115160576 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033442326980942825, + "loss": 2.94, + "theoretical_loss": 3.6118913663571206, + "tokens_seen": 1115226112 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003344132397191575, + "loss": 2.7752, + "theoretical_loss": 3.6118715728941972, + "tokens_seen": 1115291648 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033440320962888667, + "loss": 2.8586, + "theoretical_loss": 3.611851780919973, + "tokens_seen": 1115357184 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033439317953861585, + "loss": 2.798, + "theoretical_loss": 3.611831990434248, + "tokens_seen": 1115422720 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033438314944834503, + "loss": 2.6592, + "theoretical_loss": 3.611812201436823, + "tokens_seen": 1115488256 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033437311935807427, + "loss": 2.8846, + "theoretical_loss": 3.611792413927499, + "tokens_seen": 1115553792 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003343630892678034, + "loss": 2.7569, + "theoretical_loss": 3.6117726279060767, + "tokens_seen": 1115619328 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033435305917753263, + "loss": 2.96, + "theoretical_loss": 3.611752843372356, + "tokens_seen": 1115684864 + }, + { + "epoch": 0.34, + "objective/train/docs_used": 636068, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.3979570865631104, + "objective/train/theoretical_loss": 3.611733060326139, + "objective/train/tokens_used": 1136210400, + "theoretical_loss": 3.611733060326139, + "tokens_seen": 1115750400 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033434302908726175, + "loss": 3.0262, + "theoretical_loss": 3.611733060326139, + "tokens_seen": 1115750400 + }, + { + "epoch": 0.34, + "learning_rate": 0.000334332998996991, + "loss": 2.7069, + "theoretical_loss": 3.6117132787672257, + "tokens_seen": 1115815936 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033432296890672017, + "loss": 2.9201, + "theoretical_loss": 3.611693498695417, + "tokens_seen": 1115881472 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033431293881644935, + "loss": 2.7067, + "theoretical_loss": 3.6116737201105136, + "tokens_seen": 1115947008 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033430290872617853, + "loss": 2.6731, + "theoretical_loss": 3.6116539430123167, + "tokens_seen": 1116012544 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003342928786359077, + "loss": 2.6727, + "theoretical_loss": 3.611634167400627, + "tokens_seen": 1116078080 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003342828485456369, + "loss": 2.6871, + "theoretical_loss": 3.6116143932752465, + "tokens_seen": 1116143616 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033427281845536613, + "loss": 2.667, + "theoretical_loss": 3.611594620635975, + "tokens_seen": 1116209152 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033426278836509526, + "loss": 2.6889, + "theoretical_loss": 3.6115748494826136, + "tokens_seen": 1116274688 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003342527582748245, + "loss": 2.9199, + "theoretical_loss": 3.6115550798149645, + "tokens_seen": 1116340224 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003342427281845536, + "loss": 2.9199, + "theoretical_loss": 3.6115353116328275, + "tokens_seen": 1116405760 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033423269809428285, + "loss": 2.8113, + "theoretical_loss": 3.6115155449360046, + "tokens_seen": 1116471296 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033422266800401204, + "loss": 2.9831, + "theoretical_loss": 3.611495779724297, + "tokens_seen": 1116536832 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003342126379137412, + "loss": 2.6342, + "theoretical_loss": 3.6114760159975052, + "tokens_seen": 1116602368 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003342026078234704, + "loss": 2.847, + "theoretical_loss": 3.6114562537554318, + "tokens_seen": 1116667904 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033419257773319963, + "loss": 2.8187, + "theoretical_loss": 3.611436492997877, + "tokens_seen": 1116733440 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033418254764292876, + "loss": 2.9105, + "theoretical_loss": 3.611416733724643, + "tokens_seen": 1116798976 + }, + { + "epoch": 0.34, + "learning_rate": 0.000334172517552658, + "loss": 2.8531, + "theoretical_loss": 3.6113969759355307, + "tokens_seen": 1116864512 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003341624874623871, + "loss": 2.7968, + "theoretical_loss": 3.6113772196303415, + "tokens_seen": 1116930048 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033415245737211636, + "loss": 2.8352, + "theoretical_loss": 3.6113574648088766, + "tokens_seen": 1116995584 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033414242728184554, + "loss": 2.885, + "theoretical_loss": 3.611337711470939, + "tokens_seen": 1117061120 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003341323971915747, + "loss": 2.5665, + "theoretical_loss": 3.6113179596163283, + "tokens_seen": 1117126656 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003341223671013039, + "loss": 2.8546, + "theoretical_loss": 3.611298209244848, + "tokens_seen": 1117192192 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003341123370110331, + "loss": 2.8565, + "theoretical_loss": 3.611278460356298, + "tokens_seen": 1117257728 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033410230692076226, + "loss": 2.8341, + "theoretical_loss": 3.611258712950481, + "tokens_seen": 1117323264 + }, + { + "epoch": 0.34, + "objective/train/docs_used": 636687, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.787550926208496, + "objective/train/theoretical_loss": 3.611238967027199, + "objective/train/tokens_used": 1137848800, + "theoretical_loss": 3.611238967027199, + "tokens_seen": 1117388800 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003340922768304915, + "loss": 2.8336, + "theoretical_loss": 3.611238967027199, + "tokens_seen": 1117388800 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003340822467402207, + "loss": 2.9156, + "theoretical_loss": 3.6112192225862527, + "tokens_seen": 1117454336 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033407221664994986, + "loss": 3.0041, + "theoretical_loss": 3.6111994796274445, + "tokens_seen": 1117519872 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003340621865596791, + "loss": 2.8213, + "theoretical_loss": 3.611179738150576, + "tokens_seen": 1117585408 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003340521564694082, + "loss": 2.7027, + "theoretical_loss": 3.6111599981554496, + "tokens_seen": 1117650944 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033404212637913746, + "loss": 3.1244, + "theoretical_loss": 3.611140259641867, + "tokens_seen": 1117716480 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003340320962888666, + "loss": 2.8663, + "theoretical_loss": 3.61112052260963, + "tokens_seen": 1117782016 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003340220661985958, + "loss": 2.8941, + "theoretical_loss": 3.6111007870585405, + "tokens_seen": 1117847552 + }, + { + "epoch": 0.34, + "learning_rate": 0.000334012036108325, + "loss": 2.5904, + "theoretical_loss": 3.6110810529884003, + "tokens_seen": 1117913088 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003340020060180542, + "loss": 2.8001, + "theoretical_loss": 3.6110613203990116, + "tokens_seen": 1117978624 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033399197592778336, + "loss": 2.7421, + "theoretical_loss": 3.6110415892901773, + "tokens_seen": 1118044160 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033398194583751255, + "loss": 2.8302, + "theoretical_loss": 3.6110218596616988, + "tokens_seen": 1118109696 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003339719157472417, + "loss": 2.8829, + "theoretical_loss": 3.611002131513378, + "tokens_seen": 1118175232 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033396188565697096, + "loss": 2.5751, + "theoretical_loss": 3.610982404845018, + "tokens_seen": 1118240768 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003339518555667001, + "loss": 2.6088, + "theoretical_loss": 3.6109626796564203, + "tokens_seen": 1118306304 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003339418254764293, + "loss": 2.8487, + "theoretical_loss": 3.6109429559473876, + "tokens_seen": 1118371840 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033393179538615845, + "loss": 2.893, + "theoretical_loss": 3.6109232337177217, + "tokens_seen": 1118437376 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003339217652958877, + "loss": 2.9132, + "theoretical_loss": 3.6109035129672256, + "tokens_seen": 1118502912 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033391173520561687, + "loss": 2.7427, + "theoretical_loss": 3.6108837936957014, + "tokens_seen": 1118568448 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033390170511534605, + "loss": 2.9601, + "theoretical_loss": 3.6108640759029518, + "tokens_seen": 1118633984 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033389167502507523, + "loss": 2.8106, + "theoretical_loss": 3.6108443595887785, + "tokens_seen": 1118699520 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033388164493480447, + "loss": 2.9802, + "theoretical_loss": 3.6108246447529853, + "tokens_seen": 1118765056 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003338716148445336, + "loss": 2.9069, + "theoretical_loss": 3.610804931395373, + "tokens_seen": 1118830592 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033386158475426283, + "loss": 2.7738, + "theoretical_loss": 3.6107852195157464, + "tokens_seen": 1118896128 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033385155466399195, + "loss": 2.7446, + "theoretical_loss": 3.6107655091139064, + "tokens_seen": 1118961664 + }, + { + "epoch": 0.34, + "objective/train/docs_used": 638066, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6116201877593994, + "objective/train/theoretical_loss": 3.6107458001896564, + "objective/train/tokens_used": 1139487200, + "theoretical_loss": 3.6107458001896564, + "tokens_seen": 1119027200 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003338415245737212, + "loss": 2.8035, + "theoretical_loss": 3.6107458001896564, + "tokens_seen": 1119027200 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033383149448345037, + "loss": 2.6944, + "theoretical_loss": 3.6107260927427984, + "tokens_seen": 1119092736 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033382146439317955, + "loss": 2.6331, + "theoretical_loss": 3.610706386773136, + "tokens_seen": 1119158272 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033381143430290873, + "loss": 2.6877, + "theoretical_loss": 3.610686682280472, + "tokens_seen": 1119223808 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003338014042126379, + "loss": 2.6453, + "theoretical_loss": 3.6106669792646087, + "tokens_seen": 1119289344 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003337913741223671, + "loss": 2.8761, + "theoretical_loss": 3.610647277725349, + "tokens_seen": 1119354880 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033378134403209633, + "loss": 2.5685, + "theoretical_loss": 3.6106275776624956, + "tokens_seen": 1119420416 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033377131394182546, + "loss": 2.7744, + "theoretical_loss": 3.610607879075853, + "tokens_seen": 1119485952 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003337612838515547, + "loss": 2.7654, + "theoretical_loss": 3.6105881819652215, + "tokens_seen": 1119551488 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003337512537612838, + "loss": 2.5793, + "theoretical_loss": 3.6105684863304064, + "tokens_seen": 1119617024 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033374122367101306, + "loss": 2.712, + "theoretical_loss": 3.6105487921712096, + "tokens_seen": 1119682560 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033373119358074224, + "loss": 2.8562, + "theoretical_loss": 3.610529099487435, + "tokens_seen": 1119748096 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003337211634904714, + "loss": 2.6819, + "theoretical_loss": 3.6105094082788844, + "tokens_seen": 1119813632 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003337111334002006, + "loss": 2.6763, + "theoretical_loss": 3.610489718545362, + "tokens_seen": 1119879168 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033370110330992983, + "loss": 2.7431, + "theoretical_loss": 3.6104700302866712, + "tokens_seen": 1119944704 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033369107321965896, + "loss": 2.8663, + "theoretical_loss": 3.610450343502614, + "tokens_seen": 1120010240 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003336810431293882, + "loss": 2.7461, + "theoretical_loss": 3.6104306581929952, + "tokens_seen": 1120075776 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003336710130391173, + "loss": 2.9744, + "theoretical_loss": 3.6104109743576176, + "tokens_seen": 1120141312 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033366098294884656, + "loss": 2.9036, + "theoretical_loss": 3.610391291996284, + "tokens_seen": 1120206848 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033365095285857574, + "loss": 2.9433, + "theoretical_loss": 3.610371611108798, + "tokens_seen": 1120272384 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003336409227683049, + "loss": 2.5666, + "theoretical_loss": 3.6103519316949626, + "tokens_seen": 1120337920 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003336308926780341, + "loss": 2.8734, + "theoretical_loss": 3.6103322537545828, + "tokens_seen": 1120403456 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003336208625877633, + "loss": 2.547, + "theoretical_loss": 3.610312577287461, + "tokens_seen": 1120468992 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033361083249749246, + "loss": 2.916, + "theoretical_loss": 3.6102929022934003, + "tokens_seen": 1120534528 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003336008024072217, + "loss": 2.5222, + "theoretical_loss": 3.610273228772205, + "tokens_seen": 1120600064 + }, + { + "epoch": 0.34, + "objective/train/docs_used": 638756, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.5428030490875244, + "objective/train/theoretical_loss": 3.610253556723679, + "objective/train/tokens_used": 1141125600, + "theoretical_loss": 3.610253556723679, + "tokens_seen": 1120665600 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003335907723169508, + "loss": 2.7932, + "theoretical_loss": 3.610253556723679, + "tokens_seen": 1120665600 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033358074222668006, + "loss": 2.7654, + "theoretical_loss": 3.610233886147625, + "tokens_seen": 1120731136 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033357071213640924, + "loss": 2.7533, + "theoretical_loss": 3.6102142170438474, + "tokens_seen": 1120796672 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003335606820461384, + "loss": 2.6085, + "theoretical_loss": 3.61019454941215, + "tokens_seen": 1120862208 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003335506519558676, + "loss": 2.8381, + "theoretical_loss": 3.610174883252336, + "tokens_seen": 1120927744 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003335406218655968, + "loss": 2.8617, + "theoretical_loss": 3.610155218564209, + "tokens_seen": 1120993280 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033353059177532597, + "loss": 2.7272, + "theoretical_loss": 3.610135555347574, + "tokens_seen": 1121058816 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003335205616850552, + "loss": 2.5904, + "theoretical_loss": 3.610115893602234, + "tokens_seen": 1121124352 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033351053159478433, + "loss": 2.6402, + "theoretical_loss": 3.610096233327993, + "tokens_seen": 1121189888 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033350050150451356, + "loss": 2.6531, + "theoretical_loss": 3.6100765745246552, + "tokens_seen": 1121255424 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003334904714142427, + "loss": 2.794, + "theoretical_loss": 3.6100569171920247, + "tokens_seen": 1121320960 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003334804413239719, + "loss": 2.6108, + "theoretical_loss": 3.6100372613299045, + "tokens_seen": 1121386496 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003334704112337011, + "loss": 2.9138, + "theoretical_loss": 3.6100176069381003, + "tokens_seen": 1121452032 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003334603811434303, + "loss": 2.7695, + "theoretical_loss": 3.609997954016415, + "tokens_seen": 1121517568 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033345035105315947, + "loss": 2.7337, + "theoretical_loss": 3.6099783025646532, + "tokens_seen": 1121583104 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033344032096288865, + "loss": 2.6146, + "theoretical_loss": 3.609958652582619, + "tokens_seen": 1121648640 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033343029087261783, + "loss": 2.7623, + "theoretical_loss": 3.6099390040701165, + "tokens_seen": 1121714176 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033342026078234707, + "loss": 2.4816, + "theoretical_loss": 3.60991935702695, + "tokens_seen": 1121779712 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003334102306920762, + "loss": 2.7173, + "theoretical_loss": 3.6098997114529237, + "tokens_seen": 1121845248 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033340020060180543, + "loss": 2.922, + "theoretical_loss": 3.609880067347842, + "tokens_seen": 1121910784 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003333901705115346, + "loss": 2.7437, + "theoretical_loss": 3.60986042471151, + "tokens_seen": 1121976320 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003333801404212638, + "loss": 2.7785, + "theoretical_loss": 3.6098407835437305, + "tokens_seen": 1122041856 + }, + { + "epoch": 0.34, + "learning_rate": 0.000333370110330993, + "loss": 2.886, + "theoretical_loss": 3.609821143844309, + "tokens_seen": 1122107392 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033336008024072215, + "loss": 2.865, + "theoretical_loss": 3.6098015056130506, + "tokens_seen": 1122172928 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033335005015045134, + "loss": 2.8239, + "theoretical_loss": 3.6097818688497583, + "tokens_seen": 1122238464 + }, + { + "epoch": 0.34, + "objective/train/docs_used": 639983, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.5533931255340576, + "objective/train/theoretical_loss": 3.609762233554238, + "objective/train/tokens_used": 1142764000, + "theoretical_loss": 3.609762233554238, + "tokens_seen": 1122304000 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033334002006018057, + "loss": 2.801, + "theoretical_loss": 3.609762233554238, + "tokens_seen": 1122304000 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033332998996990975, + "loss": 2.7569, + "theoretical_loss": 3.6097425997262933, + "tokens_seen": 1122369536 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033331995987963893, + "loss": 2.6678, + "theoretical_loss": 3.6097229673657294, + "tokens_seen": 1122435072 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003333099297893681, + "loss": 2.9526, + "theoretical_loss": 3.6097033364723505, + "tokens_seen": 1122500608 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003332998996990973, + "loss": 2.8409, + "theoretical_loss": 3.6096837070459618, + "tokens_seen": 1122566144 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033328986960882653, + "loss": 2.7871, + "theoretical_loss": 3.609664079086368, + "tokens_seen": 1122631680 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033327983951855566, + "loss": 2.7059, + "theoretical_loss": 3.6096444525933737, + "tokens_seen": 1122697216 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003332698094282849, + "loss": 2.7491, + "theoretical_loss": 3.6096248275667833, + "tokens_seen": 1122762752 + }, + { + "epoch": 0.34, + "learning_rate": 0.000333259779338014, + "loss": 3.0377, + "theoretical_loss": 3.6096052040064026, + "tokens_seen": 1122828288 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033324974924774326, + "loss": 2.5784, + "theoretical_loss": 3.609585581912036, + "tokens_seen": 1122893824 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033323971915747244, + "loss": 2.8479, + "theoretical_loss": 3.6095659612834883, + "tokens_seen": 1122959360 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003332296890672016, + "loss": 2.6672, + "theoretical_loss": 3.6095463421205642, + "tokens_seen": 1123024896 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003332196589769308, + "loss": 2.8326, + "theoretical_loss": 3.609526724423069, + "tokens_seen": 1123090432 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033320962888666003, + "loss": 2.8255, + "theoretical_loss": 3.609507108190808, + "tokens_seen": 1123155968 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033319959879638916, + "loss": 2.7326, + "theoretical_loss": 3.609487493423586, + "tokens_seen": 1123221504 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003331895687061184, + "loss": 2.9464, + "theoretical_loss": 3.609467880121209, + "tokens_seen": 1123287040 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003331795386158475, + "loss": 2.884, + "theoretical_loss": 3.6094482682834803, + "tokens_seen": 1123352576 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033316950852557676, + "loss": 2.7343, + "theoretical_loss": 3.6094286579102066, + "tokens_seen": 1123418112 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033315947843530594, + "loss": 2.8182, + "theoretical_loss": 3.6094090490011927, + "tokens_seen": 1123483648 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003331494483450351, + "loss": 2.6158, + "theoretical_loss": 3.6093894415562437, + "tokens_seen": 1123549184 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003331394182547643, + "loss": 2.3721, + "theoretical_loss": 3.6093698355751647, + "tokens_seen": 1123614720 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003331293881644935, + "loss": 2.6278, + "theoretical_loss": 3.609350231057762, + "tokens_seen": 1123680256 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033311935807422266, + "loss": 2.469, + "theoretical_loss": 3.60933062800384, + "tokens_seen": 1123745792 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003331093279839519, + "loss": 2.8307, + "theoretical_loss": 3.609311026413204, + "tokens_seen": 1123811328 + }, + { + "epoch": 0.34, + "learning_rate": 0.000333099297893681, + "loss": 2.493, + "theoretical_loss": 3.609291426285661, + "tokens_seen": 1123876864 + }, + { + "epoch": 0.34, + "objective/train/docs_used": 640598, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6411571502685547, + "objective/train/theoretical_loss": 3.609271827621014, + "objective/train/tokens_used": 1144402400, + "theoretical_loss": 3.609271827621014, + "tokens_seen": 1123942400 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033308926780341026, + "loss": 2.6256, + "theoretical_loss": 3.609271827621014, + "tokens_seen": 1123942400 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033307923771313944, + "loss": 2.8186, + "theoretical_loss": 3.609252230419071, + "tokens_seen": 1124007936 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003330692076228686, + "loss": 2.8675, + "theoretical_loss": 3.6092326346796355, + "tokens_seen": 1124073472 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003330591775325978, + "loss": 2.7275, + "theoretical_loss": 3.6092130404025147, + "tokens_seen": 1124139008 + }, + { + "epoch": 0.34, + "learning_rate": 0.000333049147442327, + "loss": 2.7164, + "theoretical_loss": 3.6091934475875136, + "tokens_seen": 1124204544 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033303911735205617, + "loss": 2.8979, + "theoretical_loss": 3.6091738562344378, + "tokens_seen": 1124270080 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003330290872617854, + "loss": 2.8573, + "theoretical_loss": 3.6091542663430927, + "tokens_seen": 1124335616 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033301905717151453, + "loss": 2.6669, + "theoretical_loss": 3.6091346779132847, + "tokens_seen": 1124401152 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033300902708124376, + "loss": 2.8152, + "theoretical_loss": 3.6091150909448193, + "tokens_seen": 1124466688 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003329989969909729, + "loss": 2.889, + "theoretical_loss": 3.6090955054375025, + "tokens_seen": 1124532224 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003329889669007021, + "loss": 2.7453, + "theoretical_loss": 3.6090759213911396, + "tokens_seen": 1124597760 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003329789368104313, + "loss": 2.8621, + "theoretical_loss": 3.6090563388055372, + "tokens_seen": 1124663296 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003329689067201605, + "loss": 2.7673, + "theoretical_loss": 3.6090367576805007, + "tokens_seen": 1124728832 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033295887662988967, + "loss": 2.5402, + "theoretical_loss": 3.6090171780158364, + "tokens_seen": 1124794368 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033294884653961885, + "loss": 2.6932, + "theoretical_loss": 3.60899759981135, + "tokens_seen": 1124859904 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033293881644934803, + "loss": 2.991, + "theoretical_loss": 3.6089780230668476, + "tokens_seen": 1124925440 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033292878635907727, + "loss": 2.7343, + "theoretical_loss": 3.6089584477821357, + "tokens_seen": 1124990976 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003329187562688064, + "loss": 2.6751, + "theoretical_loss": 3.60893887395702, + "tokens_seen": 1125056512 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033290872617853563, + "loss": 2.6862, + "theoretical_loss": 3.608919301591307, + "tokens_seen": 1125122048 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003328986960882648, + "loss": 2.5229, + "theoretical_loss": 3.608899730684802, + "tokens_seen": 1125187584 + }, + { + "epoch": 0.34, + "learning_rate": 0.000332888665997994, + "loss": 2.7803, + "theoretical_loss": 3.6088801612373125, + "tokens_seen": 1125253120 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003328786359077232, + "loss": 2.6709, + "theoretical_loss": 3.608860593248644, + "tokens_seen": 1125318656 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033286860581745235, + "loss": 2.6893, + "theoretical_loss": 3.608841026718603, + "tokens_seen": 1125384192 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033285857572718154, + "loss": 2.597, + "theoretical_loss": 3.608821461646995, + "tokens_seen": 1125449728 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033284854563691077, + "loss": 2.7842, + "theoretical_loss": 3.6088018980336285, + "tokens_seen": 1125515264 + }, + { + "epoch": 0.34, + "objective/train/docs_used": 641290, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.591519832611084, + "objective/train/theoretical_loss": 3.608782335878308, + "objective/train/tokens_used": 1146040800, + "theoretical_loss": 3.608782335878308, + "tokens_seen": 1125580800 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003328385155466399, + "loss": 2.694, + "theoretical_loss": 3.608782335878308, + "tokens_seen": 1125580800 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033282848545636913, + "loss": 2.7903, + "theoretical_loss": 3.6087627751808404, + "tokens_seen": 1125646336 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033281845536609826, + "loss": 3.0821, + "theoretical_loss": 3.608743215941032, + "tokens_seen": 1125711872 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003328084252758275, + "loss": 2.7715, + "theoretical_loss": 3.6087236581586906, + "tokens_seen": 1125777408 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003327983951855567, + "loss": 2.6656, + "theoretical_loss": 3.608704101833621, + "tokens_seen": 1125842944 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033278836509528586, + "loss": 2.8922, + "theoretical_loss": 3.608684546965631, + "tokens_seen": 1125908480 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033277833500501504, + "loss": 2.8273, + "theoretical_loss": 3.6086649935545267, + "tokens_seen": 1125974016 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003327683049147442, + "loss": 2.6917, + "theoretical_loss": 3.608645441600115, + "tokens_seen": 1126039552 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003327582748244734, + "loss": 2.8187, + "theoretical_loss": 3.6086258911022018, + "tokens_seen": 1126105088 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033274824473420264, + "loss": 2.8183, + "theoretical_loss": 3.6086063420605954, + "tokens_seen": 1126170624 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033273821464393176, + "loss": 2.8064, + "theoretical_loss": 3.6085867944751016, + "tokens_seen": 1126236160 + }, + { + "epoch": 0.34, + "learning_rate": 0.000332728184553661, + "loss": 2.6906, + "theoretical_loss": 3.608567248345527, + "tokens_seen": 1126301696 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003327181544633902, + "loss": 2.8914, + "theoretical_loss": 3.6085477036716793, + "tokens_seen": 1126367232 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033270812437311936, + "loss": 2.811, + "theoretical_loss": 3.6085281604533646, + "tokens_seen": 1126432768 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033269809428284854, + "loss": 2.7431, + "theoretical_loss": 3.60850861869039, + "tokens_seen": 1126498304 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003326880641925777, + "loss": 2.9451, + "theoretical_loss": 3.608489078382563, + "tokens_seen": 1126563840 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003326780341023069, + "loss": 2.786, + "theoretical_loss": 3.60846953952969, + "tokens_seen": 1126629376 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033266800401203614, + "loss": 2.7559, + "theoretical_loss": 3.6084500021315784, + "tokens_seen": 1126694912 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033265797392176527, + "loss": 2.7155, + "theoretical_loss": 3.608430466188035, + "tokens_seen": 1126760448 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003326479438314945, + "loss": 2.9616, + "theoretical_loss": 3.6084109316988666, + "tokens_seen": 1126825984 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033263791374122363, + "loss": 2.5881, + "theoretical_loss": 3.6083913986638816, + "tokens_seen": 1126891520 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033262788365095286, + "loss": 2.727, + "theoretical_loss": 3.6083718670828864, + "tokens_seen": 1126957056 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033261785356068205, + "loss": 2.7538, + "theoretical_loss": 3.6083523369556874, + "tokens_seen": 1127022592 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003326078234704112, + "loss": 2.5898, + "theoretical_loss": 3.608332808282093, + "tokens_seen": 1127088128 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003325977933801404, + "loss": 2.931, + "theoretical_loss": 3.6083132810619105, + "tokens_seen": 1127153664 + }, + { + "epoch": 0.34, + "objective/train/docs_used": 642585, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.5291528701782227, + "objective/train/theoretical_loss": 3.6082937552949463, + "objective/train/tokens_used": 1147679200, + "theoretical_loss": 3.6082937552949463, + "tokens_seen": 1127219200 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033258776328986964, + "loss": 2.7102, + "theoretical_loss": 3.6082937552949463, + "tokens_seen": 1127219200 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003325777331995988, + "loss": 2.7399, + "theoretical_loss": 3.608274230981009, + "tokens_seen": 1127284736 + }, + { + "epoch": 0.34, + "learning_rate": 0.000332567703109328, + "loss": 2.8465, + "theoretical_loss": 3.608254708119905, + "tokens_seen": 1127350272 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003325576730190572, + "loss": 2.6694, + "theoretical_loss": 3.6082351867114424, + "tokens_seen": 1127415808 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033254764292878637, + "loss": 2.5065, + "theoretical_loss": 3.608215666755428, + "tokens_seen": 1127481344 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003325376128385156, + "loss": 2.8704, + "theoretical_loss": 3.60819614825167, + "tokens_seen": 1127546880 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033252758274824473, + "loss": 2.7278, + "theoretical_loss": 3.6081766311999757, + "tokens_seen": 1127612416 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033251755265797396, + "loss": 2.6342, + "theoretical_loss": 3.608157115600153, + "tokens_seen": 1127677952 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003325075225677031, + "loss": 2.6209, + "theoretical_loss": 3.6081376014520083, + "tokens_seen": 1127743488 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033249749247743233, + "loss": 2.8262, + "theoretical_loss": 3.608118088755351, + "tokens_seen": 1127809024 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003324874623871615, + "loss": 2.7084, + "theoretical_loss": 3.6080985775099874, + "tokens_seen": 1127874560 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003324774322968907, + "loss": 2.8097, + "theoretical_loss": 3.6080790677157255, + "tokens_seen": 1127940096 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033246740220661987, + "loss": 2.6076, + "theoretical_loss": 3.608059559372374, + "tokens_seen": 1128005632 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033245737211634905, + "loss": 2.5714, + "theoretical_loss": 3.60804005247974, + "tokens_seen": 1128071168 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033244734202607823, + "loss": 2.6493, + "theoretical_loss": 3.6080205470376314, + "tokens_seen": 1128136704 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033243731193580747, + "loss": 2.7445, + "theoretical_loss": 3.608001043045856, + "tokens_seen": 1128202240 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003324272818455366, + "loss": 2.6912, + "theoretical_loss": 3.607981540504222, + "tokens_seen": 1128267776 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033241725175526583, + "loss": 2.9616, + "theoretical_loss": 3.607962039412537, + "tokens_seen": 1128333312 + }, + { + "epoch": 0.34, + "learning_rate": 0.000332407221664995, + "loss": 2.4615, + "theoretical_loss": 3.607942539770609, + "tokens_seen": 1128398848 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003323971915747242, + "loss": 2.6759, + "theoretical_loss": 3.607923041578247, + "tokens_seen": 1128464384 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003323871614844534, + "loss": 2.931, + "theoretical_loss": 3.6079035448352577, + "tokens_seen": 1128529920 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033237713139418255, + "loss": 2.8097, + "theoretical_loss": 3.6078840495414495, + "tokens_seen": 1128595456 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033236710130391174, + "loss": 2.7341, + "theoretical_loss": 3.6078645556966316, + "tokens_seen": 1128660992 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033235707121364097, + "loss": 2.6044, + "theoretical_loss": 3.607845063300611, + "tokens_seen": 1128726528 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003323470411233701, + "loss": 2.7326, + "theoretical_loss": 3.6078255723531965, + "tokens_seen": 1128792064 + }, + { + "epoch": 0.34, + "objective/train/docs_used": 643131, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.9660146236419678, + "objective/train/theoretical_loss": 3.6078060828541956, + "objective/train/tokens_used": 1149317600, + "theoretical_loss": 3.6078060828541956, + "tokens_seen": 1128857600 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033233701103309933, + "loss": 2.9036, + "theoretical_loss": 3.6078060828541956, + "tokens_seen": 1128857600 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033232698094282846, + "loss": 2.9046, + "theoretical_loss": 3.607786594803418, + "tokens_seen": 1128923136 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003323169508525577, + "loss": 2.8764, + "theoretical_loss": 3.6077671082006706, + "tokens_seen": 1128988672 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003323069207622869, + "loss": 2.7548, + "theoretical_loss": 3.6077476230457624, + "tokens_seen": 1129054208 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033229689067201606, + "loss": 2.8518, + "theoretical_loss": 3.6077281393385023, + "tokens_seen": 1129119744 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033228686058174524, + "loss": 2.6993, + "theoretical_loss": 3.6077086570786974, + "tokens_seen": 1129185280 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003322768304914744, + "loss": 2.7624, + "theoretical_loss": 3.6076891762661574, + "tokens_seen": 1129250816 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003322668004012036, + "loss": 2.8877, + "theoretical_loss": 3.6076696969006905, + "tokens_seen": 1129316352 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033225677031093284, + "loss": 2.655, + "theoretical_loss": 3.6076502189821045, + "tokens_seen": 1129381888 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033224674022066196, + "loss": 2.728, + "theoretical_loss": 3.6076307425102088, + "tokens_seen": 1129447424 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003322367101303912, + "loss": 2.9008, + "theoretical_loss": 3.6076112674848115, + "tokens_seen": 1129512960 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003322266800401204, + "loss": 2.6821, + "theoretical_loss": 3.6075917939057223, + "tokens_seen": 1129578496 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033221664994984956, + "loss": 2.9441, + "theoretical_loss": 3.6075723217727487, + "tokens_seen": 1129644032 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033220661985957874, + "loss": 2.7435, + "theoretical_loss": 3.6075528510857, + "tokens_seen": 1129709568 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003321965897693079, + "loss": 2.8834, + "theoretical_loss": 3.6075333818443847, + "tokens_seen": 1129775104 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003321865596790371, + "loss": 2.6556, + "theoretical_loss": 3.6075139140486114, + "tokens_seen": 1129840640 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033217652958876634, + "loss": 2.746, + "theoretical_loss": 3.6074944476981896, + "tokens_seen": 1129906176 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033216649949849547, + "loss": 2.7611, + "theoretical_loss": 3.6074749827929278, + "tokens_seen": 1129971712 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003321564694082247, + "loss": 2.5915, + "theoretical_loss": 3.6074555193326345, + "tokens_seen": 1130037248 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033214643931795383, + "loss": 2.8962, + "theoretical_loss": 3.6074360573171194, + "tokens_seen": 1130102784 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033213640922768306, + "loss": 2.5664, + "theoretical_loss": 3.607416596746191, + "tokens_seen": 1130168320 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033212637913741225, + "loss": 2.5385, + "theoretical_loss": 3.6073971376196585, + "tokens_seen": 1130233856 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003321163490471414, + "loss": 2.8198, + "theoretical_loss": 3.607377679937331, + "tokens_seen": 1130299392 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003321063189568706, + "loss": 2.7579, + "theoretical_loss": 3.6073582236990176, + "tokens_seen": 1130364928 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033209628886659984, + "loss": 2.9061, + "theoretical_loss": 3.607338768904527, + "tokens_seen": 1130430464 + }, + { + "epoch": 0.34, + "objective/train/docs_used": 644333, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.4733054637908936, + "objective/train/theoretical_loss": 3.607319315553669, + "objective/train/tokens_used": 1150956000, + "theoretical_loss": 3.607319315553669, + "tokens_seen": 1130496000 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033208625877632897, + "loss": 2.8768, + "theoretical_loss": 3.607319315553669, + "tokens_seen": 1130496000 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003320762286860582, + "loss": 2.882, + "theoretical_loss": 3.607299863646252, + "tokens_seen": 1130561536 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033206619859578733, + "loss": 2.9346, + "theoretical_loss": 3.607280413182086, + "tokens_seen": 1130627072 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033205616850551657, + "loss": 2.8618, + "theoretical_loss": 3.60726096416098, + "tokens_seen": 1130692608 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033204613841524575, + "loss": 2.6155, + "theoretical_loss": 3.607241516582743, + "tokens_seen": 1130758144 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033203610832497493, + "loss": 2.7274, + "theoretical_loss": 3.607222070447185, + "tokens_seen": 1130823680 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003320260782347041, + "loss": 2.7617, + "theoretical_loss": 3.607202625754115, + "tokens_seen": 1130889216 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003320160481444333, + "loss": 2.7632, + "theoretical_loss": 3.6071831825033422, + "tokens_seen": 1130954752 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003320060180541625, + "loss": 2.7436, + "theoretical_loss": 3.607163740694676, + "tokens_seen": 1131020288 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003319959879638917, + "loss": 2.8807, + "theoretical_loss": 3.607144300327927, + "tokens_seen": 1131085824 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033198595787362084, + "loss": 2.7008, + "theoretical_loss": 3.6071248614029026, + "tokens_seen": 1131151360 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033197592778335007, + "loss": 2.6266, + "theoretical_loss": 3.607105423919415, + "tokens_seen": 1131216896 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003319658976930792, + "loss": 2.7287, + "theoretical_loss": 3.607085987877271, + "tokens_seen": 1131282432 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033195586760280843, + "loss": 2.8156, + "theoretical_loss": 3.607066553276283, + "tokens_seen": 1131347968 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003319458375125376, + "loss": 2.8845, + "theoretical_loss": 3.607047120116259, + "tokens_seen": 1131413504 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003319358074222668, + "loss": 2.8229, + "theoretical_loss": 3.6070276883970083, + "tokens_seen": 1131479040 + }, + { + "epoch": 0.34, + "learning_rate": 0.000331925777331996, + "loss": 2.6408, + "theoretical_loss": 3.607008258118342, + "tokens_seen": 1131544576 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003319157472417252, + "loss": 2.7129, + "theoretical_loss": 3.606988829280069, + "tokens_seen": 1131610112 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033190571715145434, + "loss": 2.8759, + "theoretical_loss": 3.606969401881999, + "tokens_seen": 1131675648 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003318956870611836, + "loss": 2.5466, + "theoretical_loss": 3.6069499759239427, + "tokens_seen": 1131741184 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003318856569709127, + "loss": 2.6498, + "theoretical_loss": 3.6069305514057097, + "tokens_seen": 1131806720 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033187562688064194, + "loss": 2.94, + "theoretical_loss": 3.606911128327109, + "tokens_seen": 1131872256 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003318655967903711, + "loss": 2.7123, + "theoretical_loss": 3.606891706687952, + "tokens_seen": 1131937792 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003318555667001003, + "loss": 2.661, + "theoretical_loss": 3.6068722864880476, + "tokens_seen": 1132003328 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003318455366098295, + "loss": 2.7786, + "theoretical_loss": 3.6068528677272065, + "tokens_seen": 1132068864 + }, + { + "epoch": 0.34, + "objective/train/docs_used": 644979, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.8492987155914307, + "objective/train/theoretical_loss": 3.6068334504052375, + "objective/train/tokens_used": 1152594400, + "theoretical_loss": 3.6068334504052375, + "tokens_seen": 1132134400 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033183550651955866, + "loss": 2.8279, + "theoretical_loss": 3.6068334504052375, + "tokens_seen": 1132134400 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003318254764292879, + "loss": 2.8731, + "theoretical_loss": 3.6068140345219524, + "tokens_seen": 1132199936 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003318154463390171, + "loss": 2.7591, + "theoretical_loss": 3.6067946200771606, + "tokens_seen": 1132265472 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033180541624874626, + "loss": 2.8935, + "theoretical_loss": 3.6067752070706725, + "tokens_seen": 1132331008 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033179538615847544, + "loss": 2.7287, + "theoretical_loss": 3.6067557955022975, + "tokens_seen": 1132396544 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003317853560682046, + "loss": 2.6252, + "theoretical_loss": 3.606736385371846, + "tokens_seen": 1132462080 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003317753259779338, + "loss": 3.073, + "theoretical_loss": 3.6067169766791296, + "tokens_seen": 1132527616 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033176529588766304, + "loss": 2.9786, + "theoretical_loss": 3.6066975694239574, + "tokens_seen": 1132593152 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033175526579739216, + "loss": 2.8354, + "theoretical_loss": 3.6066781636061402, + "tokens_seen": 1132658688 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003317452357071214, + "loss": 2.6343, + "theoretical_loss": 3.606658759225488, + "tokens_seen": 1132724224 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003317352056168506, + "loss": 2.7234, + "theoretical_loss": 3.6066393562818115, + "tokens_seen": 1132789760 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033172517552657976, + "loss": 2.6186, + "theoretical_loss": 3.6066199547749216, + "tokens_seen": 1132855296 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033171514543630894, + "loss": 2.6741, + "theoretical_loss": 3.606600554704628, + "tokens_seen": 1132920832 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003317051153460381, + "loss": 2.7224, + "theoretical_loss": 3.6065811560707415, + "tokens_seen": 1132986368 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003316950852557673, + "loss": 2.6038, + "theoretical_loss": 3.6065617588730725, + "tokens_seen": 1133051904 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033168505516549654, + "loss": 2.7552, + "theoretical_loss": 3.606542363111432, + "tokens_seen": 1133117440 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033167502507522567, + "loss": 2.5794, + "theoretical_loss": 3.6065229687856304, + "tokens_seen": 1133182976 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003316649949849549, + "loss": 2.5397, + "theoretical_loss": 3.606503575895479, + "tokens_seen": 1133248512 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033165496489468403, + "loss": 2.7015, + "theoretical_loss": 3.6064841844407876, + "tokens_seen": 1133314048 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033164493480441326, + "loss": 2.9354, + "theoretical_loss": 3.6064647944213672, + "tokens_seen": 1133379584 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033163490471414245, + "loss": 2.6921, + "theoretical_loss": 3.6064454058370288, + "tokens_seen": 1133445120 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003316248746238716, + "loss": 2.853, + "theoretical_loss": 3.6064260186875834, + "tokens_seen": 1133510656 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003316148445336008, + "loss": 2.7539, + "theoretical_loss": 3.606406632972841, + "tokens_seen": 1133576192 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033160481444333004, + "loss": 2.5276, + "theoretical_loss": 3.606387248692614, + "tokens_seen": 1133641728 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033159478435305917, + "loss": 2.7463, + "theoretical_loss": 3.6063678658467113, + "tokens_seen": 1133707264 + }, + { + "epoch": 0.34, + "objective/train/docs_used": 646384, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.829566478729248, + "objective/train/theoretical_loss": 3.6063484844349456, + "objective/train/tokens_used": 1154232800, + "theoretical_loss": 3.6063484844349456, + "tokens_seen": 1133772800 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003315847542627884, + "loss": 2.6405, + "theoretical_loss": 3.6063484844349456, + "tokens_seen": 1133772800 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033157472417251753, + "loss": 2.7125, + "theoretical_loss": 3.606329104457127, + "tokens_seen": 1133838336 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033156469408224677, + "loss": 2.6345, + "theoretical_loss": 3.606309725913067, + "tokens_seen": 1133903872 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033155466399197595, + "loss": 2.6477, + "theoretical_loss": 3.6062903488025766, + "tokens_seen": 1133969408 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033154463390170513, + "loss": 2.6936, + "theoretical_loss": 3.6062709731254663, + "tokens_seen": 1134034944 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003315346038114343, + "loss": 2.6662, + "theoretical_loss": 3.6062515988815482, + "tokens_seen": 1134100480 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003315245737211635, + "loss": 2.697, + "theoretical_loss": 3.6062322260706328, + "tokens_seen": 1134166016 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003315145436308927, + "loss": 2.7564, + "theoretical_loss": 3.606212854692531, + "tokens_seen": 1134231552 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003315045135406219, + "loss": 2.6626, + "theoretical_loss": 3.606193484747055, + "tokens_seen": 1134297088 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033149448345035104, + "loss": 2.7515, + "theoretical_loss": 3.606174116234016, + "tokens_seen": 1134362624 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033148445336008027, + "loss": 2.8018, + "theoretical_loss": 3.606154749153225, + "tokens_seen": 1134428160 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003314744232698094, + "loss": 2.7531, + "theoretical_loss": 3.6061353835044927, + "tokens_seen": 1134493696 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033146439317953863, + "loss": 2.7412, + "theoretical_loss": 3.606116019287631, + "tokens_seen": 1134559232 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003314543630892678, + "loss": 2.8854, + "theoretical_loss": 3.6060966565024524, + "tokens_seen": 1134624768 + }, + { + "epoch": 0.34, + "learning_rate": 0.000331444332998997, + "loss": 2.8525, + "theoretical_loss": 3.6060772951487667, + "tokens_seen": 1134690304 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003314343029087262, + "loss": 2.8854, + "theoretical_loss": 3.606057935226386, + "tokens_seen": 1134755840 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003314242728184554, + "loss": 2.8414, + "theoretical_loss": 3.6060385767351226, + "tokens_seen": 1134821376 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033141424272818454, + "loss": 2.9007, + "theoretical_loss": 3.6060192196747867, + "tokens_seen": 1134886912 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003314042126379138, + "loss": 2.5283, + "theoretical_loss": 3.6059998640451907, + "tokens_seen": 1134952448 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003313941825476429, + "loss": 2.8443, + "theoretical_loss": 3.605980509846147, + "tokens_seen": 1135017984 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033138415245737214, + "loss": 2.6493, + "theoretical_loss": 3.6059611570774655, + "tokens_seen": 1135083520 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003313741223671013, + "loss": 2.8762, + "theoretical_loss": 3.6059418057389596, + "tokens_seen": 1135149056 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003313640922768305, + "loss": 2.629, + "theoretical_loss": 3.6059224558304397, + "tokens_seen": 1135214592 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003313540621865597, + "loss": 2.816, + "theoretical_loss": 3.6059031073517183, + "tokens_seen": 1135280128 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033134403209628886, + "loss": 2.5454, + "theoretical_loss": 3.6058837603026075, + "tokens_seen": 1135345664 + }, + { + "epoch": 0.34, + "objective/train/docs_used": 647116, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.5147366523742676, + "objective/train/theoretical_loss": 3.6058644146829186, + "objective/train/tokens_used": 1155871200, + "theoretical_loss": 3.6058644146829186, + "tokens_seen": 1135411200 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033133400200601804, + "loss": 2.7524, + "theoretical_loss": 3.6058644146829186, + "tokens_seen": 1135411200 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003313239719157473, + "loss": 2.6299, + "theoretical_loss": 3.605845070492464, + "tokens_seen": 1135476736 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003313139418254764, + "loss": 2.8409, + "theoretical_loss": 3.6058257277310544, + "tokens_seen": 1135542272 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033130391173520564, + "loss": 2.7261, + "theoretical_loss": 3.6058063863985033, + "tokens_seen": 1135607808 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033129388164493477, + "loss": 2.7747, + "theoretical_loss": 3.605787046494622, + "tokens_seen": 1135673344 + }, + { + "epoch": 0.34, + "learning_rate": 0.000331283851554664, + "loss": 2.8166, + "theoretical_loss": 3.605767708019222, + "tokens_seen": 1135738880 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003312738214643932, + "loss": 2.992, + "theoretical_loss": 3.6057483709721168, + "tokens_seen": 1135804416 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033126379137412236, + "loss": 2.8622, + "theoretical_loss": 3.605729035353117, + "tokens_seen": 1135869952 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033125376128385155, + "loss": 2.6918, + "theoretical_loss": 3.605709701162036, + "tokens_seen": 1135935488 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003312437311935808, + "loss": 2.9525, + "theoretical_loss": 3.6056903683986854, + "tokens_seen": 1136001024 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003312337011033099, + "loss": 2.7663, + "theoretical_loss": 3.6056710370628773, + "tokens_seen": 1136066560 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033122367101303914, + "loss": 2.9066, + "theoretical_loss": 3.605651707154424, + "tokens_seen": 1136132096 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033121364092276827, + "loss": 3.0444, + "theoretical_loss": 3.6056323786731372, + "tokens_seen": 1136197632 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003312036108324975, + "loss": 2.781, + "theoretical_loss": 3.6056130516188305, + "tokens_seen": 1136263168 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003311935807422267, + "loss": 2.8249, + "theoretical_loss": 3.605593725991316, + "tokens_seen": 1136328704 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033118355065195587, + "loss": 2.795, + "theoretical_loss": 3.605574401790405, + "tokens_seen": 1136394240 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033117352056168505, + "loss": 2.7794, + "theoretical_loss": 3.605555079015911, + "tokens_seen": 1136459776 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033116349047141423, + "loss": 2.641, + "theoretical_loss": 3.6055357576676457, + "tokens_seen": 1136525312 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003311534603811434, + "loss": 2.8535, + "theoretical_loss": 3.605516437745422, + "tokens_seen": 1136590848 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033114343029087265, + "loss": 2.7815, + "theoretical_loss": 3.6054971192490526, + "tokens_seen": 1136656384 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033113340020060177, + "loss": 2.9259, + "theoretical_loss": 3.60547780217835, + "tokens_seen": 1136721920 + }, + { + "epoch": 0.34, + "learning_rate": 0.000331123370110331, + "loss": 2.6475, + "theoretical_loss": 3.6054584865331263, + "tokens_seen": 1136787456 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033111334002006013, + "loss": 2.7265, + "theoretical_loss": 3.605439172313195, + "tokens_seen": 1136852992 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033110330992978937, + "loss": 2.8477, + "theoretical_loss": 3.605419859518368, + "tokens_seen": 1136918528 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033109327983951855, + "loss": 2.7964, + "theoretical_loss": 3.6054005481484586, + "tokens_seen": 1136984064 + }, + { + "epoch": 0.34, + "objective/train/docs_used": 648555, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.897097587585449, + "objective/train/theoretical_loss": 3.605381238203279, + "objective/train/tokens_used": 1157509600, + "theoretical_loss": 3.605381238203279, + "tokens_seen": 1137049600 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033108324974924773, + "loss": 2.8466, + "theoretical_loss": 3.605381238203279, + "tokens_seen": 1137049600 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033107321965897697, + "loss": 2.827, + "theoretical_loss": 3.6053619296826422, + "tokens_seen": 1137115136 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033106318956870615, + "loss": 2.746, + "theoretical_loss": 3.6053426225863614, + "tokens_seen": 1137180672 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033105315947843533, + "loss": 2.8106, + "theoretical_loss": 3.605323316914249, + "tokens_seen": 1137246208 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003310431293881645, + "loss": 2.7255, + "theoretical_loss": 3.6053040126661173, + "tokens_seen": 1137311744 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003310330992978937, + "loss": 2.8325, + "theoretical_loss": 3.6052847098417806, + "tokens_seen": 1137377280 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003310230692076229, + "loss": 2.753, + "theoretical_loss": 3.605265408441051, + "tokens_seen": 1137442816 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003310130391173521, + "loss": 2.7575, + "theoretical_loss": 3.6052461084637413, + "tokens_seen": 1137508352 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033100300902708124, + "loss": 2.7335, + "theoretical_loss": 3.6052268099096656, + "tokens_seen": 1137573888 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033099297893681047, + "loss": 2.9484, + "theoretical_loss": 3.605207512778636, + "tokens_seen": 1137639424 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003309829488465396, + "loss": 2.7503, + "theoretical_loss": 3.6051882170704657, + "tokens_seen": 1137704960 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033097291875626883, + "loss": 2.7644, + "theoretical_loss": 3.605168922784968, + "tokens_seen": 1137770496 + }, + { + "epoch": 0.34, + "learning_rate": 0.000330962888665998, + "loss": 2.4717, + "theoretical_loss": 3.605149629921956, + "tokens_seen": 1137836032 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003309528585757272, + "loss": 2.7899, + "theoretical_loss": 3.605130338481243, + "tokens_seen": 1137901568 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003309428284854564, + "loss": 2.5483, + "theoretical_loss": 3.6051110484626423, + "tokens_seen": 1137967104 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003309327983951856, + "loss": 2.8629, + "theoretical_loss": 3.605091759865967, + "tokens_seen": 1138032640 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033092276830491474, + "loss": 2.7963, + "theoretical_loss": 3.60507247269103, + "tokens_seen": 1138098176 + }, + { + "epoch": 0.34, + "learning_rate": 0.000330912738214644, + "loss": 2.7461, + "theoretical_loss": 3.6050531869376456, + "tokens_seen": 1138163712 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003309027081243731, + "loss": 2.6356, + "theoretical_loss": 3.605033902605627, + "tokens_seen": 1138229248 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033089267803410234, + "loss": 2.7161, + "theoretical_loss": 3.6050146196947868, + "tokens_seen": 1138294784 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003308826479438315, + "loss": 2.8567, + "theoretical_loss": 3.6049953382049393, + "tokens_seen": 1138360320 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003308726178535607, + "loss": 2.7944, + "theoretical_loss": 3.6049760581358976, + "tokens_seen": 1138425856 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003308625877632899, + "loss": 2.5831, + "theoretical_loss": 3.604956779487475, + "tokens_seen": 1138491392 + }, + { + "epoch": 0.35, + "learning_rate": 0.00033085255767301906, + "loss": 2.8664, + "theoretical_loss": 3.604937502259485, + "tokens_seen": 1138556928 + }, + { + "epoch": 0.35, + "learning_rate": 0.00033084252758274824, + "loss": 2.7246, + "theoretical_loss": 3.6049182264517423, + "tokens_seen": 1138622464 + }, + { + "epoch": 0.35, + "objective/train/docs_used": 649005, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6684908866882324, + "objective/train/theoretical_loss": 3.60489895206406, + "objective/train/tokens_used": 1159148000, + "theoretical_loss": 3.60489895206406, + "tokens_seen": 1138688000 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003308324974924775, + "loss": 2.4852, + "theoretical_loss": 3.60489895206406, + "tokens_seen": 1138688000 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003308224674022066, + "loss": 2.6962, + "theoretical_loss": 3.6048796790962507, + "tokens_seen": 1138753536 + }, + { + "epoch": 0.35, + "learning_rate": 0.00033081243731193584, + "loss": 2.7425, + "theoretical_loss": 3.604860407548129, + "tokens_seen": 1138819072 + }, + { + "epoch": 0.35, + "learning_rate": 0.00033080240722166497, + "loss": 2.6033, + "theoretical_loss": 3.604841137419509, + "tokens_seen": 1138884608 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003307923771313942, + "loss": 2.6997, + "theoretical_loss": 3.6048218687102036, + "tokens_seen": 1138950144 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003307823470411234, + "loss": 2.5445, + "theoretical_loss": 3.6048026014200274, + "tokens_seen": 1139015680 + }, + { + "epoch": 0.35, + "learning_rate": 0.00033077231695085256, + "loss": 2.5808, + "theoretical_loss": 3.6047833355487935, + "tokens_seen": 1139081216 + }, + { + "epoch": 0.35, + "learning_rate": 0.00033076228686058175, + "loss": 2.936, + "theoretical_loss": 3.6047640710963162, + "tokens_seen": 1139146752 + }, + { + "epoch": 0.35, + "learning_rate": 0.000330752256770311, + "loss": 2.8448, + "theoretical_loss": 3.604744808062409, + "tokens_seen": 1139212288 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003307422266800401, + "loss": 2.4406, + "theoretical_loss": 3.604725546446887, + "tokens_seen": 1139277824 + }, + { + "epoch": 0.35, + "learning_rate": 0.00033073219658976934, + "loss": 2.8461, + "theoretical_loss": 3.604706286249563, + "tokens_seen": 1139343360 + }, + { + "epoch": 0.35, + "learning_rate": 0.00033072216649949847, + "loss": 2.3795, + "theoretical_loss": 3.604687027470251, + "tokens_seen": 1139408896 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003307121364092277, + "loss": 2.7847, + "theoretical_loss": 3.604667770108766, + "tokens_seen": 1139474432 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003307021063189569, + "loss": 2.9562, + "theoretical_loss": 3.6046485141649214, + "tokens_seen": 1139539968 + }, + { + "epoch": 0.35, + "learning_rate": 0.00033069207622868607, + "loss": 2.7052, + "theoretical_loss": 3.604629259638531, + "tokens_seen": 1139605504 + }, + { + "epoch": 0.35, + "learning_rate": 0.00033068204613841525, + "loss": 2.5585, + "theoretical_loss": 3.60461000652941, + "tokens_seen": 1139671040 + }, + { + "epoch": 0.35, + "learning_rate": 0.00033067201604814443, + "loss": 2.7782, + "theoretical_loss": 3.6045907548373712, + "tokens_seen": 1139736576 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003306619859578736, + "loss": 2.6651, + "theoretical_loss": 3.60457150456223, + "tokens_seen": 1139802112 + }, + { + "epoch": 0.35, + "learning_rate": 0.00033065195586760285, + "loss": 2.8509, + "theoretical_loss": 3.6045522557038003, + "tokens_seen": 1139867648 + }, + { + "epoch": 0.35, + "learning_rate": 0.00033064192577733197, + "loss": 2.6157, + "theoretical_loss": 3.604533008261896, + "tokens_seen": 1139933184 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003306318956870612, + "loss": 2.8576, + "theoretical_loss": 3.6045137622363326, + "tokens_seen": 1139998720 + }, + { + "epoch": 0.35, + "learning_rate": 0.00033062186559679034, + "loss": 2.8275, + "theoretical_loss": 3.6044945176269225, + "tokens_seen": 1140064256 + }, + { + "epoch": 0.35, + "learning_rate": 0.00033061183550651957, + "loss": 2.6749, + "theoretical_loss": 3.604475274433482, + "tokens_seen": 1140129792 + }, + { + "epoch": 0.35, + "learning_rate": 0.00033060180541624875, + "loss": 2.7599, + "theoretical_loss": 3.6044560326558246, + "tokens_seen": 1140195328 + }, + { + "epoch": 0.35, + "learning_rate": 0.00033059177532597793, + "loss": 2.4764, + "theoretical_loss": 3.6044367922937646, + "tokens_seen": 1140260864 + }, + { + "epoch": 0.35, + "objective/train/docs_used": 649993, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.794212579727173, + "objective/train/theoretical_loss": 3.604417553347117, + "objective/train/tokens_used": 1160786400, + "theoretical_loss": 3.604417553347117, + "tokens_seen": 1140326400 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003305817452357071, + "loss": 2.8372, + "theoretical_loss": 3.604417553347117, + "tokens_seen": 1140326400 + }, + { + "epoch": 0.35, + "learning_rate": 0.00033057171514543635, + "loss": 2.5363, + "theoretical_loss": 3.604398315815697, + "tokens_seen": 1140391936 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003305616850551655, + "loss": 2.7735, + "theoretical_loss": 3.6043790796993176, + "tokens_seen": 1140457472 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003305516549648947, + "loss": 2.7281, + "theoretical_loss": 3.604359844997794, + "tokens_seen": 1140523008 + }, + { + "epoch": 0.35, + "learning_rate": 0.00033054162487462384, + "loss": 2.8656, + "theoretical_loss": 3.604340611710941, + "tokens_seen": 1140588544 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003305315947843531, + "loss": 2.6799, + "theoretical_loss": 3.604321379838574, + "tokens_seen": 1140654080 + }, + { + "epoch": 0.35, + "learning_rate": 0.00033052156469408225, + "loss": 2.5999, + "theoretical_loss": 3.6043021493805067, + "tokens_seen": 1140719616 + }, + { + "epoch": 0.35, + "learning_rate": 0.00033051153460381144, + "loss": 2.5822, + "theoretical_loss": 3.604282920336554, + "tokens_seen": 1140785152 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003305015045135406, + "loss": 2.7812, + "theoretical_loss": 3.604263692706531, + "tokens_seen": 1140850688 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003304914744232698, + "loss": 2.8058, + "theoretical_loss": 3.6042444664902518, + "tokens_seen": 1140916224 + }, + { + "epoch": 0.35, + "learning_rate": 0.000330481444332999, + "loss": 2.4983, + "theoretical_loss": 3.604225241687532, + "tokens_seen": 1140981760 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003304714142427282, + "loss": 2.7105, + "theoretical_loss": 3.6042060182981865, + "tokens_seen": 1141047296 + }, + { + "epoch": 0.35, + "learning_rate": 0.00033046138415245734, + "loss": 2.7778, + "theoretical_loss": 3.60418679632203, + "tokens_seen": 1141112832 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003304513540621866, + "loss": 2.8218, + "theoretical_loss": 3.6041675757588774, + "tokens_seen": 1141178368 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003304413239719157, + "loss": 2.5122, + "theoretical_loss": 3.604148356608544, + "tokens_seen": 1141243904 + }, + { + "epoch": 0.35, + "learning_rate": 0.00033043129388164494, + "loss": 2.7457, + "theoretical_loss": 3.604129138870844, + "tokens_seen": 1141309440 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003304212637913741, + "loss": 2.8017, + "theoretical_loss": 3.604109922545594, + "tokens_seen": 1141374976 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003304112337011033, + "loss": 2.7728, + "theoretical_loss": 3.6040907076326074, + "tokens_seen": 1141440512 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003304012036108325, + "loss": 2.9019, + "theoretical_loss": 3.6040714941317002, + "tokens_seen": 1141506048 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003303911735205617, + "loss": 2.5523, + "theoretical_loss": 3.6040522820426877, + "tokens_seen": 1141571584 + }, + { + "epoch": 0.35, + "learning_rate": 0.00033038114343029084, + "loss": 2.8868, + "theoretical_loss": 3.6040330713653845, + "tokens_seen": 1141637120 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003303711133400201, + "loss": 2.5816, + "theoretical_loss": 3.604013862099607, + "tokens_seen": 1141702656 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003303610832497492, + "loss": 2.7731, + "theoretical_loss": 3.6039946542451684, + "tokens_seen": 1141768192 + }, + { + "epoch": 0.35, + "learning_rate": 0.00033035105315947844, + "loss": 2.8222, + "theoretical_loss": 3.6039754478018864, + "tokens_seen": 1141833728 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003303410230692076, + "loss": 2.6093, + "theoretical_loss": 3.6039562427695744, + "tokens_seen": 1141899264 + }, + { + "epoch": 0.35, + "objective/train/docs_used": 650682, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.651280164718628, + "objective/train/theoretical_loss": 3.603937039148049, + "objective/train/tokens_used": 1162424800, + "theoretical_loss": 3.603937039148049, + "tokens_seen": 1141964800 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003303309929789368, + "loss": 2.426, + "theoretical_loss": 3.603937039148049, + "tokens_seen": 1141964800 + }, + { + "epoch": 0.35, + "learning_rate": 0.00033032096288866604, + "loss": 2.8016, + "theoretical_loss": 3.603917836937125, + "tokens_seen": 1142030336 + }, + { + "epoch": 0.35, + "learning_rate": 0.00033031093279839517, + "loss": 2.8305, + "theoretical_loss": 3.603898636136618, + "tokens_seen": 1142095872 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003303009027081244, + "loss": 2.7313, + "theoretical_loss": 3.603879436746344, + "tokens_seen": 1142161408 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003302908726178536, + "loss": 2.4549, + "theoretical_loss": 3.6038602387661176, + "tokens_seen": 1142226944 + }, + { + "epoch": 0.35, + "learning_rate": 0.00033028084252758276, + "loss": 2.6843, + "theoretical_loss": 3.603841042195755, + "tokens_seen": 1142292480 + }, + { + "epoch": 0.35, + "learning_rate": 0.00033027081243731195, + "loss": 2.6087, + "theoretical_loss": 3.6038218470350714, + "tokens_seen": 1142358016 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003302607823470412, + "loss": 2.7759, + "theoretical_loss": 3.603802653283883, + "tokens_seen": 1142423552 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003302507522567703, + "loss": 2.6843, + "theoretical_loss": 3.6037834609420045, + "tokens_seen": 1142489088 + }, + { + "epoch": 0.35, + "learning_rate": 0.00033024072216649954, + "loss": 2.6573, + "theoretical_loss": 3.6037642700092523, + "tokens_seen": 1142554624 + }, + { + "epoch": 0.35, + "learning_rate": 0.00033023069207622867, + "loss": 2.8078, + "theoretical_loss": 3.603745080485442, + "tokens_seen": 1142620160 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003302206619859579, + "loss": 2.6291, + "theoretical_loss": 3.603725892370389, + "tokens_seen": 1142685696 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003302106318956871, + "loss": 2.8045, + "theoretical_loss": 3.60370670566391, + "tokens_seen": 1142751232 + }, + { + "epoch": 0.35, + "learning_rate": 0.00033020060180541627, + "loss": 3.0126, + "theoretical_loss": 3.6036875203658196, + "tokens_seen": 1142816768 + }, + { + "epoch": 0.35, + "learning_rate": 0.00033019057171514545, + "loss": 2.7668, + "theoretical_loss": 3.6036683364759345, + "tokens_seen": 1142882304 + }, + { + "epoch": 0.35, + "learning_rate": 0.00033018054162487463, + "loss": 2.8552, + "theoretical_loss": 3.6036491539940707, + "tokens_seen": 1142947840 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003301705115346038, + "loss": 2.6531, + "theoretical_loss": 3.6036299729200434, + "tokens_seen": 1143013376 + }, + { + "epoch": 0.35, + "learning_rate": 0.00033016048144433305, + "loss": 2.5897, + "theoretical_loss": 3.603610793253669, + "tokens_seen": 1143078912 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003301504513540622, + "loss": 2.656, + "theoretical_loss": 3.6035916149947638, + "tokens_seen": 1143144448 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003301404212637914, + "loss": 2.74, + "theoretical_loss": 3.6035724381431433, + "tokens_seen": 1143209984 + }, + { + "epoch": 0.35, + "learning_rate": 0.00033013039117352054, + "loss": 2.8269, + "theoretical_loss": 3.6035532626986235, + "tokens_seen": 1143275520 + }, + { + "epoch": 0.35, + "learning_rate": 0.00033012036108324977, + "loss": 2.8178, + "theoretical_loss": 3.6035340886610214, + "tokens_seen": 1143341056 + }, + { + "epoch": 0.35, + "learning_rate": 0.00033011033099297895, + "loss": 2.6579, + "theoretical_loss": 3.6035149160301523, + "tokens_seen": 1143406592 + }, + { + "epoch": 0.35, + "learning_rate": 0.00033010030090270813, + "loss": 2.8773, + "theoretical_loss": 3.6034957448058327, + "tokens_seen": 1143472128 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003300902708124373, + "loss": 2.8752, + "theoretical_loss": 3.6034765749878783, + "tokens_seen": 1143537664 + }, + { + "epoch": 0.35, + "objective/train/docs_used": 652177, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7584352493286133, + "objective/train/theoretical_loss": 3.603457406576106, + "objective/train/tokens_used": 1164063200, + "theoretical_loss": 3.603457406576106, + "tokens_seen": 1143603200 + }, + { + "epoch": 0.35, + "learning_rate": 0.00033008024072216655, + "loss": 2.7859, + "theoretical_loss": 3.603457406576106, + "tokens_seen": 1143603200 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003300702106318957, + "loss": 2.6788, + "theoretical_loss": 3.603438239570332, + "tokens_seen": 1143668736 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003300601805416249, + "loss": 2.5992, + "theoretical_loss": 3.6034190739703718, + "tokens_seen": 1143734272 + }, + { + "epoch": 0.35, + "learning_rate": 0.00033005015045135404, + "loss": 2.8617, + "theoretical_loss": 3.6033999097760434, + "tokens_seen": 1143799808 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003300401203610833, + "loss": 2.7274, + "theoretical_loss": 3.6033807469871615, + "tokens_seen": 1143865344 + }, + { + "epoch": 0.35, + "learning_rate": 0.00033003009027081245, + "loss": 2.7781, + "theoretical_loss": 3.603361585603543, + "tokens_seen": 1143930880 + }, + { + "epoch": 0.35, + "learning_rate": 0.00033002006018054164, + "loss": 2.6528, + "theoretical_loss": 3.603342425625005, + "tokens_seen": 1143996416 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003300100300902708, + "loss": 2.5909, + "theoretical_loss": 3.603323267051363, + "tokens_seen": 1144061952 + }, + { + "epoch": 0.35, + "learning_rate": 0.00033, + "loss": 2.7503, + "theoretical_loss": 3.6033041098824343, + "tokens_seen": 1144127488 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003299899699097292, + "loss": 2.7806, + "theoretical_loss": 3.603284954118035, + "tokens_seen": 1144193024 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003299799398194584, + "loss": 2.5984, + "theoretical_loss": 3.6032657997579824, + "tokens_seen": 1144258560 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032996990972918754, + "loss": 2.4974, + "theoretical_loss": 3.6032466468020923, + "tokens_seen": 1144324096 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003299598796389168, + "loss": 2.7682, + "theoretical_loss": 3.6032274952501817, + "tokens_seen": 1144389632 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003299498495486459, + "loss": 2.7423, + "theoretical_loss": 3.603208345102067, + "tokens_seen": 1144455168 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032993981945837514, + "loss": 2.6841, + "theoretical_loss": 3.6031891963575653, + "tokens_seen": 1144520704 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003299297893681043, + "loss": 2.5058, + "theoretical_loss": 3.6031700490164926, + "tokens_seen": 1144586240 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003299197592778335, + "loss": 2.6354, + "theoretical_loss": 3.6031509030786673, + "tokens_seen": 1144651776 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003299097291875627, + "loss": 2.8666, + "theoretical_loss": 3.6031317585439044, + "tokens_seen": 1144717312 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003298996990972919, + "loss": 2.6375, + "theoretical_loss": 3.603112615412022, + "tokens_seen": 1144782848 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032988966900702104, + "loss": 2.8578, + "theoretical_loss": 3.603093473682837, + "tokens_seen": 1144848384 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003298796389167503, + "loss": 2.9789, + "theoretical_loss": 3.6030743333561652, + "tokens_seen": 1144913920 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003298696088264794, + "loss": 2.6403, + "theoretical_loss": 3.6030551944318248, + "tokens_seen": 1144979456 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032985957873620864, + "loss": 2.8929, + "theoretical_loss": 3.603036056909632, + "tokens_seen": 1145044992 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003298495486459378, + "loss": 2.8432, + "theoretical_loss": 3.6030169207894045, + "tokens_seen": 1145110528 + }, + { + "epoch": 0.35, + "learning_rate": 0.000329839518555667, + "loss": 2.6046, + "theoretical_loss": 3.6029977860709588, + "tokens_seen": 1145176064 + }, + { + "epoch": 0.35, + "objective/train/docs_used": 652765, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.297375202178955, + "objective/train/theoretical_loss": 3.6029786527541123, + "objective/train/tokens_used": 1165701600, + "theoretical_loss": 3.6029786527541123, + "tokens_seen": 1145241600 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003298294884653962, + "loss": 2.9559, + "theoretical_loss": 3.6029786527541123, + "tokens_seen": 1145241600 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032981945837512537, + "loss": 2.6591, + "theoretical_loss": 3.6029595208386818, + "tokens_seen": 1145307136 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032980942828485455, + "loss": 2.8015, + "theoretical_loss": 3.6029403903244845, + "tokens_seen": 1145372672 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003297993981945838, + "loss": 2.9099, + "theoretical_loss": 3.602921261211338, + "tokens_seen": 1145438208 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003297893681043129, + "loss": 2.5436, + "theoretical_loss": 3.6029021334990596, + "tokens_seen": 1145503744 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032977933801404215, + "loss": 2.6758, + "theoretical_loss": 3.602883007187466, + "tokens_seen": 1145569280 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003297693079237713, + "loss": 2.6497, + "theoretical_loss": 3.6028638822763748, + "tokens_seen": 1145634816 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003297592778335005, + "loss": 2.8177, + "theoretical_loss": 3.6028447587656034, + "tokens_seen": 1145700352 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003297492477432297, + "loss": 2.9367, + "theoretical_loss": 3.60282563665497, + "tokens_seen": 1145765888 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032973921765295887, + "loss": 2.8167, + "theoretical_loss": 3.60280651594429, + "tokens_seen": 1145831424 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032972918756268805, + "loss": 2.6599, + "theoretical_loss": 3.6027873966333823, + "tokens_seen": 1145896960 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003297191574724173, + "loss": 2.9347, + "theoretical_loss": 3.6027682787220643, + "tokens_seen": 1145962496 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003297091273821464, + "loss": 2.4762, + "theoretical_loss": 3.6027491622101535, + "tokens_seen": 1146028032 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032969909729187565, + "loss": 2.7818, + "theoretical_loss": 3.602730047097467, + "tokens_seen": 1146093568 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003296890672016048, + "loss": 2.6407, + "theoretical_loss": 3.6027109333838228, + "tokens_seen": 1146159104 + }, + { + "epoch": 0.35, + "learning_rate": 0.000329679037111334, + "loss": 2.598, + "theoretical_loss": 3.602691821069038, + "tokens_seen": 1146224640 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003296690070210632, + "loss": 2.6162, + "theoretical_loss": 3.6026727101529312, + "tokens_seen": 1146290176 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003296589769307924, + "loss": 2.8308, + "theoretical_loss": 3.602653600635319, + "tokens_seen": 1146355712 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032964894684052155, + "loss": 2.6988, + "theoretical_loss": 3.6026344925160196, + "tokens_seen": 1146421248 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032963891675025074, + "loss": 2.6134, + "theoretical_loss": 3.602615385794851, + "tokens_seen": 1146486784 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003296288866599799, + "loss": 2.8147, + "theoretical_loss": 3.6025962804716305, + "tokens_seen": 1146552320 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032961885656970915, + "loss": 2.9491, + "theoretical_loss": 3.6025771765461765, + "tokens_seen": 1146617856 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003296088264794383, + "loss": 2.8462, + "theoretical_loss": 3.6025580740183063, + "tokens_seen": 1146683392 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003295987963891675, + "loss": 2.4403, + "theoretical_loss": 3.602538972887838, + "tokens_seen": 1146748928 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003295887662988967, + "loss": 2.7588, + "theoretical_loss": 3.6025198731545895, + "tokens_seen": 1146814464 + }, + { + "debugging/Self-BLEU-5": 0.18134929226218258, + "debugging/distinct-1-grams": 0.8412963908750427, + "debugging/distinct-2-grams": 0.9919927119294208, + "debugging/entropy-1-grams": 4.946244095376608, + "debugging/entropy-2-grams": 5.327303325558611, + "debugging/length": 450.6666666666667, + "debugging/num_segments": 3, + "debugging/score": 0.006757274686373605, + "debugging/score_std": 0.004005886028188189, + "epoch": 0.35, + "objective/train/docs_used": 652965, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6190431118011475, + "objective/train/theoretical_loss": 3.602500774818379, + "objective/train/tokens_used": 1167340000, + "theoretical_loss": 3.602500774818379, + "tokens_seen": 1146880000 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003295787362086259, + "loss": 2.8299, + "theoretical_loss": 3.602500774818379, + "tokens_seen": 1146880000 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003295687061183551, + "loss": 2.9039, + "theoretical_loss": 3.602481677879024, + "tokens_seen": 1146945536 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032955867602808424, + "loss": 2.7909, + "theoretical_loss": 3.6024625823363428, + "tokens_seen": 1147011072 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003295486459378135, + "loss": 2.722, + "theoretical_loss": 3.602443488190154, + "tokens_seen": 1147076608 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032953861584754265, + "loss": 2.605, + "theoretical_loss": 3.602424395440275, + "tokens_seen": 1147142144 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032952858575727184, + "loss": 2.5456, + "theoretical_loss": 3.602405304086524, + "tokens_seen": 1147207680 + }, + { + "epoch": 0.35, + "learning_rate": 0.000329518555667001, + "loss": 2.7842, + "theoretical_loss": 3.6023862141287193, + "tokens_seen": 1147273216 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003295085255767302, + "loss": 2.8336, + "theoretical_loss": 3.6023671255666785, + "tokens_seen": 1147338752 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003294984954864594, + "loss": 2.602, + "theoretical_loss": 3.602348038400221, + "tokens_seen": 1147404288 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003294884653961886, + "loss": 2.6961, + "theoretical_loss": 3.6023289526291644, + "tokens_seen": 1147469824 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032947843530591774, + "loss": 2.6992, + "theoretical_loss": 3.602309868253327, + "tokens_seen": 1147535360 + }, + { + "epoch": 0.35, + "learning_rate": 0.000329468405215647, + "loss": 2.6349, + "theoretical_loss": 3.602290785272527, + "tokens_seen": 1147600896 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003294583751253761, + "loss": 2.6734, + "theoretical_loss": 3.6022717036865832, + "tokens_seen": 1147666432 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032944834503510534, + "loss": 2.7111, + "theoretical_loss": 3.6022526234953136, + "tokens_seen": 1147731968 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003294383149448345, + "loss": 2.7179, + "theoretical_loss": 3.602233544698537, + "tokens_seen": 1147797504 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003294282848545637, + "loss": 2.8792, + "theoretical_loss": 3.6022144672960716, + "tokens_seen": 1147863040 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003294182547642929, + "loss": 2.8469, + "theoretical_loss": 3.602195391287736, + "tokens_seen": 1147928576 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003294082246740221, + "loss": 2.6126, + "theoretical_loss": 3.602176316673349, + "tokens_seen": 1147994112 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032939819458375124, + "loss": 2.5416, + "theoretical_loss": 3.6021572434527283, + "tokens_seen": 1148059648 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003293881644934805, + "loss": 2.7542, + "theoretical_loss": 3.6021381716256933, + "tokens_seen": 1148125184 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003293781344032096, + "loss": 2.9115, + "theoretical_loss": 3.6021191011920624, + "tokens_seen": 1148190720 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032936810431293884, + "loss": 2.7362, + "theoretical_loss": 3.6021000321516543, + "tokens_seen": 1148256256 + }, + { + "epoch": 0.35, + "learning_rate": 0.000329358074222668, + "loss": 2.679, + "theoretical_loss": 3.602080964504288, + "tokens_seen": 1148321792 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003293480441323972, + "loss": 2.7262, + "theoretical_loss": 3.602061898249781, + "tokens_seen": 1148387328 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003293380140421264, + "loss": 2.7714, + "theoretical_loss": 3.602042833387954, + "tokens_seen": 1148452864 + }, + { + "epoch": 0.35, + "objective/train/docs_used": 654364, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.9863083362579346, + "objective/train/theoretical_loss": 3.602023769918625, + "objective/train/tokens_used": 1168978400, + "theoretical_loss": 3.602023769918625, + "tokens_seen": 1148518400 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032932798395185557, + "loss": 2.8937, + "theoretical_loss": 3.602023769918625, + "tokens_seen": 1148518400 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032931795386158475, + "loss": 2.8662, + "theoretical_loss": 3.602004707841612, + "tokens_seen": 1148583936 + }, + { + "epoch": 0.35, + "learning_rate": 0.000329307923771314, + "loss": 2.6981, + "theoretical_loss": 3.6019856471567344, + "tokens_seen": 1148649472 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003292978936810431, + "loss": 2.7973, + "theoretical_loss": 3.601966587863812, + "tokens_seen": 1148715008 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032928786359077235, + "loss": 2.7422, + "theoretical_loss": 3.601947529962662, + "tokens_seen": 1148780544 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003292778335005015, + "loss": 2.701, + "theoretical_loss": 3.6019284734531056, + "tokens_seen": 1148846080 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003292678034102307, + "loss": 2.6874, + "theoretical_loss": 3.6019094183349596, + "tokens_seen": 1148911616 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003292577733199599, + "loss": 2.7553, + "theoretical_loss": 3.6018903646080447, + "tokens_seen": 1148977152 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032924774322968907, + "loss": 3.0321, + "theoretical_loss": 3.601871312272179, + "tokens_seen": 1149042688 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032923771313941825, + "loss": 2.6756, + "theoretical_loss": 3.601852261327182, + "tokens_seen": 1149108224 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003292276830491475, + "loss": 2.917, + "theoretical_loss": 3.601833211772873, + "tokens_seen": 1149173760 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003292176529588766, + "loss": 2.7465, + "theoretical_loss": 3.601814163609071, + "tokens_seen": 1149239296 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032920762286860585, + "loss": 2.7106, + "theoretical_loss": 3.6017951168355946, + "tokens_seen": 1149304832 + }, + { + "epoch": 0.35, + "learning_rate": 0.000329197592778335, + "loss": 2.6292, + "theoretical_loss": 3.6017760714522637, + "tokens_seen": 1149370368 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003291875626880642, + "loss": 2.5909, + "theoretical_loss": 3.601757027458898, + "tokens_seen": 1149435904 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003291775325977934, + "loss": 2.8127, + "theoretical_loss": 3.6017379848553164, + "tokens_seen": 1149501440 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003291675025075226, + "loss": 2.6665, + "theoretical_loss": 3.6017189436413375, + "tokens_seen": 1149566976 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032915747241725175, + "loss": 2.7374, + "theoretical_loss": 3.6016999038167814, + "tokens_seen": 1149632512 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032914744232698094, + "loss": 2.7003, + "theoretical_loss": 3.601680865381468, + "tokens_seen": 1149698048 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003291374122367101, + "loss": 2.7507, + "theoretical_loss": 3.6016618283352164, + "tokens_seen": 1149763584 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032912738214643935, + "loss": 2.6134, + "theoretical_loss": 3.601642792677845, + "tokens_seen": 1149829120 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003291173520561685, + "loss": 2.7211, + "theoretical_loss": 3.6016237584091755, + "tokens_seen": 1149894656 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003291073219658977, + "loss": 2.8167, + "theoretical_loss": 3.6016047255290253, + "tokens_seen": 1149960192 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003290972918756269, + "loss": 2.6904, + "theoretical_loss": 3.601585694037215, + "tokens_seen": 1150025728 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003290872617853561, + "loss": 2.7994, + "theoretical_loss": 3.601566663933564, + "tokens_seen": 1150091264 + }, + { + "epoch": 0.35, + "objective/train/docs_used": 654951, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.961534261703491, + "objective/train/theoretical_loss": 3.601547635217892, + "objective/train/tokens_used": 1170616800, + "theoretical_loss": 3.601547635217892, + "tokens_seen": 1150156800 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032907723169508526, + "loss": 2.743, + "theoretical_loss": 3.601547635217892, + "tokens_seen": 1150156800 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032906720160481444, + "loss": 2.9696, + "theoretical_loss": 3.6015286078900193, + "tokens_seen": 1150222336 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003290571715145436, + "loss": 2.5554, + "theoretical_loss": 3.601509581949765, + "tokens_seen": 1150287872 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032904714142427286, + "loss": 2.7514, + "theoretical_loss": 3.6014905573969482, + "tokens_seen": 1150353408 + }, + { + "epoch": 0.35, + "learning_rate": 0.000329037111334002, + "loss": 2.7805, + "theoretical_loss": 3.60147153423139, + "tokens_seen": 1150418944 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003290270812437312, + "loss": 2.7216, + "theoretical_loss": 3.6014525124529095, + "tokens_seen": 1150484480 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032901705115346034, + "loss": 2.69, + "theoretical_loss": 3.601433492061326, + "tokens_seen": 1150550016 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003290070210631896, + "loss": 2.998, + "theoretical_loss": 3.6014144730564612, + "tokens_seen": 1150615552 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032899699097291876, + "loss": 2.7891, + "theoretical_loss": 3.6013954554381336, + "tokens_seen": 1150681088 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032898696088264794, + "loss": 2.5678, + "theoretical_loss": 3.601376439206163, + "tokens_seen": 1150746624 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003289769307923771, + "loss": 2.6531, + "theoretical_loss": 3.60135742436037, + "tokens_seen": 1150812160 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003289669007021063, + "loss": 2.8238, + "theoretical_loss": 3.6013384109005746, + "tokens_seen": 1150877696 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003289568706118355, + "loss": 2.929, + "theoretical_loss": 3.601319398826597, + "tokens_seen": 1150943232 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003289468405215647, + "loss": 2.9875, + "theoretical_loss": 3.601300388138257, + "tokens_seen": 1151008768 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032893681043129385, + "loss": 2.7844, + "theoretical_loss": 3.601281378835374, + "tokens_seen": 1151074304 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003289267803410231, + "loss": 2.6614, + "theoretical_loss": 3.6012623709177696, + "tokens_seen": 1151139840 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032891675025075226, + "loss": 2.6489, + "theoretical_loss": 3.6012433643852626, + "tokens_seen": 1151205376 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032890672016048145, + "loss": 2.692, + "theoretical_loss": 3.6012243592376745, + "tokens_seen": 1151270912 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003288966900702106, + "loss": 2.7682, + "theoretical_loss": 3.601205355474825, + "tokens_seen": 1151336448 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003288866599799398, + "loss": 2.8301, + "theoretical_loss": 3.6011863530965345, + "tokens_seen": 1151401984 + }, + { + "epoch": 0.35, + "learning_rate": 0.000328876629889669, + "loss": 2.7502, + "theoretical_loss": 3.601167352102623, + "tokens_seen": 1151467520 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003288665997993982, + "loss": 2.7607, + "theoretical_loss": 3.6011483524929107, + "tokens_seen": 1151533056 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032885656970912735, + "loss": 2.7817, + "theoretical_loss": 3.601129354267219, + "tokens_seen": 1151598592 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003288465396188566, + "loss": 2.7258, + "theoretical_loss": 3.6011103574253673, + "tokens_seen": 1151664128 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003288365095285857, + "loss": 3.0391, + "theoretical_loss": 3.6010913619671765, + "tokens_seen": 1151729664 + }, + { + "epoch": 0.35, + "objective/train/docs_used": 656079, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.667681932449341, + "objective/train/theoretical_loss": 3.601072367892467, + "objective/train/tokens_used": 1172255200, + "theoretical_loss": 3.601072367892467, + "tokens_seen": 1151795200 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032882647943831495, + "loss": 2.7144, + "theoretical_loss": 3.601072367892467, + "tokens_seen": 1151795200 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003288164493480442, + "loss": 2.7221, + "theoretical_loss": 3.6010533752010594, + "tokens_seen": 1151860736 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003288064192577733, + "loss": 2.7173, + "theoretical_loss": 3.6010343838927743, + "tokens_seen": 1151926272 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032879638916750255, + "loss": 2.6291, + "theoretical_loss": 3.6010153939674323, + "tokens_seen": 1151991808 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003287863590772317, + "loss": 2.9263, + "theoretical_loss": 3.600996405424854, + "tokens_seen": 1152057344 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003287763289869609, + "loss": 2.6068, + "theoretical_loss": 3.6009774182648595, + "tokens_seen": 1152122880 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003287662988966901, + "loss": 2.6359, + "theoretical_loss": 3.6009584324872703, + "tokens_seen": 1152188416 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032875626880641927, + "loss": 2.8953, + "theoretical_loss": 3.600939448091907, + "tokens_seen": 1152253952 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032874623871614845, + "loss": 2.845, + "theoretical_loss": 3.60092046507859, + "tokens_seen": 1152319488 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003287362086258777, + "loss": 2.8206, + "theoretical_loss": 3.6009014834471404, + "tokens_seen": 1152385024 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003287261785356068, + "loss": 2.6871, + "theoretical_loss": 3.6008825031973792, + "tokens_seen": 1152450560 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032871614844533605, + "loss": 2.7891, + "theoretical_loss": 3.6008635243291263, + "tokens_seen": 1152516096 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003287061183550652, + "loss": 2.7419, + "theoretical_loss": 3.600844546842204, + "tokens_seen": 1152581632 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003286960882647944, + "loss": 2.7223, + "theoretical_loss": 3.6008255707364323, + "tokens_seen": 1152647168 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003286860581745236, + "loss": 2.7225, + "theoretical_loss": 3.600806596011632, + "tokens_seen": 1152712704 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003286760280842528, + "loss": 2.6542, + "theoretical_loss": 3.6007876226676245, + "tokens_seen": 1152778240 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032866599799398195, + "loss": 2.7688, + "theoretical_loss": 3.6007686507042314, + "tokens_seen": 1152843776 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032865596790371114, + "loss": 2.7112, + "theoretical_loss": 3.6007496801212726, + "tokens_seen": 1152909312 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003286459378134403, + "loss": 2.5245, + "theoretical_loss": 3.60073071091857, + "tokens_seen": 1152974848 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032863590772316955, + "loss": 2.8124, + "theoretical_loss": 3.6007117430959443, + "tokens_seen": 1153040384 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003286258776328987, + "loss": 2.5454, + "theoretical_loss": 3.6006927766532173, + "tokens_seen": 1153105920 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003286158475426279, + "loss": 2.8511, + "theoretical_loss": 3.6006738115902097, + "tokens_seen": 1153171456 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003286058174523571, + "loss": 2.8128, + "theoretical_loss": 3.600654847906742, + "tokens_seen": 1153236992 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003285957873620863, + "loss": 2.5728, + "theoretical_loss": 3.600635885602637, + "tokens_seen": 1153302528 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032858575727181546, + "loss": 2.7516, + "theoretical_loss": 3.6006169246777153, + "tokens_seen": 1153368064 + }, + { + "epoch": 0.35, + "objective/train/docs_used": 656588, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.829697370529175, + "objective/train/theoretical_loss": 3.6005979651317976, + "objective/train/tokens_used": 1173893600, + "theoretical_loss": 3.6005979651317976, + "tokens_seen": 1153433600 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032857572718154464, + "loss": 2.7151, + "theoretical_loss": 3.6005979651317976, + "tokens_seen": 1153433600 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003285656970912738, + "loss": 2.7781, + "theoretical_loss": 3.6005790069647063, + "tokens_seen": 1153499136 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032855566700100306, + "loss": 2.9694, + "theoretical_loss": 3.600560050176262, + "tokens_seen": 1153564672 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003285456369107322, + "loss": 2.7466, + "theoretical_loss": 3.6005410947662866, + "tokens_seen": 1153630208 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003285356068204614, + "loss": 2.6679, + "theoretical_loss": 3.6005221407346015, + "tokens_seen": 1153695744 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032852557673019054, + "loss": 2.949, + "theoretical_loss": 3.600503188081028, + "tokens_seen": 1153761280 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003285155466399198, + "loss": 2.6139, + "theoretical_loss": 3.6004842368053875, + "tokens_seen": 1153826816 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032850551654964896, + "loss": 2.6435, + "theoretical_loss": 3.600465286907502, + "tokens_seen": 1153892352 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032849548645937814, + "loss": 2.7226, + "theoretical_loss": 3.6004463383871927, + "tokens_seen": 1153957888 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003284854563691073, + "loss": 2.689, + "theoretical_loss": 3.600427391244282, + "tokens_seen": 1154023424 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003284754262788365, + "loss": 2.5612, + "theoretical_loss": 3.6004084454785907, + "tokens_seen": 1154088960 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003284653961885657, + "loss": 2.6997, + "theoretical_loss": 3.600389501089941, + "tokens_seen": 1154154496 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003284553660982949, + "loss": 2.6546, + "theoretical_loss": 3.600370558078154, + "tokens_seen": 1154220032 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032844533600802405, + "loss": 3.0301, + "theoretical_loss": 3.600351616443052, + "tokens_seen": 1154285568 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003284353059177533, + "loss": 2.7329, + "theoretical_loss": 3.6003326761844567, + "tokens_seen": 1154351104 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032842527582748246, + "loss": 2.513, + "theoretical_loss": 3.60031373730219, + "tokens_seen": 1154416640 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032841524573721165, + "loss": 3.0042, + "theoretical_loss": 3.6002947997960737, + "tokens_seen": 1154482176 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003284052156469408, + "loss": 2.8915, + "theoretical_loss": 3.6002758636659293, + "tokens_seen": 1154547712 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032839518555667, + "loss": 2.5617, + "theoretical_loss": 3.6002569289115796, + "tokens_seen": 1154613248 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003283851554663992, + "loss": 2.983, + "theoretical_loss": 3.6002379955328454, + "tokens_seen": 1154678784 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003283751253761284, + "loss": 2.8475, + "theoretical_loss": 3.6002190635295497, + "tokens_seen": 1154744320 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032836509528585755, + "loss": 2.8399, + "theoretical_loss": 3.600200132901514, + "tokens_seen": 1154809856 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003283550651955868, + "loss": 2.9751, + "theoretical_loss": 3.6001812036485608, + "tokens_seen": 1154875392 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003283450351053159, + "loss": 2.7469, + "theoretical_loss": 3.600162275770512, + "tokens_seen": 1154940928 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032833500501504515, + "loss": 2.7254, + "theoretical_loss": 3.6001433492671895, + "tokens_seen": 1155006464 + }, + { + "epoch": 0.35, + "objective/train/docs_used": 657761, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6493988037109375, + "objective/train/theoretical_loss": 3.600124424138415, + "objective/train/tokens_used": 1175532000, + "theoretical_loss": 3.600124424138415, + "tokens_seen": 1155072000 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032832497492477433, + "loss": 2.8278, + "theoretical_loss": 3.600124424138415, + "tokens_seen": 1155072000 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003283149448345035, + "loss": 2.8443, + "theoretical_loss": 3.600105500384012, + "tokens_seen": 1155137536 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003283049147442327, + "loss": 2.7201, + "theoretical_loss": 3.6000865780038023, + "tokens_seen": 1155203072 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003282948846539619, + "loss": 2.7431, + "theoretical_loss": 3.600067656997607, + "tokens_seen": 1155268608 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032828485456369105, + "loss": 2.5909, + "theoretical_loss": 3.60004873736525, + "tokens_seen": 1155334144 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003282748244734203, + "loss": 2.7176, + "theoretical_loss": 3.6000298191065525, + "tokens_seen": 1155399680 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003282647943831494, + "loss": 2.7691, + "theoretical_loss": 3.600010902221337, + "tokens_seen": 1155465216 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032825476429287865, + "loss": 2.8113, + "theoretical_loss": 3.5999919867094268, + "tokens_seen": 1155530752 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032824473420260783, + "loss": 2.6789, + "theoretical_loss": 3.599973072570643, + "tokens_seen": 1155596288 + }, + { + "epoch": 0.35, + "learning_rate": 0.000328234704112337, + "loss": 2.9085, + "theoretical_loss": 3.5999541598048097, + "tokens_seen": 1155661824 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003282246740220662, + "loss": 2.6583, + "theoretical_loss": 3.5999352484117475, + "tokens_seen": 1155727360 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003282146439317954, + "loss": 2.6084, + "theoretical_loss": 3.59991633839128, + "tokens_seen": 1155792896 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032820461384152456, + "loss": 2.8408, + "theoretical_loss": 3.59989742974323, + "tokens_seen": 1155858432 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003281945837512538, + "loss": 2.8312, + "theoretical_loss": 3.599878522467419, + "tokens_seen": 1155923968 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003281845536609829, + "loss": 2.714, + "theoretical_loss": 3.5998596165636707, + "tokens_seen": 1155989504 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032817452357071215, + "loss": 2.722, + "theoretical_loss": 3.599840712031807, + "tokens_seen": 1156055040 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003281644934804413, + "loss": 2.8259, + "theoretical_loss": 3.599821808871652, + "tokens_seen": 1156120576 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003281544633901705, + "loss": 2.8276, + "theoretical_loss": 3.5998029070830264, + "tokens_seen": 1156186112 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003281444332998997, + "loss": 2.7623, + "theoretical_loss": 3.5997840066657543, + "tokens_seen": 1156251648 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003281344032096289, + "loss": 2.9237, + "theoretical_loss": 3.5997651076196577, + "tokens_seen": 1156317184 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032812437311935806, + "loss": 2.5922, + "theoretical_loss": 3.59974620994456, + "tokens_seen": 1156382720 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003281143430290873, + "loss": 2.7064, + "theoretical_loss": 3.599727313640284, + "tokens_seen": 1156448256 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003281043129388164, + "loss": 2.8833, + "theoretical_loss": 3.599708418706653, + "tokens_seen": 1156513792 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032809428284854566, + "loss": 2.8831, + "theoretical_loss": 3.599689525143489, + "tokens_seen": 1156579328 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003280842527582748, + "loss": 2.4101, + "theoretical_loss": 3.599670632950615, + "tokens_seen": 1156644864 + }, + { + "epoch": 0.35, + "objective/train/docs_used": 658495, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.794929027557373, + "objective/train/theoretical_loss": 3.599651742127855, + "objective/train/tokens_used": 1177170400, + "theoretical_loss": 3.599651742127855, + "tokens_seen": 1156710400 + }, + { + "epoch": 0.35, + "learning_rate": 0.000328074222668004, + "loss": 2.6775, + "theoretical_loss": 3.599651742127855, + "tokens_seen": 1156710400 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032806419257773326, + "loss": 2.7651, + "theoretical_loss": 3.5996328526750307, + "tokens_seen": 1156775936 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003280541624874624, + "loss": 2.7172, + "theoretical_loss": 3.5996139645919656, + "tokens_seen": 1156841472 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003280441323971916, + "loss": 2.8091, + "theoretical_loss": 3.5995950778784835, + "tokens_seen": 1156907008 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032803410230692074, + "loss": 2.828, + "theoretical_loss": 3.599576192534407, + "tokens_seen": 1156972544 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032802407221665, + "loss": 2.7442, + "theoretical_loss": 3.5995573085595587, + "tokens_seen": 1157038080 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032801404212637916, + "loss": 2.6926, + "theoretical_loss": 3.599538425953763, + "tokens_seen": 1157103616 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032800401203610834, + "loss": 2.7818, + "theoretical_loss": 3.5995195447168418, + "tokens_seen": 1157169152 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003279939819458375, + "loss": 2.5837, + "theoretical_loss": 3.5995006648486196, + "tokens_seen": 1157234688 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003279839518555667, + "loss": 2.6159, + "theoretical_loss": 3.5994817863489184, + "tokens_seen": 1157300224 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003279739217652959, + "loss": 2.7135, + "theoretical_loss": 3.5994629092175625, + "tokens_seen": 1157365760 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003279638916750251, + "loss": 2.7423, + "theoretical_loss": 3.5994440334543754, + "tokens_seen": 1157431296 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032795386158475425, + "loss": 2.723, + "theoretical_loss": 3.5994251590591793, + "tokens_seen": 1157496832 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003279438314944835, + "loss": 2.755, + "theoretical_loss": 3.5994062860317984, + "tokens_seen": 1157562368 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032793380140421266, + "loss": 2.7236, + "theoretical_loss": 3.5993874143720563, + "tokens_seen": 1157627904 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032792377131394185, + "loss": 2.5149, + "theoretical_loss": 3.599368544079776, + "tokens_seen": 1157693440 + }, + { + "epoch": 0.35, + "learning_rate": 0.000327913741223671, + "loss": 2.7442, + "theoretical_loss": 3.5993496751547815, + "tokens_seen": 1157758976 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003279037111334002, + "loss": 2.9412, + "theoretical_loss": 3.599330807596896, + "tokens_seen": 1157824512 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003278936810431294, + "loss": 2.8474, + "theoretical_loss": 3.5993119414059427, + "tokens_seen": 1157890048 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003278836509528586, + "loss": 2.5435, + "theoretical_loss": 3.5992930765817466, + "tokens_seen": 1157955584 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032787362086258775, + "loss": 2.8543, + "theoretical_loss": 3.5992742131241293, + "tokens_seen": 1158021120 + }, + { + "epoch": 0.35, + "learning_rate": 0.000327863590772317, + "loss": 2.6591, + "theoretical_loss": 3.599255351032917, + "tokens_seen": 1158086656 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003278535606820461, + "loss": 2.7708, + "theoretical_loss": 3.5992364903079306, + "tokens_seen": 1158152192 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032784353059177535, + "loss": 2.8862, + "theoretical_loss": 3.599217630948996, + "tokens_seen": 1158217728 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032783350050150453, + "loss": 2.8184, + "theoretical_loss": 3.599198772955936, + "tokens_seen": 1158283264 + }, + { + "epoch": 0.35, + "objective/train/docs_used": 659118, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.748365640640259, + "objective/train/theoretical_loss": 3.5991799163285743, + "objective/train/tokens_used": 1178808800, + "theoretical_loss": 3.5991799163285743, + "tokens_seen": 1158348800 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003278234704112337, + "loss": 2.7749, + "theoretical_loss": 3.5991799163285743, + "tokens_seen": 1158348800 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003278134403209629, + "loss": 2.7205, + "theoretical_loss": 3.5991610610667353, + "tokens_seen": 1158414336 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032780341023069213, + "loss": 2.7382, + "theoretical_loss": 3.599142207170243, + "tokens_seen": 1158479872 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032779338014042125, + "loss": 2.9044, + "theoretical_loss": 3.5991233546389205, + "tokens_seen": 1158545408 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003277833500501505, + "loss": 2.7899, + "theoretical_loss": 3.599104503472592, + "tokens_seen": 1158610944 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003277733199598796, + "loss": 2.9274, + "theoretical_loss": 3.599085653671082, + "tokens_seen": 1158676480 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032776328986960885, + "loss": 2.8176, + "theoretical_loss": 3.599066805234214, + "tokens_seen": 1158742016 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032775325977933803, + "loss": 2.7207, + "theoretical_loss": 3.5990479581618118, + "tokens_seen": 1158807552 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003277432296890672, + "loss": 2.8243, + "theoretical_loss": 3.5990291124537004, + "tokens_seen": 1158873088 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003277331995987964, + "loss": 2.7413, + "theoretical_loss": 3.5990102681097027, + "tokens_seen": 1158938624 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003277231695085256, + "loss": 2.6875, + "theoretical_loss": 3.5989914251296438, + "tokens_seen": 1159004160 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032771313941825476, + "loss": 2.7396, + "theoretical_loss": 3.5989725835133473, + "tokens_seen": 1159069696 + }, + { + "epoch": 0.35, + "learning_rate": 0.000327703109327984, + "loss": 2.8195, + "theoretical_loss": 3.5989537432606378, + "tokens_seen": 1159135232 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003276930792377131, + "loss": 2.5246, + "theoretical_loss": 3.5989349043713394, + "tokens_seen": 1159200768 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032768304914744235, + "loss": 2.809, + "theoretical_loss": 3.598916066845276, + "tokens_seen": 1159266304 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003276730190571715, + "loss": 2.5284, + "theoretical_loss": 3.5988972306822724, + "tokens_seen": 1159331840 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003276629889669007, + "loss": 2.6285, + "theoretical_loss": 3.5988783958821524, + "tokens_seen": 1159397376 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003276529588766299, + "loss": 2.9043, + "theoretical_loss": 3.598859562444741, + "tokens_seen": 1159462912 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003276429287863591, + "loss": 2.6758, + "theoretical_loss": 3.5988407303698624, + "tokens_seen": 1159528448 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032763289869608826, + "loss": 2.7666, + "theoretical_loss": 3.5988218996573407, + "tokens_seen": 1159593984 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003276228686058175, + "loss": 2.5635, + "theoretical_loss": 3.5988030703070004, + "tokens_seen": 1159659520 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003276128385155466, + "loss": 2.9303, + "theoretical_loss": 3.5987842423186662, + "tokens_seen": 1159725056 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032760280842527586, + "loss": 2.7379, + "theoretical_loss": 3.5987654156921627, + "tokens_seen": 1159790592 + }, + { + "epoch": 0.35, + "learning_rate": 0.000327592778335005, + "loss": 2.7169, + "theoretical_loss": 3.5987465904273144, + "tokens_seen": 1159856128 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003275827482447342, + "loss": 2.6846, + "theoretical_loss": 3.598727766523945, + "tokens_seen": 1159921664 + }, + { + "epoch": 0.35, + "objective/train/docs_used": 660623, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7210474014282227, + "objective/train/theoretical_loss": 3.5987089439818805, + "objective/train/tokens_used": 1180447200, + "theoretical_loss": 3.5987089439818805, + "tokens_seen": 1159987200 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003275727181544634, + "loss": 2.7058, + "theoretical_loss": 3.5987089439818805, + "tokens_seen": 1159987200 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003275626880641926, + "loss": 2.8083, + "theoretical_loss": 3.5986901228009454, + "tokens_seen": 1160052736 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032755265797392176, + "loss": 2.6412, + "theoretical_loss": 3.5986713029809634, + "tokens_seen": 1160118272 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032754262788365094, + "loss": 2.7109, + "theoretical_loss": 3.59865248452176, + "tokens_seen": 1160183808 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003275325977933801, + "loss": 2.6844, + "theoretical_loss": 3.59863366742316, + "tokens_seen": 1160249344 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032752256770310936, + "loss": 2.5496, + "theoretical_loss": 3.598614851684987, + "tokens_seen": 1160314880 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003275125376128385, + "loss": 2.7591, + "theoretical_loss": 3.5985960373070673, + "tokens_seen": 1160380416 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003275025075225677, + "loss": 2.7228, + "theoretical_loss": 3.598577224289225, + "tokens_seen": 1160445952 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032749247743229685, + "loss": 2.5548, + "theoretical_loss": 3.5985584126312853, + "tokens_seen": 1160511488 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003274824473420261, + "loss": 2.5781, + "theoretical_loss": 3.598539602333073, + "tokens_seen": 1160577024 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032747241725175527, + "loss": 2.5299, + "theoretical_loss": 3.598520793394413, + "tokens_seen": 1160642560 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032746238716148445, + "loss": 2.6975, + "theoretical_loss": 3.59850198581513, + "tokens_seen": 1160708096 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032745235707121363, + "loss": 2.5308, + "theoretical_loss": 3.59848317959505, + "tokens_seen": 1160773632 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032744232698094286, + "loss": 2.7449, + "theoretical_loss": 3.5984643747339966, + "tokens_seen": 1160839168 + }, + { + "epoch": 0.35, + "learning_rate": 0.000327432296890672, + "loss": 2.5889, + "theoretical_loss": 3.5984455712317955, + "tokens_seen": 1160904704 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003274222668004012, + "loss": 2.837, + "theoretical_loss": 3.598426769088272, + "tokens_seen": 1160970240 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032741223671013035, + "loss": 2.7549, + "theoretical_loss": 3.5984079683032517, + "tokens_seen": 1161035776 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003274022066198596, + "loss": 2.7987, + "theoretical_loss": 3.5983891688765586, + "tokens_seen": 1161101312 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032739217652958877, + "loss": 2.6975, + "theoretical_loss": 3.598370370808019, + "tokens_seen": 1161166848 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032738214643931795, + "loss": 2.6799, + "theoretical_loss": 3.598351574097457, + "tokens_seen": 1161232384 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032737211634904713, + "loss": 2.6518, + "theoretical_loss": 3.598332778744699, + "tokens_seen": 1161297920 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003273620862587763, + "loss": 2.7545, + "theoretical_loss": 3.59831398474957, + "tokens_seen": 1161363456 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003273520561685055, + "loss": 2.6724, + "theoretical_loss": 3.5982951921118946, + "tokens_seen": 1161428992 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032734202607823473, + "loss": 2.9403, + "theoretical_loss": 3.598276400831499, + "tokens_seen": 1161494528 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032733199598796386, + "loss": 2.5596, + "theoretical_loss": 3.5982576109082083, + "tokens_seen": 1161560064 + }, + { + "epoch": 0.35, + "objective/train/docs_used": 661405, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.5079736709594727, + "objective/train/theoretical_loss": 3.598238822341848, + "objective/train/tokens_used": 1182085600, + "theoretical_loss": 3.598238822341848, + "tokens_seen": 1161625600 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003273219658976931, + "loss": 2.7365, + "theoretical_loss": 3.598238822341848, + "tokens_seen": 1161625600 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032731193580742233, + "loss": 2.7177, + "theoretical_loss": 3.5982200351322433, + "tokens_seen": 1161691136 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032730190571715145, + "loss": 2.7762, + "theoretical_loss": 3.59820124927922, + "tokens_seen": 1161756672 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003272918756268807, + "loss": 2.4883, + "theoretical_loss": 3.598182464782604, + "tokens_seen": 1161822208 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003272818455366098, + "loss": 2.811, + "theoretical_loss": 3.5981636816422196, + "tokens_seen": 1161887744 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032727181544633905, + "loss": 2.8697, + "theoretical_loss": 3.5981448998578935, + "tokens_seen": 1161953280 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032726178535606823, + "loss": 2.8787, + "theoretical_loss": 3.598126119429451, + "tokens_seen": 1162018816 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003272517552657974, + "loss": 2.6289, + "theoretical_loss": 3.5981073403567176, + "tokens_seen": 1162084352 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003272417251755266, + "loss": 2.5937, + "theoretical_loss": 3.5980885626395196, + "tokens_seen": 1162149888 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003272316950852558, + "loss": 2.6052, + "theoretical_loss": 3.598069786277682, + "tokens_seen": 1162215424 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032722166499498496, + "loss": 2.9094, + "theoretical_loss": 3.598051011271031, + "tokens_seen": 1162280960 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003272116349047142, + "loss": 2.8303, + "theoretical_loss": 3.5980322376193916, + "tokens_seen": 1162346496 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003272016048144433, + "loss": 2.6611, + "theoretical_loss": 3.598013465322591, + "tokens_seen": 1162412032 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032719157472417256, + "loss": 2.8298, + "theoretical_loss": 3.5979946943804535, + "tokens_seen": 1162477568 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003271815446339017, + "loss": 2.6802, + "theoretical_loss": 3.5979759247928067, + "tokens_seen": 1162543104 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003271715145436309, + "loss": 2.8374, + "theoretical_loss": 3.597957156559475, + "tokens_seen": 1162608640 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003271614844533601, + "loss": 2.8161, + "theoretical_loss": 3.5979383896802846, + "tokens_seen": 1162674176 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003271514543630893, + "loss": 2.7147, + "theoretical_loss": 3.597919624155062, + "tokens_seen": 1162739712 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032714142427281846, + "loss": 2.715, + "theoretical_loss": 3.597900859983633, + "tokens_seen": 1162805248 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003271313941825477, + "loss": 2.5887, + "theoretical_loss": 3.5978820971658236, + "tokens_seen": 1162870784 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003271213640922768, + "loss": 2.5987, + "theoretical_loss": 3.5978633357014598, + "tokens_seen": 1162936320 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032711133400200606, + "loss": 2.835, + "theoretical_loss": 3.5978445755903685, + "tokens_seen": 1163001856 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003271013039117352, + "loss": 2.875, + "theoretical_loss": 3.5978258168323745, + "tokens_seen": 1163067392 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003270912738214644, + "loss": 2.3998, + "theoretical_loss": 3.5978070594273044, + "tokens_seen": 1163132928 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003270812437311936, + "loss": 2.8252, + "theoretical_loss": 3.597788303374985, + "tokens_seen": 1163198464 + }, + { + "epoch": 0.35, + "objective/train/docs_used": 662532, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.417668104171753, + "objective/train/theoretical_loss": 3.5977695486752426, + "objective/train/tokens_used": 1183724000, + "theoretical_loss": 3.5977695486752426, + "tokens_seen": 1163264000 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003270712136409228, + "loss": 2.466, + "theoretical_loss": 3.5977695486752426, + "tokens_seen": 1163264000 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032706118355065196, + "loss": 2.796, + "theoretical_loss": 3.597750795327902, + "tokens_seen": 1163329536 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032705115346038114, + "loss": 2.8811, + "theoretical_loss": 3.5977320433327913, + "tokens_seen": 1163395072 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003270411233701103, + "loss": 2.8114, + "theoretical_loss": 3.597713292689736, + "tokens_seen": 1163460608 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032703109327983956, + "loss": 2.9529, + "theoretical_loss": 3.5976945433985623, + "tokens_seen": 1163526144 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003270210631895687, + "loss": 2.7068, + "theoretical_loss": 3.5976757954590965, + "tokens_seen": 1163591680 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003270110330992979, + "loss": 2.7642, + "theoretical_loss": 3.597657048871166, + "tokens_seen": 1163657216 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032700100300902705, + "loss": 2.9212, + "theoretical_loss": 3.5976383036345965, + "tokens_seen": 1163722752 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003269909729187563, + "loss": 2.7017, + "theoretical_loss": 3.597619559749214, + "tokens_seen": 1163788288 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032698094282848547, + "loss": 2.9861, + "theoretical_loss": 3.5976008172148464, + "tokens_seen": 1163853824 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032697091273821465, + "loss": 2.7336, + "theoretical_loss": 3.597582076031319, + "tokens_seen": 1163919360 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032696088264794383, + "loss": 2.4907, + "theoretical_loss": 3.5975633361984594, + "tokens_seen": 1163984896 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032695085255767306, + "loss": 2.8516, + "theoretical_loss": 3.5975445977160927, + "tokens_seen": 1164050432 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003269408224674022, + "loss": 2.7507, + "theoretical_loss": 3.597525860584047, + "tokens_seen": 1164115968 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003269307923771314, + "loss": 2.6513, + "theoretical_loss": 3.597507124802149, + "tokens_seen": 1164181504 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032692076228686055, + "loss": 2.7964, + "theoretical_loss": 3.5974883903702244, + "tokens_seen": 1164247040 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003269107321965898, + "loss": 2.6115, + "theoretical_loss": 3.5974696572881006, + "tokens_seen": 1164312576 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032690070210631897, + "loss": 2.5491, + "theoretical_loss": 3.597450925555604, + "tokens_seen": 1164378112 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032689067201604815, + "loss": 2.6008, + "theoretical_loss": 3.597432195172562, + "tokens_seen": 1164443648 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032688064192577733, + "loss": 2.5754, + "theoretical_loss": 3.597413466138801, + "tokens_seen": 1164509184 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003268706118355065, + "loss": 2.6417, + "theoretical_loss": 3.597394738454148, + "tokens_seen": 1164574720 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003268605817452357, + "loss": 2.5774, + "theoretical_loss": 3.59737601211843, + "tokens_seen": 1164640256 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032685055165496493, + "loss": 2.585, + "theoretical_loss": 3.597357287131474, + "tokens_seen": 1164705792 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032684052156469406, + "loss": 2.8026, + "theoretical_loss": 3.597338563493106, + "tokens_seen": 1164771328 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003268304914744233, + "loss": 2.8464, + "theoretical_loss": 3.5973198412031544, + "tokens_seen": 1164836864 + }, + { + "epoch": 0.35, + "objective/train/docs_used": 663185, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7970621585845947, + "objective/train/theoretical_loss": 3.5973011202614456, + "objective/train/tokens_used": 1185362400, + "theoretical_loss": 3.5973011202614456, + "tokens_seen": 1164902400 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003268204613841524, + "loss": 2.7157, + "theoretical_loss": 3.5973011202614456, + "tokens_seen": 1164902400 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032681043129388165, + "loss": 2.7111, + "theoretical_loss": 3.5972824006678064, + "tokens_seen": 1164967936 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032680040120361084, + "loss": 2.528, + "theoretical_loss": 3.5972636824220645, + "tokens_seen": 1165033472 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032679037111334, + "loss": 2.9649, + "theoretical_loss": 3.5972449655240473, + "tokens_seen": 1165099008 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003267803410230692, + "loss": 2.6128, + "theoretical_loss": 3.5972262499735805, + "tokens_seen": 1165164544 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032677031093279843, + "loss": 2.8148, + "theoretical_loss": 3.597207535770493, + "tokens_seen": 1165230080 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032676028084252756, + "loss": 2.8291, + "theoretical_loss": 3.5971888229146103, + "tokens_seen": 1165295616 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003267502507522568, + "loss": 2.6058, + "theoretical_loss": 3.5971701114057613, + "tokens_seen": 1165361152 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003267402206619859, + "loss": 2.7603, + "theoretical_loss": 3.5971514012437718, + "tokens_seen": 1165426688 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032673019057171516, + "loss": 2.8156, + "theoretical_loss": 3.5971326924284703, + "tokens_seen": 1165492224 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032672016048144434, + "loss": 2.8549, + "theoretical_loss": 3.5971139849596843, + "tokens_seen": 1165557760 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003267101303911735, + "loss": 2.54, + "theoretical_loss": 3.59709527883724, + "tokens_seen": 1165623296 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003267001003009027, + "loss": 2.8396, + "theoretical_loss": 3.5970765740609654, + "tokens_seen": 1165688832 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003266900702106319, + "loss": 2.8055, + "theoretical_loss": 3.5970578706306884, + "tokens_seen": 1165754368 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032668004012036106, + "loss": 2.6769, + "theoretical_loss": 3.597039168546236, + "tokens_seen": 1165819904 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003266700100300903, + "loss": 2.7902, + "theoretical_loss": 3.597020467807436, + "tokens_seen": 1165885440 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003266599799398194, + "loss": 2.6034, + "theoretical_loss": 3.597001768414115, + "tokens_seen": 1165950976 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032664994984954866, + "loss": 2.7535, + "theoretical_loss": 3.596983070366102, + "tokens_seen": 1166016512 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032663991975927784, + "loss": 2.6095, + "theoretical_loss": 3.5969643736632237, + "tokens_seen": 1166082048 + }, + { + "epoch": 0.35, + "learning_rate": 0.000326629889669007, + "loss": 2.7329, + "theoretical_loss": 3.5969456783053078, + "tokens_seen": 1166147584 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003266198595787362, + "loss": 2.6486, + "theoretical_loss": 3.5969269842921823, + "tokens_seen": 1166213120 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003266098294884654, + "loss": 2.7354, + "theoretical_loss": 3.5969082916236745, + "tokens_seen": 1166278656 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032659979939819457, + "loss": 2.4818, + "theoretical_loss": 3.5968896002996127, + "tokens_seen": 1166344192 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003265897693079238, + "loss": 2.7617, + "theoretical_loss": 3.596870910319824, + "tokens_seen": 1166409728 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032657973921765293, + "loss": 2.769, + "theoretical_loss": 3.596852221684137, + "tokens_seen": 1166475264 + }, + { + "epoch": 0.35, + "objective/train/docs_used": 664585, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.8181874752044678, + "objective/train/theoretical_loss": 3.596833534392379, + "objective/train/tokens_used": 1187000800, + "theoretical_loss": 3.596833534392379, + "tokens_seen": 1166540800 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032656970912738216, + "loss": 2.8626, + "theoretical_loss": 3.596833534392379, + "tokens_seen": 1166540800 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032655967903711135, + "loss": 2.9086, + "theoretical_loss": 3.596814848444378, + "tokens_seen": 1166606336 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003265496489468405, + "loss": 2.7945, + "theoretical_loss": 3.5967961638399615, + "tokens_seen": 1166671872 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032653961885656976, + "loss": 2.5797, + "theoretical_loss": 3.596777480578958, + "tokens_seen": 1166737408 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003265295887662989, + "loss": 2.9325, + "theoretical_loss": 3.596758798661195, + "tokens_seen": 1166802944 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003265195586760281, + "loss": 2.7996, + "theoretical_loss": 3.5967401180865006, + "tokens_seen": 1166868480 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032650952858575725, + "loss": 2.7275, + "theoretical_loss": 3.5967214388547033, + "tokens_seen": 1166934016 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003264994984954865, + "loss": 2.9005, + "theoretical_loss": 3.5967027609656306, + "tokens_seen": 1166999552 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032648946840521567, + "loss": 2.6037, + "theoretical_loss": 3.5966840844191106, + "tokens_seen": 1167065088 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032647943831494485, + "loss": 2.6582, + "theoretical_loss": 3.596665409214972, + "tokens_seen": 1167130624 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032646940822467403, + "loss": 2.7315, + "theoretical_loss": 3.5966467353530422, + "tokens_seen": 1167196160 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032645937813440326, + "loss": 2.5, + "theoretical_loss": 3.5966280628331493, + "tokens_seen": 1167261696 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003264493480441324, + "loss": 2.6969, + "theoretical_loss": 3.5966093916551225, + "tokens_seen": 1167327232 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003264393179538616, + "loss": 2.6808, + "theoretical_loss": 3.5965907218187896, + "tokens_seen": 1167392768 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032642928786359075, + "loss": 2.6917, + "theoretical_loss": 3.5965720533239782, + "tokens_seen": 1167458304 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032641925777332, + "loss": 2.6729, + "theoretical_loss": 3.596553386170518, + "tokens_seen": 1167523840 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032640922768304917, + "loss": 2.5809, + "theoretical_loss": 3.5965347203582354, + "tokens_seen": 1167589376 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032639919759277835, + "loss": 2.6824, + "theoretical_loss": 3.59651605588696, + "tokens_seen": 1167654912 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032638916750250753, + "loss": 2.5108, + "theoretical_loss": 3.596497392756521, + "tokens_seen": 1167720448 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003263791374122367, + "loss": 2.77, + "theoretical_loss": 3.596478730966745, + "tokens_seen": 1167785984 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003263691073219659, + "loss": 2.7199, + "theoretical_loss": 3.5964600705174616, + "tokens_seen": 1167851520 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032635907723169513, + "loss": 2.6957, + "theoretical_loss": 3.596441411408499, + "tokens_seen": 1167917056 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032634904714142426, + "loss": 2.7849, + "theoretical_loss": 3.5964227536396858, + "tokens_seen": 1167982592 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003263390170511535, + "loss": 2.5002, + "theoretical_loss": 3.5964040972108506, + "tokens_seen": 1168048128 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003263289869608826, + "loss": 2.531, + "theoretical_loss": 3.5963854421218215, + "tokens_seen": 1168113664 + }, + { + "epoch": 0.35, + "objective/train/docs_used": 665171, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7857608795166016, + "objective/train/theoretical_loss": 3.596366788372428, + "objective/train/tokens_used": 1188639200, + "theoretical_loss": 3.596366788372428, + "tokens_seen": 1168179200 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032631895687061185, + "loss": 2.767, + "theoretical_loss": 3.596366788372428, + "tokens_seen": 1168179200 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032630892678034104, + "loss": 2.846, + "theoretical_loss": 3.596348135962498, + "tokens_seen": 1168244736 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003262988966900702, + "loss": 2.6529, + "theoretical_loss": 3.5963294848918608, + "tokens_seen": 1168310272 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003262888665997994, + "loss": 2.7984, + "theoretical_loss": 3.5963108351603443, + "tokens_seen": 1168375808 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032627883650952863, + "loss": 2.7841, + "theoretical_loss": 3.596292186767778, + "tokens_seen": 1168441344 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032626880641925776, + "loss": 2.8222, + "theoretical_loss": 3.596273539713991, + "tokens_seen": 1168506880 + }, + { + "epoch": 0.35, + "learning_rate": 0.000326258776328987, + "loss": 2.7222, + "theoretical_loss": 3.5962548939988106, + "tokens_seen": 1168572416 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003262487462387161, + "loss": 2.8464, + "theoretical_loss": 3.5962362496220672, + "tokens_seen": 1168637952 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032623871614844536, + "loss": 2.7701, + "theoretical_loss": 3.5962176065835885, + "tokens_seen": 1168703488 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032622868605817454, + "loss": 2.8669, + "theoretical_loss": 3.5961989648832042, + "tokens_seen": 1168769024 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003262186559679037, + "loss": 2.6719, + "theoretical_loss": 3.596180324520743, + "tokens_seen": 1168834560 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003262086258776329, + "loss": 2.482, + "theoretical_loss": 3.5961616854960337, + "tokens_seen": 1168900096 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003261985957873621, + "loss": 2.685, + "theoretical_loss": 3.596143047808906, + "tokens_seen": 1168965632 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032618856569709126, + "loss": 2.8602, + "theoretical_loss": 3.5961244114591877, + "tokens_seen": 1169031168 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003261785356068205, + "loss": 2.7298, + "theoretical_loss": 3.596105776446709, + "tokens_seen": 1169096704 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003261685055165496, + "loss": 2.6926, + "theoretical_loss": 3.596087142771298, + "tokens_seen": 1169162240 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032615847542627886, + "loss": 2.8971, + "theoretical_loss": 3.5960685104327847, + "tokens_seen": 1169227776 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032614844533600804, + "loss": 2.9593, + "theoretical_loss": 3.596049879430998, + "tokens_seen": 1169293312 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003261384152457372, + "loss": 2.6746, + "theoretical_loss": 3.596031249765767, + "tokens_seen": 1169358848 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003261283851554664, + "loss": 2.5234, + "theoretical_loss": 3.5960126214369206, + "tokens_seen": 1169424384 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003261183550651956, + "loss": 2.8026, + "theoretical_loss": 3.5959939944442887, + "tokens_seen": 1169489920 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032610832497492477, + "loss": 2.6867, + "theoretical_loss": 3.5959753687877005, + "tokens_seen": 1169555456 + }, + { + "epoch": 0.35, + "learning_rate": 0.000326098294884654, + "loss": 2.7826, + "theoretical_loss": 3.5959567444669847, + "tokens_seen": 1169620992 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032608826479438313, + "loss": 2.9279, + "theoretical_loss": 3.595938121481971, + "tokens_seen": 1169686528 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032607823470411236, + "loss": 2.7413, + "theoretical_loss": 3.5959194998324886, + "tokens_seen": 1169752064 + }, + { + "epoch": 0.35, + "objective/train/docs_used": 666471, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.473968267440796, + "objective/train/theoretical_loss": 3.595900879518368, + "objective/train/tokens_used": 1190277600, + "theoretical_loss": 3.595900879518368, + "tokens_seen": 1169817600 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003260682046138415, + "loss": 2.5522, + "theoretical_loss": 3.595900879518368, + "tokens_seen": 1169817600 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003260581745235707, + "loss": 2.6265, + "theoretical_loss": 3.5958822605394367, + "tokens_seen": 1169883136 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003260481444332999, + "loss": 2.6278, + "theoretical_loss": 3.5958636428955257, + "tokens_seen": 1169948672 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003260381143430291, + "loss": 2.691, + "theoretical_loss": 3.595845026586464, + "tokens_seen": 1170014208 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032602808425275827, + "loss": 2.6306, + "theoretical_loss": 3.595826411612081, + "tokens_seen": 1170079744 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032601805416248745, + "loss": 2.9739, + "theoretical_loss": 3.5958077979722063, + "tokens_seen": 1170145280 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032600802407221663, + "loss": 2.7628, + "theoretical_loss": 3.59578918566667, + "tokens_seen": 1170210816 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032599799398194587, + "loss": 2.5597, + "theoretical_loss": 3.595770574695301, + "tokens_seen": 1170276352 + }, + { + "epoch": 0.35, + "learning_rate": 0.000325987963891675, + "loss": 2.6749, + "theoretical_loss": 3.5957519650579295, + "tokens_seen": 1170341888 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032597793380140423, + "loss": 2.8345, + "theoretical_loss": 3.5957333567543848, + "tokens_seen": 1170407424 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003259679037111334, + "loss": 2.7553, + "theoretical_loss": 3.595714749784497, + "tokens_seen": 1170472960 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003259578736208626, + "loss": 2.6303, + "theoretical_loss": 3.5956961441480955, + "tokens_seen": 1170538496 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003259478435305918, + "loss": 2.5847, + "theoretical_loss": 3.5956775398450103, + "tokens_seen": 1170604032 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032593781344032095, + "loss": 2.5965, + "theoretical_loss": 3.595658936875071, + "tokens_seen": 1170669568 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032592778335005014, + "loss": 2.7786, + "theoretical_loss": 3.595640335238108, + "tokens_seen": 1170735104 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032591775325977937, + "loss": 2.6617, + "theoretical_loss": 3.5956217349339505, + "tokens_seen": 1170800640 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003259077231695085, + "loss": 2.851, + "theoretical_loss": 3.5956031359624285, + "tokens_seen": 1170866176 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032589769307923773, + "loss": 2.5711, + "theoretical_loss": 3.595584538323372, + "tokens_seen": 1170931712 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032588766298896686, + "loss": 2.7575, + "theoretical_loss": 3.595565942016612, + "tokens_seen": 1170997248 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003258776328986961, + "loss": 2.6161, + "theoretical_loss": 3.5955473470419763, + "tokens_seen": 1171062784 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003258676028084253, + "loss": 2.7198, + "theoretical_loss": 3.5955287533992975, + "tokens_seen": 1171128320 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032585757271815446, + "loss": 2.6666, + "theoretical_loss": 3.5955101610884035, + "tokens_seen": 1171193856 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032584754262788364, + "loss": 2.6485, + "theoretical_loss": 3.5954915701091257, + "tokens_seen": 1171259392 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003258375125376128, + "loss": 2.6865, + "theoretical_loss": 3.5954729804612935, + "tokens_seen": 1171324928 + }, + { + "epoch": 0.35, + "learning_rate": 0.000325827482447342, + "loss": 2.8037, + "theoretical_loss": 3.5954543921447377, + "tokens_seen": 1171390464 + }, + { + "epoch": 0.35, + "objective/train/docs_used": 666886, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6622915267944336, + "objective/train/theoretical_loss": 3.595435805159288, + "objective/train/tokens_used": 1191916000, + "theoretical_loss": 3.595435805159288, + "tokens_seen": 1171456000 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032581745235707124, + "loss": 2.6703, + "theoretical_loss": 3.595435805159288, + "tokens_seen": 1171456000 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003258074222668004, + "loss": 2.6833, + "theoretical_loss": 3.5954172195047747, + "tokens_seen": 1171521536 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003257973921765296, + "loss": 2.8674, + "theoretical_loss": 3.5953986351810285, + "tokens_seen": 1171587072 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032578736208625883, + "loss": 2.6857, + "theoretical_loss": 3.5953800521878794, + "tokens_seen": 1171652608 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032577733199598796, + "loss": 2.5176, + "theoretical_loss": 3.595361470525157, + "tokens_seen": 1171718144 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003257673019057172, + "loss": 2.8121, + "theoretical_loss": 3.5953428901926925, + "tokens_seen": 1171783680 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003257572718154463, + "loss": 2.5657, + "theoretical_loss": 3.595324311190316, + "tokens_seen": 1171849216 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032574724172517556, + "loss": 2.8293, + "theoretical_loss": 3.595305733517858, + "tokens_seen": 1171914752 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032573721163490474, + "loss": 2.7937, + "theoretical_loss": 3.5952871571751492, + "tokens_seen": 1171980288 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003257271815446339, + "loss": 2.7349, + "theoretical_loss": 3.5952685821620194, + "tokens_seen": 1172045824 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003257171514543631, + "loss": 3.0255, + "theoretical_loss": 3.5952500084782995, + "tokens_seen": 1172111360 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003257071213640923, + "loss": 2.8878, + "theoretical_loss": 3.59523143612382, + "tokens_seen": 1172176896 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032569709127382146, + "loss": 2.7275, + "theoretical_loss": 3.5952128650984116, + "tokens_seen": 1172242432 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003256870611835507, + "loss": 2.7538, + "theoretical_loss": 3.595194295401905, + "tokens_seen": 1172307968 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003256770310932798, + "loss": 2.8547, + "theoretical_loss": 3.59517572703413, + "tokens_seen": 1172373504 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032566700100300906, + "loss": 2.7892, + "theoretical_loss": 3.595157159994918, + "tokens_seen": 1172439040 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032565697091273824, + "loss": 2.581, + "theoretical_loss": 3.5951385942840997, + "tokens_seen": 1172504576 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003256469408224674, + "loss": 2.6886, + "theoretical_loss": 3.595120029901506, + "tokens_seen": 1172570112 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003256369107321966, + "loss": 2.7399, + "theoretical_loss": 3.595101466846966, + "tokens_seen": 1172635648 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003256268806419258, + "loss": 2.6991, + "theoretical_loss": 3.595082905120313, + "tokens_seen": 1172701184 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032561685055165497, + "loss": 2.6965, + "theoretical_loss": 3.595064344721376, + "tokens_seen": 1172766720 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003256068204613842, + "loss": 2.7385, + "theoretical_loss": 3.5950457856499867, + "tokens_seen": 1172832256 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032559679037111333, + "loss": 2.8294, + "theoretical_loss": 3.595027227905976, + "tokens_seen": 1172897792 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032558676028084256, + "loss": 2.8067, + "theoretical_loss": 3.5950086714891736, + "tokens_seen": 1172963328 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003255767301905717, + "loss": 2.8619, + "theoretical_loss": 3.594990116399412, + "tokens_seen": 1173028864 + }, + { + "epoch": 0.36, + "objective/train/docs_used": 667905, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.458381414413452, + "objective/train/theoretical_loss": 3.594971562636521, + "objective/train/tokens_used": 1193554400, + "theoretical_loss": 3.594971562636521, + "tokens_seen": 1173094400 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003255667001003009, + "loss": 2.4993, + "theoretical_loss": 3.594971562636521, + "tokens_seen": 1173094400 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003255566700100301, + "loss": 2.7605, + "theoretical_loss": 3.5949530102003324, + "tokens_seen": 1173159936 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003255466399197593, + "loss": 2.8099, + "theoretical_loss": 3.594934459090677, + "tokens_seen": 1173225472 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032553660982948847, + "loss": 2.7545, + "theoretical_loss": 3.5949159093073857, + "tokens_seen": 1173291008 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032552657973921765, + "loss": 2.636, + "theoretical_loss": 3.5948973608502897, + "tokens_seen": 1173356544 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032551654964894683, + "loss": 2.7529, + "theoretical_loss": 3.5948788137192196, + "tokens_seen": 1173422080 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032550651955867607, + "loss": 2.6515, + "theoretical_loss": 3.594860267914008, + "tokens_seen": 1173487616 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003254964894684052, + "loss": 2.6996, + "theoretical_loss": 3.5948417234344845, + "tokens_seen": 1173553152 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032548645937813443, + "loss": 2.8955, + "theoretical_loss": 3.594823180280481, + "tokens_seen": 1173618688 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003254764292878636, + "loss": 2.723, + "theoretical_loss": 3.5948046384518286, + "tokens_seen": 1173684224 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003254663991975928, + "loss": 2.8945, + "theoretical_loss": 3.594786097948359, + "tokens_seen": 1173749760 + }, + { + "epoch": 0.36, + "learning_rate": 0.000325456369107322, + "loss": 2.6284, + "theoretical_loss": 3.5947675587699024, + "tokens_seen": 1173815296 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032544633901705115, + "loss": 2.8716, + "theoretical_loss": 3.594749020916292, + "tokens_seen": 1173880832 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032543630892678034, + "loss": 2.779, + "theoretical_loss": 3.594730484387357, + "tokens_seen": 1173946368 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032542627883650957, + "loss": 2.6489, + "theoretical_loss": 3.5947119491829307, + "tokens_seen": 1174011904 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003254162487462387, + "loss": 2.8877, + "theoretical_loss": 3.594693415302843, + "tokens_seen": 1174077440 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032540621865596793, + "loss": 2.3157, + "theoretical_loss": 3.5946748827469266, + "tokens_seen": 1174142976 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032539618856569706, + "loss": 2.6529, + "theoretical_loss": 3.5946563515150123, + "tokens_seen": 1174208512 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003253861584754263, + "loss": 2.5519, + "theoretical_loss": 3.5946378216069315, + "tokens_seen": 1174274048 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003253761283851555, + "loss": 2.8236, + "theoretical_loss": 3.5946192930225163, + "tokens_seen": 1174339584 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032536609829488466, + "loss": 2.7744, + "theoretical_loss": 3.594600765761598, + "tokens_seen": 1174405120 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032535606820461384, + "loss": 2.71, + "theoretical_loss": 3.5945822398240077, + "tokens_seen": 1174470656 + }, + { + "epoch": 0.36, + "learning_rate": 0.000325346038114343, + "loss": 2.8725, + "theoretical_loss": 3.5945637152095786, + "tokens_seen": 1174536192 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003253360080240722, + "loss": 2.7474, + "theoretical_loss": 3.59454519191814, + "tokens_seen": 1174601728 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032532597793380144, + "loss": 2.7489, + "theoretical_loss": 3.594526669949526, + "tokens_seen": 1174667264 + }, + { + "epoch": 0.36, + "objective/train/docs_used": 668590, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6248531341552734, + "objective/train/theoretical_loss": 3.594508149303567, + "objective/train/tokens_used": 1195192800, + "theoretical_loss": 3.594508149303567, + "tokens_seen": 1174732800 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032531594784353056, + "loss": 2.5856, + "theoretical_loss": 3.594508149303567, + "tokens_seen": 1174732800 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003253059177532598, + "loss": 2.5755, + "theoretical_loss": 3.594489629980095, + "tokens_seen": 1174798336 + }, + { + "epoch": 0.36, + "learning_rate": 0.000325295887662989, + "loss": 3.1093, + "theoretical_loss": 3.594471111978942, + "tokens_seen": 1174863872 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032528585757271816, + "loss": 2.7317, + "theoretical_loss": 3.5944525952999395, + "tokens_seen": 1174929408 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032527582748244734, + "loss": 2.9013, + "theoretical_loss": 3.5944340799429195, + "tokens_seen": 1174994944 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003252657973921765, + "loss": 2.6894, + "theoretical_loss": 3.594415565907714, + "tokens_seen": 1175060480 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003252557673019057, + "loss": 2.8523, + "theoretical_loss": 3.5943970531941556, + "tokens_seen": 1175126016 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032524573721163494, + "loss": 2.7566, + "theoretical_loss": 3.5943785418020746, + "tokens_seen": 1175191552 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032523570712136407, + "loss": 2.6102, + "theoretical_loss": 3.594360031731304, + "tokens_seen": 1175257088 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003252256770310933, + "loss": 2.7248, + "theoretical_loss": 3.5943415229816758, + "tokens_seen": 1175322624 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032521564694082243, + "loss": 2.3498, + "theoretical_loss": 3.594323015553022, + "tokens_seen": 1175388160 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032520561685055166, + "loss": 2.605, + "theoretical_loss": 3.5943045094451747, + "tokens_seen": 1175453696 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032519558676028084, + "loss": 2.7856, + "theoretical_loss": 3.594286004657966, + "tokens_seen": 1175519232 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032518555667001, + "loss": 2.9256, + "theoretical_loss": 3.5942675011912275, + "tokens_seen": 1175584768 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003251755265797392, + "loss": 2.871, + "theoretical_loss": 3.594248999044792, + "tokens_seen": 1175650304 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032516549648946844, + "loss": 2.6653, + "theoretical_loss": 3.594230498218492, + "tokens_seen": 1175715840 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032515546639919757, + "loss": 2.9802, + "theoretical_loss": 3.594211998712159, + "tokens_seen": 1175781376 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003251454363089268, + "loss": 2.8207, + "theoretical_loss": 3.5941935005256256, + "tokens_seen": 1175846912 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032513540621865593, + "loss": 2.8212, + "theoretical_loss": 3.594175003658724, + "tokens_seen": 1175912448 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032512537612838517, + "loss": 2.6903, + "theoretical_loss": 3.5941565081112863, + "tokens_seen": 1175977984 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032511534603811435, + "loss": 2.8107, + "theoretical_loss": 3.5941380138831454, + "tokens_seen": 1176043520 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032510531594784353, + "loss": 2.6854, + "theoretical_loss": 3.5941195209741332, + "tokens_seen": 1176109056 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003250952858575727, + "loss": 2.6984, + "theoretical_loss": 3.5941010293840825, + "tokens_seen": 1176174592 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003250852557673019, + "loss": 2.6319, + "theoretical_loss": 3.5940825391128257, + "tokens_seen": 1176240128 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032507522567703107, + "loss": 2.7233, + "theoretical_loss": 3.5940640501601946, + "tokens_seen": 1176305664 + }, + { + "epoch": 0.36, + "objective/train/docs_used": 669752, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6123292446136475, + "objective/train/theoretical_loss": 3.5940455625260226, + "objective/train/tokens_used": 1196831200, + "theoretical_loss": 3.5940455625260226, + "tokens_seen": 1176371200 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003250651955867603, + "loss": 2.8031, + "theoretical_loss": 3.5940455625260226, + "tokens_seen": 1176371200 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003250551654964895, + "loss": 2.7889, + "theoretical_loss": 3.594027076210142, + "tokens_seen": 1176436736 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032504513540621867, + "loss": 2.6558, + "theoretical_loss": 3.5940085912123845, + "tokens_seen": 1176502272 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032503510531594785, + "loss": 2.7257, + "theoretical_loss": 3.593990107532584, + "tokens_seen": 1176567808 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032502507522567703, + "loss": 2.7879, + "theoretical_loss": 3.593971625170572, + "tokens_seen": 1176633344 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032501504513540627, + "loss": 2.7707, + "theoretical_loss": 3.593953144126182, + "tokens_seen": 1176698880 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003250050150451354, + "loss": 2.6011, + "theoretical_loss": 3.5939346643992467, + "tokens_seen": 1176764416 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032499498495486463, + "loss": 2.8586, + "theoretical_loss": 3.5939161859895985, + "tokens_seen": 1176829952 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003249849548645938, + "loss": 2.7627, + "theoretical_loss": 3.59389770889707, + "tokens_seen": 1176895488 + }, + { + "epoch": 0.36, + "learning_rate": 0.000324974924774323, + "loss": 2.7929, + "theoretical_loss": 3.593879233121494, + "tokens_seen": 1176961024 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003249648946840522, + "loss": 2.7041, + "theoretical_loss": 3.5938607586627036, + "tokens_seen": 1177026560 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032495486459378135, + "loss": 2.8375, + "theoretical_loss": 3.5938422855205316, + "tokens_seen": 1177092096 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032494483450351054, + "loss": 2.5431, + "theoretical_loss": 3.5938238136948106, + "tokens_seen": 1177157632 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032493480441323977, + "loss": 2.89, + "theoretical_loss": 3.5938053431853736, + "tokens_seen": 1177223168 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003249247743229689, + "loss": 2.7269, + "theoretical_loss": 3.5937868739920535, + "tokens_seen": 1177288704 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032491474423269813, + "loss": 2.9435, + "theoretical_loss": 3.593768406114684, + "tokens_seen": 1177354240 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032490471414242726, + "loss": 2.5726, + "theoretical_loss": 3.593749939553097, + "tokens_seen": 1177419776 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003248946840521565, + "loss": 2.8623, + "theoretical_loss": 3.593731474307126, + "tokens_seen": 1177485312 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003248846539618857, + "loss": 2.675, + "theoretical_loss": 3.5937130103766046, + "tokens_seen": 1177550848 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032487462387161486, + "loss": 2.7829, + "theoretical_loss": 3.5936945477613644, + "tokens_seen": 1177616384 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032486459378134404, + "loss": 2.8055, + "theoretical_loss": 3.59367608646124, + "tokens_seen": 1177681920 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003248545636910732, + "loss": 2.6682, + "theoretical_loss": 3.593657626476064, + "tokens_seen": 1177747456 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003248445336008024, + "loss": 3.0486, + "theoretical_loss": 3.5936391678056694, + "tokens_seen": 1177812992 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032483450351053164, + "loss": 2.63, + "theoretical_loss": 3.5936207104498896, + "tokens_seen": 1177878528 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032482447342026076, + "loss": 2.7713, + "theoretical_loss": 3.5936022544085575, + "tokens_seen": 1177944064 + }, + { + "epoch": 0.36, + "objective/train/docs_used": 670392, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.4856088161468506, + "objective/train/theoretical_loss": 3.593583799681507, + "objective/train/tokens_used": 1198469600, + "theoretical_loss": 3.593583799681507, + "tokens_seen": 1178009600 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032481444332999, + "loss": 2.5958, + "theoretical_loss": 3.593583799681507, + "tokens_seen": 1178009600 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003248044132397192, + "loss": 2.7008, + "theoretical_loss": 3.593565346268571, + "tokens_seen": 1178075136 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032479438314944836, + "loss": 2.4381, + "theoretical_loss": 3.593546894169583, + "tokens_seen": 1178140672 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032478435305917754, + "loss": 2.9318, + "theoretical_loss": 3.5935284433843764, + "tokens_seen": 1178206208 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003247743229689067, + "loss": 2.6757, + "theoretical_loss": 3.593509993912784, + "tokens_seen": 1178271744 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003247642928786359, + "loss": 2.7036, + "theoretical_loss": 3.593491545754641, + "tokens_seen": 1178337280 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032475426278836514, + "loss": 2.7314, + "theoretical_loss": 3.5934730989097776, + "tokens_seen": 1178402816 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032474423269809427, + "loss": 2.7616, + "theoretical_loss": 3.5934546533780303, + "tokens_seen": 1178468352 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003247342026078235, + "loss": 2.7629, + "theoretical_loss": 3.593436209159232, + "tokens_seen": 1178533888 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032472417251755263, + "loss": 2.6825, + "theoretical_loss": 3.5934177662532147, + "tokens_seen": 1178599424 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032471414242728186, + "loss": 2.5104, + "theoretical_loss": 3.5933993246598135, + "tokens_seen": 1178664960 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032470411233701104, + "loss": 2.7605, + "theoretical_loss": 3.5933808843788615, + "tokens_seen": 1178730496 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003246940822467402, + "loss": 2.8385, + "theoretical_loss": 3.5933624454101922, + "tokens_seen": 1178796032 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003246840521564694, + "loss": 2.7494, + "theoretical_loss": 3.5933440077536396, + "tokens_seen": 1178861568 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032467402206619864, + "loss": 2.7928, + "theoretical_loss": 3.5933255714090375, + "tokens_seen": 1178927104 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032466399197592777, + "loss": 2.8499, + "theoretical_loss": 3.593307136376219, + "tokens_seen": 1178992640 + }, + { + "epoch": 0.36, + "learning_rate": 0.000324653961885657, + "loss": 2.8963, + "theoretical_loss": 3.5932887026550184, + "tokens_seen": 1179058176 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032464393179538613, + "loss": 2.7095, + "theoretical_loss": 3.5932702702452692, + "tokens_seen": 1179123712 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032463390170511537, + "loss": 2.7182, + "theoretical_loss": 3.5932518391468053, + "tokens_seen": 1179189248 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032462387161484455, + "loss": 2.7637, + "theoretical_loss": 3.5932334093594607, + "tokens_seen": 1179254784 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032461384152457373, + "loss": 2.8736, + "theoretical_loss": 3.5932149808830687, + "tokens_seen": 1179320320 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003246038114343029, + "loss": 2.6913, + "theoretical_loss": 3.593196553717464, + "tokens_seen": 1179385856 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003245937813440321, + "loss": 2.7872, + "theoretical_loss": 3.5931781278624806, + "tokens_seen": 1179451392 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032458375125376127, + "loss": 2.9188, + "theoretical_loss": 3.5931597033179514, + "tokens_seen": 1179516928 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003245737211634905, + "loss": 2.6885, + "theoretical_loss": 3.593141280083711, + "tokens_seen": 1179582464 + }, + { + "epoch": 0.36, + "objective/train/docs_used": 671618, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.696552276611328, + "objective/train/theoretical_loss": 3.5931228581595938, + "objective/train/tokens_used": 1200108000, + "theoretical_loss": 3.5931228581595938, + "tokens_seen": 1179648000 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032456369107321963, + "loss": 2.7531, + "theoretical_loss": 3.5931228581595938, + "tokens_seen": 1179648000 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032455366098294887, + "loss": 2.522, + "theoretical_loss": 3.593104437545433, + "tokens_seen": 1179713536 + }, + { + "epoch": 0.36, + "learning_rate": 0.000324543630892678, + "loss": 2.7627, + "theoretical_loss": 3.5930860182410638, + "tokens_seen": 1179779072 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032453360080240723, + "loss": 2.6755, + "theoretical_loss": 3.5930676002463198, + "tokens_seen": 1179844608 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003245235707121364, + "loss": 2.9214, + "theoretical_loss": 3.5930491835610345, + "tokens_seen": 1179910144 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003245135406218656, + "loss": 2.7724, + "theoretical_loss": 3.593030768185043, + "tokens_seen": 1179975680 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003245035105315948, + "loss": 2.602, + "theoretical_loss": 3.593012354118179, + "tokens_seen": 1180041216 + }, + { + "epoch": 0.36, + "learning_rate": 0.000324493480441324, + "loss": 2.6166, + "theoretical_loss": 3.5929939413602767, + "tokens_seen": 1180106752 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032448345035105314, + "loss": 2.9177, + "theoretical_loss": 3.592975529911171, + "tokens_seen": 1180172288 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003244734202607824, + "loss": 2.5322, + "theoretical_loss": 3.5929571197706958, + "tokens_seen": 1180237824 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003244633901705115, + "loss": 2.7514, + "theoretical_loss": 3.592938710938685, + "tokens_seen": 1180303360 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032445336008024074, + "loss": 2.5351, + "theoretical_loss": 3.5929203034149735, + "tokens_seen": 1180368896 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003244433299899699, + "loss": 2.8907, + "theoretical_loss": 3.5929018971993956, + "tokens_seen": 1180434432 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003244332998996991, + "loss": 2.8717, + "theoretical_loss": 3.5928834922917856, + "tokens_seen": 1180499968 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003244232698094283, + "loss": 2.4774, + "theoretical_loss": 3.592865088691978, + "tokens_seen": 1180565504 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032441323971915746, + "loss": 2.3557, + "theoretical_loss": 3.592846686399807, + "tokens_seen": 1180631040 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032440320962888664, + "loss": 2.8044, + "theoretical_loss": 3.592828285415108, + "tokens_seen": 1180696576 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003243931795386159, + "loss": 2.7205, + "theoretical_loss": 3.5928098857377146, + "tokens_seen": 1180762112 + }, + { + "epoch": 0.36, + "learning_rate": 0.000324383149448345, + "loss": 2.8443, + "theoretical_loss": 3.592791487367462, + "tokens_seen": 1180827648 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032437311935807424, + "loss": 2.7376, + "theoretical_loss": 3.592773090304184, + "tokens_seen": 1180893184 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032436308926780337, + "loss": 2.613, + "theoretical_loss": 3.592754694547716, + "tokens_seen": 1180958720 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003243530591775326, + "loss": 2.5542, + "theoretical_loss": 3.5927363000978927, + "tokens_seen": 1181024256 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003243430290872618, + "loss": 2.465, + "theoretical_loss": 3.592717906954548, + "tokens_seen": 1181089792 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032433299899699096, + "loss": 2.9728, + "theoretical_loss": 3.592699515117517, + "tokens_seen": 1181155328 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003243229689067202, + "loss": 2.8271, + "theoretical_loss": 3.592681124586635, + "tokens_seen": 1181220864 + }, + { + "epoch": 0.36, + "objective/train/docs_used": 672229, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.09159255027771, + "objective/train/theoretical_loss": 3.592662735361736, + "objective/train/tokens_used": 1201746400, + "theoretical_loss": 3.592662735361736, + "tokens_seen": 1181286400 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003243129388164494, + "loss": 2.8637, + "theoretical_loss": 3.592662735361736, + "tokens_seen": 1181286400 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032430290872617856, + "loss": 2.6741, + "theoretical_loss": 3.5926443474426555, + "tokens_seen": 1181351936 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032429287863590774, + "loss": 2.6063, + "theoretical_loss": 3.5926259608292277, + "tokens_seen": 1181417472 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003242828485456369, + "loss": 2.6751, + "theoretical_loss": 3.592607575521288, + "tokens_seen": 1181483008 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003242728184553661, + "loss": 2.5301, + "theoretical_loss": 3.592589191518671, + "tokens_seen": 1181548544 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032426278836509534, + "loss": 2.7418, + "theoretical_loss": 3.5925708088212116, + "tokens_seen": 1181614080 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032425275827482447, + "loss": 2.6605, + "theoretical_loss": 3.592552427428745, + "tokens_seen": 1181679616 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003242427281845537, + "loss": 2.9327, + "theoretical_loss": 3.592534047341106, + "tokens_seen": 1181745152 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032423269809428283, + "loss": 2.7073, + "theoretical_loss": 3.5925156685581294, + "tokens_seen": 1181810688 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032422266800401206, + "loss": 2.8191, + "theoretical_loss": 3.592497291079651, + "tokens_seen": 1181876224 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032421263791374125, + "loss": 2.8166, + "theoretical_loss": 3.592478914905505, + "tokens_seen": 1181941760 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003242026078234704, + "loss": 2.6835, + "theoretical_loss": 3.592460540035527, + "tokens_seen": 1182007296 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003241925777331996, + "loss": 2.8089, + "theoretical_loss": 3.592442166469552, + "tokens_seen": 1182072832 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032418254764292884, + "loss": 2.4676, + "theoretical_loss": 3.5924237942074155, + "tokens_seen": 1182138368 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032417251755265797, + "loss": 3.0168, + "theoretical_loss": 3.5924054232489517, + "tokens_seen": 1182203904 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003241624874623872, + "loss": 3.059, + "theoretical_loss": 3.5923870535939972, + "tokens_seen": 1182269440 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032415245737211633, + "loss": 2.7371, + "theoretical_loss": 3.592368685242387, + "tokens_seen": 1182334976 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032414242728184557, + "loss": 3.004, + "theoretical_loss": 3.5923503181939553, + "tokens_seen": 1182400512 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032413239719157475, + "loss": 2.9665, + "theoretical_loss": 3.5923319524485384, + "tokens_seen": 1182466048 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032412236710130393, + "loss": 2.724, + "theoretical_loss": 3.592313588005971, + "tokens_seen": 1182531584 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003241123370110331, + "loss": 2.5917, + "theoretical_loss": 3.5922952248660893, + "tokens_seen": 1182597120 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003241023069207623, + "loss": 2.8491, + "theoretical_loss": 3.592276863028728, + "tokens_seen": 1182662656 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032409227683049147, + "loss": 2.6928, + "theoretical_loss": 3.5922585024937232, + "tokens_seen": 1182728192 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003240822467402207, + "loss": 2.8204, + "theoretical_loss": 3.5922401432609092, + "tokens_seen": 1182793728 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032407221664994984, + "loss": 2.6993, + "theoretical_loss": 3.592221785330123, + "tokens_seen": 1182859264 + }, + { + "epoch": 0.36, + "objective/train/docs_used": 673021, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.9995357990264893, + "objective/train/theoretical_loss": 3.5922034287011995, + "objective/train/tokens_used": 1203384800, + "theoretical_loss": 3.5922034287011995, + "tokens_seen": 1182924800 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032406218655967907, + "loss": 2.7431, + "theoretical_loss": 3.5922034287011995, + "tokens_seen": 1182924800 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003240521564694082, + "loss": 2.5346, + "theoretical_loss": 3.5921850733739733, + "tokens_seen": 1182990336 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032404212637913743, + "loss": 2.3793, + "theoretical_loss": 3.592166719348282, + "tokens_seen": 1183055872 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003240320962888666, + "loss": 2.6941, + "theoretical_loss": 3.5921483666239595, + "tokens_seen": 1183121408 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003240220661985958, + "loss": 2.7855, + "theoretical_loss": 3.592130015200842, + "tokens_seen": 1183186944 + }, + { + "epoch": 0.36, + "learning_rate": 0.000324012036108325, + "loss": 2.8426, + "theoretical_loss": 3.592111665078765, + "tokens_seen": 1183252480 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003240020060180542, + "loss": 2.8268, + "theoretical_loss": 3.592093316257565, + "tokens_seen": 1183318016 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032399197592778334, + "loss": 2.6832, + "theoretical_loss": 3.5920749687370765, + "tokens_seen": 1183383552 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003239819458375126, + "loss": 2.5622, + "theoretical_loss": 3.5920566225171364, + "tokens_seen": 1183449088 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003239719157472417, + "loss": 2.9001, + "theoretical_loss": 3.5920382775975797, + "tokens_seen": 1183514624 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032396188565697094, + "loss": 2.7952, + "theoretical_loss": 3.592019933978243, + "tokens_seen": 1183580160 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003239518555667001, + "loss": 2.9058, + "theoretical_loss": 3.592001591658961, + "tokens_seen": 1183645696 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003239418254764293, + "loss": 2.8782, + "theoretical_loss": 3.5919832506395712, + "tokens_seen": 1183711232 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003239317953861585, + "loss": 2.808, + "theoretical_loss": 3.591964910919908, + "tokens_seen": 1183776768 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032392176529588766, + "loss": 2.7334, + "theoretical_loss": 3.5919465724998076, + "tokens_seen": 1183842304 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032391173520561684, + "loss": 2.8147, + "theoretical_loss": 3.591928235379107, + "tokens_seen": 1183907840 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003239017051153461, + "loss": 2.8321, + "theoretical_loss": 3.5919098995576415, + "tokens_seen": 1183973376 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003238916750250752, + "loss": 2.5386, + "theoretical_loss": 3.5918915650352474, + "tokens_seen": 1184038912 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032388164493480444, + "loss": 2.733, + "theoretical_loss": 3.59187323181176, + "tokens_seen": 1184104448 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032387161484453357, + "loss": 2.6981, + "theoretical_loss": 3.591854899887016, + "tokens_seen": 1184169984 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003238615847542628, + "loss": 2.931, + "theoretical_loss": 3.5918365692608516, + "tokens_seen": 1184235520 + }, + { + "epoch": 0.36, + "learning_rate": 0.000323851554663992, + "loss": 2.7046, + "theoretical_loss": 3.5918182399331036, + "tokens_seen": 1184301056 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032384152457372116, + "loss": 2.8066, + "theoretical_loss": 3.5917999119036064, + "tokens_seen": 1184366592 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032383149448345034, + "loss": 2.6677, + "theoretical_loss": 3.5917815851721975, + "tokens_seen": 1184432128 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003238214643931796, + "loss": 2.6666, + "theoretical_loss": 3.5917632597387126, + "tokens_seen": 1184497664 + }, + { + "epoch": 0.36, + "objective/train/docs_used": 674157, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.753328323364258, + "objective/train/theoretical_loss": 3.591744935602989, + "objective/train/tokens_used": 1205023200, + "theoretical_loss": 3.591744935602989, + "tokens_seen": 1184563200 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003238114343029087, + "loss": 2.7799, + "theoretical_loss": 3.591744935602989, + "tokens_seen": 1184563200 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032380140421263794, + "loss": 2.7652, + "theoretical_loss": 3.5917266127648615, + "tokens_seen": 1184628736 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032379137412236707, + "loss": 2.8815, + "theoretical_loss": 3.591708291224168, + "tokens_seen": 1184694272 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003237813440320963, + "loss": 2.9294, + "theoretical_loss": 3.5916899709807435, + "tokens_seen": 1184759808 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003237713139418255, + "loss": 2.838, + "theoretical_loss": 3.5916716520344254, + "tokens_seen": 1184825344 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032376128385155467, + "loss": 2.9081, + "theoretical_loss": 3.591653334385049, + "tokens_seen": 1184890880 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032375125376128385, + "loss": 2.7063, + "theoretical_loss": 3.591635018032452, + "tokens_seen": 1184956416 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032374122367101303, + "loss": 2.8805, + "theoretical_loss": 3.59161670297647, + "tokens_seen": 1185021952 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003237311935807422, + "loss": 2.7009, + "theoretical_loss": 3.59159838921694, + "tokens_seen": 1185087488 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032372116349047145, + "loss": 2.7593, + "theoretical_loss": 3.591580076753698, + "tokens_seen": 1185153024 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032371113340020057, + "loss": 2.7228, + "theoretical_loss": 3.5915617655865812, + "tokens_seen": 1185218560 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003237011033099298, + "loss": 2.5533, + "theoretical_loss": 3.591543455715426, + "tokens_seen": 1185284096 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032369107321965893, + "loss": 2.8218, + "theoretical_loss": 3.591525147140069, + "tokens_seen": 1185349632 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032368104312938817, + "loss": 2.6372, + "theoretical_loss": 3.591506839860347, + "tokens_seen": 1185415168 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032367101303911735, + "loss": 2.9183, + "theoretical_loss": 3.5914885338760962, + "tokens_seen": 1185480704 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032366098294884653, + "loss": 2.7873, + "theoretical_loss": 3.591470229187154, + "tokens_seen": 1185546240 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003236509528585757, + "loss": 2.6702, + "theoretical_loss": 3.5914519257933564, + "tokens_seen": 1185611776 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032364092276830495, + "loss": 2.7908, + "theoretical_loss": 3.5914336236945408, + "tokens_seen": 1185677312 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003236308926780341, + "loss": 2.7056, + "theoretical_loss": 3.5914153228905437, + "tokens_seen": 1185742848 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003236208625877633, + "loss": 2.9517, + "theoretical_loss": 3.5913970233812016, + "tokens_seen": 1185808384 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032361083249749244, + "loss": 2.5798, + "theoretical_loss": 3.5913787251663525, + "tokens_seen": 1185873920 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003236008024072217, + "loss": 2.8316, + "theoretical_loss": 3.5913604282458325, + "tokens_seen": 1185939456 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032359077231695085, + "loss": 2.928, + "theoretical_loss": 3.591342132619478, + "tokens_seen": 1186004992 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032358074222668004, + "loss": 2.5723, + "theoretical_loss": 3.5913238382871273, + "tokens_seen": 1186070528 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032357071213640927, + "loss": 2.7917, + "theoretical_loss": 3.5913055452486162, + "tokens_seen": 1186136064 + }, + { + "epoch": 0.36, + "objective/train/docs_used": 674881, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.588200569152832, + "objective/train/theoretical_loss": 3.5912872535037828, + "objective/train/tokens_used": 1206661600, + "theoretical_loss": 3.5912872535037828, + "tokens_seen": 1186201600 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003235606820461384, + "loss": 2.7793, + "theoretical_loss": 3.5912872535037828, + "tokens_seen": 1186201600 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032355065195586763, + "loss": 2.9582, + "theoretical_loss": 3.591268963052463, + "tokens_seen": 1186267136 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003235406218655968, + "loss": 2.8331, + "theoretical_loss": 3.591250673894494, + "tokens_seen": 1186332672 + }, + { + "epoch": 0.36, + "learning_rate": 0.000323530591775326, + "loss": 2.7296, + "theoretical_loss": 3.5912323860297137, + "tokens_seen": 1186398208 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003235205616850552, + "loss": 2.5977, + "theoretical_loss": 3.591214099457959, + "tokens_seen": 1186463744 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003235105315947844, + "loss": 2.5173, + "theoretical_loss": 3.5911958141790667, + "tokens_seen": 1186529280 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032350050150451354, + "loss": 2.7797, + "theoretical_loss": 3.5911775301928746, + "tokens_seen": 1186594816 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003234904714142428, + "loss": 2.7668, + "theoretical_loss": 3.5911592474992187, + "tokens_seen": 1186660352 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003234804413239719, + "loss": 2.583, + "theoretical_loss": 3.591140966097938, + "tokens_seen": 1186725888 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032347041123370114, + "loss": 2.8898, + "theoretical_loss": 3.591122685988868, + "tokens_seen": 1186791424 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003234603811434303, + "loss": 2.6947, + "theoretical_loss": 3.5911044071718474, + "tokens_seen": 1186856960 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003234503510531595, + "loss": 2.9274, + "theoretical_loss": 3.5910861296467127, + "tokens_seen": 1186922496 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003234403209628887, + "loss": 2.7148, + "theoretical_loss": 3.5910678534133016, + "tokens_seen": 1186988032 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032343029087261786, + "loss": 2.5666, + "theoretical_loss": 3.591049578471452, + "tokens_seen": 1187053568 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032342026078234704, + "loss": 2.55, + "theoretical_loss": 3.591031304821, + "tokens_seen": 1187119104 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003234102306920763, + "loss": 2.5686, + "theoretical_loss": 3.5910130324617846, + "tokens_seen": 1187184640 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003234002006018054, + "loss": 2.4367, + "theoretical_loss": 3.5909947613936417, + "tokens_seen": 1187250176 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032339017051153464, + "loss": 2.7289, + "theoretical_loss": 3.59097649161641, + "tokens_seen": 1187315712 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032338014042126377, + "loss": 2.8538, + "theoretical_loss": 3.5909582231299266, + "tokens_seen": 1187381248 + }, + { + "epoch": 0.36, + "learning_rate": 0.000323370110330993, + "loss": 2.6959, + "theoretical_loss": 3.5909399559340294, + "tokens_seen": 1187446784 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003233600802407222, + "loss": 2.8246, + "theoretical_loss": 3.590921690028556, + "tokens_seen": 1187512320 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032335005015045136, + "loss": 2.6255, + "theoretical_loss": 3.5909034254133436, + "tokens_seen": 1187577856 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032334002006018054, + "loss": 2.6157, + "theoretical_loss": 3.59088516208823, + "tokens_seen": 1187643392 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003233299899699098, + "loss": 2.7292, + "theoretical_loss": 3.590866900053053, + "tokens_seen": 1187708928 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003233199598796389, + "loss": 2.7256, + "theoretical_loss": 3.5908486393076506, + "tokens_seen": 1187774464 + }, + { + "epoch": 0.36, + "objective/train/docs_used": 676005, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.952125310897827, + "objective/train/theoretical_loss": 3.5908303798518597, + "objective/train/tokens_used": 1208300000, + "theoretical_loss": 3.5908303798518597, + "tokens_seen": 1187840000 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032330992978936814, + "loss": 2.7462, + "theoretical_loss": 3.5908303798518597, + "tokens_seen": 1187840000 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032329989969909727, + "loss": 2.5309, + "theoretical_loss": 3.590812121685519, + "tokens_seen": 1187905536 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003232898696088265, + "loss": 2.6665, + "theoretical_loss": 3.590793864808466, + "tokens_seen": 1187971072 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003232798395185557, + "loss": 2.5084, + "theoretical_loss": 3.590775609220538, + "tokens_seen": 1188036608 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032326980942828487, + "loss": 2.8898, + "theoretical_loss": 3.590757354921574, + "tokens_seen": 1188102144 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032325977933801405, + "loss": 2.7834, + "theoretical_loss": 3.590739101911411, + "tokens_seen": 1188167680 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032324974924774323, + "loss": 2.6955, + "theoretical_loss": 3.5907208501898866, + "tokens_seen": 1188233216 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003232397191574724, + "loss": 2.7121, + "theoretical_loss": 3.59070259975684, + "tokens_seen": 1188298752 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032322968906720165, + "loss": 2.6889, + "theoretical_loss": 3.5906843506121087, + "tokens_seen": 1188364288 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032321965897693077, + "loss": 2.6691, + "theoretical_loss": 3.59066610275553, + "tokens_seen": 1188429824 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032320962888666, + "loss": 2.5605, + "theoretical_loss": 3.5906478561869424, + "tokens_seen": 1188495360 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032319959879638913, + "loss": 2.7183, + "theoretical_loss": 3.590629610906184, + "tokens_seen": 1188560896 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032318956870611837, + "loss": 2.5778, + "theoretical_loss": 3.5906113669130937, + "tokens_seen": 1188626432 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032317953861584755, + "loss": 2.616, + "theoretical_loss": 3.5905931242075084, + "tokens_seen": 1188691968 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032316950852557673, + "loss": 2.7378, + "theoretical_loss": 3.5905748827892667, + "tokens_seen": 1188757504 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003231594784353059, + "loss": 2.8914, + "theoretical_loss": 3.5905566426582074, + "tokens_seen": 1188823040 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032314944834503515, + "loss": 2.8342, + "theoretical_loss": 3.5905384038141674, + "tokens_seen": 1188888576 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003231394182547643, + "loss": 2.5128, + "theoretical_loss": 3.5905201662569866, + "tokens_seen": 1188954112 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003231293881644935, + "loss": 2.7463, + "theoretical_loss": 3.5905019299865018, + "tokens_seen": 1189019648 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032311935807422264, + "loss": 2.9151, + "theoretical_loss": 3.590483695002552, + "tokens_seen": 1189085184 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003231093279839519, + "loss": 2.5926, + "theoretical_loss": 3.5904654613049756, + "tokens_seen": 1189150720 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032309929789368105, + "loss": 2.8375, + "theoretical_loss": 3.5904472288936105, + "tokens_seen": 1189216256 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032308926780341024, + "loss": 2.7233, + "theoretical_loss": 3.5904289977682957, + "tokens_seen": 1189281792 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003230792377131394, + "loss": 2.5502, + "theoretical_loss": 3.590410767928869, + "tokens_seen": 1189347328 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003230692076228686, + "loss": 2.9868, + "theoretical_loss": 3.5903925393751694, + "tokens_seen": 1189412864 + }, + { + "epoch": 0.36, + "objective/train/docs_used": 676669, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7727530002593994, + "objective/train/theoretical_loss": 3.590374312107035, + "objective/train/tokens_used": 1209938400, + "theoretical_loss": 3.590374312107035, + "tokens_seen": 1189478400 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003230591775325978, + "loss": 2.7673, + "theoretical_loss": 3.590374312107035, + "tokens_seen": 1189478400 + }, + { + "epoch": 0.36, + "learning_rate": 0.000323049147442327, + "loss": 2.6961, + "theoretical_loss": 3.590356086124305, + "tokens_seen": 1189543936 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032303911735205614, + "loss": 2.7749, + "theoretical_loss": 3.590337861426817, + "tokens_seen": 1189609472 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003230290872617854, + "loss": 2.6582, + "theoretical_loss": 3.59031963801441, + "tokens_seen": 1189675008 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003230190571715145, + "loss": 2.6485, + "theoretical_loss": 3.5903014158869224, + "tokens_seen": 1189740544 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032300902708124374, + "loss": 2.8059, + "theoretical_loss": 3.5902831950441936, + "tokens_seen": 1189806080 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003229989969909729, + "loss": 2.7799, + "theoretical_loss": 3.590264975486061, + "tokens_seen": 1189871616 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003229889669007021, + "loss": 2.8217, + "theoretical_loss": 3.5902467572123644, + "tokens_seen": 1189937152 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003229789368104313, + "loss": 2.6195, + "theoretical_loss": 3.5902285402229426, + "tokens_seen": 1190002688 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003229689067201605, + "loss": 2.8864, + "theoretical_loss": 3.590210324517633, + "tokens_seen": 1190068224 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032295887662988964, + "loss": 2.7658, + "theoretical_loss": 3.5901921100962753, + "tokens_seen": 1190133760 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003229488465396189, + "loss": 2.6175, + "theoretical_loss": 3.590173896958709, + "tokens_seen": 1190199296 + }, + { + "epoch": 0.36, + "learning_rate": 0.000322938816449348, + "loss": 2.4766, + "theoretical_loss": 3.5901556851047713, + "tokens_seen": 1190264832 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032292878635907724, + "loss": 2.631, + "theoretical_loss": 3.590137474534302, + "tokens_seen": 1190330368 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003229187562688064, + "loss": 2.7503, + "theoretical_loss": 3.5901192652471403, + "tokens_seen": 1190395904 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003229087261785356, + "loss": 2.6481, + "theoretical_loss": 3.5901010572431247, + "tokens_seen": 1190461440 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003228986960882648, + "loss": 2.8611, + "theoretical_loss": 3.5900828505220943, + "tokens_seen": 1190526976 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032288866599799397, + "loss": 2.8685, + "theoretical_loss": 3.5900646450838876, + "tokens_seen": 1190592512 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032287863590772315, + "loss": 2.9023, + "theoretical_loss": 3.590046440928344, + "tokens_seen": 1190658048 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003228686058174524, + "loss": 2.6635, + "theoretical_loss": 3.590028238055303, + "tokens_seen": 1190723584 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003228585757271815, + "loss": 2.7458, + "theoretical_loss": 3.5900100364646024, + "tokens_seen": 1190789120 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032284854563691074, + "loss": 2.7923, + "theoretical_loss": 3.5899918361560825, + "tokens_seen": 1190854656 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003228385155466399, + "loss": 2.6967, + "theoretical_loss": 3.5899736371295825, + "tokens_seen": 1190920192 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003228284854563691, + "loss": 2.8096, + "theoretical_loss": 3.5899554393849398, + "tokens_seen": 1190985728 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032281845536609834, + "loss": 2.7782, + "theoretical_loss": 3.589937242921996, + "tokens_seen": 1191051264 + }, + { + "epoch": 0.36, + "objective/train/docs_used": 677774, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.1389665603637695, + "objective/train/theoretical_loss": 3.5899190477405885, + "objective/train/tokens_used": 1211576800, + "theoretical_loss": 3.5899190477405885, + "tokens_seen": 1191116800 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032280842527582747, + "loss": 2.8327, + "theoretical_loss": 3.5899190477405885, + "tokens_seen": 1191116800 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003227983951855567, + "loss": 2.6059, + "theoretical_loss": 3.5899008538405575, + "tokens_seen": 1191182336 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003227883650952859, + "loss": 2.5885, + "theoretical_loss": 3.5898826612217416, + "tokens_seen": 1191247872 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032277833500501507, + "loss": 2.8266, + "theoretical_loss": 3.589864469883981, + "tokens_seen": 1191313408 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032276830491474425, + "loss": 2.6714, + "theoretical_loss": 3.589846279827114, + "tokens_seen": 1191378944 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032275827482447343, + "loss": 2.6103, + "theoretical_loss": 3.5898280910509808, + "tokens_seen": 1191444480 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003227482447342026, + "loss": 2.771, + "theoretical_loss": 3.5898099035554196, + "tokens_seen": 1191510016 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032273821464393185, + "loss": 2.7896, + "theoretical_loss": 3.5897917173402716, + "tokens_seen": 1191575552 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032272818455366097, + "loss": 2.7812, + "theoretical_loss": 3.589773532405375, + "tokens_seen": 1191641088 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003227181544633902, + "loss": 2.8264, + "theoretical_loss": 3.589755348750569, + "tokens_seen": 1191706624 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032270812437311933, + "loss": 2.7027, + "theoretical_loss": 3.5897371663756945, + "tokens_seen": 1191772160 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032269809428284857, + "loss": 2.8258, + "theoretical_loss": 3.58971898528059, + "tokens_seen": 1191837696 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032268806419257775, + "loss": 2.5562, + "theoretical_loss": 3.5897008054650947, + "tokens_seen": 1191903232 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032267803410230693, + "loss": 2.6826, + "theoretical_loss": 3.589682626929049, + "tokens_seen": 1191968768 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003226680040120361, + "loss": 2.8682, + "theoretical_loss": 3.5896644496722927, + "tokens_seen": 1192034304 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032265797392176535, + "loss": 2.692, + "theoretical_loss": 3.5896462736946644, + "tokens_seen": 1192099840 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003226479438314945, + "loss": 2.7942, + "theoretical_loss": 3.5896280989960045, + "tokens_seen": 1192165376 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003226379137412237, + "loss": 2.7608, + "theoretical_loss": 3.589609925576153, + "tokens_seen": 1192230912 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032262788365095284, + "loss": 2.7706, + "theoretical_loss": 3.5895917534349486, + "tokens_seen": 1192296448 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003226178535606821, + "loss": 2.753, + "theoretical_loss": 3.589573582572232, + "tokens_seen": 1192361984 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032260782347041125, + "loss": 2.9643, + "theoretical_loss": 3.589555412987843, + "tokens_seen": 1192427520 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032259779338014044, + "loss": 2.5482, + "theoretical_loss": 3.5895372446816203, + "tokens_seen": 1192493056 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003225877632898696, + "loss": 2.7198, + "theoretical_loss": 3.5895190776534047, + "tokens_seen": 1192558592 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003225777331995988, + "loss": 2.6879, + "theoretical_loss": 3.5895009119030368, + "tokens_seen": 1192624128 + }, + { + "epoch": 0.36, + "learning_rate": 0.000322567703109328, + "loss": 2.7795, + "theoretical_loss": 3.5894827474303543, + "tokens_seen": 1192689664 + }, + { + "epoch": 0.36, + "objective/train/docs_used": 678420, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.5659327507019043, + "objective/train/theoretical_loss": 3.5894645842351993, + "objective/train/tokens_used": 1213215200, + "theoretical_loss": 3.5894645842351993, + "tokens_seen": 1192755200 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003225576730190572, + "loss": 2.6216, + "theoretical_loss": 3.5894645842351993, + "tokens_seen": 1192755200 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032254764292878634, + "loss": 2.6556, + "theoretical_loss": 3.589446422317411, + "tokens_seen": 1192820736 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003225376128385156, + "loss": 2.6758, + "theoretical_loss": 3.5894282616768285, + "tokens_seen": 1192886272 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003225275827482447, + "loss": 2.6687, + "theoretical_loss": 3.5894101023132934, + "tokens_seen": 1192951808 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032251755265797394, + "loss": 2.8191, + "theoretical_loss": 3.5893919442266444, + "tokens_seen": 1193017344 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003225075225677031, + "loss": 2.923, + "theoretical_loss": 3.5893737874167226, + "tokens_seen": 1193082880 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003224974924774323, + "loss": 2.808, + "theoretical_loss": 3.5893556318833673, + "tokens_seen": 1193148416 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003224874623871615, + "loss": 2.8, + "theoretical_loss": 3.5893374776264197, + "tokens_seen": 1193213952 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003224774322968907, + "loss": 2.585, + "theoretical_loss": 3.589319324645719, + "tokens_seen": 1193279488 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032246740220661984, + "loss": 2.7193, + "theoretical_loss": 3.589301172941105, + "tokens_seen": 1193345024 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003224573721163491, + "loss": 3.0235, + "theoretical_loss": 3.5892830225124186, + "tokens_seen": 1193410560 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003224473420260782, + "loss": 2.9273, + "theoretical_loss": 3.5892648733595007, + "tokens_seen": 1193476096 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032243731193580744, + "loss": 2.8087, + "theoretical_loss": 3.589246725482191, + "tokens_seen": 1193541632 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003224272818455366, + "loss": 2.804, + "theoretical_loss": 3.589228578880329, + "tokens_seen": 1193607168 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003224172517552658, + "loss": 2.7281, + "theoretical_loss": 3.589210433553756, + "tokens_seen": 1193672704 + }, + { + "epoch": 0.36, + "learning_rate": 0.000322407221664995, + "loss": 2.6717, + "theoretical_loss": 3.5891922895023125, + "tokens_seen": 1193738240 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032239719157472417, + "loss": 2.6629, + "theoretical_loss": 3.5891741467258385, + "tokens_seen": 1193803776 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032238716148445335, + "loss": 2.8336, + "theoretical_loss": 3.589156005224174, + "tokens_seen": 1193869312 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003223771313941826, + "loss": 2.674, + "theoretical_loss": 3.58913786499716, + "tokens_seen": 1193934848 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003223671013039117, + "loss": 2.7804, + "theoretical_loss": 3.589119726044637, + "tokens_seen": 1194000384 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032235707121364095, + "loss": 2.6856, + "theoretical_loss": 3.5891015883664457, + "tokens_seen": 1194065920 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003223470411233701, + "loss": 2.6529, + "theoretical_loss": 3.5890834519624257, + "tokens_seen": 1194131456 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003223370110330993, + "loss": 2.856, + "theoretical_loss": 3.5890653168324187, + "tokens_seen": 1194196992 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003223269809428285, + "loss": 2.6831, + "theoretical_loss": 3.5890471829762647, + "tokens_seen": 1194262528 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032231695085255767, + "loss": 2.7468, + "theoretical_loss": 3.5890290503938047, + "tokens_seen": 1194328064 + }, + { + "epoch": 0.36, + "objective/train/docs_used": 679445, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.5847508907318115, + "objective/train/theoretical_loss": 3.5890109190848785, + "objective/train/tokens_used": 1214853600, + "theoretical_loss": 3.5890109190848785, + "tokens_seen": 1194393600 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032230692076228685, + "loss": 2.6491, + "theoretical_loss": 3.5890109190848785, + "tokens_seen": 1194393600 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003222968906720161, + "loss": 2.8563, + "theoretical_loss": 3.5889927890493274, + "tokens_seen": 1194459136 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003222868605817452, + "loss": 2.6557, + "theoretical_loss": 3.5889746602869925, + "tokens_seen": 1194524672 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032227683049147445, + "loss": 2.6066, + "theoretical_loss": 3.588956532797714, + "tokens_seen": 1194590208 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003222668004012036, + "loss": 2.7765, + "theoretical_loss": 3.5889384065813323, + "tokens_seen": 1194655744 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003222567703109328, + "loss": 2.6519, + "theoretical_loss": 3.588920281637689, + "tokens_seen": 1194721280 + }, + { + "epoch": 0.36, + "learning_rate": 0.000322246740220662, + "loss": 2.7527, + "theoretical_loss": 3.5889021579666247, + "tokens_seen": 1194786816 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032223671013039117, + "loss": 2.7397, + "theoretical_loss": 3.58888403556798, + "tokens_seen": 1194852352 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032222668004012035, + "loss": 2.8825, + "theoretical_loss": 3.5888659144415964, + "tokens_seen": 1194917888 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032221664994984953, + "loss": 2.594, + "theoretical_loss": 3.5888477945873136, + "tokens_seen": 1194983424 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003222066198595787, + "loss": 2.7697, + "theoretical_loss": 3.5888296760049743, + "tokens_seen": 1195048960 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032219658976930795, + "loss": 2.6984, + "theoretical_loss": 3.5888115586944176, + "tokens_seen": 1195114496 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003221865596790371, + "loss": 2.4567, + "theoretical_loss": 3.5887934426554855, + "tokens_seen": 1195180032 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003221765295887663, + "loss": 2.7545, + "theoretical_loss": 3.588775327888019, + "tokens_seen": 1195245568 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003221664994984955, + "loss": 2.7027, + "theoretical_loss": 3.588757214391859, + "tokens_seen": 1195311104 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003221564694082247, + "loss": 2.622, + "theoretical_loss": 3.588739102166847, + "tokens_seen": 1195376640 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032214643931795386, + "loss": 2.6229, + "theoretical_loss": 3.5887209912128233, + "tokens_seen": 1195442176 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032213640922768304, + "loss": 2.6863, + "theoretical_loss": 3.5887028815296294, + "tokens_seen": 1195507712 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003221263791374122, + "loss": 2.7944, + "theoretical_loss": 3.588684773117107, + "tokens_seen": 1195573248 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032211634904714145, + "loss": 2.676, + "theoretical_loss": 3.5886666659750963, + "tokens_seen": 1195638784 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003221063189568706, + "loss": 2.6474, + "theoretical_loss": 3.5886485601034397, + "tokens_seen": 1195704320 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003220962888665998, + "loss": 2.7304, + "theoretical_loss": 3.5886304555019772, + "tokens_seen": 1195769856 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032208625877632894, + "loss": 2.8768, + "theoretical_loss": 3.588612352170551, + "tokens_seen": 1195835392 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003220762286860582, + "loss": 2.7806, + "theoretical_loss": 3.5885942501090025, + "tokens_seen": 1195900928 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003220661985957874, + "loss": 3.05, + "theoretical_loss": 3.588576149317172, + "tokens_seen": 1195966464 + }, + { + "epoch": 0.36, + "objective/train/docs_used": 680084, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6942007541656494, + "objective/train/theoretical_loss": 3.588558049794902, + "objective/train/tokens_used": 1216492000, + "theoretical_loss": 3.588558049794902, + "tokens_seen": 1196032000 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032205616850551654, + "loss": 2.8924, + "theoretical_loss": 3.588558049794902, + "tokens_seen": 1196032000 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003220461384152458, + "loss": 2.6538, + "theoretical_loss": 3.588539951542033, + "tokens_seen": 1196097536 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003220361083249749, + "loss": 2.6872, + "theoretical_loss": 3.588521854558407, + "tokens_seen": 1196163072 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032202607823470414, + "loss": 3.039, + "theoretical_loss": 3.5885037588438657, + "tokens_seen": 1196228608 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003220160481444333, + "loss": 2.7117, + "theoretical_loss": 3.5884856643982497, + "tokens_seen": 1196294144 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003220060180541625, + "loss": 2.7353, + "theoretical_loss": 3.5884675712214014, + "tokens_seen": 1196359680 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003219959879638917, + "loss": 2.8084, + "theoretical_loss": 3.588449479313162, + "tokens_seen": 1196425216 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003219859578736209, + "loss": 2.7322, + "theoretical_loss": 3.5884313886733725, + "tokens_seen": 1196490752 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032197592778335004, + "loss": 2.5804, + "theoretical_loss": 3.5884132993018754, + "tokens_seen": 1196556288 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003219658976930793, + "loss": 2.758, + "theoretical_loss": 3.588395211198512, + "tokens_seen": 1196621824 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003219558676028084, + "loss": 2.875, + "theoretical_loss": 3.5883771243631237, + "tokens_seen": 1196687360 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032194583751253764, + "loss": 2.6237, + "theoretical_loss": 3.588359038795552, + "tokens_seen": 1196752896 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003219358074222668, + "loss": 2.7649, + "theoretical_loss": 3.5883409544956395, + "tokens_seen": 1196818432 + }, + { + "epoch": 0.36, + "learning_rate": 0.000321925777331996, + "loss": 3.0428, + "theoretical_loss": 3.5883228714632276, + "tokens_seen": 1196883968 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003219157472417252, + "loss": 2.4955, + "theoretical_loss": 3.5883047896981575, + "tokens_seen": 1196949504 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032190571715145437, + "loss": 2.6275, + "theoretical_loss": 3.588286709200271, + "tokens_seen": 1197015040 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032189568706118355, + "loss": 2.7047, + "theoretical_loss": 3.588268629969411, + "tokens_seen": 1197080576 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003218856569709128, + "loss": 2.8675, + "theoretical_loss": 3.5882505520054186, + "tokens_seen": 1197146112 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003218756268806419, + "loss": 2.7177, + "theoretical_loss": 3.5882324753081356, + "tokens_seen": 1197211648 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032186559679037115, + "loss": 2.8709, + "theoretical_loss": 3.588214399877404, + "tokens_seen": 1197277184 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003218555667001003, + "loss": 2.7974, + "theoretical_loss": 3.588196325713066, + "tokens_seen": 1197342720 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003218455366098295, + "loss": 2.731, + "theoretical_loss": 3.588178252814963, + "tokens_seen": 1197408256 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003218355065195587, + "loss": 2.7529, + "theoretical_loss": 3.5881601811829373, + "tokens_seen": 1197473792 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032182547642928787, + "loss": 2.6668, + "theoretical_loss": 3.5881421108168317, + "tokens_seen": 1197539328 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032181544633901705, + "loss": 2.9087, + "theoretical_loss": 3.588124041716487, + "tokens_seen": 1197604864 + }, + { + "epoch": 0.36, + "objective/train/docs_used": 681502, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.553053140640259, + "objective/train/theoretical_loss": 3.5881059738817456, + "objective/train/tokens_used": 1218130400, + "theoretical_loss": 3.5881059738817456, + "tokens_seen": 1197670400 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003218054162487463, + "loss": 2.5903, + "theoretical_loss": 3.5881059738817456, + "tokens_seen": 1197670400 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003217953861584754, + "loss": 2.7995, + "theoretical_loss": 3.58808790731245, + "tokens_seen": 1197735936 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032178535606820465, + "loss": 2.847, + "theoretical_loss": 3.588069842008442, + "tokens_seen": 1197801472 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003217753259779338, + "loss": 2.9581, + "theoretical_loss": 3.5880517779695644, + "tokens_seen": 1197867008 + }, + { + "epoch": 0.36, + "learning_rate": 0.000321765295887663, + "loss": 2.6084, + "theoretical_loss": 3.5880337151956585, + "tokens_seen": 1197932544 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003217552657973922, + "loss": 2.6442, + "theoretical_loss": 3.588015653686567, + "tokens_seen": 1197998080 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032174523570712137, + "loss": 2.2631, + "theoretical_loss": 3.5879975934421324, + "tokens_seen": 1198063616 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032173520561685055, + "loss": 2.7295, + "theoretical_loss": 3.587979534462196, + "tokens_seen": 1198129152 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032172517552657974, + "loss": 2.6757, + "theoretical_loss": 3.5879614767466013, + "tokens_seen": 1198194688 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003217151454363089, + "loss": 2.6473, + "theoretical_loss": 3.58794342029519, + "tokens_seen": 1198260224 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032170511534603815, + "loss": 2.6126, + "theoretical_loss": 3.5879253651078047, + "tokens_seen": 1198325760 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003216950852557673, + "loss": 2.9768, + "theoretical_loss": 3.5879073111842876, + "tokens_seen": 1198391296 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003216850551654965, + "loss": 2.8209, + "theoretical_loss": 3.5878892585244806, + "tokens_seen": 1198456832 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003216750250752257, + "loss": 2.7116, + "theoretical_loss": 3.5878712071282273, + "tokens_seen": 1198522368 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003216649949849549, + "loss": 2.6613, + "theoretical_loss": 3.5878531569953696, + "tokens_seen": 1198587904 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032165496489468406, + "loss": 2.6501, + "theoretical_loss": 3.58783510812575, + "tokens_seen": 1198653440 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032164493480441324, + "loss": 3.0588, + "theoretical_loss": 3.5878170605192103, + "tokens_seen": 1198718976 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003216349047141424, + "loss": 2.6298, + "theoretical_loss": 3.5877990141755944, + "tokens_seen": 1198784512 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032162487462387165, + "loss": 2.7194, + "theoretical_loss": 3.5877809690947444, + "tokens_seen": 1198850048 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003216148445336008, + "loss": 2.8236, + "theoretical_loss": 3.5877629252765026, + "tokens_seen": 1198915584 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032160481444333, + "loss": 2.825, + "theoretical_loss": 3.587744882720712, + "tokens_seen": 1198981120 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032159478435305914, + "loss": 2.7786, + "theoretical_loss": 3.587726841427215, + "tokens_seen": 1199046656 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003215847542627884, + "loss": 2.7598, + "theoretical_loss": 3.5877088013958547, + "tokens_seen": 1199112192 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032157472417251756, + "loss": 2.7222, + "theoretical_loss": 3.5876907626264734, + "tokens_seen": 1199177728 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032156469408224674, + "loss": 2.6758, + "theoretical_loss": 3.5876727251189138, + "tokens_seen": 1199243264 + }, + { + "epoch": 0.36, + "objective/train/docs_used": 682215, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6359903812408447, + "objective/train/theoretical_loss": 3.5876546888730187, + "objective/train/tokens_used": 1219768800, + "theoretical_loss": 3.5876546888730187, + "tokens_seen": 1199308800 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003215546639919759, + "loss": 2.8608, + "theoretical_loss": 3.5876546888730187, + "tokens_seen": 1199308800 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003215446339017051, + "loss": 2.582, + "theoretical_loss": 3.5876366538886315, + "tokens_seen": 1199374336 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003215346038114343, + "loss": 2.6765, + "theoretical_loss": 3.5876186201655944, + "tokens_seen": 1199439872 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003215245737211635, + "loss": 2.7514, + "theoretical_loss": 3.5876005877037507, + "tokens_seen": 1199505408 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032151454363089265, + "loss": 2.6795, + "theoretical_loss": 3.587582556502943, + "tokens_seen": 1199570944 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003215045135406219, + "loss": 2.7177, + "theoretical_loss": 3.5875645265630145, + "tokens_seen": 1199636480 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032149448345035106, + "loss": 2.6905, + "theoretical_loss": 3.5875464978838076, + "tokens_seen": 1199702016 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032148445336008024, + "loss": 2.6339, + "theoretical_loss": 3.587528470465166, + "tokens_seen": 1199767552 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003214744232698094, + "loss": 2.5989, + "theoretical_loss": 3.5875104443069326, + "tokens_seen": 1199833088 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003214643931795386, + "loss": 2.5567, + "theoretical_loss": 3.5874924194089504, + "tokens_seen": 1199898624 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003214543630892678, + "loss": 2.7454, + "theoretical_loss": 3.5874743957710615, + "tokens_seen": 1199964160 + }, + { + "epoch": 0.36, + "learning_rate": 0.000321444332998997, + "loss": 2.7147, + "theoretical_loss": 3.5874563733931106, + "tokens_seen": 1200029696 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032143430290872615, + "loss": 2.7347, + "theoretical_loss": 3.5874383522749396, + "tokens_seen": 1200095232 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003214242728184554, + "loss": 2.6554, + "theoretical_loss": 3.587420332416392, + "tokens_seen": 1200160768 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003214142427281845, + "loss": 2.626, + "theoretical_loss": 3.5874023138173117, + "tokens_seen": 1200226304 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032140421263791375, + "loss": 2.7496, + "theoretical_loss": 3.5873842964775404, + "tokens_seen": 1200291840 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032139418254764293, + "loss": 2.8013, + "theoretical_loss": 3.587366280396923, + "tokens_seen": 1200357376 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003213841524573721, + "loss": 2.9206, + "theoretical_loss": 3.5873482655753013, + "tokens_seen": 1200422912 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003213741223671013, + "loss": 2.603, + "theoretical_loss": 3.5873302520125194, + "tokens_seen": 1200488448 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003213640922768305, + "loss": 2.7167, + "theoretical_loss": 3.587312239708421, + "tokens_seen": 1200553984 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032135406218655965, + "loss": 2.7846, + "theoretical_loss": 3.587294228662848, + "tokens_seen": 1200619520 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003213440320962889, + "loss": 2.6252, + "theoretical_loss": 3.5872762188756453, + "tokens_seen": 1200685056 + }, + { + "epoch": 0.36, + "learning_rate": 0.000321334002006018, + "loss": 2.726, + "theoretical_loss": 3.587258210346656, + "tokens_seen": 1200750592 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032132397191574725, + "loss": 2.9211, + "theoretical_loss": 3.5872402030757224, + "tokens_seen": 1200816128 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003213139418254765, + "loss": 2.715, + "theoretical_loss": 3.5872221970626894, + "tokens_seen": 1200881664 + }, + { + "epoch": 0.36, + "objective/train/docs_used": 683386, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.906690835952759, + "objective/train/theoretical_loss": 3.5872041923074, + "objective/train/tokens_used": 1221407200, + "theoretical_loss": 3.5872041923074, + "tokens_seen": 1200947200 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003213039117352056, + "loss": 2.8264, + "theoretical_loss": 3.5872041923074, + "tokens_seen": 1200947200 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032129388164493485, + "loss": 2.6141, + "theoretical_loss": 3.587186188809697, + "tokens_seen": 1201012736 + }, + { + "epoch": 0.36, + "learning_rate": 0.000321283851554664, + "loss": 2.6112, + "theoretical_loss": 3.587168186569425, + "tokens_seen": 1201078272 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003212738214643932, + "loss": 2.6227, + "theoretical_loss": 3.5871501855864274, + "tokens_seen": 1201143808 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003212637913741224, + "loss": 2.7832, + "theoretical_loss": 3.587132185860547, + "tokens_seen": 1201209344 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003212537612838516, + "loss": 2.6502, + "theoretical_loss": 3.587114187391628, + "tokens_seen": 1201274880 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032124373119358075, + "loss": 2.7239, + "theoretical_loss": 3.587096190179514, + "tokens_seen": 1201340416 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032123370110330994, + "loss": 2.5984, + "theoretical_loss": 3.5870781942240493, + "tokens_seen": 1201405952 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003212236710130391, + "loss": 2.7779, + "theoretical_loss": 3.5870601995250766, + "tokens_seen": 1201471488 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032121364092276835, + "loss": 2.5959, + "theoretical_loss": 3.58704220608244, + "tokens_seen": 1201537024 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003212036108324975, + "loss": 2.6358, + "theoretical_loss": 3.587024213895983, + "tokens_seen": 1201602560 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003211935807422267, + "loss": 2.9084, + "theoretical_loss": 3.58700622296555, + "tokens_seen": 1201668096 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003211835506519559, + "loss": 2.4517, + "theoretical_loss": 3.5869882332909846, + "tokens_seen": 1201733632 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003211735205616851, + "loss": 2.8324, + "theoretical_loss": 3.5869702448721306, + "tokens_seen": 1201799168 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032116349047141426, + "loss": 2.6307, + "theoretical_loss": 3.5869522577088317, + "tokens_seen": 1201864704 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032115346038114344, + "loss": 2.8055, + "theoretical_loss": 3.5869342718009323, + "tokens_seen": 1201930240 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003211434302908726, + "loss": 2.8236, + "theoretical_loss": 3.586916287148276, + "tokens_seen": 1201995776 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032113340020060185, + "loss": 2.7232, + "theoretical_loss": 3.5868983037507065, + "tokens_seen": 1202061312 + }, + { + "epoch": 0.36, + "learning_rate": 0.000321123370110331, + "loss": 2.687, + "theoretical_loss": 3.586880321608068, + "tokens_seen": 1202126848 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003211133400200602, + "loss": 2.5957, + "theoretical_loss": 3.586862340720205, + "tokens_seen": 1202192384 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032110330992978934, + "loss": 2.8207, + "theoretical_loss": 3.5868443610869605, + "tokens_seen": 1202257920 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003210932798395186, + "loss": 2.5863, + "theoretical_loss": 3.58682638270818, + "tokens_seen": 1202323456 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032108324974924776, + "loss": 2.5331, + "theoretical_loss": 3.5868084055837066, + "tokens_seen": 1202388992 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032107321965897694, + "loss": 2.51, + "theoretical_loss": 3.5867904297133846, + "tokens_seen": 1202454528 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003210631895687061, + "loss": 2.6814, + "theoretical_loss": 3.5867724550970577, + "tokens_seen": 1202520064 + }, + { + "epoch": 0.36, + "objective/train/docs_used": 683937, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.582972288131714, + "objective/train/theoretical_loss": 3.5867544817345713, + "objective/train/tokens_used": 1223045600, + "theoretical_loss": 3.5867544817345713, + "tokens_seen": 1202585600 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003210531594784353, + "loss": 2.6984, + "theoretical_loss": 3.5867544817345713, + "tokens_seen": 1202585600 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003210431293881645, + "loss": 2.6719, + "theoretical_loss": 3.5867365096257684, + "tokens_seen": 1202651136 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003210330992978937, + "loss": 2.7544, + "theoretical_loss": 3.5867185387704943, + "tokens_seen": 1202716672 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032102306920762285, + "loss": 2.7843, + "theoretical_loss": 3.586700569168592, + "tokens_seen": 1202782208 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003210130391173521, + "loss": 2.4785, + "theoretical_loss": 3.5866826008199073, + "tokens_seen": 1202847744 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032100300902708126, + "loss": 2.7565, + "theoretical_loss": 3.5866646337242836, + "tokens_seen": 1202913280 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032099297893681044, + "loss": 2.9654, + "theoretical_loss": 3.586646667881565, + "tokens_seen": 1202978816 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003209829488465396, + "loss": 2.6209, + "theoretical_loss": 3.586628703291596, + "tokens_seen": 1203044352 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003209729187562688, + "loss": 2.7488, + "theoretical_loss": 3.5866107399542217, + "tokens_seen": 1203109888 + }, + { + "epoch": 0.36, + "learning_rate": 0.000320962888665998, + "loss": 2.6568, + "theoretical_loss": 3.5865927778692868, + "tokens_seen": 1203175424 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003209528585757272, + "loss": 2.7736, + "theoretical_loss": 3.5865748170366345, + "tokens_seen": 1203240960 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032094282848545635, + "loss": 2.8178, + "theoretical_loss": 3.5865568574561095, + "tokens_seen": 1203306496 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003209327983951856, + "loss": 2.7047, + "theoretical_loss": 3.5865388991275573, + "tokens_seen": 1203372032 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003209227683049147, + "loss": 2.9743, + "theoretical_loss": 3.5865209420508215, + "tokens_seen": 1203437568 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032091273821464395, + "loss": 2.8188, + "theoretical_loss": 3.5865029862257476, + "tokens_seen": 1203503104 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032090270812437313, + "loss": 2.8384, + "theoretical_loss": 3.5864850316521792, + "tokens_seen": 1203568640 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003208926780341023, + "loss": 2.6097, + "theoretical_loss": 3.586467078329961, + "tokens_seen": 1203634176 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003208826479438315, + "loss": 2.6191, + "theoretical_loss": 3.586449126258939, + "tokens_seen": 1203699712 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003208726178535607, + "loss": 2.8123, + "theoretical_loss": 3.586431175438956, + "tokens_seen": 1203765248 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032086258776328985, + "loss": 2.79, + "theoretical_loss": 3.5864132258698582, + "tokens_seen": 1203830784 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003208525576730191, + "loss": 2.6712, + "theoretical_loss": 3.58639527755149, + "tokens_seen": 1203896320 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003208425275827482, + "loss": 2.6898, + "theoretical_loss": 3.5863773304836952, + "tokens_seen": 1203961856 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032083249749247745, + "loss": 2.7514, + "theoretical_loss": 3.58635938466632, + "tokens_seen": 1204027392 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032082246740220663, + "loss": 2.7075, + "theoretical_loss": 3.586341440099208, + "tokens_seen": 1204092928 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003208124373119358, + "loss": 2.7048, + "theoretical_loss": 3.586323496782205, + "tokens_seen": 1204158464 + }, + { + "epoch": 0.36, + "objective/train/docs_used": 684175, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.688260793685913, + "objective/train/theoretical_loss": 3.5863055547151554, + "objective/train/tokens_used": 1224684000, + "theoretical_loss": 3.5863055547151554, + "tokens_seen": 1204224000 + }, + { + "epoch": 0.36, + "learning_rate": 0.000320802407221665, + "loss": 2.7724, + "theoretical_loss": 3.5863055547151554, + "tokens_seen": 1204224000 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003207923771313942, + "loss": 2.911, + "theoretical_loss": 3.586287613897904, + "tokens_seen": 1204289536 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032078234704112336, + "loss": 2.7658, + "theoretical_loss": 3.586269674330296, + "tokens_seen": 1204355072 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003207723169508526, + "loss": 2.7935, + "theoretical_loss": 3.5862517360121773, + "tokens_seen": 1204420608 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003207622868605817, + "loss": 2.7706, + "theoretical_loss": 3.5862337989433906, + "tokens_seen": 1204486144 + }, + { + "epoch": 0.37, + "learning_rate": 0.00032075225677031095, + "loss": 2.84, + "theoretical_loss": 3.5862158631237833, + "tokens_seen": 1204551680 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003207422266800401, + "loss": 2.9132, + "theoretical_loss": 3.5861979285531986, + "tokens_seen": 1204617216 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003207321965897693, + "loss": 2.9229, + "theoretical_loss": 3.5861799952314826, + "tokens_seen": 1204682752 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003207221664994985, + "loss": 2.5738, + "theoretical_loss": 3.58616206315848, + "tokens_seen": 1204748288 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003207121364092277, + "loss": 2.6475, + "theoretical_loss": 3.586144132334036, + "tokens_seen": 1204813824 + }, + { + "epoch": 0.37, + "learning_rate": 0.00032070210631895686, + "loss": 2.8866, + "theoretical_loss": 3.5861262027579963, + "tokens_seen": 1204879360 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003206920762286861, + "loss": 2.9794, + "theoretical_loss": 3.5861082744302055, + "tokens_seen": 1204944896 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003206820461384152, + "loss": 2.4647, + "theoretical_loss": 3.586090347350509, + "tokens_seen": 1205010432 + }, + { + "epoch": 0.37, + "learning_rate": 0.00032067201604814446, + "loss": 2.6889, + "theoretical_loss": 3.586072421518752, + "tokens_seen": 1205075968 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003206619859578736, + "loss": 2.8833, + "theoretical_loss": 3.5860544969347794, + "tokens_seen": 1205141504 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003206519558676028, + "loss": 2.9821, + "theoretical_loss": 3.586036573598437, + "tokens_seen": 1205207040 + }, + { + "epoch": 0.37, + "learning_rate": 0.000320641925777332, + "loss": 2.7933, + "theoretical_loss": 3.58601865150957, + "tokens_seen": 1205272576 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003206318956870612, + "loss": 2.7788, + "theoretical_loss": 3.586000730668024, + "tokens_seen": 1205338112 + }, + { + "epoch": 0.37, + "learning_rate": 0.00032062186559679036, + "loss": 2.9698, + "theoretical_loss": 3.585982811073644, + "tokens_seen": 1205403648 + }, + { + "epoch": 0.37, + "learning_rate": 0.00032061183550651954, + "loss": 2.886, + "theoretical_loss": 3.5859648927262753, + "tokens_seen": 1205469184 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003206018054162487, + "loss": 2.8944, + "theoretical_loss": 3.5859469756257636, + "tokens_seen": 1205534720 + }, + { + "epoch": 0.37, + "learning_rate": 0.00032059177532597796, + "loss": 2.8251, + "theoretical_loss": 3.5859290597719546, + "tokens_seen": 1205600256 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003205817452357071, + "loss": 2.7099, + "theoretical_loss": 3.585911145164693, + "tokens_seen": 1205665792 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003205717151454363, + "loss": 2.4502, + "theoretical_loss": 3.5858932318038255, + "tokens_seen": 1205731328 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003205616850551655, + "loss": 2.674, + "theoretical_loss": 3.585875319689196, + "tokens_seen": 1205796864 + }, + { + "epoch": 0.37, + "objective/train/docs_used": 684684, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.0819106101989746, + "objective/train/theoretical_loss": 3.585857408820652, + "objective/train/tokens_used": 1226322400, + "theoretical_loss": 3.585857408820652, + "tokens_seen": 1205862400 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003205516549648947, + "loss": 2.8436, + "theoretical_loss": 3.585857408820652, + "tokens_seen": 1205862400 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003205416248746239, + "loss": 2.9324, + "theoretical_loss": 3.5858394991980376, + "tokens_seen": 1205927936 + }, + { + "epoch": 0.37, + "learning_rate": 0.00032053159478435305, + "loss": 2.8579, + "theoretical_loss": 3.585821590821199, + "tokens_seen": 1205993472 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003205215646940823, + "loss": 2.6322, + "theoretical_loss": 3.5858036836899814, + "tokens_seen": 1206059008 + }, + { + "epoch": 0.37, + "learning_rate": 0.00032051153460381146, + "loss": 2.7064, + "theoretical_loss": 3.5857857778042312, + "tokens_seen": 1206124544 + }, + { + "epoch": 0.37, + "learning_rate": 0.00032050150451354064, + "loss": 2.558, + "theoretical_loss": 3.5857678731637943, + "tokens_seen": 1206190080 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003204914744232698, + "loss": 2.7874, + "theoretical_loss": 3.585749969768515, + "tokens_seen": 1206255616 + }, + { + "epoch": 0.37, + "learning_rate": 0.000320481444332999, + "loss": 2.797, + "theoretical_loss": 3.585732067618241, + "tokens_seen": 1206321152 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003204714142427282, + "loss": 2.8371, + "theoretical_loss": 3.5857141667128163, + "tokens_seen": 1206386688 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003204613841524574, + "loss": 2.859, + "theoretical_loss": 3.5856962670520884, + "tokens_seen": 1206452224 + }, + { + "epoch": 0.37, + "learning_rate": 0.00032045135406218655, + "loss": 2.8396, + "theoretical_loss": 3.5856783686359015, + "tokens_seen": 1206517760 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003204413239719158, + "loss": 2.4909, + "theoretical_loss": 3.5856604714641023, + "tokens_seen": 1206583296 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003204312938816449, + "loss": 2.6788, + "theoretical_loss": 3.5856425755365366, + "tokens_seen": 1206648832 + }, + { + "epoch": 0.37, + "learning_rate": 0.00032042126379137415, + "loss": 2.9046, + "theoretical_loss": 3.585624680853051, + "tokens_seen": 1206714368 + }, + { + "epoch": 0.37, + "learning_rate": 0.00032041123370110333, + "loss": 2.6234, + "theoretical_loss": 3.5856067874134903, + "tokens_seen": 1206779904 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003204012036108325, + "loss": 2.988, + "theoretical_loss": 3.5855888952177013, + "tokens_seen": 1206845440 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003203911735205617, + "loss": 2.8971, + "theoretical_loss": 3.585571004265529, + "tokens_seen": 1206910976 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003203811434302909, + "loss": 2.6037, + "theoretical_loss": 3.585553114556821, + "tokens_seen": 1206976512 + }, + { + "epoch": 0.37, + "learning_rate": 0.00032037111334002005, + "loss": 2.8941, + "theoretical_loss": 3.585535226091422, + "tokens_seen": 1207042048 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003203610832497493, + "loss": 2.7382, + "theoretical_loss": 3.585517338869179, + "tokens_seen": 1207107584 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003203510531594784, + "loss": 2.9165, + "theoretical_loss": 3.5854994528899375, + "tokens_seen": 1207173120 + }, + { + "epoch": 0.37, + "learning_rate": 0.00032034102306920765, + "loss": 2.6588, + "theoretical_loss": 3.5854815681535435, + "tokens_seen": 1207238656 + }, + { + "epoch": 0.37, + "learning_rate": 0.00032033099297893683, + "loss": 2.8222, + "theoretical_loss": 3.585463684659844, + "tokens_seen": 1207304192 + }, + { + "epoch": 0.37, + "learning_rate": 0.000320320962888666, + "loss": 2.7888, + "theoretical_loss": 3.5854458024086853, + "tokens_seen": 1207369728 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003203109327983952, + "loss": 3.0465, + "theoretical_loss": 3.585427921399912, + "tokens_seen": 1207435264 + }, + { + "epoch": 0.37, + "objective/train/docs_used": 686035, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.806461811065674, + "objective/train/theoretical_loss": 3.585410041633372, + "objective/train/tokens_used": 1227960800, + "theoretical_loss": 3.585410041633372, + "tokens_seen": 1207500800 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003203009027081244, + "loss": 2.9656, + "theoretical_loss": 3.585410041633372, + "tokens_seen": 1207500800 + }, + { + "epoch": 0.37, + "learning_rate": 0.00032029087261785356, + "loss": 2.9102, + "theoretical_loss": 3.585392163108911, + "tokens_seen": 1207566336 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003202808425275828, + "loss": 2.6581, + "theoretical_loss": 3.5853742858263753, + "tokens_seen": 1207631872 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003202708124373119, + "loss": 2.8673, + "theoretical_loss": 3.5853564097856117, + "tokens_seen": 1207697408 + }, + { + "epoch": 0.37, + "learning_rate": 0.00032026078234704115, + "loss": 2.6675, + "theoretical_loss": 3.585338534986466, + "tokens_seen": 1207762944 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003202507522567703, + "loss": 3.206, + "theoretical_loss": 3.585320661428784, + "tokens_seen": 1207828480 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003202407221664995, + "loss": 2.7435, + "theoretical_loss": 3.585302789112413, + "tokens_seen": 1207894016 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003202306920762287, + "loss": 2.6711, + "theoretical_loss": 3.5852849180372006, + "tokens_seen": 1207959552 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003202206619859579, + "loss": 3.0009, + "theoretical_loss": 3.585267048202991, + "tokens_seen": 1208025088 + }, + { + "epoch": 0.37, + "learning_rate": 0.00032021063189568706, + "loss": 2.7715, + "theoretical_loss": 3.5852491796096317, + "tokens_seen": 1208090624 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003202006018054163, + "loss": 2.7248, + "theoretical_loss": 3.5852313122569694, + "tokens_seen": 1208156160 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003201905717151454, + "loss": 2.8686, + "theoretical_loss": 3.5852134461448504, + "tokens_seen": 1208221696 + }, + { + "epoch": 0.37, + "learning_rate": 0.00032018054162487466, + "loss": 2.9115, + "theoretical_loss": 3.5851955812731218, + "tokens_seen": 1208287232 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003201705115346038, + "loss": 2.8475, + "theoretical_loss": 3.585177717641629, + "tokens_seen": 1208352768 + }, + { + "epoch": 0.37, + "learning_rate": 0.000320160481444333, + "loss": 2.9015, + "theoretical_loss": 3.58515985525022, + "tokens_seen": 1208418304 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003201504513540622, + "loss": 2.851, + "theoretical_loss": 3.585141994098741, + "tokens_seen": 1208483840 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003201404212637914, + "loss": 2.7081, + "theoretical_loss": 3.585124134187038, + "tokens_seen": 1208549376 + }, + { + "epoch": 0.37, + "learning_rate": 0.00032013039117352056, + "loss": 2.6988, + "theoretical_loss": 3.585106275514959, + "tokens_seen": 1208614912 + }, + { + "epoch": 0.37, + "learning_rate": 0.00032012036108324974, + "loss": 3.017, + "theoretical_loss": 3.58508841808235, + "tokens_seen": 1208680448 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003201103309929789, + "loss": 2.886, + "theoretical_loss": 3.5850705618890575, + "tokens_seen": 1208745984 + }, + { + "epoch": 0.37, + "learning_rate": 0.00032010030090270816, + "loss": 2.8448, + "theoretical_loss": 3.5850527069349285, + "tokens_seen": 1208811520 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003200902708124373, + "loss": 2.7701, + "theoretical_loss": 3.5850348532198106, + "tokens_seen": 1208877056 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003200802407221665, + "loss": 2.7868, + "theoretical_loss": 3.58501700074355, + "tokens_seen": 1208942592 + }, + { + "epoch": 0.37, + "learning_rate": 0.00032007021063189565, + "loss": 2.8145, + "theoretical_loss": 3.5849991495059927, + "tokens_seen": 1209008128 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003200601805416249, + "loss": 2.7509, + "theoretical_loss": 3.584981299506987, + "tokens_seen": 1209073664 + }, + { + "epoch": 0.37, + "objective/train/docs_used": 686549, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.8526084423065186, + "objective/train/theoretical_loss": 3.58496345074638, + "objective/train/tokens_used": 1229599200, + "theoretical_loss": 3.58496345074638, + "tokens_seen": 1209139200 + }, + { + "epoch": 0.37, + "learning_rate": 0.00032005015045135407, + "loss": 2.647, + "theoretical_loss": 3.58496345074638, + "tokens_seen": 1209139200 + }, + { + "epoch": 0.37, + "learning_rate": 0.00032004012036108325, + "loss": 2.5969, + "theoretical_loss": 3.5849456032240172, + "tokens_seen": 1209204736 + }, + { + "epoch": 0.37, + "learning_rate": 0.00032003009027081243, + "loss": 2.9251, + "theoretical_loss": 3.584927756939747, + "tokens_seen": 1209270272 + }, + { + "epoch": 0.37, + "learning_rate": 0.00032002006018054166, + "loss": 3.0758, + "theoretical_loss": 3.5849099118934156, + "tokens_seen": 1209335808 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003200100300902708, + "loss": 3.0173, + "theoretical_loss": 3.58489206808487, + "tokens_seen": 1209401344 + }, + { + "epoch": 0.37, + "learning_rate": 0.00032, + "loss": 2.8673, + "theoretical_loss": 3.5848742255139583, + "tokens_seen": 1209466880 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031998996990972915, + "loss": 2.6436, + "theoretical_loss": 3.5848563841805268, + "tokens_seen": 1209532416 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003199799398194584, + "loss": 2.913, + "theoretical_loss": 3.5848385440844224, + "tokens_seen": 1209597952 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031996990972918757, + "loss": 2.8213, + "theoretical_loss": 3.584820705225493, + "tokens_seen": 1209663488 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031995987963891675, + "loss": 2.7557, + "theoretical_loss": 3.584802867603585, + "tokens_seen": 1209729024 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031994984954864593, + "loss": 2.8644, + "theoretical_loss": 3.584785031218546, + "tokens_seen": 1209794560 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003199398194583751, + "loss": 2.7644, + "theoretical_loss": 3.584767196070223, + "tokens_seen": 1209860096 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003199297893681043, + "loss": 2.7202, + "theoretical_loss": 3.5847493621584636, + "tokens_seen": 1209925632 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031991975927783353, + "loss": 2.8793, + "theoretical_loss": 3.5847315294831152, + "tokens_seen": 1209991168 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031990972918756266, + "loss": 2.7777, + "theoretical_loss": 3.584713698044025, + "tokens_seen": 1210056704 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003198996990972919, + "loss": 2.8378, + "theoretical_loss": 3.58469586784104, + "tokens_seen": 1210122240 + }, + { + "epoch": 0.37, + "learning_rate": 0.000319889669007021, + "loss": 2.6985, + "theoretical_loss": 3.5846780388740074, + "tokens_seen": 1210187776 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031987963891675025, + "loss": 2.8938, + "theoretical_loss": 3.5846602111427757, + "tokens_seen": 1210253312 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031986960882647943, + "loss": 2.801, + "theoretical_loss": 3.5846423846471915, + "tokens_seen": 1210318848 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003198595787362086, + "loss": 2.9614, + "theoretical_loss": 3.5846245593871027, + "tokens_seen": 1210384384 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003198495486459378, + "loss": 2.5257, + "theoretical_loss": 3.584606735362356, + "tokens_seen": 1210449920 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031983951855566703, + "loss": 2.5252, + "theoretical_loss": 3.5845889125727997, + "tokens_seen": 1210515456 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031982948846539616, + "loss": 2.8372, + "theoretical_loss": 3.5845710910182804, + "tokens_seen": 1210580992 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003198194583751254, + "loss": 2.8013, + "theoretical_loss": 3.584553270698647, + "tokens_seen": 1210646528 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003198094282848546, + "loss": 2.9884, + "theoretical_loss": 3.584535451613746, + "tokens_seen": 1210712064 + }, + { + "epoch": 0.37, + "objective/train/docs_used": 687794, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.944929838180542, + "objective/train/theoretical_loss": 3.5845176337634257, + "objective/train/tokens_used": 1231237600, + "theoretical_loss": 3.5845176337634257, + "tokens_seen": 1210777600 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031979939819458376, + "loss": 2.9006, + "theoretical_loss": 3.5845176337634257, + "tokens_seen": 1210777600 + }, + { + "epoch": 0.37, + "learning_rate": 0.000319789368104313, + "loss": 2.7008, + "theoretical_loss": 3.5844998171475333, + "tokens_seen": 1210843136 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003197793380140421, + "loss": 2.9909, + "theoretical_loss": 3.5844820017659167, + "tokens_seen": 1210908672 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031976930792377135, + "loss": 2.9641, + "theoretical_loss": 3.5844641876184236, + "tokens_seen": 1210974208 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003197592778335005, + "loss": 2.699, + "theoretical_loss": 3.5844463747049016, + "tokens_seen": 1211039744 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003197492477432297, + "loss": 2.9374, + "theoretical_loss": 3.5844285630251984, + "tokens_seen": 1211105280 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003197392176529589, + "loss": 2.6553, + "theoretical_loss": 3.5844107525791618, + "tokens_seen": 1211170816 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003197291875626881, + "loss": 2.8757, + "theoretical_loss": 3.58439294336664, + "tokens_seen": 1211236352 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031971915747241726, + "loss": 2.6855, + "theoretical_loss": 3.5843751353874804, + "tokens_seen": 1211301888 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003197091273821465, + "loss": 2.6565, + "theoretical_loss": 3.584357328641531, + "tokens_seen": 1211367424 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003196990972918756, + "loss": 2.6015, + "theoretical_loss": 3.58433952312864, + "tokens_seen": 1211432960 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031968906720160486, + "loss": 2.9114, + "theoretical_loss": 3.5843217188486545, + "tokens_seen": 1211498496 + }, + { + "epoch": 0.37, + "learning_rate": 0.000319679037111334, + "loss": 2.8052, + "theoretical_loss": 3.584303915801423, + "tokens_seen": 1211564032 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003196690070210632, + "loss": 2.7927, + "theoretical_loss": 3.584286113986794, + "tokens_seen": 1211629568 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003196589769307924, + "loss": 2.7372, + "theoretical_loss": 3.584268313404614, + "tokens_seen": 1211695104 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003196489468405216, + "loss": 2.9052, + "theoretical_loss": 3.584250514054732, + "tokens_seen": 1211760640 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031963891675025076, + "loss": 3.0027, + "theoretical_loss": 3.584232715936997, + "tokens_seen": 1211826176 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031962888665997994, + "loss": 2.7698, + "theoretical_loss": 3.584214919051255, + "tokens_seen": 1211891712 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003196188565697091, + "loss": 2.8338, + "theoretical_loss": 3.584197123397356, + "tokens_seen": 1211957248 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031960882647943836, + "loss": 2.5987, + "theoretical_loss": 3.5841793289751465, + "tokens_seen": 1212022784 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003195987963891675, + "loss": 2.7552, + "theoretical_loss": 3.5841615357844763, + "tokens_seen": 1212088320 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003195887662988967, + "loss": 2.664, + "theoretical_loss": 3.584143743825192, + "tokens_seen": 1212153856 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031957873620862585, + "loss": 2.7219, + "theoretical_loss": 3.5841259530971428, + "tokens_seen": 1212219392 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003195687061183551, + "loss": 2.8271, + "theoretical_loss": 3.584108163600177, + "tokens_seen": 1212284928 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031955867602808427, + "loss": 2.7763, + "theoretical_loss": 3.584090375334142, + "tokens_seen": 1212350464 + }, + { + "epoch": 0.37, + "objective/train/docs_used": 688453, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.8370773792266846, + "objective/train/theoretical_loss": 3.5840725882988873, + "objective/train/tokens_used": 1232876000, + "theoretical_loss": 3.5840725882988873, + "tokens_seen": 1212416000 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031954864593781345, + "loss": 2.9026, + "theoretical_loss": 3.5840725882988873, + "tokens_seen": 1212416000 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031953861584754263, + "loss": 2.6775, + "theoretical_loss": 3.5840548024942604, + "tokens_seen": 1212481536 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031952858575727186, + "loss": 2.716, + "theoretical_loss": 3.5840370179201093, + "tokens_seen": 1212547072 + }, + { + "epoch": 0.37, + "learning_rate": 0.000319518555667001, + "loss": 2.6303, + "theoretical_loss": 3.5840192345762834, + "tokens_seen": 1212612608 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003195085255767302, + "loss": 2.6831, + "theoretical_loss": 3.5840014524626307, + "tokens_seen": 1212678144 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031949849548645935, + "loss": 2.7063, + "theoretical_loss": 3.5839836715789994, + "tokens_seen": 1212743680 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003194884653961886, + "loss": 2.7718, + "theoretical_loss": 3.5839658919252377, + "tokens_seen": 1212809216 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031947843530591777, + "loss": 2.6907, + "theoretical_loss": 3.5839481135011946, + "tokens_seen": 1212874752 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031946840521564695, + "loss": 2.7321, + "theoretical_loss": 3.583930336306719, + "tokens_seen": 1212940288 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031945837512537613, + "loss": 2.789, + "theoretical_loss": 3.583912560341658, + "tokens_seen": 1213005824 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003194483450351053, + "loss": 2.9261, + "theoretical_loss": 3.583894785605862, + "tokens_seen": 1213071360 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003194383149448345, + "loss": 2.787, + "theoretical_loss": 3.583877012099178, + "tokens_seen": 1213136896 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031942828485456373, + "loss": 2.7963, + "theoretical_loss": 3.5838592398214555, + "tokens_seen": 1213202432 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031941825476429286, + "loss": 2.8111, + "theoretical_loss": 3.583841468772543, + "tokens_seen": 1213267968 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003194082246740221, + "loss": 2.7889, + "theoretical_loss": 3.5838236989522887, + "tokens_seen": 1213333504 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003193981945837512, + "loss": 2.6584, + "theoretical_loss": 3.5838059303605423, + "tokens_seen": 1213399040 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031938816449348045, + "loss": 2.9764, + "theoretical_loss": 3.583788162997151, + "tokens_seen": 1213464576 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031937813440320964, + "loss": 2.7443, + "theoretical_loss": 3.5837703968619654, + "tokens_seen": 1213530112 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003193681043129388, + "loss": 2.5527, + "theoretical_loss": 3.583752631954833, + "tokens_seen": 1213595648 + }, + { + "epoch": 0.37, + "learning_rate": 0.000319358074222668, + "loss": 2.679, + "theoretical_loss": 3.5837348682756027, + "tokens_seen": 1213661184 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031934804413239723, + "loss": 2.7923, + "theoretical_loss": 3.5837171058241237, + "tokens_seen": 1213726720 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031933801404212636, + "loss": 2.6277, + "theoretical_loss": 3.5836993446002445, + "tokens_seen": 1213792256 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003193279839518556, + "loss": 2.879, + "theoretical_loss": 3.5836815846038146, + "tokens_seen": 1213857792 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003193179538615847, + "loss": 2.4895, + "theoretical_loss": 3.5836638258346825, + "tokens_seen": 1213923328 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031930792377131396, + "loss": 2.7958, + "theoretical_loss": 3.5836460682926967, + "tokens_seen": 1213988864 + }, + { + "epoch": 0.37, + "objective/train/docs_used": 689723, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.0348711013793945, + "objective/train/theoretical_loss": 3.583628311977707, + "objective/train/tokens_used": 1234514400, + "theoretical_loss": 3.583628311977707, + "tokens_seen": 1214054400 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031929789368104314, + "loss": 2.7753, + "theoretical_loss": 3.583628311977707, + "tokens_seen": 1214054400 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003192878635907723, + "loss": 2.7369, + "theoretical_loss": 3.583610556889562, + "tokens_seen": 1214119936 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003192778335005015, + "loss": 2.8185, + "theoretical_loss": 3.5835928030281106, + "tokens_seen": 1214185472 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003192678034102307, + "loss": 2.7077, + "theoretical_loss": 3.583575050393202, + "tokens_seen": 1214251008 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031925777331995986, + "loss": 2.6866, + "theoretical_loss": 3.583557298984685, + "tokens_seen": 1214316544 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003192477432296891, + "loss": 2.6541, + "theoretical_loss": 3.583539548802409, + "tokens_seen": 1214382080 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003192377131394182, + "loss": 2.6914, + "theoretical_loss": 3.583521799846223, + "tokens_seen": 1214447616 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031922768304914746, + "loss": 2.9408, + "theoretical_loss": 3.5835040521159764, + "tokens_seen": 1214513152 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003192176529588766, + "loss": 2.5991, + "theoretical_loss": 3.583486305611518, + "tokens_seen": 1214578688 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003192076228686058, + "loss": 2.7752, + "theoretical_loss": 3.5834685603326975, + "tokens_seen": 1214644224 + }, + { + "epoch": 0.37, + "learning_rate": 0.000319197592778335, + "loss": 2.6786, + "theoretical_loss": 3.5834508162793637, + "tokens_seen": 1214709760 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003191875626880642, + "loss": 2.895, + "theoretical_loss": 3.583433073451366, + "tokens_seen": 1214775296 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031917753259779337, + "loss": 2.6524, + "theoretical_loss": 3.5834153318485535, + "tokens_seen": 1214840832 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003191675025075226, + "loss": 2.7445, + "theoretical_loss": 3.5833975914707756, + "tokens_seen": 1214906368 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031915747241725173, + "loss": 2.6166, + "theoretical_loss": 3.583379852317882, + "tokens_seen": 1214971904 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031914744232698096, + "loss": 2.7943, + "theoretical_loss": 3.5833621143897214, + "tokens_seen": 1215037440 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003191374122367101, + "loss": 2.6957, + "theoretical_loss": 3.583344377686144, + "tokens_seen": 1215102976 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003191273821464393, + "loss": 2.8581, + "theoretical_loss": 3.5833266422069983, + "tokens_seen": 1215168512 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003191173520561685, + "loss": 2.6712, + "theoretical_loss": 3.5833089079521345, + "tokens_seen": 1215234048 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003191073219658977, + "loss": 2.6871, + "theoretical_loss": 3.583291174921402, + "tokens_seen": 1215299584 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031909729187562687, + "loss": 2.7912, + "theoretical_loss": 3.5832734431146496, + "tokens_seen": 1215365120 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031908726178535605, + "loss": 2.6529, + "theoretical_loss": 3.5832557125317273, + "tokens_seen": 1215430656 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031907723169508523, + "loss": 2.5985, + "theoretical_loss": 3.5832379831724848, + "tokens_seen": 1215496192 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031906720160481447, + "loss": 2.8988, + "theoretical_loss": 3.583220255036772, + "tokens_seen": 1215561728 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031905717151454365, + "loss": 2.7182, + "theoretical_loss": 3.5832025281244375, + "tokens_seen": 1215627264 + }, + { + "epoch": 0.37, + "objective/train/docs_used": 690230, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.9925808906555176, + "objective/train/theoretical_loss": 3.5831848024353317, + "objective/train/tokens_used": 1236152800, + "theoretical_loss": 3.5831848024353317, + "tokens_seen": 1215692800 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031904714142427283, + "loss": 2.7008, + "theoretical_loss": 3.5831848024353317, + "tokens_seen": 1215692800 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031903711133400206, + "loss": 2.8074, + "theoretical_loss": 3.5831670779693034, + "tokens_seen": 1215758336 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003190270812437312, + "loss": 2.728, + "theoretical_loss": 3.5831493547262037, + "tokens_seen": 1215823872 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003190170511534604, + "loss": 2.7477, + "theoretical_loss": 3.5831316327058813, + "tokens_seen": 1215889408 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031900702106318955, + "loss": 2.7347, + "theoretical_loss": 3.5831139119081863, + "tokens_seen": 1215954944 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003189969909729188, + "loss": 2.707, + "theoretical_loss": 3.583096192332968, + "tokens_seen": 1216020480 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031898696088264797, + "loss": 2.8895, + "theoretical_loss": 3.583078473980077, + "tokens_seen": 1216086016 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031897693079237715, + "loss": 2.8246, + "theoretical_loss": 3.5830607568493624, + "tokens_seen": 1216151552 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031896690070210633, + "loss": 2.6538, + "theoretical_loss": 3.583043040940674, + "tokens_seen": 1216217088 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003189568706118355, + "loss": 2.7352, + "theoretical_loss": 3.5830253262538623, + "tokens_seen": 1216282624 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003189468405215647, + "loss": 2.6387, + "theoretical_loss": 3.5830076127887764, + "tokens_seen": 1216348160 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031893681043129393, + "loss": 2.7032, + "theoretical_loss": 3.582989900545267, + "tokens_seen": 1216413696 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031892678034102306, + "loss": 2.5455, + "theoretical_loss": 3.5829721895231836, + "tokens_seen": 1216479232 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003189167502507523, + "loss": 2.9358, + "theoretical_loss": 3.5829544797223765, + "tokens_seen": 1216544768 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003189067201604814, + "loss": 2.5639, + "theoretical_loss": 3.582936771142695, + "tokens_seen": 1216610304 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031889669007021065, + "loss": 2.9865, + "theoretical_loss": 3.58291906378399, + "tokens_seen": 1216675840 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031888665997993984, + "loss": 2.5474, + "theoretical_loss": 3.582901357646111, + "tokens_seen": 1216741376 + }, + { + "epoch": 0.37, + "learning_rate": 0.000318876629889669, + "loss": 2.6514, + "theoretical_loss": 3.582883652728908, + "tokens_seen": 1216806912 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003188665997993982, + "loss": 2.7149, + "theoretical_loss": 3.5828659490322323, + "tokens_seen": 1216872448 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031885656970912743, + "loss": 2.6781, + "theoretical_loss": 3.5828482465559324, + "tokens_seen": 1216937984 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031884653961885656, + "loss": 2.8493, + "theoretical_loss": 3.582830545299859, + "tokens_seen": 1217003520 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003188365095285858, + "loss": 2.8369, + "theoretical_loss": 3.582812845263863, + "tokens_seen": 1217069056 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003188264794383149, + "loss": 2.9377, + "theoretical_loss": 3.5827951464477934, + "tokens_seen": 1217134592 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031881644934804416, + "loss": 2.5862, + "theoretical_loss": 3.582777448851502, + "tokens_seen": 1217200128 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031880641925777334, + "loss": 2.7915, + "theoretical_loss": 3.5827597524748374, + "tokens_seen": 1217265664 + }, + { + "epoch": 0.37, + "objective/train/docs_used": 691368, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.8258609771728516, + "objective/train/theoretical_loss": 3.5827420573176507, + "objective/train/tokens_used": 1237791200, + "theoretical_loss": 3.5827420573176507, + "tokens_seen": 1217331200 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003187963891675025, + "loss": 2.8035, + "theoretical_loss": 3.5827420573176507, + "tokens_seen": 1217331200 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003187863590772317, + "loss": 2.8493, + "theoretical_loss": 3.582724363379792, + "tokens_seen": 1217396736 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003187763289869609, + "loss": 2.8281, + "theoretical_loss": 3.5827066706611124, + "tokens_seen": 1217462272 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031876629889669006, + "loss": 2.8478, + "theoretical_loss": 3.5826889791614613, + "tokens_seen": 1217527808 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003187562688064193, + "loss": 2.9892, + "theoretical_loss": 3.5826712888806895, + "tokens_seen": 1217593344 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003187462387161484, + "loss": 2.8904, + "theoretical_loss": 3.5826535998186477, + "tokens_seen": 1217658880 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031873620862587766, + "loss": 2.6822, + "theoretical_loss": 3.582635911975186, + "tokens_seen": 1217724416 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003187261785356068, + "loss": 2.9108, + "theoretical_loss": 3.5826182253501546, + "tokens_seen": 1217789952 + }, + { + "epoch": 0.37, + "learning_rate": 0.000318716148445336, + "loss": 2.7595, + "theoretical_loss": 3.582600539943405, + "tokens_seen": 1217855488 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003187061183550652, + "loss": 2.7544, + "theoretical_loss": 3.5825828557547865, + "tokens_seen": 1217921024 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003186960882647944, + "loss": 2.7713, + "theoretical_loss": 3.5825651727841503, + "tokens_seen": 1217986560 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031868605817452357, + "loss": 2.6062, + "theoretical_loss": 3.582547491031347, + "tokens_seen": 1218052096 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003186760280842528, + "loss": 2.8221, + "theoretical_loss": 3.582529810496227, + "tokens_seen": 1218117632 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031866599799398193, + "loss": 2.9624, + "theoretical_loss": 3.5825121311786416, + "tokens_seen": 1218183168 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031865596790371116, + "loss": 2.8517, + "theoretical_loss": 3.58249445307844, + "tokens_seen": 1218248704 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003186459378134403, + "loss": 2.7643, + "theoretical_loss": 3.5824767761954748, + "tokens_seen": 1218314240 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003186359077231695, + "loss": 2.4802, + "theoretical_loss": 3.5824591005295954, + "tokens_seen": 1218379776 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003186258776328987, + "loss": 2.822, + "theoretical_loss": 3.5824414260806527, + "tokens_seen": 1218445312 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003186158475426279, + "loss": 2.6846, + "theoretical_loss": 3.5824237528484977, + "tokens_seen": 1218510848 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031860581745235707, + "loss": 2.7188, + "theoretical_loss": 3.582406080832981, + "tokens_seen": 1218576384 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031859578736208625, + "loss": 2.6338, + "theoretical_loss": 3.5823884100339543, + "tokens_seen": 1218641920 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031858575727181543, + "loss": 2.6056, + "theoretical_loss": 3.582370740451267, + "tokens_seen": 1218707456 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031857572718154467, + "loss": 2.6008, + "theoretical_loss": 3.582353072084771, + "tokens_seen": 1218772992 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003185656970912738, + "loss": 2.543, + "theoretical_loss": 3.582335404934317, + "tokens_seen": 1218838528 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031855566700100303, + "loss": 2.8125, + "theoretical_loss": 3.5823177389997554, + "tokens_seen": 1218904064 + }, + { + "epoch": 0.37, + "objective/train/docs_used": 692060, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7530789375305176, + "objective/train/theoretical_loss": 3.5823000742809374, + "objective/train/tokens_used": 1239429600, + "theoretical_loss": 3.5823000742809374, + "tokens_seen": 1218969600 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003185456369107322, + "loss": 2.7468, + "theoretical_loss": 3.5823000742809374, + "tokens_seen": 1218969600 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003185356068204614, + "loss": 2.628, + "theoretical_loss": 3.5822824107777143, + "tokens_seen": 1219035136 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031852557673019057, + "loss": 2.8262, + "theoretical_loss": 3.582264748489937, + "tokens_seen": 1219100672 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031851554663991975, + "loss": 2.6352, + "theoretical_loss": 3.582247087417457, + "tokens_seen": 1219166208 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031850551654964893, + "loss": 2.7667, + "theoretical_loss": 3.582229427560124, + "tokens_seen": 1219231744 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031849548645937817, + "loss": 2.8181, + "theoretical_loss": 3.5822117689177904, + "tokens_seen": 1219297280 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003184854563691073, + "loss": 2.7416, + "theoretical_loss": 3.582194111490307, + "tokens_seen": 1219362816 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031847542627883653, + "loss": 2.7048, + "theoretical_loss": 3.582176455277524, + "tokens_seen": 1219428352 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031846539618856566, + "loss": 2.7708, + "theoretical_loss": 3.582158800279294, + "tokens_seen": 1219493888 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003184553660982949, + "loss": 2.723, + "theoretical_loss": 3.5821411464954673, + "tokens_seen": 1219559424 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003184453360080241, + "loss": 2.5896, + "theoretical_loss": 3.5821234939258955, + "tokens_seen": 1219624960 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031843530591775326, + "loss": 2.761, + "theoretical_loss": 3.5821058425704293, + "tokens_seen": 1219690496 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031842527582748244, + "loss": 2.7691, + "theoretical_loss": 3.5820881924289205, + "tokens_seen": 1219756032 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003184152457372116, + "loss": 2.8376, + "theoretical_loss": 3.5820705435012203, + "tokens_seen": 1219821568 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003184052156469408, + "loss": 2.6366, + "theoretical_loss": 3.58205289578718, + "tokens_seen": 1219887104 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031839518555667004, + "loss": 2.8383, + "theoretical_loss": 3.5820352492866507, + "tokens_seen": 1219952640 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031838515546639916, + "loss": 2.5699, + "theoretical_loss": 3.5820176039994838, + "tokens_seen": 1220018176 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003183751253761284, + "loss": 2.7132, + "theoretical_loss": 3.5819999599255317, + "tokens_seen": 1220083712 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003183650952858576, + "loss": 2.7842, + "theoretical_loss": 3.581982317064644, + "tokens_seen": 1220149248 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031835506519558676, + "loss": 2.5959, + "theoretical_loss": 3.581964675416674, + "tokens_seen": 1220214784 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031834503510531594, + "loss": 2.6686, + "theoretical_loss": 3.5819470349814715, + "tokens_seen": 1220280320 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003183350050150451, + "loss": 2.5781, + "theoretical_loss": 3.5819293957588894, + "tokens_seen": 1220345856 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003183249749247743, + "loss": 2.6658, + "theoretical_loss": 3.5819117577487782, + "tokens_seen": 1220411392 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031831494483450354, + "loss": 2.5958, + "theoretical_loss": 3.5818941209509902, + "tokens_seen": 1220476928 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003183049147442327, + "loss": 2.7592, + "theoretical_loss": 3.5818764853653766, + "tokens_seen": 1220542464 + }, + { + "epoch": 0.37, + "objective/train/docs_used": 693613, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.952972650527954, + "objective/train/theoretical_loss": 3.581858850991789, + "objective/train/tokens_used": 1241068000, + "theoretical_loss": 3.581858850991789, + "tokens_seen": 1220608000 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003182948846539619, + "loss": 2.7592, + "theoretical_loss": 3.581858850991789, + "tokens_seen": 1220608000 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003182848545636911, + "loss": 2.7642, + "theoretical_loss": 3.581841217830079, + "tokens_seen": 1220673536 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031827482447342026, + "loss": 2.593, + "theoretical_loss": 3.581823585880099, + "tokens_seen": 1220739072 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003182647943831495, + "loss": 3.0433, + "theoretical_loss": 3.5818059551416996, + "tokens_seen": 1220804608 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003182547642928786, + "loss": 2.7097, + "theoretical_loss": 3.581788325614733, + "tokens_seen": 1220870144 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031824473420260786, + "loss": 2.5867, + "theoretical_loss": 3.581770697299051, + "tokens_seen": 1220935680 + }, + { + "epoch": 0.37, + "learning_rate": 0.000318234704112337, + "loss": 2.4571, + "theoretical_loss": 3.5817530701945053, + "tokens_seen": 1221001216 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003182246740220662, + "loss": 2.9921, + "theoretical_loss": 3.5817354443009473, + "tokens_seen": 1221066752 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003182146439317954, + "loss": 2.8108, + "theoretical_loss": 3.5817178196182295, + "tokens_seen": 1221132288 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003182046138415246, + "loss": 2.5708, + "theoretical_loss": 3.581700196146204, + "tokens_seen": 1221197824 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031819458375125377, + "loss": 2.9105, + "theoretical_loss": 3.5816825738847213, + "tokens_seen": 1221263360 + }, + { + "epoch": 0.37, + "learning_rate": 0.000318184553660983, + "loss": 2.7966, + "theoretical_loss": 3.581664952833634, + "tokens_seen": 1221328896 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031817452357071213, + "loss": 2.8952, + "theoretical_loss": 3.581647332992795, + "tokens_seen": 1221394432 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031816449348044136, + "loss": 2.8357, + "theoretical_loss": 3.5816297143620544, + "tokens_seen": 1221459968 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003181544633901705, + "loss": 2.5428, + "theoretical_loss": 3.581612096941266, + "tokens_seen": 1221525504 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003181444332998997, + "loss": 2.8147, + "theoretical_loss": 3.5815944807302804, + "tokens_seen": 1221591040 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003181344032096289, + "loss": 2.8515, + "theoretical_loss": 3.58157686572895, + "tokens_seen": 1221656576 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003181243731193581, + "loss": 2.7033, + "theoretical_loss": 3.5815592519371275, + "tokens_seen": 1221722112 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031811434302908727, + "loss": 2.8021, + "theoretical_loss": 3.581541639354664, + "tokens_seen": 1221787648 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031810431293881645, + "loss": 2.79, + "theoretical_loss": 3.5815240279814127, + "tokens_seen": 1221853184 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031809428284854563, + "loss": 2.8131, + "theoretical_loss": 3.5815064178172245, + "tokens_seen": 1221918720 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031808425275827487, + "loss": 2.7604, + "theoretical_loss": 3.5814888088619528, + "tokens_seen": 1221984256 + }, + { + "epoch": 0.37, + "learning_rate": 0.000318074222668004, + "loss": 2.8455, + "theoretical_loss": 3.5814712011154484, + "tokens_seen": 1222049792 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031806419257773323, + "loss": 2.7688, + "theoretical_loss": 3.5814535945775647, + "tokens_seen": 1222115328 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003180541624874624, + "loss": 2.7503, + "theoretical_loss": 3.5814359892481535, + "tokens_seen": 1222180864 + }, + { + "epoch": 0.37, + "objective/train/docs_used": 694027, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7478246688842773, + "objective/train/theoretical_loss": 3.5814183851270673, + "objective/train/tokens_used": 1242706400, + "theoretical_loss": 3.5814183851270673, + "tokens_seen": 1222246400 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003180441323971916, + "loss": 2.8509, + "theoretical_loss": 3.5814183851270673, + "tokens_seen": 1222246400 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031803410230692077, + "loss": 2.7125, + "theoretical_loss": 3.581400782214158, + "tokens_seen": 1222311936 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031802407221664995, + "loss": 2.7841, + "theoretical_loss": 3.5813831805092775, + "tokens_seen": 1222377472 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031801404212637913, + "loss": 2.6587, + "theoretical_loss": 3.581365580012279, + "tokens_seen": 1222443008 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031800401203610837, + "loss": 2.6367, + "theoretical_loss": 3.581347980723015, + "tokens_seen": 1222508544 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003179939819458375, + "loss": 2.8854, + "theoretical_loss": 3.5813303826413367, + "tokens_seen": 1222574080 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031798395185556673, + "loss": 2.672, + "theoretical_loss": 3.5813127857670977, + "tokens_seen": 1222639616 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031797392176529586, + "loss": 2.6939, + "theoretical_loss": 3.5812951901001497, + "tokens_seen": 1222705152 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003179638916750251, + "loss": 2.4925, + "theoretical_loss": 3.581277595640346, + "tokens_seen": 1222770688 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003179538615847543, + "loss": 2.6682, + "theoretical_loss": 3.581260002387538, + "tokens_seen": 1222836224 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031794383149448346, + "loss": 2.6739, + "theoretical_loss": 3.581242410341579, + "tokens_seen": 1222901760 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031793380140421264, + "loss": 2.7415, + "theoretical_loss": 3.5812248195023213, + "tokens_seen": 1222967296 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003179237713139418, + "loss": 2.6468, + "theoretical_loss": 3.5812072298696176, + "tokens_seen": 1223032832 + }, + { + "epoch": 0.37, + "learning_rate": 0.000317913741223671, + "loss": 2.85, + "theoretical_loss": 3.58118964144332, + "tokens_seen": 1223098368 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031790371113340024, + "loss": 2.6927, + "theoretical_loss": 3.581172054223282, + "tokens_seen": 1223163904 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031789368104312936, + "loss": 2.675, + "theoretical_loss": 3.5811544682093555, + "tokens_seen": 1223229440 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003178836509528586, + "loss": 2.7626, + "theoretical_loss": 3.5811368834013937, + "tokens_seen": 1223294976 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003178736208625878, + "loss": 2.5028, + "theoretical_loss": 3.5811192997992487, + "tokens_seen": 1223360512 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031786359077231696, + "loss": 2.7043, + "theoretical_loss": 3.581101717402773, + "tokens_seen": 1223426048 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031785356068204614, + "loss": 2.6433, + "theoretical_loss": 3.581084136211821, + "tokens_seen": 1223491584 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003178435305917753, + "loss": 2.7921, + "theoretical_loss": 3.5810665562262436, + "tokens_seen": 1223557120 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003178335005015045, + "loss": 2.8965, + "theoretical_loss": 3.581048977445895, + "tokens_seen": 1223622656 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031782347041123374, + "loss": 2.7155, + "theoretical_loss": 3.581031399870627, + "tokens_seen": 1223688192 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031781344032096287, + "loss": 2.5742, + "theoretical_loss": 3.5810138235002933, + "tokens_seen": 1223753728 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003178034102306921, + "loss": 2.8308, + "theoretical_loss": 3.580996248334746, + "tokens_seen": 1223819264 + }, + { + "epoch": 0.37, + "objective/train/docs_used": 695222, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.5498063564300537, + "objective/train/theoretical_loss": 3.580978674373838, + "objective/train/tokens_used": 1244344800, + "theoretical_loss": 3.580978674373838, + "tokens_seen": 1223884800 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031779338014042123, + "loss": 2.5351, + "theoretical_loss": 3.580978674373838, + "tokens_seen": 1223884800 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031778335005015046, + "loss": 2.6223, + "theoretical_loss": 3.580961101617423, + "tokens_seen": 1223950336 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031777331995987964, + "loss": 2.8908, + "theoretical_loss": 3.580943530065354, + "tokens_seen": 1224015872 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003177632898696088, + "loss": 3.0291, + "theoretical_loss": 3.580925959717483, + "tokens_seen": 1224081408 + }, + { + "epoch": 0.37, + "learning_rate": 0.000317753259779338, + "loss": 2.7347, + "theoretical_loss": 3.580908390573663, + "tokens_seen": 1224146944 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003177432296890672, + "loss": 2.7372, + "theoretical_loss": 3.580890822633749, + "tokens_seen": 1224212480 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031773319959879637, + "loss": 2.5423, + "theoretical_loss": 3.5808732558975915, + "tokens_seen": 1224278016 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003177231695085256, + "loss": 2.6118, + "theoretical_loss": 3.580855690365045, + "tokens_seen": 1224343552 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031771313941825473, + "loss": 2.8752, + "theoretical_loss": 3.580838126035962, + "tokens_seen": 1224409088 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031770310932798397, + "loss": 2.8439, + "theoretical_loss": 3.580820562910197, + "tokens_seen": 1224474624 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031769307923771315, + "loss": 2.6814, + "theoretical_loss": 3.5808030009876015, + "tokens_seen": 1224540160 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031768304914744233, + "loss": 2.8132, + "theoretical_loss": 3.580785440268029, + "tokens_seen": 1224605696 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003176730190571715, + "loss": 2.5512, + "theoretical_loss": 3.5807678807513335, + "tokens_seen": 1224671232 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003176629889669007, + "loss": 2.7133, + "theoretical_loss": 3.5807503224373676, + "tokens_seen": 1224736768 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031765295887662987, + "loss": 2.67, + "theoretical_loss": 3.580732765325985, + "tokens_seen": 1224802304 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003176429287863591, + "loss": 2.6659, + "theoretical_loss": 3.5807152094170385, + "tokens_seen": 1224867840 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031763289869608823, + "loss": 2.8027, + "theoretical_loss": 3.5806976547103817, + "tokens_seen": 1224933376 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031762286860581747, + "loss": 2.7546, + "theoretical_loss": 3.5806801012058678, + "tokens_seen": 1224998912 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003176128385155466, + "loss": 2.8184, + "theoretical_loss": 3.5806625489033506, + "tokens_seen": 1225064448 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031760280842527583, + "loss": 2.6431, + "theoretical_loss": 3.580644997802683, + "tokens_seen": 1225129984 + }, + { + "epoch": 0.37, + "learning_rate": 0.000317592778335005, + "loss": 2.8167, + "theoretical_loss": 3.580627447903719, + "tokens_seen": 1225195520 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003175827482447342, + "loss": 2.8409, + "theoretical_loss": 3.580609899206311, + "tokens_seen": 1225261056 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003175727181544634, + "loss": 2.3932, + "theoretical_loss": 3.5805923517103135, + "tokens_seen": 1225326592 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003175626880641926, + "loss": 2.7776, + "theoretical_loss": 3.58057480541558, + "tokens_seen": 1225392128 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003175526579739218, + "loss": 2.5691, + "theoretical_loss": 3.580557260321963, + "tokens_seen": 1225457664 + }, + { + "epoch": 0.37, + "objective/train/docs_used": 695595, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.866990804672241, + "objective/train/theoretical_loss": 3.5805397164293167, + "objective/train/tokens_used": 1245983200, + "theoretical_loss": 3.5805397164293167, + "tokens_seen": 1225523200 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031754262788365097, + "loss": 2.6176, + "theoretical_loss": 3.5805397164293167, + "tokens_seen": 1225523200 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031753259779338015, + "loss": 2.7066, + "theoretical_loss": 3.580522173737495, + "tokens_seen": 1225588736 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031752256770310933, + "loss": 2.8313, + "theoretical_loss": 3.580504632246351, + "tokens_seen": 1225654272 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031751253761283857, + "loss": 2.9266, + "theoretical_loss": 3.5804870919557388, + "tokens_seen": 1225719808 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003175025075225677, + "loss": 2.7095, + "theoretical_loss": 3.5804695528655115, + "tokens_seen": 1225785344 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031749247743229693, + "loss": 2.8812, + "theoretical_loss": 3.5804520149755232, + "tokens_seen": 1225850880 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031748244734202606, + "loss": 2.7803, + "theoretical_loss": 3.5804344782856274, + "tokens_seen": 1225916416 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003174724172517553, + "loss": 2.8249, + "theoretical_loss": 3.580416942795678, + "tokens_seen": 1225981952 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003174623871614845, + "loss": 2.7154, + "theoretical_loss": 3.5803994085055284, + "tokens_seen": 1226047488 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031745235707121366, + "loss": 2.6114, + "theoretical_loss": 3.580381875415033, + "tokens_seen": 1226113024 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031744232698094284, + "loss": 2.6531, + "theoretical_loss": 3.5803643435240446, + "tokens_seen": 1226178560 + }, + { + "epoch": 0.37, + "learning_rate": 0.000317432296890672, + "loss": 2.9103, + "theoretical_loss": 3.580346812832418, + "tokens_seen": 1226244096 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003174222668004012, + "loss": 2.7653, + "theoretical_loss": 3.5803292833400064, + "tokens_seen": 1226309632 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031741223671013044, + "loss": 2.6331, + "theoretical_loss": 3.5803117550466643, + "tokens_seen": 1226375168 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031740220661985956, + "loss": 2.8433, + "theoretical_loss": 3.580294227952246, + "tokens_seen": 1226440704 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003173921765295888, + "loss": 2.7442, + "theoretical_loss": 3.5802767020566035, + "tokens_seen": 1226506240 + }, + { + "epoch": 0.37, + "learning_rate": 0.000317382146439318, + "loss": 2.8399, + "theoretical_loss": 3.5802591773595926, + "tokens_seen": 1226571776 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031737211634904716, + "loss": 2.7011, + "theoretical_loss": 3.5802416538610666, + "tokens_seen": 1226637312 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031736208625877634, + "loss": 2.7757, + "theoretical_loss": 3.58022413156088, + "tokens_seen": 1226702848 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003173520561685055, + "loss": 2.7515, + "theoretical_loss": 3.580206610458886, + "tokens_seen": 1226768384 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003173420260782347, + "loss": 2.7187, + "theoretical_loss": 3.5801890905549394, + "tokens_seen": 1226833920 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031733199598796394, + "loss": 2.7913, + "theoretical_loss": 3.5801715718488936, + "tokens_seen": 1226899456 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031732196589769307, + "loss": 2.9512, + "theoretical_loss": 3.5801540543406034, + "tokens_seen": 1226964992 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003173119358074223, + "loss": 2.7667, + "theoretical_loss": 3.580136538029923, + "tokens_seen": 1227030528 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031730190571715143, + "loss": 2.7695, + "theoretical_loss": 3.5801190229167057, + "tokens_seen": 1227096064 + }, + { + "epoch": 0.37, + "objective/train/docs_used": 696474, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.5584464073181152, + "objective/train/theoretical_loss": 3.580101509000806, + "objective/train/tokens_used": 1247621600, + "theoretical_loss": 3.580101509000806, + "tokens_seen": 1227161600 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031729187562688066, + "loss": 2.8262, + "theoretical_loss": 3.580101509000806, + "tokens_seen": 1227161600 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031728184553660984, + "loss": 2.7832, + "theoretical_loss": 3.5800839962820787, + "tokens_seen": 1227227136 + }, + { + "epoch": 0.37, + "learning_rate": 0.000317271815446339, + "loss": 2.8166, + "theoretical_loss": 3.580066484760378, + "tokens_seen": 1227292672 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003172617853560682, + "loss": 2.8897, + "theoretical_loss": 3.5800489744355573, + "tokens_seen": 1227358208 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003172517552657974, + "loss": 2.8483, + "theoretical_loss": 3.5800314653074716, + "tokens_seen": 1227423744 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031724172517552657, + "loss": 2.7649, + "theoretical_loss": 3.580013957375975, + "tokens_seen": 1227489280 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003172316950852558, + "loss": 2.9125, + "theoretical_loss": 3.5799964506409214, + "tokens_seen": 1227554816 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031722166499498493, + "loss": 2.4246, + "theoretical_loss": 3.5799789451021664, + "tokens_seen": 1227620352 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031721163490471417, + "loss": 2.6073, + "theoretical_loss": 3.579961440759564, + "tokens_seen": 1227685888 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031720160481444335, + "loss": 2.8044, + "theoretical_loss": 3.579943937612967, + "tokens_seen": 1227751424 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031719157472417253, + "loss": 2.7498, + "theoretical_loss": 3.579926435662232, + "tokens_seen": 1227816960 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003171815446339017, + "loss": 2.8842, + "theoretical_loss": 3.5799089349072126, + "tokens_seen": 1227882496 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003171715145436309, + "loss": 2.7453, + "theoretical_loss": 3.579891435347763, + "tokens_seen": 1227948032 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031716148445336007, + "loss": 2.7949, + "theoretical_loss": 3.5798739369837382, + "tokens_seen": 1228013568 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003171514543630893, + "loss": 2.7968, + "theoretical_loss": 3.5798564398149924, + "tokens_seen": 1228079104 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031714142427281843, + "loss": 2.7971, + "theoretical_loss": 3.5798389438413807, + "tokens_seen": 1228144640 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031713139418254767, + "loss": 2.6749, + "theoretical_loss": 3.5798214490627567, + "tokens_seen": 1228210176 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003171213640922768, + "loss": 2.4327, + "theoretical_loss": 3.5798039554789765, + "tokens_seen": 1228275712 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031711133400200603, + "loss": 2.7057, + "theoretical_loss": 3.579786463089893, + "tokens_seen": 1228341248 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003171013039117352, + "loss": 2.593, + "theoretical_loss": 3.5797689718953625, + "tokens_seen": 1228406784 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003170912738214644, + "loss": 2.6124, + "theoretical_loss": 3.5797514818952383, + "tokens_seen": 1228472320 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003170812437311936, + "loss": 2.7862, + "theoretical_loss": 3.579733993089377, + "tokens_seen": 1228537856 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003170712136409228, + "loss": 2.7434, + "theoretical_loss": 3.579716505477631, + "tokens_seen": 1228603392 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031706118355065194, + "loss": 2.8087, + "theoretical_loss": 3.5796990190598565, + "tokens_seen": 1228668928 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003170511534603812, + "loss": 2.9854, + "theoretical_loss": 3.5796815338359087, + "tokens_seen": 1228734464 + }, + { + "epoch": 0.37, + "objective/train/docs_used": 696846, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.774714708328247, + "objective/train/theoretical_loss": 3.5796640498056407, + "objective/train/tokens_used": 1249260000, + "theoretical_loss": 3.5796640498056407, + "tokens_seen": 1228800000 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003170411233701103, + "loss": 2.6356, + "theoretical_loss": 3.5796640498056407, + "tokens_seen": 1228800000 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031703109327983954, + "loss": 2.6116, + "theoretical_loss": 3.5796465669689095, + "tokens_seen": 1228865536 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003170210631895687, + "loss": 2.8912, + "theoretical_loss": 3.579629085325568, + "tokens_seen": 1228931072 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003170110330992979, + "loss": 2.8717, + "theoretical_loss": 3.5796116048754727, + "tokens_seen": 1228996608 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003170010030090271, + "loss": 2.3401, + "theoretical_loss": 3.5795941256184776, + "tokens_seen": 1229062144 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031699097291875626, + "loss": 2.6425, + "theoretical_loss": 3.579576647554438, + "tokens_seen": 1229127680 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031698094282848544, + "loss": 2.6069, + "theoretical_loss": 3.5795591706832086, + "tokens_seen": 1229193216 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003169709127382147, + "loss": 2.4638, + "theoretical_loss": 3.5795416950046444, + "tokens_seen": 1229258752 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003169608826479438, + "loss": 2.901, + "theoretical_loss": 3.579524220518601, + "tokens_seen": 1229324288 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031695085255767304, + "loss": 2.7456, + "theoretical_loss": 3.579506747224933, + "tokens_seen": 1229389824 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031694082246740217, + "loss": 2.7213, + "theoretical_loss": 3.5794892751234952, + "tokens_seen": 1229455360 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003169307923771314, + "loss": 2.7787, + "theoretical_loss": 3.579471804214143, + "tokens_seen": 1229520896 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003169207622868606, + "loss": 2.7746, + "theoretical_loss": 3.579454334496732, + "tokens_seen": 1229586432 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031691073219658976, + "loss": 2.7683, + "theoretical_loss": 3.579436865971117, + "tokens_seen": 1229651968 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031690070210631894, + "loss": 2.8152, + "theoretical_loss": 3.5794193986371527, + "tokens_seen": 1229717504 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003168906720160482, + "loss": 2.7741, + "theoretical_loss": 3.579401932494695, + "tokens_seen": 1229783040 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003168806419257773, + "loss": 2.8848, + "theoretical_loss": 3.579384467543599, + "tokens_seen": 1229848576 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031687061183550654, + "loss": 2.8695, + "theoretical_loss": 3.5793670037837195, + "tokens_seen": 1229914112 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031686058174523567, + "loss": 2.8008, + "theoretical_loss": 3.5793495412149126, + "tokens_seen": 1229979648 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003168505516549649, + "loss": 2.5332, + "theoretical_loss": 3.579332079837033, + "tokens_seen": 1230045184 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003168405215646941, + "loss": 2.6315, + "theoretical_loss": 3.579314619649936, + "tokens_seen": 1230110720 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031683049147442327, + "loss": 2.9045, + "theoretical_loss": 3.5792971606534767, + "tokens_seen": 1230176256 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031682046138415245, + "loss": 2.5323, + "theoretical_loss": 3.5792797028475114, + "tokens_seen": 1230241792 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031681043129388163, + "loss": 2.7788, + "theoretical_loss": 3.579262246231895, + "tokens_seen": 1230307328 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031680040120361086, + "loss": 2.8543, + "theoretical_loss": 3.5792447908064826, + "tokens_seen": 1230372864 + }, + { + "epoch": 0.37, + "objective/train/docs_used": 698095, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.757563829421997, + "objective/train/theoretical_loss": 3.5792273365711305, + "objective/train/tokens_used": 1250898400, + "theoretical_loss": 3.5792273365711305, + "tokens_seen": 1230438400 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031679037111334004, + "loss": 2.78, + "theoretical_loss": 3.5792273365711305, + "tokens_seen": 1230438400 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003167803410230692, + "loss": 2.6563, + "theoretical_loss": 3.5792098835256936, + "tokens_seen": 1230503936 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003167703109327984, + "loss": 2.6935, + "theoretical_loss": 3.5791924316700277, + "tokens_seen": 1230569472 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003167602808425276, + "loss": 2.602, + "theoretical_loss": 3.5791749810039875, + "tokens_seen": 1230635008 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031675025075225677, + "loss": 2.8063, + "theoretical_loss": 3.5791575315274295, + "tokens_seen": 1230700544 + }, + { + "epoch": 0.37, + "learning_rate": 0.000316740220661986, + "loss": 2.8801, + "theoretical_loss": 3.579140083240209, + "tokens_seen": 1230766080 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031673019057171513, + "loss": 2.6375, + "theoretical_loss": 3.5791226361421815, + "tokens_seen": 1230831616 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031672016048144437, + "loss": 2.7324, + "theoretical_loss": 3.579105190233203, + "tokens_seen": 1230897152 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031671013039117355, + "loss": 2.6578, + "theoretical_loss": 3.579087745513129, + "tokens_seen": 1230962688 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031670010030090273, + "loss": 2.6397, + "theoretical_loss": 3.579070301981815, + "tokens_seen": 1231028224 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003166900702106319, + "loss": 2.7666, + "theoretical_loss": 3.5790528596391167, + "tokens_seen": 1231093760 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003166800401203611, + "loss": 2.9134, + "theoretical_loss": 3.5790354184848896, + "tokens_seen": 1231159296 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031667001003009027, + "loss": 2.8504, + "theoretical_loss": 3.5790179785189906, + "tokens_seen": 1231224832 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003166599799398195, + "loss": 2.7306, + "theoretical_loss": 3.579000539741274, + "tokens_seen": 1231290368 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031664994984954863, + "loss": 2.8047, + "theoretical_loss": 3.5789831021515965, + "tokens_seen": 1231355904 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031663991975927787, + "loss": 2.6695, + "theoretical_loss": 3.5789656657498137, + "tokens_seen": 1231421440 + }, + { + "epoch": 0.37, + "learning_rate": 0.000316629889669007, + "loss": 2.733, + "theoretical_loss": 3.578948230535782, + "tokens_seen": 1231486976 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031661985957873623, + "loss": 2.6415, + "theoretical_loss": 3.5789307965093564, + "tokens_seen": 1231552512 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003166098294884654, + "loss": 2.5906, + "theoretical_loss": 3.5789133636703934, + "tokens_seen": 1231618048 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003165997993981946, + "loss": 2.6422, + "theoretical_loss": 3.5788959320187486, + "tokens_seen": 1231683584 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003165897693079238, + "loss": 2.7335, + "theoretical_loss": 3.5788785015542786, + "tokens_seen": 1231749120 + }, + { + "epoch": 0.37, + "learning_rate": 0.000316579739217653, + "loss": 2.7721, + "theoretical_loss": 3.578861072276838, + "tokens_seen": 1231814656 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031656970912738214, + "loss": 2.6464, + "theoretical_loss": 3.5788436441862843, + "tokens_seen": 1231880192 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003165596790371114, + "loss": 2.7423, + "theoretical_loss": 3.578826217282473, + "tokens_seen": 1231945728 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003165496489468405, + "loss": 2.6788, + "theoretical_loss": 3.57880879156526, + "tokens_seen": 1232011264 + }, + { + "epoch": 0.37, + "objective/train/docs_used": 698724, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.614719867706299, + "objective/train/theoretical_loss": 3.5787913670345013, + "objective/train/tokens_used": 1252536800, + "theoretical_loss": 3.5787913670345013, + "tokens_seen": 1232076800 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031653961885656974, + "loss": 2.8085, + "theoretical_loss": 3.5787913670345013, + "tokens_seen": 1232076800 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003165295887662989, + "loss": 2.7896, + "theoretical_loss": 3.578773943690054, + "tokens_seen": 1232142336 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003165195586760281, + "loss": 2.7228, + "theoretical_loss": 3.5787565215317727, + "tokens_seen": 1232207872 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003165095285857573, + "loss": 2.6504, + "theoretical_loss": 3.578739100559515, + "tokens_seen": 1232273408 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031649949849548646, + "loss": 2.8084, + "theoretical_loss": 3.578721680773136, + "tokens_seen": 1232338944 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031648946840521564, + "loss": 2.7866, + "theoretical_loss": 3.5787042621724927, + "tokens_seen": 1232404480 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003164794383149449, + "loss": 2.699, + "theoretical_loss": 3.5786868447574403, + "tokens_seen": 1232470016 + }, + { + "epoch": 0.37, + "learning_rate": 0.000316469408224674, + "loss": 2.6889, + "theoretical_loss": 3.5786694285278364, + "tokens_seen": 1232535552 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031645937813440324, + "loss": 2.849, + "theoretical_loss": 3.578652013483537, + "tokens_seen": 1232601088 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031644934804413237, + "loss": 2.826, + "theoretical_loss": 3.5786345996243973, + "tokens_seen": 1232666624 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003164393179538616, + "loss": 2.5725, + "theoretical_loss": 3.5786171869502743, + "tokens_seen": 1232732160 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003164292878635908, + "loss": 2.7976, + "theoretical_loss": 3.578599775461025, + "tokens_seen": 1232797696 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031641925777331996, + "loss": 2.8037, + "theoretical_loss": 3.578582365156505, + "tokens_seen": 1232863232 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031640922768304914, + "loss": 2.5788, + "theoretical_loss": 3.5785649560365718, + "tokens_seen": 1232928768 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003163991975927784, + "loss": 2.8014, + "theoretical_loss": 3.5785475481010796, + "tokens_seen": 1232994304 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003163891675025075, + "loss": 2.6799, + "theoretical_loss": 3.5785301413498876, + "tokens_seen": 1233059840 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031637913741223674, + "loss": 2.8265, + "theoretical_loss": 3.57851273578285, + "tokens_seen": 1233125376 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031636910732196587, + "loss": 2.9292, + "theoretical_loss": 3.578495331399825, + "tokens_seen": 1233190912 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003163590772316951, + "loss": 2.8932, + "theoretical_loss": 3.5784779282006682, + "tokens_seen": 1233256448 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003163490471414243, + "loss": 2.6298, + "theoretical_loss": 3.5784605261852365, + "tokens_seen": 1233321984 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031633901705115347, + "loss": 2.5248, + "theoretical_loss": 3.5784431253533864, + "tokens_seen": 1233387520 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031632898696088265, + "loss": 2.6509, + "theoretical_loss": 3.5784257257049745, + "tokens_seen": 1233453056 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031631895687061183, + "loss": 2.6523, + "theoretical_loss": 3.578408327239857, + "tokens_seen": 1233518592 + }, + { + "epoch": 0.37, + "learning_rate": 0.000316308926780341, + "loss": 2.5436, + "theoretical_loss": 3.5783909299578918, + "tokens_seen": 1233584128 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031629889669007024, + "loss": 2.8018, + "theoretical_loss": 3.578373533858935, + "tokens_seen": 1233649664 + }, + { + "epoch": 0.37, + "objective/train/docs_used": 699861, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6158342361450195, + "objective/train/theoretical_loss": 3.578356138942842, + "objective/train/tokens_used": 1254175200, + "theoretical_loss": 3.578356138942842, + "tokens_seen": 1233715200 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031628886659979937, + "loss": 2.6698, + "theoretical_loss": 3.578356138942842, + "tokens_seen": 1233715200 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003162788365095286, + "loss": 2.6286, + "theoretical_loss": 3.5783387452094715, + "tokens_seen": 1233780736 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031626880641925773, + "loss": 2.8959, + "theoretical_loss": 3.578321352658679, + "tokens_seen": 1233846272 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031625877632898697, + "loss": 2.7886, + "theoretical_loss": 3.578303961290322, + "tokens_seen": 1233911808 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031624874623871615, + "loss": 2.7025, + "theoretical_loss": 3.5782865711042575, + "tokens_seen": 1233977344 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031623871614844533, + "loss": 2.7395, + "theoretical_loss": 3.5782691821003407, + "tokens_seen": 1234042880 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003162286860581745, + "loss": 2.4506, + "theoretical_loss": 3.578251794278431, + "tokens_seen": 1234108416 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031621865596790375, + "loss": 2.7471, + "theoretical_loss": 3.578234407638383, + "tokens_seen": 1234173952 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003162086258776329, + "loss": 2.7344, + "theoretical_loss": 3.5782170221800556, + "tokens_seen": 1234239488 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003161985957873621, + "loss": 2.7312, + "theoretical_loss": 3.5781996379033036, + "tokens_seen": 1234305024 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031618856569709124, + "loss": 2.5937, + "theoretical_loss": 3.5781822548079854, + "tokens_seen": 1234370560 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031617853560682047, + "loss": 2.8285, + "theoretical_loss": 3.578164872893958, + "tokens_seen": 1234436096 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031616850551654965, + "loss": 2.5336, + "theoretical_loss": 3.5781474921610776, + "tokens_seen": 1234501632 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031615847542627883, + "loss": 2.7779, + "theoretical_loss": 3.578130112609202, + "tokens_seen": 1234567168 + }, + { + "epoch": 0.37, + "learning_rate": 0.000316148445336008, + "loss": 2.7912, + "theoretical_loss": 3.5781127342381884, + "tokens_seen": 1234632704 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003161384152457372, + "loss": 2.8465, + "theoretical_loss": 3.578095357047893, + "tokens_seen": 1234698240 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003161283851554664, + "loss": 2.8298, + "theoretical_loss": 3.5780779810381733, + "tokens_seen": 1234763776 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003161183550651956, + "loss": 2.6493, + "theoretical_loss": 3.578060606208887, + "tokens_seen": 1234829312 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031610832497492474, + "loss": 2.7252, + "theoretical_loss": 3.5780432325598905, + "tokens_seen": 1234894848 + }, + { + "epoch": 0.37, + "learning_rate": 0.000316098294884654, + "loss": 2.5919, + "theoretical_loss": 3.5780258600910413, + "tokens_seen": 1234960384 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003160882647943831, + "loss": 2.7243, + "theoretical_loss": 3.5780084888021966, + "tokens_seen": 1235025920 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031607823470411234, + "loss": 2.5558, + "theoretical_loss": 3.5779911186932134, + "tokens_seen": 1235091456 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003160682046138415, + "loss": 2.7178, + "theoretical_loss": 3.5779737497639497, + "tokens_seen": 1235156992 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003160581745235707, + "loss": 2.8568, + "theoretical_loss": 3.5779563820142624, + "tokens_seen": 1235222528 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031604814443329994, + "loss": 2.7768, + "theoretical_loss": 3.577939015444008, + "tokens_seen": 1235288064 + }, + { + "epoch": 0.37, + "objective/train/docs_used": 700550, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.8100011348724365, + "objective/train/theoretical_loss": 3.577921650053045, + "objective/train/tokens_used": 1255813600, + "theoretical_loss": 3.577921650053045, + "tokens_seen": 1235353600 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003160381143430291, + "loss": 2.9162, + "theoretical_loss": 3.577921650053045, + "tokens_seen": 1235353600 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003160280842527583, + "loss": 2.6416, + "theoretical_loss": 3.57790428584123, + "tokens_seen": 1235419136 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003160180541624875, + "loss": 2.7175, + "theoretical_loss": 3.577886922808421, + "tokens_seen": 1235484672 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031600802407221666, + "loss": 2.8634, + "theoretical_loss": 3.5778695609544755, + "tokens_seen": 1235550208 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031599799398194584, + "loss": 2.4183, + "theoretical_loss": 3.57785220027925, + "tokens_seen": 1235615744 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003159879638916751, + "loss": 2.944, + "theoretical_loss": 3.577834840782603, + "tokens_seen": 1235681280 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003159779338014042, + "loss": 2.9418, + "theoretical_loss": 3.5778174824643907, + "tokens_seen": 1235746816 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031596790371113344, + "loss": 2.6143, + "theoretical_loss": 3.577800125324472, + "tokens_seen": 1235812352 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031595787362086257, + "loss": 2.8111, + "theoretical_loss": 3.577782769362704, + "tokens_seen": 1235877888 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003159478435305918, + "loss": 2.5698, + "theoretical_loss": 3.577765414578944, + "tokens_seen": 1235943424 + }, + { + "epoch": 0.37, + "learning_rate": 0.000315937813440321, + "loss": 2.8099, + "theoretical_loss": 3.5777480609730494, + "tokens_seen": 1236008960 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031592778335005016, + "loss": 2.6613, + "theoretical_loss": 3.577730708544878, + "tokens_seen": 1236074496 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031591775325977934, + "loss": 2.6853, + "theoretical_loss": 3.577713357294288, + "tokens_seen": 1236140032 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003159077231695086, + "loss": 2.8404, + "theoretical_loss": 3.5776960072211366, + "tokens_seen": 1236205568 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003158976930792377, + "loss": 2.6782, + "theoretical_loss": 3.577678658325281, + "tokens_seen": 1236271104 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031588766298896694, + "loss": 2.6525, + "theoretical_loss": 3.5776613106065804, + "tokens_seen": 1236336640 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031587763289869607, + "loss": 2.8669, + "theoretical_loss": 3.5776439640648907, + "tokens_seen": 1236402176 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003158676028084253, + "loss": 2.9581, + "theoretical_loss": 3.5776266187000707, + "tokens_seen": 1236467712 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003158575727181545, + "loss": 2.7408, + "theoretical_loss": 3.5776092745119783, + "tokens_seen": 1236533248 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031584754262788367, + "loss": 2.712, + "theoretical_loss": 3.5775919315004705, + "tokens_seen": 1236598784 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031583751253761285, + "loss": 2.804, + "theoretical_loss": 3.5775745896654056, + "tokens_seen": 1236664320 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031582748244734203, + "loss": 2.8401, + "theoretical_loss": 3.5775572490066425, + "tokens_seen": 1236729856 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003158174523570712, + "loss": 2.7507, + "theoretical_loss": 3.577539909524037, + "tokens_seen": 1236795392 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031580742226680044, + "loss": 2.7191, + "theoretical_loss": 3.5775225712174485, + "tokens_seen": 1236860928 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031579739217652957, + "loss": 2.7457, + "theoretical_loss": 3.5775052340867344, + "tokens_seen": 1236926464 + }, + { + "epoch": 0.37, + "objective/train/docs_used": 701918, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7977521419525146, + "objective/train/theoretical_loss": 3.577487898131753, + "objective/train/tokens_used": 1257452000, + "theoretical_loss": 3.577487898131753, + "tokens_seen": 1236992000 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003157873620862588, + "loss": 2.717, + "theoretical_loss": 3.577487898131753, + "tokens_seen": 1236992000 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031577733199598793, + "loss": 2.6878, + "theoretical_loss": 3.577470563352362, + "tokens_seen": 1237057536 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031576730190571717, + "loss": 2.7427, + "theoretical_loss": 3.5774532297484196, + "tokens_seen": 1237123072 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031575727181544635, + "loss": 2.6341, + "theoretical_loss": 3.5774358973197833, + "tokens_seen": 1237188608 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031574724172517553, + "loss": 2.8447, + "theoretical_loss": 3.577418566066312, + "tokens_seen": 1237254144 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003157372116349047, + "loss": 2.6626, + "theoretical_loss": 3.577401235987863, + "tokens_seen": 1237319680 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031572718154463395, + "loss": 2.8817, + "theoretical_loss": 3.577383907084295, + "tokens_seen": 1237385216 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003157171514543631, + "loss": 2.7036, + "theoretical_loss": 3.5773665793554654, + "tokens_seen": 1237450752 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003157071213640923, + "loss": 2.7825, + "theoretical_loss": 3.577349252801233, + "tokens_seen": 1237516288 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031569709127382144, + "loss": 2.8014, + "theoretical_loss": 3.577331927421456, + "tokens_seen": 1237581824 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031568706118355067, + "loss": 2.5994, + "theoretical_loss": 3.5773146032159926, + "tokens_seen": 1237647360 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031567703109327985, + "loss": 2.9362, + "theoretical_loss": 3.5772972801847005, + "tokens_seen": 1237712896 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031566700100300903, + "loss": 2.9725, + "theoretical_loss": 3.5772799583274386, + "tokens_seen": 1237778432 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003156569709127382, + "loss": 2.8823, + "theoretical_loss": 3.577262637644065, + "tokens_seen": 1237843968 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003156469408224674, + "loss": 2.603, + "theoretical_loss": 3.5772453181344375, + "tokens_seen": 1237909504 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003156369107321966, + "loss": 2.7227, + "theoretical_loss": 3.5772279997984144, + "tokens_seen": 1237975040 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003156268806419258, + "loss": 2.709, + "theoretical_loss": 3.5772106826358554, + "tokens_seen": 1238040576 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031561685055165494, + "loss": 2.5907, + "theoretical_loss": 3.5771933666466174, + "tokens_seen": 1238106112 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003156068204613842, + "loss": 2.7243, + "theoretical_loss": 3.5771760518305595, + "tokens_seen": 1238171648 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003155967903711133, + "loss": 2.7612, + "theoretical_loss": 3.57715873818754, + "tokens_seen": 1238237184 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031558676028084254, + "loss": 2.8014, + "theoretical_loss": 3.577141425717417, + "tokens_seen": 1238302720 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003155767301905717, + "loss": 2.6374, + "theoretical_loss": 3.5771241144200494, + "tokens_seen": 1238368256 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003155667001003009, + "loss": 2.6185, + "theoretical_loss": 3.577106804295296, + "tokens_seen": 1238433792 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003155566700100301, + "loss": 2.7642, + "theoretical_loss": 3.5770894953430146, + "tokens_seen": 1238499328 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003155466399197593, + "loss": 2.5761, + "theoretical_loss": 3.577072187563064, + "tokens_seen": 1238564864 + }, + { + "epoch": 0.38, + "objective/train/docs_used": 702757, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.736950397491455, + "objective/train/theoretical_loss": 3.577054880955303, + "objective/train/tokens_used": 1259090400, + "theoretical_loss": 3.577054880955303, + "tokens_seen": 1238630400 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031553660982948844, + "loss": 2.6497, + "theoretical_loss": 3.577054880955303, + "tokens_seen": 1238630400 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003155265797392177, + "loss": 2.7407, + "theoretical_loss": 3.57703757551959, + "tokens_seen": 1238695936 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003155165496489468, + "loss": 2.8513, + "theoretical_loss": 3.5770202712557833, + "tokens_seen": 1238761472 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031550651955867604, + "loss": 2.6862, + "theoretical_loss": 3.5770029681637423, + "tokens_seen": 1238827008 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003154964894684052, + "loss": 2.6621, + "theoretical_loss": 3.576985666243325, + "tokens_seen": 1238892544 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003154864593781344, + "loss": 2.7253, + "theoretical_loss": 3.5769683654943907, + "tokens_seen": 1238958080 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003154764292878636, + "loss": 2.9655, + "theoretical_loss": 3.5769510659167976, + "tokens_seen": 1239023616 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031546639919759277, + "loss": 2.5667, + "theoretical_loss": 3.576933767510405, + "tokens_seen": 1239089152 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031545636910732195, + "loss": 2.7462, + "theoretical_loss": 3.5769164702750706, + "tokens_seen": 1239154688 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003154463390170512, + "loss": 2.84, + "theoretical_loss": 3.576899174210655, + "tokens_seen": 1239220224 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003154363089267803, + "loss": 2.7286, + "theoretical_loss": 3.576881879317015, + "tokens_seen": 1239285760 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031542627883650954, + "loss": 2.7654, + "theoretical_loss": 3.57686458559401, + "tokens_seen": 1239351296 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003154162487462387, + "loss": 2.7899, + "theoretical_loss": 3.5768472930415003, + "tokens_seen": 1239416832 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003154062186559679, + "loss": 2.833, + "theoretical_loss": 3.576830001659343, + "tokens_seen": 1239482368 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003153961885656971, + "loss": 2.7274, + "theoretical_loss": 3.576812711447398, + "tokens_seen": 1239547904 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031538615847542627, + "loss": 2.7528, + "theoretical_loss": 3.5767954224055236, + "tokens_seen": 1239613440 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031537612838515545, + "loss": 2.5753, + "theoretical_loss": 3.57677813453358, + "tokens_seen": 1239678976 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003153660982948847, + "loss": 2.6806, + "theoretical_loss": 3.576760847831425, + "tokens_seen": 1239744512 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003153560682046138, + "loss": 2.7516, + "theoretical_loss": 3.576743562298917, + "tokens_seen": 1239810048 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031534603811434305, + "loss": 2.8501, + "theoretical_loss": 3.576726277935917, + "tokens_seen": 1239875584 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003153360080240722, + "loss": 2.7325, + "theoretical_loss": 3.5767089947422828, + "tokens_seen": 1239941120 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003153259779338014, + "loss": 2.676, + "theoretical_loss": 3.5766917127178735, + "tokens_seen": 1240006656 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003153159478435306, + "loss": 2.9092, + "theoretical_loss": 3.576674431862549, + "tokens_seen": 1240072192 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031530591775325977, + "loss": 2.5906, + "theoretical_loss": 3.5766571521761676, + "tokens_seen": 1240137728 + }, + { + "epoch": 0.38, + "learning_rate": 0.000315295887662989, + "loss": 2.7998, + "theoretical_loss": 3.5766398736585883, + "tokens_seen": 1240203264 + }, + { + "epoch": 0.38, + "objective/train/docs_used": 703958, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6864168643951416, + "objective/train/theoretical_loss": 3.5766225963096714, + "objective/train/tokens_used": 1260728800, + "theoretical_loss": 3.5766225963096714, + "tokens_seen": 1240268800 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031528585757271813, + "loss": 2.7544, + "theoretical_loss": 3.5766225963096714, + "tokens_seen": 1240268800 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031527582748244737, + "loss": 2.5222, + "theoretical_loss": 3.576605320129275, + "tokens_seen": 1240334336 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031526579739217655, + "loss": 2.6686, + "theoretical_loss": 3.576588045117259, + "tokens_seen": 1240399872 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031525576730190573, + "loss": 2.8487, + "theoretical_loss": 3.576570771273482, + "tokens_seen": 1240465408 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003152457372116349, + "loss": 2.6475, + "theoretical_loss": 3.5765534985978045, + "tokens_seen": 1240530944 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031523570712136415, + "loss": 2.6343, + "theoretical_loss": 3.5765362270900845, + "tokens_seen": 1240596480 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003152256770310933, + "loss": 2.7477, + "theoretical_loss": 3.576518956750182, + "tokens_seen": 1240662016 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003152156469408225, + "loss": 2.6575, + "theoretical_loss": 3.576501687577956, + "tokens_seen": 1240727552 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031520561685055164, + "loss": 2.4754, + "theoretical_loss": 3.5764844195732666, + "tokens_seen": 1240793088 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031519558676028087, + "loss": 2.9164, + "theoretical_loss": 3.576467152735972, + "tokens_seen": 1240858624 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031518555667001005, + "loss": 2.6808, + "theoretical_loss": 3.5764498870659325, + "tokens_seen": 1240924160 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031517552657973924, + "loss": 2.668, + "theoretical_loss": 3.576432622563008, + "tokens_seen": 1240989696 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003151654964894684, + "loss": 2.8285, + "theoretical_loss": 3.576415359227057, + "tokens_seen": 1241055232 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003151554663991976, + "loss": 2.4292, + "theoretical_loss": 3.5763980970579388, + "tokens_seen": 1241120768 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003151454363089268, + "loss": 2.9155, + "theoretical_loss": 3.576380836055514, + "tokens_seen": 1241186304 + }, + { + "epoch": 0.38, + "learning_rate": 0.000315135406218656, + "loss": 2.747, + "theoretical_loss": 3.5763635762196415, + "tokens_seen": 1241251840 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031512537612838514, + "loss": 2.9143, + "theoretical_loss": 3.576346317550181, + "tokens_seen": 1241317376 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003151153460381144, + "loss": 2.498, + "theoretical_loss": 3.5763290600469926, + "tokens_seen": 1241382912 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003151053159478435, + "loss": 2.9911, + "theoretical_loss": 3.5763118037099346, + "tokens_seen": 1241448448 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031509528585757274, + "loss": 2.8794, + "theoretical_loss": 3.5762945485388675, + "tokens_seen": 1241513984 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003150852557673019, + "loss": 2.7325, + "theoretical_loss": 3.576277294533652, + "tokens_seen": 1241579520 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003150752256770311, + "loss": 2.7359, + "theoretical_loss": 3.5762600416941455, + "tokens_seen": 1241645056 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003150651955867603, + "loss": 2.8889, + "theoretical_loss": 3.57624279002021, + "tokens_seen": 1241710592 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003150551654964895, + "loss": 2.6088, + "theoretical_loss": 3.5762255395117037, + "tokens_seen": 1241776128 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031504513540621864, + "loss": 3.0109, + "theoretical_loss": 3.576208290168487, + "tokens_seen": 1241841664 + }, + { + "epoch": 0.38, + "objective/train/docs_used": 704662, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.1910390853881836, + "objective/train/theoretical_loss": 3.5761910419904193, + "objective/train/tokens_used": 1262367200, + "theoretical_loss": 3.5761910419904193, + "tokens_seen": 1241907200 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003150351053159479, + "loss": 2.9116, + "theoretical_loss": 3.5761910419904193, + "tokens_seen": 1241907200 + }, + { + "epoch": 0.38, + "learning_rate": 0.000315025075225677, + "loss": 2.8317, + "theoretical_loss": 3.576173794977361, + "tokens_seen": 1241972736 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031501504513540624, + "loss": 2.8007, + "theoretical_loss": 3.5761565491291716, + "tokens_seen": 1242038272 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003150050150451354, + "loss": 2.6628, + "theoretical_loss": 3.576139304445711, + "tokens_seen": 1242103808 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003149949849548646, + "loss": 2.8661, + "theoretical_loss": 3.5761220609268385, + "tokens_seen": 1242169344 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003149849548645938, + "loss": 2.8519, + "theoretical_loss": 3.5761048185724156, + "tokens_seen": 1242234880 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031497492477432297, + "loss": 2.687, + "theoretical_loss": 3.5760875773823004, + "tokens_seen": 1242300416 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031496489468405215, + "loss": 2.6528, + "theoretical_loss": 3.5760703373563545, + "tokens_seen": 1242365952 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003149548645937814, + "loss": 2.406, + "theoretical_loss": 3.5760530984944365, + "tokens_seen": 1242431488 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003149448345035105, + "loss": 2.5904, + "theoretical_loss": 3.5760358607964076, + "tokens_seen": 1242497024 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031493480441323974, + "loss": 2.7906, + "theoretical_loss": 3.576018624262127, + "tokens_seen": 1242562560 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003149247743229689, + "loss": 2.5988, + "theoretical_loss": 3.5760013888914544, + "tokens_seen": 1242628096 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003149147442326981, + "loss": 2.7711, + "theoretical_loss": 3.5759841546842512, + "tokens_seen": 1242693632 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003149047141424273, + "loss": 3.0101, + "theoretical_loss": 3.575966921640376, + "tokens_seen": 1242759168 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031489468405215647, + "loss": 2.7869, + "theoretical_loss": 3.5759496897596907, + "tokens_seen": 1242824704 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031488465396188565, + "loss": 2.824, + "theoretical_loss": 3.575932459042054, + "tokens_seen": 1242890240 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003148746238716149, + "loss": 2.6195, + "theoretical_loss": 3.5759152294873267, + "tokens_seen": 1242955776 + }, + { + "epoch": 0.38, + "learning_rate": 0.000314864593781344, + "loss": 2.6907, + "theoretical_loss": 3.5758980010953687, + "tokens_seen": 1243021312 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031485456369107325, + "loss": 2.8226, + "theoretical_loss": 3.5758807738660403, + "tokens_seen": 1243086848 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003148445336008024, + "loss": 2.6886, + "theoretical_loss": 3.5758635477992025, + "tokens_seen": 1243152384 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003148345035105316, + "loss": 2.67, + "theoretical_loss": 3.575846322894715, + "tokens_seen": 1243217920 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003148244734202608, + "loss": 2.8636, + "theoretical_loss": 3.575829099152437, + "tokens_seen": 1243283456 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031481444332998997, + "loss": 2.7808, + "theoretical_loss": 3.5758118765722307, + "tokens_seen": 1243348992 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031480441323971915, + "loss": 2.6436, + "theoretical_loss": 3.5757946551539552, + "tokens_seen": 1243414528 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031479438314944833, + "loss": 2.8704, + "theoretical_loss": 3.5757774348974714, + "tokens_seen": 1243480064 + }, + { + "epoch": 0.38, + "objective/train/docs_used": 705833, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.8021810054779053, + "objective/train/theoretical_loss": 3.5757602158026396, + "objective/train/tokens_used": 1264005600, + "theoretical_loss": 3.5757602158026396, + "tokens_seen": 1243545600 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003147843530591775, + "loss": 2.6747, + "theoretical_loss": 3.5757602158026396, + "tokens_seen": 1243545600 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031477432296890675, + "loss": 2.5947, + "theoretical_loss": 3.5757429978693205, + "tokens_seen": 1243611136 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003147642928786359, + "loss": 2.6812, + "theoretical_loss": 3.5757257810973737, + "tokens_seen": 1243676672 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003147542627883651, + "loss": 2.9325, + "theoretical_loss": 3.5757085654866607, + "tokens_seen": 1243742208 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003147442326980943, + "loss": 2.8036, + "theoretical_loss": 3.575691351037041, + "tokens_seen": 1243807744 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003147342026078235, + "loss": 2.7406, + "theoretical_loss": 3.5756741377483765, + "tokens_seen": 1243873280 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031472417251755266, + "loss": 2.5949, + "theoretical_loss": 3.5756569256205264, + "tokens_seen": 1243938816 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031471414242728184, + "loss": 2.7765, + "theoretical_loss": 3.5756397146533514, + "tokens_seen": 1244004352 + }, + { + "epoch": 0.38, + "learning_rate": 0.000314704112337011, + "loss": 2.6569, + "theoretical_loss": 3.575622504846713, + "tokens_seen": 1244069888 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031469408224674025, + "loss": 2.8402, + "theoretical_loss": 3.5756052962004707, + "tokens_seen": 1244135424 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003146840521564694, + "loss": 2.6024, + "theoretical_loss": 3.575588088714486, + "tokens_seen": 1244200960 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003146740220661986, + "loss": 2.7853, + "theoretical_loss": 3.57557088238862, + "tokens_seen": 1244266496 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031466399197592774, + "loss": 2.8178, + "theoretical_loss": 3.5755536772227314, + "tokens_seen": 1244332032 + }, + { + "epoch": 0.38, + "learning_rate": 0.000314653961885657, + "loss": 2.5323, + "theoretical_loss": 3.5755364732166828, + "tokens_seen": 1244397568 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031464393179538616, + "loss": 2.8476, + "theoretical_loss": 3.5755192703703345, + "tokens_seen": 1244463104 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031463390170511534, + "loss": 2.8456, + "theoretical_loss": 3.5755020686835466, + "tokens_seen": 1244528640 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003146238716148445, + "loss": 2.7552, + "theoretical_loss": 3.5754848681561806, + "tokens_seen": 1244594176 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003146138415245737, + "loss": 2.6825, + "theoretical_loss": 3.575467668788097, + "tokens_seen": 1244659712 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003146038114343029, + "loss": 2.6266, + "theoretical_loss": 3.575450470579157, + "tokens_seen": 1244725248 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003145937813440321, + "loss": 2.6688, + "theoretical_loss": 3.575433273529221, + "tokens_seen": 1244790784 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031458375125376125, + "loss": 2.6962, + "theoretical_loss": 3.5754160776381494, + "tokens_seen": 1244856320 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003145737211634905, + "loss": 2.6526, + "theoretical_loss": 3.5753988829058043, + "tokens_seen": 1244921856 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003145636910732197, + "loss": 2.8873, + "theoretical_loss": 3.575381689332046, + "tokens_seen": 1244987392 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031455366098294884, + "loss": 2.9126, + "theoretical_loss": 3.5753644969167353, + "tokens_seen": 1245052928 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003145436308926781, + "loss": 2.7203, + "theoretical_loss": 3.575347305659734, + "tokens_seen": 1245118464 + }, + { + "epoch": 0.38, + "objective/train/docs_used": 706471, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.4460039138793945, + "objective/train/theoretical_loss": 3.5753301155609014, + "objective/train/tokens_used": 1265644000, + "theoretical_loss": 3.5753301155609014, + "tokens_seen": 1245184000 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003145336008024072, + "loss": 2.8651, + "theoretical_loss": 3.5753301155609014, + "tokens_seen": 1245184000 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031452357071213644, + "loss": 2.7651, + "theoretical_loss": 3.5753129266201005, + "tokens_seen": 1245249536 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003145135406218656, + "loss": 2.7051, + "theoretical_loss": 3.575295738837191, + "tokens_seen": 1245315072 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003145035105315948, + "loss": 2.9014, + "theoretical_loss": 3.5752785522120347, + "tokens_seen": 1245380608 + }, + { + "epoch": 0.38, + "learning_rate": 0.000314493480441324, + "loss": 2.7662, + "theoretical_loss": 3.5752613667444924, + "tokens_seen": 1245446144 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031448345035105317, + "loss": 2.7545, + "theoretical_loss": 3.5752441824344254, + "tokens_seen": 1245511680 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031447342026078235, + "loss": 2.7877, + "theoretical_loss": 3.5752269992816945, + "tokens_seen": 1245577216 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003144633901705116, + "loss": 2.7236, + "theoretical_loss": 3.5752098172861606, + "tokens_seen": 1245642752 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003144533600802407, + "loss": 2.8654, + "theoretical_loss": 3.575192636447686, + "tokens_seen": 1245708288 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031444332998996994, + "loss": 2.532, + "theoretical_loss": 3.5751754567661314, + "tokens_seen": 1245773824 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003144332998996991, + "loss": 2.6695, + "theoretical_loss": 3.575158278241357, + "tokens_seen": 1245839360 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003144232698094283, + "loss": 2.7142, + "theoretical_loss": 3.5751411008732257, + "tokens_seen": 1245904896 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003144132397191575, + "loss": 2.8451, + "theoretical_loss": 3.575123924661598, + "tokens_seen": 1245970432 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031440320962888667, + "loss": 2.7109, + "theoretical_loss": 3.575106749606335, + "tokens_seen": 1246035968 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031439317953861585, + "loss": 2.7424, + "theoretical_loss": 3.575089575707299, + "tokens_seen": 1246101504 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003143831494483451, + "loss": 2.6203, + "theoretical_loss": 3.5750724029643495, + "tokens_seen": 1246167040 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003143731193580742, + "loss": 2.5532, + "theoretical_loss": 3.57505523137735, + "tokens_seen": 1246232576 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031436308926780345, + "loss": 2.6625, + "theoretical_loss": 3.5750380609461607, + "tokens_seen": 1246298112 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003143530591775326, + "loss": 2.7454, + "theoretical_loss": 3.5750208916706434, + "tokens_seen": 1246363648 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003143430290872618, + "loss": 2.6388, + "theoretical_loss": 3.575003723550659, + "tokens_seen": 1246429184 + }, + { + "epoch": 0.38, + "learning_rate": 0.000314332998996991, + "loss": 2.8257, + "theoretical_loss": 3.5749865565860697, + "tokens_seen": 1246494720 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031432296890672017, + "loss": 2.9266, + "theoretical_loss": 3.5749693907767366, + "tokens_seen": 1246560256 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031431293881644935, + "loss": 2.7399, + "theoretical_loss": 3.5749522261225217, + "tokens_seen": 1246625792 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031430290872617853, + "loss": 2.7846, + "theoretical_loss": 3.574935062623286, + "tokens_seen": 1246691328 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003142928786359077, + "loss": 2.7793, + "theoretical_loss": 3.5749179002788916, + "tokens_seen": 1246756864 + }, + { + "epoch": 0.38, + "objective/train/docs_used": 707914, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.3902783393859863, + "objective/train/theoretical_loss": 3.574900739089199, + "objective/train/tokens_used": 1267282400, + "theoretical_loss": 3.574900739089199, + "tokens_seen": 1246822400 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031428284854563695, + "loss": 2.6447, + "theoretical_loss": 3.574900739089199, + "tokens_seen": 1246822400 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003142728184553661, + "loss": 2.7842, + "theoretical_loss": 3.574883579054071, + "tokens_seen": 1246887936 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003142627883650953, + "loss": 2.9528, + "theoretical_loss": 3.574866420173369, + "tokens_seen": 1246953472 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003142527582748245, + "loss": 2.9081, + "theoretical_loss": 3.5748492624469543, + "tokens_seen": 1247019008 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003142427281845537, + "loss": 2.7662, + "theoretical_loss": 3.5748321058746892, + "tokens_seen": 1247084544 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031423269809428286, + "loss": 2.6078, + "theoretical_loss": 3.5748149504564344, + "tokens_seen": 1247150080 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031422266800401204, + "loss": 2.6315, + "theoretical_loss": 3.574797796192053, + "tokens_seen": 1247215616 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003142126379137412, + "loss": 2.6541, + "theoretical_loss": 3.5747806430814055, + "tokens_seen": 1247281152 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031420260782347045, + "loss": 2.6325, + "theoretical_loss": 3.5747634911243544, + "tokens_seen": 1247346688 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003141925777331996, + "loss": 2.6209, + "theoretical_loss": 3.574746340320761, + "tokens_seen": 1247412224 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003141825476429288, + "loss": 2.7134, + "theoretical_loss": 3.574729190670488, + "tokens_seen": 1247477760 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031417251755265794, + "loss": 2.5593, + "theoretical_loss": 3.574712042173396, + "tokens_seen": 1247543296 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003141624874623872, + "loss": 2.8382, + "theoretical_loss": 3.5746948948293484, + "tokens_seen": 1247608832 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031415245737211636, + "loss": 2.7268, + "theoretical_loss": 3.5746777486382064, + "tokens_seen": 1247674368 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031414242728184554, + "loss": 2.7844, + "theoretical_loss": 3.574660603599831, + "tokens_seen": 1247739904 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003141323971915747, + "loss": 2.7938, + "theoretical_loss": 3.574643459714086, + "tokens_seen": 1247805440 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003141223671013039, + "loss": 2.8802, + "theoretical_loss": 3.574626316980832, + "tokens_seen": 1247870976 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003141123370110331, + "loss": 2.7911, + "theoretical_loss": 3.574609175399931, + "tokens_seen": 1247936512 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003141023069207623, + "loss": 2.7464, + "theoretical_loss": 3.5745920349712463, + "tokens_seen": 1248002048 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031409227683049145, + "loss": 2.6687, + "theoretical_loss": 3.574574895694638, + "tokens_seen": 1248067584 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003140822467402207, + "loss": 2.7603, + "theoretical_loss": 3.57455775756997, + "tokens_seen": 1248133120 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031407221664994986, + "loss": 2.7885, + "theoretical_loss": 3.5745406205971033, + "tokens_seen": 1248198656 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031406218655967904, + "loss": 2.6757, + "theoretical_loss": 3.5745234847759004, + "tokens_seen": 1248264192 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003140521564694082, + "loss": 2.6635, + "theoretical_loss": 3.5745063501062235, + "tokens_seen": 1248329728 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003140421263791374, + "loss": 2.5838, + "theoretical_loss": 3.574489216587935, + "tokens_seen": 1248395264 + }, + { + "epoch": 0.38, + "objective/train/docs_used": 708590, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.0909862518310547, + "objective/train/theoretical_loss": 3.574472084220896, + "objective/train/tokens_used": 1268920800, + "theoretical_loss": 3.574472084220896, + "tokens_seen": 1248460800 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003140320962888666, + "loss": 2.6555, + "theoretical_loss": 3.574472084220896, + "tokens_seen": 1248460800 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003140220661985958, + "loss": 2.5521, + "theoretical_loss": 3.5744549530049703, + "tokens_seen": 1248526336 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031401203610832495, + "loss": 2.8127, + "theoretical_loss": 3.574437822940019, + "tokens_seen": 1248591872 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003140020060180542, + "loss": 2.747, + "theoretical_loss": 3.574420694025905, + "tokens_seen": 1248657408 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003139919759277833, + "loss": 2.8705, + "theoretical_loss": 3.5744035662624896, + "tokens_seen": 1248722944 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031398194583751255, + "loss": 2.646, + "theoretical_loss": 3.5743864396496363, + "tokens_seen": 1248788480 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031397191574724173, + "loss": 2.7242, + "theoretical_loss": 3.574369314187207, + "tokens_seen": 1248854016 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003139618856569709, + "loss": 2.774, + "theoretical_loss": 3.5743521898750643, + "tokens_seen": 1248919552 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003139518555667001, + "loss": 2.8655, + "theoretical_loss": 3.5743350667130698, + "tokens_seen": 1248985088 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003139418254764293, + "loss": 2.8577, + "theoretical_loss": 3.574317944701086, + "tokens_seen": 1249050624 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031393179538615845, + "loss": 2.686, + "theoretical_loss": 3.5743008238389766, + "tokens_seen": 1249116160 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003139217652958877, + "loss": 2.6138, + "theoretical_loss": 3.5742837041266027, + "tokens_seen": 1249181696 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003139117352056168, + "loss": 2.7764, + "theoretical_loss": 3.5742665855638274, + "tokens_seen": 1249247232 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031390170511534605, + "loss": 2.7975, + "theoretical_loss": 3.5742494681505126, + "tokens_seen": 1249312768 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031389167502507523, + "loss": 2.8303, + "theoretical_loss": 3.574232351886522, + "tokens_seen": 1249378304 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003138816449348044, + "loss": 2.8249, + "theoretical_loss": 3.5742152367717166, + "tokens_seen": 1249443840 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003138716148445336, + "loss": 2.8505, + "theoretical_loss": 3.5741981228059605, + "tokens_seen": 1249509376 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003138615847542628, + "loss": 2.702, + "theoretical_loss": 3.5741810099891156, + "tokens_seen": 1249574912 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031385155466399196, + "loss": 2.8252, + "theoretical_loss": 3.5741638983210438, + "tokens_seen": 1249640448 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003138415245737212, + "loss": 2.6705, + "theoretical_loss": 3.574146787801609, + "tokens_seen": 1249705984 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003138314944834503, + "loss": 2.76, + "theoretical_loss": 3.5741296784306735, + "tokens_seen": 1249771520 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031382146439317955, + "loss": 2.6244, + "theoretical_loss": 3.5741125702081, + "tokens_seen": 1249837056 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031381143430290873, + "loss": 2.7279, + "theoretical_loss": 3.57409546313375, + "tokens_seen": 1249902592 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003138014042126379, + "loss": 2.7587, + "theoretical_loss": 3.5740783572074877, + "tokens_seen": 1249968128 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031379137412236715, + "loss": 2.8205, + "theoretical_loss": 3.5740612524291757, + "tokens_seen": 1250033664 + }, + { + "epoch": 0.38, + "objective/train/docs_used": 709757, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.5754740238189697, + "objective/train/theoretical_loss": 3.5740441487986767, + "objective/train/tokens_used": 1270559200, + "theoretical_loss": 3.5740441487986767, + "tokens_seen": 1250099200 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003137813440320963, + "loss": 2.6312, + "theoretical_loss": 3.5740441487986767, + "tokens_seen": 1250099200 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003137713139418255, + "loss": 2.8333, + "theoretical_loss": 3.574027046315853, + "tokens_seen": 1250164736 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003137612838515547, + "loss": 2.7055, + "theoretical_loss": 3.574009944980568, + "tokens_seen": 1250230272 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003137512537612839, + "loss": 2.6905, + "theoretical_loss": 3.573992844792684, + "tokens_seen": 1250295808 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031374122367101306, + "loss": 2.689, + "theoretical_loss": 3.5739757457520644, + "tokens_seen": 1250361344 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031373119358074224, + "loss": 2.6307, + "theoretical_loss": 3.5739586478585714, + "tokens_seen": 1250426880 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003137211634904714, + "loss": 2.8768, + "theoretical_loss": 3.5739415511120693, + "tokens_seen": 1250492416 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031371113340020065, + "loss": 2.7915, + "theoretical_loss": 3.573924455512419, + "tokens_seen": 1250557952 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003137011033099298, + "loss": 2.9292, + "theoretical_loss": 3.5739073610594856, + "tokens_seen": 1250623488 + }, + { + "epoch": 0.38, + "learning_rate": 0.000313691073219659, + "loss": 2.6701, + "theoretical_loss": 3.5738902677531312, + "tokens_seen": 1250689024 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031368104312938814, + "loss": 2.7098, + "theoretical_loss": 3.573873175593218, + "tokens_seen": 1250754560 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003136710130391174, + "loss": 2.5726, + "theoretical_loss": 3.5738560845796106, + "tokens_seen": 1250820096 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031366098294884656, + "loss": 2.6597, + "theoretical_loss": 3.5738389947121707, + "tokens_seen": 1250885632 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031365095285857574, + "loss": 2.9234, + "theoretical_loss": 3.573821905990762, + "tokens_seen": 1250951168 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003136409227683049, + "loss": 2.7965, + "theoretical_loss": 3.573804818415248, + "tokens_seen": 1251016704 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003136308926780341, + "loss": 2.6368, + "theoretical_loss": 3.573787731985491, + "tokens_seen": 1251082240 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003136208625877633, + "loss": 2.8108, + "theoretical_loss": 3.5737706467013552, + "tokens_seen": 1251147776 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003136108324974925, + "loss": 2.7396, + "theoretical_loss": 3.5737535625627026, + "tokens_seen": 1251213312 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031360080240722165, + "loss": 2.5298, + "theoretical_loss": 3.573736479569397, + "tokens_seen": 1251278848 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003135907723169509, + "loss": 2.6241, + "theoretical_loss": 3.5737193977213018, + "tokens_seen": 1251344384 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031358074222668006, + "loss": 2.9928, + "theoretical_loss": 3.57370231701828, + "tokens_seen": 1251409920 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031357071213640924, + "loss": 2.7514, + "theoretical_loss": 3.573685237460195, + "tokens_seen": 1251475456 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003135606820461384, + "loss": 2.7552, + "theoretical_loss": 3.5736681590469104, + "tokens_seen": 1251540992 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003135506519558676, + "loss": 2.5887, + "theoretical_loss": 3.5736510817782885, + "tokens_seen": 1251606528 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003135406218655968, + "loss": 2.5781, + "theoretical_loss": 3.5736340056541933, + "tokens_seen": 1251672064 + }, + { + "epoch": 0.38, + "objective/train/docs_used": 710221, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.8493094444274902, + "objective/train/theoretical_loss": 3.5736169306744885, + "objective/train/tokens_used": 1272197600, + "theoretical_loss": 3.5736169306744885, + "tokens_seen": 1251737600 + }, + { + "epoch": 0.38, + "learning_rate": 0.000313530591775326, + "loss": 2.8457, + "theoretical_loss": 3.5736169306744885, + "tokens_seen": 1251737600 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031352056168505515, + "loss": 2.9271, + "theoretical_loss": 3.5735998568390372, + "tokens_seen": 1251803136 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003135105315947844, + "loss": 2.6061, + "theoretical_loss": 3.573582784147703, + "tokens_seen": 1251868672 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003135005015045135, + "loss": 2.6745, + "theoretical_loss": 3.573565712600349, + "tokens_seen": 1251934208 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031349047141424275, + "loss": 2.7123, + "theoretical_loss": 3.5735486421968385, + "tokens_seen": 1251999744 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031348044132397193, + "loss": 2.6102, + "theoretical_loss": 3.5735315729370356, + "tokens_seen": 1252065280 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003134704112337011, + "loss": 2.8354, + "theoretical_loss": 3.5735145048208032, + "tokens_seen": 1252130816 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003134603811434303, + "loss": 2.8523, + "theoretical_loss": 3.5734974378480056, + "tokens_seen": 1252196352 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003134503510531595, + "loss": 2.4044, + "theoretical_loss": 3.5734803720185058, + "tokens_seen": 1252261888 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031344032096288865, + "loss": 2.5294, + "theoretical_loss": 3.573463307332168, + "tokens_seen": 1252327424 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003134302908726179, + "loss": 2.8539, + "theoretical_loss": 3.5734462437888546, + "tokens_seen": 1252392960 + }, + { + "epoch": 0.38, + "learning_rate": 0.000313420260782347, + "loss": 2.6632, + "theoretical_loss": 3.57342918138843, + "tokens_seen": 1252458496 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031341023069207625, + "loss": 2.718, + "theoretical_loss": 3.573412120130758, + "tokens_seen": 1252524032 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031340020060180543, + "loss": 2.7968, + "theoretical_loss": 3.5733950600157023, + "tokens_seen": 1252589568 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003133901705115346, + "loss": 2.8905, + "theoretical_loss": 3.573378001043126, + "tokens_seen": 1252655104 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003133801404212638, + "loss": 2.6838, + "theoretical_loss": 3.5733609432128937, + "tokens_seen": 1252720640 + }, + { + "epoch": 0.38, + "learning_rate": 0.000313370110330993, + "loss": 2.7829, + "theoretical_loss": 3.573343886524868, + "tokens_seen": 1252786176 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031336008024072216, + "loss": 2.7799, + "theoretical_loss": 3.5733268309789135, + "tokens_seen": 1252851712 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003133500501504514, + "loss": 2.5961, + "theoretical_loss": 3.5733097765748942, + "tokens_seen": 1252917248 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003133400200601805, + "loss": 2.819, + "theoretical_loss": 3.5732927233126732, + "tokens_seen": 1252982784 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031332998996990975, + "loss": 2.6495, + "theoretical_loss": 3.573275671192115, + "tokens_seen": 1253048320 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003133199598796389, + "loss": 2.6261, + "theoretical_loss": 3.5732586202130827, + "tokens_seen": 1253113856 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003133099297893681, + "loss": 2.7582, + "theoretical_loss": 3.5732415703754405, + "tokens_seen": 1253179392 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003132998996990973, + "loss": 2.7958, + "theoretical_loss": 3.573224521679053, + "tokens_seen": 1253244928 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003132898696088265, + "loss": 2.9179, + "theoretical_loss": 3.5732074741237834, + "tokens_seen": 1253310464 + }, + { + "epoch": 0.38, + "objective/train/docs_used": 710893, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.205178737640381, + "objective/train/theoretical_loss": 3.5731904277094957, + "objective/train/tokens_used": 1273836000, + "theoretical_loss": 3.5731904277094957, + "tokens_seen": 1253376000 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031327983951855566, + "loss": 2.452, + "theoretical_loss": 3.5731904277094957, + "tokens_seen": 1253376000 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003132698094282849, + "loss": 3.0096, + "theoretical_loss": 3.573173382436054, + "tokens_seen": 1253441536 + }, + { + "epoch": 0.38, + "learning_rate": 0.000313259779338014, + "loss": 2.5985, + "theoretical_loss": 3.5731563383033227, + "tokens_seen": 1253507072 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031324974924774326, + "loss": 2.7022, + "theoretical_loss": 3.573139295311165, + "tokens_seen": 1253572608 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003132397191574724, + "loss": 2.5898, + "theoretical_loss": 3.573122253459446, + "tokens_seen": 1253638144 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003132296890672016, + "loss": 2.6613, + "theoretical_loss": 3.5731052127480285, + "tokens_seen": 1253703680 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003132196589769308, + "loss": 2.8081, + "theoretical_loss": 3.573088173176778, + "tokens_seen": 1253769216 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031320962888666, + "loss": 2.7532, + "theoretical_loss": 3.5730711347455575, + "tokens_seen": 1253834752 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031319959879638916, + "loss": 2.7048, + "theoretical_loss": 3.5730540974542313, + "tokens_seen": 1253900288 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031318956870611834, + "loss": 2.5604, + "theoretical_loss": 3.573037061302664, + "tokens_seen": 1253965824 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003131795386158475, + "loss": 2.6507, + "theoretical_loss": 3.5730200262907195, + "tokens_seen": 1254031360 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031316950852557676, + "loss": 2.7859, + "theoretical_loss": 3.5730029924182625, + "tokens_seen": 1254096896 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003131594784353059, + "loss": 2.738, + "theoretical_loss": 3.5729859596851563, + "tokens_seen": 1254162432 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003131494483450351, + "loss": 2.5597, + "theoretical_loss": 3.5729689280912664, + "tokens_seen": 1254227968 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031313941825476425, + "loss": 2.8666, + "theoretical_loss": 3.5729518976364556, + "tokens_seen": 1254293504 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003131293881644935, + "loss": 2.8194, + "theoretical_loss": 3.5729348683205897, + "tokens_seen": 1254359040 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031311935807422267, + "loss": 2.6254, + "theoretical_loss": 3.572917840143532, + "tokens_seen": 1254424576 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031310932798395185, + "loss": 2.5781, + "theoretical_loss": 3.5729008131051474, + "tokens_seen": 1254490112 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031309929789368103, + "loss": 2.9429, + "theoretical_loss": 3.5728837872053, + "tokens_seen": 1254555648 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031308926780341026, + "loss": 2.6755, + "theoretical_loss": 3.572866762443854, + "tokens_seen": 1254621184 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003130792377131394, + "loss": 2.8386, + "theoretical_loss": 3.5728497388206746, + "tokens_seen": 1254686720 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003130692076228686, + "loss": 2.93, + "theoretical_loss": 3.5728327163356255, + "tokens_seen": 1254752256 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003130591775325978, + "loss": 2.7123, + "theoretical_loss": 3.572815694988572, + "tokens_seen": 1254817792 + }, + { + "epoch": 0.38, + "learning_rate": 0.000313049147442327, + "loss": 2.8437, + "theoretical_loss": 3.572798674779377, + "tokens_seen": 1254883328 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003130391173520562, + "loss": 2.9013, + "theoretical_loss": 3.5727816557079066, + "tokens_seen": 1254948864 + }, + { + "epoch": 0.38, + "objective/train/docs_used": 712142, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.497066020965576, + "objective/train/theoretical_loss": 3.572764637774024, + "objective/train/tokens_used": 1275474400, + "theoretical_loss": 3.572764637774024, + "tokens_seen": 1255014400 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031302908726178535, + "loss": 2.8457, + "theoretical_loss": 3.572764637774024, + "tokens_seen": 1255014400 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003130190571715146, + "loss": 2.7893, + "theoretical_loss": 3.5727476209775952, + "tokens_seen": 1255079936 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003130090270812437, + "loss": 2.7621, + "theoretical_loss": 3.572730605318484, + "tokens_seen": 1255145472 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031299899699097295, + "loss": 2.6635, + "theoretical_loss": 3.572713590796555, + "tokens_seen": 1255211008 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031298896690070213, + "loss": 2.7087, + "theoretical_loss": 3.5726965774116737, + "tokens_seen": 1255276544 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003129789368104313, + "loss": 2.7332, + "theoretical_loss": 3.5726795651637033, + "tokens_seen": 1255342080 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003129689067201605, + "loss": 2.9796, + "theoretical_loss": 3.572662554052509, + "tokens_seen": 1255407616 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003129588766298897, + "loss": 2.9703, + "theoretical_loss": 3.5726455440779565, + "tokens_seen": 1255473152 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031294884653961885, + "loss": 2.5694, + "theoretical_loss": 3.572628535239909, + "tokens_seen": 1255538688 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003129388164493481, + "loss": 2.6606, + "theoretical_loss": 3.572611527538232, + "tokens_seen": 1255604224 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003129287863590772, + "loss": 2.8348, + "theoretical_loss": 3.5725945209727907, + "tokens_seen": 1255669760 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031291875626880645, + "loss": 2.9385, + "theoretical_loss": 3.5725775155434496, + "tokens_seen": 1255735296 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031290872617853563, + "loss": 2.6273, + "theoretical_loss": 3.5725605112500727, + "tokens_seen": 1255800832 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003128986960882648, + "loss": 2.7236, + "theoretical_loss": 3.572543508092526, + "tokens_seen": 1255866368 + }, + { + "epoch": 0.38, + "learning_rate": 0.000312888665997994, + "loss": 2.6249, + "theoretical_loss": 3.5725265060706732, + "tokens_seen": 1255931904 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003128786359077232, + "loss": 2.7438, + "theoretical_loss": 3.5725095051843803, + "tokens_seen": 1255997440 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031286860581745236, + "loss": 2.7848, + "theoretical_loss": 3.5724925054335115, + "tokens_seen": 1256062976 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003128585757271816, + "loss": 2.574, + "theoretical_loss": 3.5724755068179324, + "tokens_seen": 1256128512 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003128485456369107, + "loss": 2.7463, + "theoretical_loss": 3.5724585093375074, + "tokens_seen": 1256194048 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031283851554663995, + "loss": 2.9462, + "theoretical_loss": 3.5724415129921017, + "tokens_seen": 1256259584 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003128284854563691, + "loss": 2.5761, + "theoretical_loss": 3.5724245177815797, + "tokens_seen": 1256325120 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003128184553660983, + "loss": 2.6303, + "theoretical_loss": 3.5724075237058077, + "tokens_seen": 1256390656 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003128084252758275, + "loss": 2.8675, + "theoretical_loss": 3.5723905307646495, + "tokens_seen": 1256456192 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003127983951855567, + "loss": 2.7932, + "theoretical_loss": 3.5723735389579705, + "tokens_seen": 1256521728 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031278836509528586, + "loss": 2.7735, + "theoretical_loss": 3.5723565482856365, + "tokens_seen": 1256587264 + }, + { + "epoch": 0.38, + "objective/train/docs_used": 712613, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.5070910453796387, + "objective/train/theoretical_loss": 3.5723395587475117, + "objective/train/tokens_used": 1277112800, + "theoretical_loss": 3.5723395587475117, + "tokens_seen": 1256652800 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003127783350050151, + "loss": 2.6511, + "theoretical_loss": 3.5723395587475117, + "tokens_seen": 1256652800 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003127683049147442, + "loss": 2.5948, + "theoretical_loss": 3.572322570343462, + "tokens_seen": 1256718336 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031275827482447346, + "loss": 2.8476, + "theoretical_loss": 3.572305583073352, + "tokens_seen": 1256783872 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003127482447342026, + "loss": 2.7921, + "theoretical_loss": 3.5722885969370473, + "tokens_seen": 1256849408 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003127382146439318, + "loss": 2.7557, + "theoretical_loss": 3.5722716119344122, + "tokens_seen": 1256914944 + }, + { + "epoch": 0.38, + "learning_rate": 0.000312728184553661, + "loss": 2.7158, + "theoretical_loss": 3.5722546280653136, + "tokens_seen": 1256980480 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003127181544633902, + "loss": 2.8592, + "theoretical_loss": 3.572237645329615, + "tokens_seen": 1257046016 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031270812437311936, + "loss": 2.8413, + "theoretical_loss": 3.572220663727183, + "tokens_seen": 1257111552 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031269809428284854, + "loss": 2.8053, + "theoretical_loss": 3.5722036832578823, + "tokens_seen": 1257177088 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003126880641925777, + "loss": 2.6893, + "theoretical_loss": 3.572186703921578, + "tokens_seen": 1257242624 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031267803410230696, + "loss": 2.7776, + "theoretical_loss": 3.572169725718136, + "tokens_seen": 1257308160 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003126680040120361, + "loss": 2.6939, + "theoretical_loss": 3.5721527486474214, + "tokens_seen": 1257373696 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003126579739217653, + "loss": 2.6743, + "theoretical_loss": 3.5721357727092995, + "tokens_seen": 1257439232 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031264794383149445, + "loss": 2.5672, + "theoretical_loss": 3.5721187979036353, + "tokens_seen": 1257504768 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003126379137412237, + "loss": 2.6085, + "theoretical_loss": 3.5721018242302955, + "tokens_seen": 1257570304 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031262788365095287, + "loss": 2.5576, + "theoretical_loss": 3.572084851689145, + "tokens_seen": 1257635840 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031261785356068205, + "loss": 2.7838, + "theoretical_loss": 3.5720678802800485, + "tokens_seen": 1257701376 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031260782347041123, + "loss": 2.6142, + "theoretical_loss": 3.572050910002872, + "tokens_seen": 1257766912 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031259779338014046, + "loss": 2.9182, + "theoretical_loss": 3.572033940857482, + "tokens_seen": 1257832448 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003125877632898696, + "loss": 2.9302, + "theoretical_loss": 3.572016972843743, + "tokens_seen": 1257897984 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003125777331995988, + "loss": 2.8813, + "theoretical_loss": 3.5720000059615202, + "tokens_seen": 1257963520 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031256770310932795, + "loss": 2.6708, + "theoretical_loss": 3.5719830402106805, + "tokens_seen": 1258029056 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003125576730190572, + "loss": 2.8109, + "theoretical_loss": 3.5719660755910883, + "tokens_seen": 1258094592 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031254764292878637, + "loss": 2.8447, + "theoretical_loss": 3.57194911210261, + "tokens_seen": 1258160128 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031253761283851555, + "loss": 2.7007, + "theoretical_loss": 3.571932149745111, + "tokens_seen": 1258225664 + }, + { + "epoch": 0.38, + "objective/train/docs_used": 713934, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.0430784225463867, + "objective/train/theoretical_loss": 3.571915188518457, + "objective/train/tokens_used": 1278751200, + "theoretical_loss": 3.571915188518457, + "tokens_seen": 1258291200 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031252758274824473, + "loss": 2.8127, + "theoretical_loss": 3.571915188518457, + "tokens_seen": 1258291200 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003125175526579739, + "loss": 2.8602, + "theoretical_loss": 3.571898228422514, + "tokens_seen": 1258356736 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003125075225677031, + "loss": 2.5883, + "theoretical_loss": 3.571881269457147, + "tokens_seen": 1258422272 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031249749247743233, + "loss": 2.9639, + "theoretical_loss": 3.571864311622223, + "tokens_seen": 1258487808 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031248746238716146, + "loss": 2.9191, + "theoretical_loss": 3.5718473549176064, + "tokens_seen": 1258553344 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003124774322968907, + "loss": 2.9422, + "theoretical_loss": 3.5718303993431637, + "tokens_seen": 1258618880 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003124674022066198, + "loss": 2.7452, + "theoretical_loss": 3.571813444898761, + "tokens_seen": 1258684416 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031245737211634905, + "loss": 2.5434, + "theoretical_loss": 3.5717964915842635, + "tokens_seen": 1258749952 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031244734202607823, + "loss": 2.6992, + "theoretical_loss": 3.5717795393995377, + "tokens_seen": 1258815488 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003124373119358074, + "loss": 2.7118, + "theoretical_loss": 3.5717625883444493, + "tokens_seen": 1258881024 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003124272818455366, + "loss": 2.6215, + "theoretical_loss": 3.5717456384188635, + "tokens_seen": 1258946560 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031241725175526583, + "loss": 2.8427, + "theoretical_loss": 3.5717286896226472, + "tokens_seen": 1259012096 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031240722166499496, + "loss": 2.8976, + "theoretical_loss": 3.5717117419556663, + "tokens_seen": 1259077632 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003123971915747242, + "loss": 2.8602, + "theoretical_loss": 3.5716947954177867, + "tokens_seen": 1259143168 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003123871614844533, + "loss": 2.7266, + "theoretical_loss": 3.5716778500088737, + "tokens_seen": 1259208704 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031237713139418256, + "loss": 2.7473, + "theoretical_loss": 3.5716609057287942, + "tokens_seen": 1259274240 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031236710130391174, + "loss": 2.6921, + "theoretical_loss": 3.571643962577414, + "tokens_seen": 1259339776 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003123570712136409, + "loss": 2.9223, + "theoretical_loss": 3.5716270205545992, + "tokens_seen": 1259405312 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003123470411233701, + "loss": 2.8816, + "theoretical_loss": 3.571610079660215, + "tokens_seen": 1259470848 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003123370110330993, + "loss": 2.62, + "theoretical_loss": 3.571593139894129, + "tokens_seen": 1259536384 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031232698094282846, + "loss": 2.5366, + "theoretical_loss": 3.571576201256207, + "tokens_seen": 1259601920 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003123169508525577, + "loss": 2.4943, + "theoretical_loss": 3.571559263746315, + "tokens_seen": 1259667456 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003123069207622869, + "loss": 2.6743, + "theoretical_loss": 3.5715423273643188, + "tokens_seen": 1259732992 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031229689067201606, + "loss": 2.8286, + "theoretical_loss": 3.5715253921100847, + "tokens_seen": 1259798528 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003122868605817453, + "loss": 2.6891, + "theoretical_loss": 3.5715084579834793, + "tokens_seen": 1259864064 + }, + { + "epoch": 0.38, + "objective/train/docs_used": 714588, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6527364253997803, + "objective/train/theoretical_loss": 3.571491524984369, + "objective/train/tokens_used": 1280389600, + "theoretical_loss": 3.571491524984369, + "tokens_seen": 1259929600 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003122768304914744, + "loss": 2.7491, + "theoretical_loss": 3.571491524984369, + "tokens_seen": 1259929600 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031226680040120366, + "loss": 2.7829, + "theoretical_loss": 3.5714745931126193, + "tokens_seen": 1259995136 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003122567703109328, + "loss": 2.9062, + "theoretical_loss": 3.5714576623680974, + "tokens_seen": 1260060672 + }, + { + "epoch": 0.38, + "learning_rate": 0.000312246740220662, + "loss": 2.8737, + "theoretical_loss": 3.571440732750669, + "tokens_seen": 1260126208 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003122367101303912, + "loss": 2.8319, + "theoretical_loss": 3.571423804260201, + "tokens_seen": 1260191744 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003122266800401204, + "loss": 2.8841, + "theoretical_loss": 3.5714068768965594, + "tokens_seen": 1260257280 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031221664994984956, + "loss": 2.55, + "theoretical_loss": 3.5713899506596105, + "tokens_seen": 1260322816 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031220661985957874, + "loss": 2.9387, + "theoretical_loss": 3.571373025549221, + "tokens_seen": 1260388352 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003121965897693079, + "loss": 2.9852, + "theoretical_loss": 3.5713561015652573, + "tokens_seen": 1260453888 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031218655967903716, + "loss": 2.6796, + "theoretical_loss": 3.5713391787075857, + "tokens_seen": 1260519424 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003121765295887663, + "loss": 2.7173, + "theoretical_loss": 3.571322256976073, + "tokens_seen": 1260584960 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003121664994984955, + "loss": 2.6888, + "theoretical_loss": 3.571305336370586, + "tokens_seen": 1260650496 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031215646940822465, + "loss": 2.8263, + "theoretical_loss": 3.57128841689099, + "tokens_seen": 1260716032 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003121464393179539, + "loss": 2.578, + "theoretical_loss": 3.5712714985371528, + "tokens_seen": 1260781568 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031213640922768307, + "loss": 2.9614, + "theoretical_loss": 3.5712545813089402, + "tokens_seen": 1260847104 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031212637913741225, + "loss": 2.7321, + "theoretical_loss": 3.57123766520622, + "tokens_seen": 1260912640 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031211634904714143, + "loss": 2.3339, + "theoretical_loss": 3.571220750228857, + "tokens_seen": 1260978176 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031210631895687066, + "loss": 2.8782, + "theoretical_loss": 3.571203836376719, + "tokens_seen": 1261043712 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003120962888665998, + "loss": 2.6384, + "theoretical_loss": 3.5711869236496727, + "tokens_seen": 1261109248 + }, + { + "epoch": 0.38, + "learning_rate": 0.000312086258776329, + "loss": 2.7173, + "theoretical_loss": 3.5711700120475847, + "tokens_seen": 1261174784 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031207622868605815, + "loss": 2.7892, + "theoretical_loss": 3.5711531015703217, + "tokens_seen": 1261240320 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003120661985957874, + "loss": 2.8776, + "theoretical_loss": 3.5711361922177502, + "tokens_seen": 1261305856 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031205616850551657, + "loss": 2.6509, + "theoretical_loss": 3.571119283989737, + "tokens_seen": 1261371392 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031204613841524575, + "loss": 2.6117, + "theoretical_loss": 3.5711023768861487, + "tokens_seen": 1261436928 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031203610832497493, + "loss": 2.8423, + "theoretical_loss": 3.571085470906853, + "tokens_seen": 1261502464 + }, + { + "epoch": 0.38, + "objective/train/docs_used": 715954, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.514625310897827, + "objective/train/theoretical_loss": 3.571068566051716, + "objective/train/tokens_used": 1282028000, + "theoretical_loss": 3.571068566051716, + "tokens_seen": 1261568000 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003120260782347041, + "loss": 2.5932, + "theoretical_loss": 3.571068566051716, + "tokens_seen": 1261568000 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003120160481444333, + "loss": 2.8702, + "theoretical_loss": 3.571051662320605, + "tokens_seen": 1261633536 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031200601805416253, + "loss": 3.0874, + "theoretical_loss": 3.5710347597133865, + "tokens_seen": 1261699072 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031199598796389166, + "loss": 2.4622, + "theoretical_loss": 3.5710178582299275, + "tokens_seen": 1261764608 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003119859578736209, + "loss": 2.6018, + "theoretical_loss": 3.571000957870094, + "tokens_seen": 1261830144 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031197592778335, + "loss": 2.6065, + "theoretical_loss": 3.570984058633755, + "tokens_seen": 1261895680 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031196589769307925, + "loss": 2.8123, + "theoretical_loss": 3.5709671605207762, + "tokens_seen": 1261961216 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031195586760280843, + "loss": 2.7899, + "theoretical_loss": 3.5709502635310244, + "tokens_seen": 1262026752 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003119458375125376, + "loss": 2.7357, + "theoretical_loss": 3.5709333676643675, + "tokens_seen": 1262092288 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003119358074222668, + "loss": 2.8243, + "theoretical_loss": 3.570916472920671, + "tokens_seen": 1262157824 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031192577733199603, + "loss": 2.7048, + "theoretical_loss": 3.570899579299804, + "tokens_seen": 1262223360 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031191574724172516, + "loss": 2.5905, + "theoretical_loss": 3.570882686801632, + "tokens_seen": 1262288896 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003119057171514544, + "loss": 2.7387, + "theoretical_loss": 3.570865795426023, + "tokens_seen": 1262354432 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003118956870611835, + "loss": 2.6992, + "theoretical_loss": 3.570848905172843, + "tokens_seen": 1262419968 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031188565697091276, + "loss": 2.6291, + "theoretical_loss": 3.570832016041961, + "tokens_seen": 1262485504 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031187562688064194, + "loss": 2.7681, + "theoretical_loss": 3.5708151280332423, + "tokens_seen": 1262551040 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003118655967903711, + "loss": 2.5868, + "theoretical_loss": 3.5707982411465555, + "tokens_seen": 1262616576 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003118555667001003, + "loss": 2.7705, + "theoretical_loss": 3.570781355381767, + "tokens_seen": 1262682112 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003118455366098295, + "loss": 2.8227, + "theoretical_loss": 3.570764470738744, + "tokens_seen": 1262747648 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031183550651955866, + "loss": 2.9151, + "theoretical_loss": 3.5707475872173546, + "tokens_seen": 1262813184 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003118254764292879, + "loss": 2.8036, + "theoretical_loss": 3.570730704817465, + "tokens_seen": 1262878720 + }, + { + "epoch": 0.38, + "learning_rate": 0.000311815446339017, + "loss": 2.6038, + "theoretical_loss": 3.5707138235389433, + "tokens_seen": 1262944256 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031180541624874626, + "loss": 2.5376, + "theoretical_loss": 3.570696943381656, + "tokens_seen": 1263009792 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003117953861584754, + "loss": 2.7394, + "theoretical_loss": 3.5706800643454715, + "tokens_seen": 1263075328 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003117853560682046, + "loss": 2.7348, + "theoretical_loss": 3.5706631864302567, + "tokens_seen": 1263140864 + }, + { + "epoch": 0.38, + "objective/train/docs_used": 716702, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.0328049659729004, + "objective/train/theoretical_loss": 3.5706463096358787, + "objective/train/tokens_used": 1283666400, + "theoretical_loss": 3.5706463096358787, + "tokens_seen": 1263206400 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003117753259779338, + "loss": 2.9081, + "theoretical_loss": 3.5706463096358787, + "tokens_seen": 1263206400 + }, + { + "epoch": 0.38, + "learning_rate": 0.000311765295887663, + "loss": 2.915, + "theoretical_loss": 3.570629433962206, + "tokens_seen": 1263271936 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031175526579739217, + "loss": 2.4379, + "theoretical_loss": 3.5706125594091045, + "tokens_seen": 1263337472 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003117452357071214, + "loss": 2.6849, + "theoretical_loss": 3.5705956859764427, + "tokens_seen": 1263403008 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031173520561685053, + "loss": 2.428, + "theoretical_loss": 3.5705788136640875, + "tokens_seen": 1263468544 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031172517552657976, + "loss": 2.7665, + "theoretical_loss": 3.5705619424719073, + "tokens_seen": 1263534080 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003117151454363089, + "loss": 2.5992, + "theoretical_loss": 3.5705450723997685, + "tokens_seen": 1263599616 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003117051153460381, + "loss": 2.681, + "theoretical_loss": 3.5705282034475396, + "tokens_seen": 1263665152 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003116950852557673, + "loss": 2.6824, + "theoretical_loss": 3.5705113356150875, + "tokens_seen": 1263730688 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003116850551654965, + "loss": 2.8354, + "theoretical_loss": 3.5704944689022806, + "tokens_seen": 1263796224 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031167502507522567, + "loss": 2.4251, + "theoretical_loss": 3.570477603308986, + "tokens_seen": 1263861760 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031166499498495485, + "loss": 2.6582, + "theoretical_loss": 3.570460738835071, + "tokens_seen": 1263927296 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031165496489468403, + "loss": 2.6208, + "theoretical_loss": 3.5704438754804038, + "tokens_seen": 1263992832 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031164493480441327, + "loss": 2.634, + "theoretical_loss": 3.570427013244852, + "tokens_seen": 1264058368 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003116349047141424, + "loss": 2.7092, + "theoretical_loss": 3.570410152128283, + "tokens_seen": 1264123904 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031162487462387163, + "loss": 2.8892, + "theoretical_loss": 3.570393292130565, + "tokens_seen": 1264189440 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003116148445336008, + "loss": 2.593, + "theoretical_loss": 3.5703764332515657, + "tokens_seen": 1264254976 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031160481444333, + "loss": 2.7545, + "theoretical_loss": 3.5703595754911523, + "tokens_seen": 1264320512 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031159478435305917, + "loss": 2.7343, + "theoretical_loss": 3.5703427188491936, + "tokens_seen": 1264386048 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031158475426278835, + "loss": 2.673, + "theoretical_loss": 3.570325863325557, + "tokens_seen": 1264451584 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031157472417251753, + "loss": 2.7361, + "theoretical_loss": 3.5703090089201095, + "tokens_seen": 1264517120 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031156469408224677, + "loss": 2.7468, + "theoretical_loss": 3.5702921556327203, + "tokens_seen": 1264582656 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031155466399197595, + "loss": 2.7378, + "theoretical_loss": 3.5702753034632564, + "tokens_seen": 1264648192 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031154463390170513, + "loss": 2.785, + "theoretical_loss": 3.5702584524115863, + "tokens_seen": 1264713728 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003115346038114343, + "loss": 2.7403, + "theoretical_loss": 3.5702416024775774, + "tokens_seen": 1264779264 + }, + { + "epoch": 0.38, + "objective/train/docs_used": 717941, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.8332362174987793, + "objective/train/theoretical_loss": 3.5702247536610976, + "objective/train/tokens_used": 1285304800, + "theoretical_loss": 3.5702247536610976, + "tokens_seen": 1264844800 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003115245737211635, + "loss": 2.7479, + "theoretical_loss": 3.5702247536610976, + "tokens_seen": 1264844800 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031151454363089273, + "loss": 2.6861, + "theoretical_loss": 3.570207905962016, + "tokens_seen": 1264910336 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031150451354062186, + "loss": 2.6501, + "theoretical_loss": 3.570191059380199, + "tokens_seen": 1264975872 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003114944834503511, + "loss": 2.5488, + "theoretical_loss": 3.570174213915516, + "tokens_seen": 1265041408 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003114844533600802, + "loss": 2.7075, + "theoretical_loss": 3.5701573695678346, + "tokens_seen": 1265106944 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031147442326980945, + "loss": 2.5057, + "theoretical_loss": 3.5701405263370223, + "tokens_seen": 1265172480 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031146439317953863, + "loss": 2.6261, + "theoretical_loss": 3.5701236842229482, + "tokens_seen": 1265238016 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003114543630892678, + "loss": 2.8022, + "theoretical_loss": 3.5701068432254797, + "tokens_seen": 1265303552 + }, + { + "epoch": 0.38, + "learning_rate": 0.000311444332998997, + "loss": 2.8013, + "theoretical_loss": 3.570090003344485, + "tokens_seen": 1265369088 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031143430290872623, + "loss": 2.5714, + "theoretical_loss": 3.5700731645798323, + "tokens_seen": 1265434624 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031142427281845536, + "loss": 2.7805, + "theoretical_loss": 3.57005632693139, + "tokens_seen": 1265500160 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003114142427281846, + "loss": 2.7987, + "theoretical_loss": 3.570039490399026, + "tokens_seen": 1265565696 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003114042126379137, + "loss": 2.5803, + "theoretical_loss": 3.5700226549826093, + "tokens_seen": 1265631232 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031139418254764296, + "loss": 2.7565, + "theoretical_loss": 3.570005820682007, + "tokens_seen": 1265696768 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031138415245737214, + "loss": 2.7881, + "theoretical_loss": 3.569988987497088, + "tokens_seen": 1265762304 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003113741223671013, + "loss": 2.6601, + "theoretical_loss": 3.569972155427721, + "tokens_seen": 1265827840 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003113640922768305, + "loss": 2.7649, + "theoretical_loss": 3.5699553244737734, + "tokens_seen": 1265893376 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003113540621865597, + "loss": 2.7011, + "theoretical_loss": 3.5699384946351147, + "tokens_seen": 1265958912 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031134403209628886, + "loss": 2.8021, + "theoretical_loss": 3.569921665911612, + "tokens_seen": 1266024448 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003113340020060181, + "loss": 2.7674, + "theoretical_loss": 3.569904838303134, + "tokens_seen": 1266089984 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003113239719157472, + "loss": 2.8126, + "theoretical_loss": 3.56988801180955, + "tokens_seen": 1266155520 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031131394182547646, + "loss": 2.7785, + "theoretical_loss": 3.5698711864307278, + "tokens_seen": 1266221056 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003113039117352056, + "loss": 2.5682, + "theoretical_loss": 3.5698543621665353, + "tokens_seen": 1266286592 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003112938816449348, + "loss": 2.7238, + "theoretical_loss": 3.5698375390168424, + "tokens_seen": 1266352128 + }, + { + "epoch": 0.38, + "learning_rate": 0.000311283851554664, + "loss": 2.605, + "theoretical_loss": 3.569820716981516, + "tokens_seen": 1266417664 + }, + { + "epoch": 0.38, + "objective/train/docs_used": 718481, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.777399778366089, + "objective/train/theoretical_loss": 3.5698038960604257, + "objective/train/tokens_used": 1286943200, + "theoretical_loss": 3.5698038960604257, + "tokens_seen": 1266483200 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003112738214643932, + "loss": 2.6243, + "theoretical_loss": 3.5698038960604257, + "tokens_seen": 1266483200 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031126379137412237, + "loss": 2.7629, + "theoretical_loss": 3.56978707625344, + "tokens_seen": 1266548736 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003112537612838516, + "loss": 2.5234, + "theoretical_loss": 3.5697702575604264, + "tokens_seen": 1266614272 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031124373119358073, + "loss": 2.7678, + "theoretical_loss": 3.5697534399812545, + "tokens_seen": 1266679808 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031123370110330996, + "loss": 2.7107, + "theoretical_loss": 3.5697366235157935, + "tokens_seen": 1266745344 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003112236710130391, + "loss": 2.6637, + "theoretical_loss": 3.56971980816391, + "tokens_seen": 1266810880 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003112136409227683, + "loss": 2.567, + "theoretical_loss": 3.5697029939254747, + "tokens_seen": 1266876416 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003112036108324975, + "loss": 2.7601, + "theoretical_loss": 3.5696861808003555, + "tokens_seen": 1266941952 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003111935807422267, + "loss": 2.5782, + "theoretical_loss": 3.5696693687884204, + "tokens_seen": 1267007488 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031118355065195587, + "loss": 2.7886, + "theoretical_loss": 3.5696525578895395, + "tokens_seen": 1267073024 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031117352056168505, + "loss": 2.8024, + "theoretical_loss": 3.5696357481035808, + "tokens_seen": 1267138560 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031116349047141423, + "loss": 2.9076, + "theoretical_loss": 3.5696189394304128, + "tokens_seen": 1267204096 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031115346038114347, + "loss": 2.902, + "theoretical_loss": 3.5696021318699045, + "tokens_seen": 1267269632 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003111434302908726, + "loss": 2.9842, + "theoretical_loss": 3.569585325421925, + "tokens_seen": 1267335168 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031113340020060183, + "loss": 2.6774, + "theoretical_loss": 3.5695685200863427, + "tokens_seen": 1267400704 + }, + { + "epoch": 0.38, + "learning_rate": 0.000311123370110331, + "loss": 2.7262, + "theoretical_loss": 3.5695517158630268, + "tokens_seen": 1267466240 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003111133400200602, + "loss": 2.7173, + "theoretical_loss": 3.569534912751846, + "tokens_seen": 1267531776 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031110330992978937, + "loss": 2.8264, + "theoretical_loss": 3.5695181107526697, + "tokens_seen": 1267597312 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031109327983951855, + "loss": 2.8763, + "theoretical_loss": 3.569501309865366, + "tokens_seen": 1267662848 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031108324974924773, + "loss": 2.6793, + "theoretical_loss": 3.5694845100898043, + "tokens_seen": 1267728384 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031107321965897697, + "loss": 2.9453, + "theoretical_loss": 3.5694677114258533, + "tokens_seen": 1267793920 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003110631895687061, + "loss": 2.6301, + "theoretical_loss": 3.569450913873382, + "tokens_seen": 1267859456 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031105315947843533, + "loss": 2.8281, + "theoretical_loss": 3.5694341174322606, + "tokens_seen": 1267924992 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031104312938816446, + "loss": 2.6992, + "theoretical_loss": 3.5694173221023564, + "tokens_seen": 1267990528 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003110330992978937, + "loss": 2.7683, + "theoretical_loss": 3.5694005278835395, + "tokens_seen": 1268056064 + }, + { + "epoch": 0.38, + "objective/train/docs_used": 719716, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7441720962524414, + "objective/train/theoretical_loss": 3.5693837347756783, + "objective/train/tokens_used": 1288581600, + "theoretical_loss": 3.5693837347756783, + "tokens_seen": 1268121600 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003110230692076229, + "loss": 2.7084, + "theoretical_loss": 3.5693837347756783, + "tokens_seen": 1268121600 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031101303911735206, + "loss": 3.0588, + "theoretical_loss": 3.5693669427786423, + "tokens_seen": 1268187136 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031100300902708124, + "loss": 2.4956, + "theoretical_loss": 3.5693501518923005, + "tokens_seen": 1268252672 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003109929789368104, + "loss": 2.7418, + "theoretical_loss": 3.569333362116523, + "tokens_seen": 1268318208 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003109829488465396, + "loss": 2.8078, + "theoretical_loss": 3.5693165734511774, + "tokens_seen": 1268383744 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031097291875626883, + "loss": 2.7301, + "theoretical_loss": 3.569299785896134, + "tokens_seen": 1268449280 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031096288866599796, + "loss": 2.8808, + "theoretical_loss": 3.5692829994512607, + "tokens_seen": 1268514816 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003109528585757272, + "loss": 2.692, + "theoretical_loss": 3.5692662141164284, + "tokens_seen": 1268580352 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003109428284854564, + "loss": 2.5746, + "theoretical_loss": 3.5692494298915056, + "tokens_seen": 1268645888 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031093279839518556, + "loss": 2.6475, + "theoretical_loss": 3.5692326467763618, + "tokens_seen": 1268711424 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031092276830491474, + "loss": 2.94, + "theoretical_loss": 3.5692158647708654, + "tokens_seen": 1268776960 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003109127382146439, + "loss": 2.8347, + "theoretical_loss": 3.569199083874887, + "tokens_seen": 1268842496 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003109027081243731, + "loss": 2.6905, + "theoretical_loss": 3.5691823040882946, + "tokens_seen": 1268908032 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031089267803410234, + "loss": 2.9192, + "theoretical_loss": 3.569165525410959, + "tokens_seen": 1268973568 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031088264794383147, + "loss": 2.6637, + "theoretical_loss": 3.569148747842749, + "tokens_seen": 1269039104 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003108726178535607, + "loss": 2.8468, + "theoretical_loss": 3.569131971383533, + "tokens_seen": 1269104640 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031086258776328983, + "loss": 2.7285, + "theoretical_loss": 3.5691151960331826, + "tokens_seen": 1269170176 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031085255767301906, + "loss": 2.7732, + "theoretical_loss": 3.5690984217915656, + "tokens_seen": 1269235712 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031084252758274824, + "loss": 2.6586, + "theoretical_loss": 3.5690816486585515, + "tokens_seen": 1269301248 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003108324974924774, + "loss": 2.6795, + "theoretical_loss": 3.56906487663401, + "tokens_seen": 1269366784 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003108224674022066, + "loss": 2.9773, + "theoretical_loss": 3.5690481057178114, + "tokens_seen": 1269432320 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003108124373119358, + "loss": 2.818, + "theoretical_loss": 3.5690313359098242, + "tokens_seen": 1269497856 + }, + { + "epoch": 0.38, + "learning_rate": 0.000310802407221665, + "loss": 2.3667, + "theoretical_loss": 3.5690145672099187, + "tokens_seen": 1269563392 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003107923771313942, + "loss": 2.6479, + "theoretical_loss": 3.5689977996179643, + "tokens_seen": 1269628928 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003107823470411234, + "loss": 2.8636, + "theoretical_loss": 3.56898103313383, + "tokens_seen": 1269694464 + }, + { + "epoch": 0.38, + "objective/train/docs_used": 720440, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.0873234272003174, + "objective/train/theoretical_loss": 3.5689642677573863, + "objective/train/tokens_used": 1290220000, + "theoretical_loss": 3.5689642677573863, + "tokens_seen": 1269760000 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031077231695085257, + "loss": 2.8214, + "theoretical_loss": 3.5689642677573863, + "tokens_seen": 1269760000 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003107622868605818, + "loss": 2.6872, + "theoretical_loss": 3.568947503488503, + "tokens_seen": 1269825536 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031075225677031093, + "loss": 2.7963, + "theoretical_loss": 3.5689307403270485, + "tokens_seen": 1269891072 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031074222668004016, + "loss": 2.6595, + "theoretical_loss": 3.568913978272893, + "tokens_seen": 1269956608 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003107321965897693, + "loss": 2.7834, + "theoretical_loss": 3.5688972173259073, + "tokens_seen": 1270022144 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003107221664994985, + "loss": 2.7302, + "theoretical_loss": 3.5688804574859603, + "tokens_seen": 1270087680 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003107121364092277, + "loss": 2.4257, + "theoretical_loss": 3.5688636987529216, + "tokens_seen": 1270153216 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003107021063189569, + "loss": 2.8125, + "theoretical_loss": 3.568846941126661, + "tokens_seen": 1270218752 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031069207622868607, + "loss": 3.0101, + "theoretical_loss": 3.5688301846070485, + "tokens_seen": 1270284288 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031068204613841525, + "loss": 2.7311, + "theoretical_loss": 3.568813429193954, + "tokens_seen": 1270349824 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031067201604814443, + "loss": 2.5605, + "theoretical_loss": 3.5687966748872473, + "tokens_seen": 1270415360 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031066198595787367, + "loss": 2.8293, + "theoretical_loss": 3.568779921686798, + "tokens_seen": 1270480896 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003106519558676028, + "loss": 2.6844, + "theoretical_loss": 3.5687631695924766, + "tokens_seen": 1270546432 + }, + { + "epoch": 0.39, + "learning_rate": 0.00031064192577733203, + "loss": 2.6834, + "theoretical_loss": 3.568746418604153, + "tokens_seen": 1270611968 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003106318956870612, + "loss": 2.7331, + "theoretical_loss": 3.5687296687216965, + "tokens_seen": 1270677504 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003106218655967904, + "loss": 2.5208, + "theoretical_loss": 3.568712919944977, + "tokens_seen": 1270743040 + }, + { + "epoch": 0.39, + "learning_rate": 0.00031061183550651957, + "loss": 2.7247, + "theoretical_loss": 3.5686961722738655, + "tokens_seen": 1270808576 + }, + { + "epoch": 0.39, + "learning_rate": 0.00031060180541624875, + "loss": 2.9474, + "theoretical_loss": 3.568679425708231, + "tokens_seen": 1270874112 + }, + { + "epoch": 0.39, + "learning_rate": 0.00031059177532597793, + "loss": 2.6887, + "theoretical_loss": 3.568662680247944, + "tokens_seen": 1270939648 + }, + { + "epoch": 0.39, + "learning_rate": 0.00031058174523570717, + "loss": 2.814, + "theoretical_loss": 3.5686459358928744, + "tokens_seen": 1271005184 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003105717151454363, + "loss": 2.7156, + "theoretical_loss": 3.568629192642892, + "tokens_seen": 1271070720 + }, + { + "epoch": 0.39, + "learning_rate": 0.00031056168505516553, + "loss": 2.6847, + "theoretical_loss": 3.5686124504978673, + "tokens_seen": 1271136256 + }, + { + "epoch": 0.39, + "learning_rate": 0.00031055165496489466, + "loss": 2.8616, + "theoretical_loss": 3.5685957094576706, + "tokens_seen": 1271201792 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003105416248746239, + "loss": 2.8411, + "theoretical_loss": 3.5685789695221715, + "tokens_seen": 1271267328 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003105315947843531, + "loss": 2.7664, + "theoretical_loss": 3.5685622306912403, + "tokens_seen": 1271332864 + }, + { + "epoch": 0.39, + "objective/train/docs_used": 721295, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.334505558013916, + "objective/train/theoretical_loss": 3.5685454929647475, + "objective/train/tokens_used": 1291858400, + "theoretical_loss": 3.5685454929647475, + "tokens_seen": 1271398400 + }, + { + "epoch": 0.39, + "learning_rate": 0.00031052156469408226, + "loss": 2.6288, + "theoretical_loss": 3.5685454929647475, + "tokens_seen": 1271398400 + }, + { + "epoch": 0.39, + "learning_rate": 0.00031051153460381144, + "loss": 3.0007, + "theoretical_loss": 3.5685287563425634, + "tokens_seen": 1271463936 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003105015045135406, + "loss": 2.9023, + "theoretical_loss": 3.5685120208245578, + "tokens_seen": 1271529472 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003104914744232698, + "loss": 2.5194, + "theoretical_loss": 3.568495286410601, + "tokens_seen": 1271595008 + }, + { + "epoch": 0.39, + "learning_rate": 0.00031048144433299904, + "loss": 2.8795, + "theoretical_loss": 3.5684785531005634, + "tokens_seen": 1271660544 + }, + { + "epoch": 0.39, + "learning_rate": 0.00031047141424272816, + "loss": 2.7327, + "theoretical_loss": 3.5684618208943153, + "tokens_seen": 1271726080 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003104613841524574, + "loss": 2.8101, + "theoretical_loss": 3.5684450897917266, + "tokens_seen": 1271791616 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003104513540621866, + "loss": 2.5526, + "theoretical_loss": 3.5684283597926685, + "tokens_seen": 1271857152 + }, + { + "epoch": 0.39, + "learning_rate": 0.00031044132397191576, + "loss": 2.8037, + "theoretical_loss": 3.5684116308970104, + "tokens_seen": 1271922688 + }, + { + "epoch": 0.39, + "learning_rate": 0.00031043129388164494, + "loss": 2.5559, + "theoretical_loss": 3.5683949031046236, + "tokens_seen": 1271988224 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003104212637913741, + "loss": 2.6487, + "theoretical_loss": 3.5683781764153775, + "tokens_seen": 1272053760 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003104112337011033, + "loss": 3.0733, + "theoretical_loss": 3.5683614508291432, + "tokens_seen": 1272119296 + }, + { + "epoch": 0.39, + "learning_rate": 0.00031040120361083254, + "loss": 2.5462, + "theoretical_loss": 3.5683447263457913, + "tokens_seen": 1272184832 + }, + { + "epoch": 0.39, + "learning_rate": 0.00031039117352056167, + "loss": 2.564, + "theoretical_loss": 3.568328002965192, + "tokens_seen": 1272250368 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003103811434302909, + "loss": 2.8355, + "theoretical_loss": 3.5683112806872157, + "tokens_seen": 1272315904 + }, + { + "epoch": 0.39, + "learning_rate": 0.00031037111334002003, + "loss": 2.6935, + "theoretical_loss": 3.568294559511733, + "tokens_seen": 1272381440 + }, + { + "epoch": 0.39, + "learning_rate": 0.00031036108324974926, + "loss": 2.9034, + "theoretical_loss": 3.568277839438614, + "tokens_seen": 1272446976 + }, + { + "epoch": 0.39, + "learning_rate": 0.00031035105315947844, + "loss": 2.7654, + "theoretical_loss": 3.5682611204677297, + "tokens_seen": 1272512512 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003103410230692076, + "loss": 2.4637, + "theoretical_loss": 3.568244402598951, + "tokens_seen": 1272578048 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003103309929789368, + "loss": 2.7611, + "theoretical_loss": 3.5682276858321478, + "tokens_seen": 1272643584 + }, + { + "epoch": 0.39, + "learning_rate": 0.000310320962888666, + "loss": 2.6624, + "theoretical_loss": 3.5682109701671916, + "tokens_seen": 1272709120 + }, + { + "epoch": 0.39, + "learning_rate": 0.00031031093279839517, + "loss": 2.8263, + "theoretical_loss": 3.568194255603952, + "tokens_seen": 1272774656 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003103009027081244, + "loss": 2.5522, + "theoretical_loss": 3.5681775421423003, + "tokens_seen": 1272840192 + }, + { + "epoch": 0.39, + "learning_rate": 0.00031029087261785353, + "loss": 2.8726, + "theoretical_loss": 3.5681608297821072, + "tokens_seen": 1272905728 + }, + { + "epoch": 0.39, + "learning_rate": 0.00031028084252758277, + "loss": 2.7274, + "theoretical_loss": 3.568144118523243, + "tokens_seen": 1272971264 + }, + { + "epoch": 0.39, + "objective/train/docs_used": 722001, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.8035433292388916, + "objective/train/theoretical_loss": 3.5681274083655787, + "objective/train/tokens_used": 1293496800, + "theoretical_loss": 3.5681274083655787, + "tokens_seen": 1273036800 + }, + { + "epoch": 0.39, + "learning_rate": 0.00031027081243731195, + "loss": 2.7223, + "theoretical_loss": 3.5681274083655787, + "tokens_seen": 1273036800 + }, + { + "epoch": 0.39, + "learning_rate": 0.00031026078234704113, + "loss": 2.7063, + "theoretical_loss": 3.5681106993089853, + "tokens_seen": 1273102336 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003102507522567703, + "loss": 2.9061, + "theoretical_loss": 3.568093991353333, + "tokens_seen": 1273167872 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003102407221664995, + "loss": 2.587, + "theoretical_loss": 3.568077284498493, + "tokens_seen": 1273233408 + }, + { + "epoch": 0.39, + "learning_rate": 0.00031023069207622867, + "loss": 2.6797, + "theoretical_loss": 3.568060578744336, + "tokens_seen": 1273298944 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003102206619859579, + "loss": 2.6825, + "theoretical_loss": 3.5680438740907334, + "tokens_seen": 1273364480 + }, + { + "epoch": 0.39, + "learning_rate": 0.00031021063189568703, + "loss": 2.6534, + "theoretical_loss": 3.5680271705375546, + "tokens_seen": 1273430016 + }, + { + "epoch": 0.39, + "learning_rate": 0.00031020060180541627, + "loss": 2.9128, + "theoretical_loss": 3.5680104680846725, + "tokens_seen": 1273495552 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003101905717151454, + "loss": 2.9194, + "theoretical_loss": 3.5679937667319557, + "tokens_seen": 1273561088 + }, + { + "epoch": 0.39, + "learning_rate": 0.00031018054162487463, + "loss": 2.8142, + "theoretical_loss": 3.567977066479277, + "tokens_seen": 1273626624 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003101705115346038, + "loss": 2.6751, + "theoretical_loss": 3.567960367326507, + "tokens_seen": 1273692160 + }, + { + "epoch": 0.39, + "learning_rate": 0.000310160481444333, + "loss": 2.8027, + "theoretical_loss": 3.5679436692735154, + "tokens_seen": 1273757696 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003101504513540622, + "loss": 2.9134, + "theoretical_loss": 3.567926972320175, + "tokens_seen": 1273823232 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003101404212637914, + "loss": 2.7984, + "theoretical_loss": 3.5679102764663555, + "tokens_seen": 1273888768 + }, + { + "epoch": 0.39, + "learning_rate": 0.00031013039117352054, + "loss": 2.5213, + "theoretical_loss": 3.567893581711929, + "tokens_seen": 1273954304 + }, + { + "epoch": 0.39, + "learning_rate": 0.00031012036108324977, + "loss": 2.7889, + "theoretical_loss": 3.567876888056765, + "tokens_seen": 1274019840 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003101103309929789, + "loss": 2.6634, + "theoretical_loss": 3.567860195500736, + "tokens_seen": 1274085376 + }, + { + "epoch": 0.39, + "learning_rate": 0.00031010030090270813, + "loss": 2.7211, + "theoretical_loss": 3.5678435040437124, + "tokens_seen": 1274150912 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003100902708124373, + "loss": 2.8899, + "theoretical_loss": 3.567826813685566, + "tokens_seen": 1274216448 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003100802407221665, + "loss": 2.8838, + "theoretical_loss": 3.5678101244261677, + "tokens_seen": 1274281984 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003100702106318957, + "loss": 2.812, + "theoretical_loss": 3.5677934362653874, + "tokens_seen": 1274347520 + }, + { + "epoch": 0.39, + "learning_rate": 0.00031006018054162486, + "loss": 2.739, + "theoretical_loss": 3.567776749203098, + "tokens_seen": 1274413056 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003100501504513541, + "loss": 2.6876, + "theoretical_loss": 3.56776006323917, + "tokens_seen": 1274478592 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003100401203610833, + "loss": 2.8152, + "theoretical_loss": 3.567743378373475, + "tokens_seen": 1274544128 + }, + { + "epoch": 0.39, + "learning_rate": 0.00031003009027081246, + "loss": 2.7654, + "theoretical_loss": 3.5677266946058834, + "tokens_seen": 1274609664 + }, + { + "epoch": 0.39, + "objective/train/docs_used": 723183, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.1512532234191895, + "objective/train/theoretical_loss": 3.5677100119362675, + "objective/train/tokens_used": 1295135200, + "theoretical_loss": 3.5677100119362675, + "tokens_seen": 1274675200 + }, + { + "epoch": 0.39, + "learning_rate": 0.00031002006018054164, + "loss": 2.6428, + "theoretical_loss": 3.5677100119362675, + "tokens_seen": 1274675200 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003100100300902708, + "loss": 2.7878, + "theoretical_loss": 3.5676933303644978, + "tokens_seen": 1274740736 + }, + { + "epoch": 0.39, + "learning_rate": 0.00031, + "loss": 2.7997, + "theoretical_loss": 3.567676649890446, + "tokens_seen": 1274806272 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030998996990972924, + "loss": 2.9171, + "theoretical_loss": 3.5676599705139833, + "tokens_seen": 1274871808 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030997993981945836, + "loss": 2.8792, + "theoretical_loss": 3.5676432922349814, + "tokens_seen": 1274937344 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003099699097291876, + "loss": 2.891, + "theoretical_loss": 3.567626615053311, + "tokens_seen": 1275002880 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003099598796389168, + "loss": 2.7614, + "theoretical_loss": 3.567609938968844, + "tokens_seen": 1275068416 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030994984954864596, + "loss": 2.7732, + "theoretical_loss": 3.567593263981452, + "tokens_seen": 1275133952 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030993981945837514, + "loss": 2.8733, + "theoretical_loss": 3.5675765900910057, + "tokens_seen": 1275199488 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003099297893681043, + "loss": 2.8266, + "theoretical_loss": 3.5675599172973778, + "tokens_seen": 1275265024 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003099197592778335, + "loss": 2.7804, + "theoretical_loss": 3.5675432456004383, + "tokens_seen": 1275330560 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030990972918756274, + "loss": 2.7475, + "theoretical_loss": 3.5675265750000595, + "tokens_seen": 1275396096 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030989969909729187, + "loss": 2.7238, + "theoretical_loss": 3.567509905496113, + "tokens_seen": 1275461632 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003098896690070211, + "loss": 2.5513, + "theoretical_loss": 3.5674932370884704, + "tokens_seen": 1275527168 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030987963891675023, + "loss": 2.5637, + "theoretical_loss": 3.5674765697770034, + "tokens_seen": 1275592704 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030986960882647946, + "loss": 2.7458, + "theoretical_loss": 3.5674599035615824, + "tokens_seen": 1275658240 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030985957873620864, + "loss": 2.6523, + "theoretical_loss": 3.5674432384420807, + "tokens_seen": 1275723776 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003098495486459378, + "loss": 2.8965, + "theoretical_loss": 3.567426574418369, + "tokens_seen": 1275789312 + }, + { + "epoch": 0.39, + "learning_rate": 0.000309839518555667, + "loss": 2.8219, + "theoretical_loss": 3.5674099114903193, + "tokens_seen": 1275854848 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003098294884653962, + "loss": 2.9198, + "theoretical_loss": 3.567393249657803, + "tokens_seen": 1275920384 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030981945837512537, + "loss": 2.8861, + "theoretical_loss": 3.567376588920692, + "tokens_seen": 1275985920 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003098094282848546, + "loss": 2.7833, + "theoretical_loss": 3.567359929278858, + "tokens_seen": 1276051456 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030979939819458373, + "loss": 2.6114, + "theoretical_loss": 3.567343270732172, + "tokens_seen": 1276116992 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030978936810431297, + "loss": 2.863, + "theoretical_loss": 3.5673266132805073, + "tokens_seen": 1276182528 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030977933801404215, + "loss": 2.756, + "theoretical_loss": 3.5673099569237348, + "tokens_seen": 1276248064 + }, + { + "epoch": 0.39, + "objective/train/docs_used": 723876, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.4731040000915527, + "objective/train/theoretical_loss": 3.567293301661726, + "objective/train/tokens_used": 1296773600, + "theoretical_loss": 3.567293301661726, + "tokens_seen": 1276313600 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030976930792377133, + "loss": 2.6582, + "theoretical_loss": 3.567293301661726, + "tokens_seen": 1276313600 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003097592778335005, + "loss": 2.9534, + "theoretical_loss": 3.567276647494354, + "tokens_seen": 1276379136 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003097492477432297, + "loss": 2.6146, + "theoretical_loss": 3.567259994421489, + "tokens_seen": 1276444672 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030973921765295887, + "loss": 3.0731, + "theoretical_loss": 3.5672433424430037, + "tokens_seen": 1276510208 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003097291875626881, + "loss": 2.5678, + "theoretical_loss": 3.56722669155877, + "tokens_seen": 1276575744 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030971915747241723, + "loss": 2.4663, + "theoretical_loss": 3.56721004176866, + "tokens_seen": 1276641280 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030970912738214647, + "loss": 2.9143, + "theoretical_loss": 3.5671933930725457, + "tokens_seen": 1276706816 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003096990972918756, + "loss": 2.7186, + "theoretical_loss": 3.5671767454702983, + "tokens_seen": 1276772352 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030968906720160483, + "loss": 2.7449, + "theoretical_loss": 3.56716009896179, + "tokens_seen": 1276837888 + }, + { + "epoch": 0.39, + "learning_rate": 0.000309679037111334, + "loss": 2.584, + "theoretical_loss": 3.567143453546894, + "tokens_seen": 1276903424 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003096690070210632, + "loss": 2.6674, + "theoretical_loss": 3.567126809225481, + "tokens_seen": 1276968960 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003096589769307924, + "loss": 3.0791, + "theoretical_loss": 3.5671101659974234, + "tokens_seen": 1277034496 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003096489468405216, + "loss": 2.688, + "theoretical_loss": 3.567093523862593, + "tokens_seen": 1277100032 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030963891675025074, + "loss": 2.6521, + "theoretical_loss": 3.567076882820863, + "tokens_seen": 1277165568 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030962888665997997, + "loss": 2.8105, + "theoretical_loss": 3.5670602428721043, + "tokens_seen": 1277231104 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003096188565697091, + "loss": 2.6421, + "theoretical_loss": 3.56704360401619, + "tokens_seen": 1277296640 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030960882647943833, + "loss": 2.8575, + "theoretical_loss": 3.5670269662529916, + "tokens_seen": 1277362176 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003095987963891675, + "loss": 2.6462, + "theoretical_loss": 3.5670103295823816, + "tokens_seen": 1277427712 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003095887662988967, + "loss": 2.5327, + "theoretical_loss": 3.5669936940042315, + "tokens_seen": 1277493248 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003095787362086259, + "loss": 3.0035, + "theoretical_loss": 3.5669770595184143, + "tokens_seen": 1277558784 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030956870611835506, + "loss": 3.0216, + "theoretical_loss": 3.566960426124802, + "tokens_seen": 1277624320 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030955867602808424, + "loss": 2.7338, + "theoretical_loss": 3.566943793823267, + "tokens_seen": 1277689856 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003095486459378135, + "loss": 2.6627, + "theoretical_loss": 3.5669271626136814, + "tokens_seen": 1277755392 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003095386158475426, + "loss": 2.8628, + "theoretical_loss": 3.5669105324959176, + "tokens_seen": 1277820928 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030952858575727184, + "loss": 2.776, + "theoretical_loss": 3.566893903469848, + "tokens_seen": 1277886464 + }, + { + "epoch": 0.39, + "objective/train/docs_used": 725220, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6839542388916016, + "objective/train/theoretical_loss": 3.566877275535345, + "objective/train/tokens_used": 1298412000, + "theoretical_loss": 3.566877275535345, + "tokens_seen": 1277952000 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030951855566700096, + "loss": 2.5356, + "theoretical_loss": 3.566877275535345, + "tokens_seen": 1277952000 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003095085255767302, + "loss": 2.7823, + "theoretical_loss": 3.5668606486922805, + "tokens_seen": 1278017536 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003094984954864594, + "loss": 2.7454, + "theoretical_loss": 3.566844022940528, + "tokens_seen": 1278083072 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030948846539618856, + "loss": 2.9112, + "theoretical_loss": 3.5668273982799583, + "tokens_seen": 1278148608 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030947843530591774, + "loss": 2.8234, + "theoretical_loss": 3.5668107747104454, + "tokens_seen": 1278214144 + }, + { + "epoch": 0.39, + "learning_rate": 0.000309468405215647, + "loss": 2.5028, + "theoretical_loss": 3.5667941522318607, + "tokens_seen": 1278279680 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003094583751253761, + "loss": 2.4679, + "theoretical_loss": 3.566777530844077, + "tokens_seen": 1278345216 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030944834503510534, + "loss": 2.6337, + "theoretical_loss": 3.5667609105469666, + "tokens_seen": 1278410752 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030943831494483447, + "loss": 2.6138, + "theoretical_loss": 3.5667442913404024, + "tokens_seen": 1278476288 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003094282848545637, + "loss": 2.6465, + "theoretical_loss": 3.5667276732242574, + "tokens_seen": 1278541824 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003094182547642929, + "loss": 2.7473, + "theoretical_loss": 3.566711056198403, + "tokens_seen": 1278607360 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030940822467402207, + "loss": 2.6446, + "theoretical_loss": 3.5666944402627125, + "tokens_seen": 1278672896 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030939819458375125, + "loss": 2.7966, + "theoretical_loss": 3.5666778254170586, + "tokens_seen": 1278738432 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030938816449348043, + "loss": 2.5215, + "theoretical_loss": 3.5666612116613137, + "tokens_seen": 1278803968 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003093781344032096, + "loss": 2.7873, + "theoretical_loss": 3.5666445989953504, + "tokens_seen": 1278869504 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030936810431293884, + "loss": 2.6912, + "theoretical_loss": 3.566627987419041, + "tokens_seen": 1278935040 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030935807422266797, + "loss": 2.604, + "theoretical_loss": 3.5666113769322596, + "tokens_seen": 1279000576 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003093480441323972, + "loss": 2.7246, + "theoretical_loss": 3.5665947675348777, + "tokens_seen": 1279066112 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030933801404212633, + "loss": 2.8881, + "theoretical_loss": 3.5665781592267676, + "tokens_seen": 1279131648 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030932798395185557, + "loss": 2.5632, + "theoretical_loss": 3.5665615520078036, + "tokens_seen": 1279197184 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030931795386158475, + "loss": 2.6241, + "theoretical_loss": 3.5665449458778573, + "tokens_seen": 1279262720 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030930792377131393, + "loss": 2.7755, + "theoretical_loss": 3.566528340836802, + "tokens_seen": 1279328256 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030929789368104317, + "loss": 2.7532, + "theoretical_loss": 3.56651173688451, + "tokens_seen": 1279393792 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030928786359077235, + "loss": 2.88, + "theoretical_loss": 3.5664951340208546, + "tokens_seen": 1279459328 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030927783350050153, + "loss": 2.7093, + "theoretical_loss": 3.566478532245709, + "tokens_seen": 1279524864 + }, + { + "epoch": 0.39, + "objective/train/docs_used": 725568, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.038022518157959, + "objective/train/theoretical_loss": 3.566461931558945, + "objective/train/tokens_used": 1300050400, + "theoretical_loss": 3.566461931558945, + "tokens_seen": 1279590400 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003092678034102307, + "loss": 2.8837, + "theoretical_loss": 3.566461931558945, + "tokens_seen": 1279590400 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003092577733199599, + "loss": 2.7899, + "theoretical_loss": 3.566445331960437, + "tokens_seen": 1279655936 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030924774322968907, + "loss": 2.6906, + "theoretical_loss": 3.5664287334500564, + "tokens_seen": 1279721472 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003092377131394183, + "loss": 2.7888, + "theoretical_loss": 3.5664121360276777, + "tokens_seen": 1279787008 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030922768304914743, + "loss": 2.9558, + "theoretical_loss": 3.566395539693172, + "tokens_seen": 1279852544 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030921765295887667, + "loss": 2.7238, + "theoretical_loss": 3.566378944446414, + "tokens_seen": 1279918080 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003092076228686058, + "loss": 2.4656, + "theoretical_loss": 3.5663623502872763, + "tokens_seen": 1279983616 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030919759277833503, + "loss": 2.6224, + "theoretical_loss": 3.5663457572156307, + "tokens_seen": 1280049152 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003091875626880642, + "loss": 2.7599, + "theoretical_loss": 3.5663291652313522, + "tokens_seen": 1280114688 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003091775325977934, + "loss": 2.9582, + "theoretical_loss": 3.5663125743343125, + "tokens_seen": 1280180224 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003091675025075226, + "loss": 2.9001, + "theoretical_loss": 3.566295984524386, + "tokens_seen": 1280245760 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003091574724172518, + "loss": 2.7023, + "theoretical_loss": 3.566279395801444, + "tokens_seen": 1280311296 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030914744232698094, + "loss": 3.0427, + "theoretical_loss": 3.5662628081653605, + "tokens_seen": 1280376832 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030913741223671017, + "loss": 2.6504, + "theoretical_loss": 3.566246221616009, + "tokens_seen": 1280442368 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003091273821464393, + "loss": 2.5941, + "theoretical_loss": 3.566229636153263, + "tokens_seen": 1280507904 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030911735205616853, + "loss": 2.7546, + "theoretical_loss": 3.5662130517769945, + "tokens_seen": 1280573440 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003091073219658977, + "loss": 2.6621, + "theoretical_loss": 3.5661964684870777, + "tokens_seen": 1280638976 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003090972918756269, + "loss": 2.6886, + "theoretical_loss": 3.5661798862833853, + "tokens_seen": 1280704512 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003090872617853561, + "loss": 2.7497, + "theoretical_loss": 3.5661633051657917, + "tokens_seen": 1280770048 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030907723169508526, + "loss": 2.6808, + "theoretical_loss": 3.5661467251341685, + "tokens_seen": 1280835584 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030906720160481444, + "loss": 2.9142, + "theoretical_loss": 3.5661301461883896, + "tokens_seen": 1280901120 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003090571715145437, + "loss": 2.603, + "theoretical_loss": 3.566113568328329, + "tokens_seen": 1280966656 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003090471414242728, + "loss": 2.7435, + "theoretical_loss": 3.5660969915538594, + "tokens_seen": 1281032192 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030903711133400204, + "loss": 2.859, + "theoretical_loss": 3.566080415864855, + "tokens_seen": 1281097728 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030902708124373116, + "loss": 2.7963, + "theoretical_loss": 3.566063841261188, + "tokens_seen": 1281163264 + }, + { + "epoch": 0.39, + "objective/train/docs_used": 726917, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.13041615486145, + "objective/train/theoretical_loss": 3.566047267742733, + "objective/train/tokens_used": 1301688800, + "theoretical_loss": 3.566047267742733, + "tokens_seen": 1281228800 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003090170511534604, + "loss": 3.0033, + "theoretical_loss": 3.566047267742733, + "tokens_seen": 1281228800 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003090070210631896, + "loss": 2.6401, + "theoretical_loss": 3.5660306953093626, + "tokens_seen": 1281294336 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030899699097291876, + "loss": 2.6882, + "theoretical_loss": 3.56601412396095, + "tokens_seen": 1281359872 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030898696088264794, + "loss": 2.6836, + "theoretical_loss": 3.56599755369737, + "tokens_seen": 1281425408 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003089769307923772, + "loss": 2.902, + "theoretical_loss": 3.565980984518495, + "tokens_seen": 1281490944 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003089669007021063, + "loss": 2.9099, + "theoretical_loss": 3.5659644164241984, + "tokens_seen": 1281556480 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030895687061183554, + "loss": 2.8642, + "theoretical_loss": 3.565947849414355, + "tokens_seen": 1281622016 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030894684052156467, + "loss": 2.5986, + "theoretical_loss": 3.565931283488837, + "tokens_seen": 1281687552 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003089368104312939, + "loss": 2.6451, + "theoretical_loss": 3.5659147186475186, + "tokens_seen": 1281753088 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003089267803410231, + "loss": 2.897, + "theoretical_loss": 3.5658981548902737, + "tokens_seen": 1281818624 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030891675025075227, + "loss": 2.7386, + "theoretical_loss": 3.565881592216975, + "tokens_seen": 1281884160 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030890672016048145, + "loss": 2.6905, + "theoretical_loss": 3.5658650306274975, + "tokens_seen": 1281949696 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030889669007021063, + "loss": 2.6541, + "theoretical_loss": 3.565848470121714, + "tokens_seen": 1282015232 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003088866599799398, + "loss": 2.5688, + "theoretical_loss": 3.5658319106994982, + "tokens_seen": 1282080768 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030887662988966904, + "loss": 2.8599, + "theoretical_loss": 3.5658153523607234, + "tokens_seen": 1282146304 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030886659979939817, + "loss": 2.8099, + "theoretical_loss": 3.5657987951052648, + "tokens_seen": 1282211840 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003088565697091274, + "loss": 2.722, + "theoretical_loss": 3.5657822389329947, + "tokens_seen": 1282277376 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030884653961885653, + "loss": 2.9849, + "theoretical_loss": 3.5657656838437877, + "tokens_seen": 1282342912 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030883650952858577, + "loss": 2.7494, + "theoretical_loss": 3.565749129837517, + "tokens_seen": 1282408448 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030882647943831495, + "loss": 2.7136, + "theoretical_loss": 3.565732576914057, + "tokens_seen": 1282473984 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030881644934804413, + "loss": 2.9537, + "theoretical_loss": 3.5657160250732813, + "tokens_seen": 1282539520 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003088064192577733, + "loss": 2.9209, + "theoretical_loss": 3.5656994743150636, + "tokens_seen": 1282605056 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030879638916750255, + "loss": 2.67, + "theoretical_loss": 3.565682924639278, + "tokens_seen": 1282670592 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003087863590772317, + "loss": 2.8581, + "theoretical_loss": 3.5656663760457987, + "tokens_seen": 1282736128 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003087763289869609, + "loss": 2.783, + "theoretical_loss": 3.565649828534499, + "tokens_seen": 1282801664 + }, + { + "epoch": 0.39, + "objective/train/docs_used": 727506, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.008744239807129, + "objective/train/theoretical_loss": 3.565633282105253, + "objective/train/tokens_used": 1303327200, + "theoretical_loss": 3.565633282105253, + "tokens_seen": 1282867200 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030876629889669004, + "loss": 2.937, + "theoretical_loss": 3.565633282105253, + "tokens_seen": 1282867200 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030875626880641927, + "loss": 2.8008, + "theoretical_loss": 3.565616736757935, + "tokens_seen": 1282932736 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030874623871614845, + "loss": 2.8456, + "theoretical_loss": 3.565600192492419, + "tokens_seen": 1282998272 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030873620862587763, + "loss": 2.906, + "theoretical_loss": 3.5655836493085786, + "tokens_seen": 1283063808 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003087261785356068, + "loss": 2.7121, + "theoretical_loss": 3.565567107206288, + "tokens_seen": 1283129344 + }, + { + "epoch": 0.39, + "learning_rate": 0.000308716148445336, + "loss": 2.7818, + "theoretical_loss": 3.5655505661854217, + "tokens_seen": 1283194880 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003087061183550652, + "loss": 2.9383, + "theoretical_loss": 3.565534026245853, + "tokens_seen": 1283260416 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003086960882647944, + "loss": 2.5752, + "theoretical_loss": 3.5655174873874564, + "tokens_seen": 1283325952 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030868605817452354, + "loss": 2.6737, + "theoretical_loss": 3.565500949610106, + "tokens_seen": 1283391488 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003086760280842528, + "loss": 2.8999, + "theoretical_loss": 3.5654844129136762, + "tokens_seen": 1283457024 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003086659979939819, + "loss": 3.0214, + "theoretical_loss": 3.565467877298041, + "tokens_seen": 1283522560 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030865596790371114, + "loss": 2.9509, + "theoretical_loss": 3.5654513427630743, + "tokens_seen": 1283588096 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003086459378134403, + "loss": 2.6614, + "theoretical_loss": 3.56543480930865, + "tokens_seen": 1283653632 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003086359077231695, + "loss": 2.8541, + "theoretical_loss": 3.5654182769346434, + "tokens_seen": 1283719168 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003086258776328987, + "loss": 2.6348, + "theoretical_loss": 3.565401745640928, + "tokens_seen": 1283784704 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003086158475426279, + "loss": 2.8541, + "theoretical_loss": 3.565385215427378, + "tokens_seen": 1283850240 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030860581745235704, + "loss": 2.8181, + "theoretical_loss": 3.5653686862938687, + "tokens_seen": 1283915776 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003085957873620863, + "loss": 2.8355, + "theoretical_loss": 3.565352158240273, + "tokens_seen": 1283981312 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003085857572718154, + "loss": 2.8306, + "theoretical_loss": 3.565335631266466, + "tokens_seen": 1284046848 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030857572718154464, + "loss": 2.5394, + "theoretical_loss": 3.565319105372322, + "tokens_seen": 1284112384 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003085656970912738, + "loss": 2.3912, + "theoretical_loss": 3.565302580557715, + "tokens_seen": 1284177920 + }, + { + "epoch": 0.39, + "learning_rate": 0.000308555667001003, + "loss": 2.6112, + "theoretical_loss": 3.5652860568225204, + "tokens_seen": 1284243456 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030854563691073224, + "loss": 2.6667, + "theoretical_loss": 3.565269534166611, + "tokens_seen": 1284308992 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030853560682046137, + "loss": 2.7875, + "theoretical_loss": 3.5652530125898627, + "tokens_seen": 1284374528 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003085255767301906, + "loss": 2.7755, + "theoretical_loss": 3.565236492092149, + "tokens_seen": 1284440064 + }, + { + "epoch": 0.39, + "objective/train/docs_used": 728764, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.9708006381988525, + "objective/train/theoretical_loss": 3.5652199726733453, + "objective/train/tokens_used": 1304965600, + "theoretical_loss": 3.5652199726733453, + "tokens_seen": 1284505600 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003085155466399198, + "loss": 2.7818, + "theoretical_loss": 3.5652199726733453, + "tokens_seen": 1284505600 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030850551654964896, + "loss": 2.5844, + "theoretical_loss": 3.565203454333325, + "tokens_seen": 1284571136 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030849548645937814, + "loss": 2.5834, + "theoretical_loss": 3.5651869370719633, + "tokens_seen": 1284636672 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003084854563691074, + "loss": 2.7849, + "theoretical_loss": 3.565170420889135, + "tokens_seen": 1284702208 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003084754262788365, + "loss": 2.8815, + "theoretical_loss": 3.565153905784714, + "tokens_seen": 1284767744 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030846539618856574, + "loss": 2.7876, + "theoretical_loss": 3.565137391758575, + "tokens_seen": 1284833280 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030845536609829487, + "loss": 2.6952, + "theoretical_loss": 3.565120878810593, + "tokens_seen": 1284898816 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003084453360080241, + "loss": 2.5779, + "theoretical_loss": 3.565104366940642, + "tokens_seen": 1284964352 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003084353059177533, + "loss": 2.638, + "theoretical_loss": 3.565087856148598, + "tokens_seen": 1285029888 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030842527582748247, + "loss": 2.857, + "theoretical_loss": 3.5650713464343338, + "tokens_seen": 1285095424 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030841524573721165, + "loss": 2.7629, + "theoretical_loss": 3.5650548377977254, + "tokens_seen": 1285160960 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030840521564694083, + "loss": 2.7156, + "theoretical_loss": 3.5650383302386466, + "tokens_seen": 1285226496 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030839518555667, + "loss": 2.5168, + "theoretical_loss": 3.5650218237569735, + "tokens_seen": 1285292032 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030838515546639924, + "loss": 2.72, + "theoretical_loss": 3.565005318352579, + "tokens_seen": 1285357568 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030837512537612837, + "loss": 2.7989, + "theoretical_loss": 3.5649888140253396, + "tokens_seen": 1285423104 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003083650952858576, + "loss": 2.7286, + "theoretical_loss": 3.564972310775129, + "tokens_seen": 1285488640 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030835506519558673, + "loss": 2.6591, + "theoretical_loss": 3.5649558086018223, + "tokens_seen": 1285554176 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030834503510531597, + "loss": 2.9037, + "theoretical_loss": 3.564939307505295, + "tokens_seen": 1285619712 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030833500501504515, + "loss": 2.8205, + "theoretical_loss": 3.564922807485421, + "tokens_seen": 1285685248 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030832497492477433, + "loss": 2.7974, + "theoretical_loss": 3.564906308542075, + "tokens_seen": 1285750784 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003083149448345035, + "loss": 2.7585, + "theoretical_loss": 3.5648898106751328, + "tokens_seen": 1285816320 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030830491474423275, + "loss": 2.7403, + "theoretical_loss": 3.564873313884469, + "tokens_seen": 1285881856 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003082948846539619, + "loss": 2.7961, + "theoretical_loss": 3.5648568181699583, + "tokens_seen": 1285947392 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003082848545636911, + "loss": 2.678, + "theoretical_loss": 3.5648403235314756, + "tokens_seen": 1286012928 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030827482447342024, + "loss": 2.7683, + "theoretical_loss": 3.564823829968897, + "tokens_seen": 1286078464 + }, + { + "epoch": 0.39, + "objective/train/docs_used": 729512, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.5554006099700928, + "objective/train/theoretical_loss": 3.5648073374820957, + "objective/train/tokens_used": 1306604000, + "theoretical_loss": 3.5648073374820957, + "tokens_seen": 1286144000 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030826479438314947, + "loss": 2.7696, + "theoretical_loss": 3.5648073374820957, + "tokens_seen": 1286144000 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030825476429287865, + "loss": 2.705, + "theoretical_loss": 3.564790846070948, + "tokens_seen": 1286209536 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030824473420260783, + "loss": 2.8292, + "theoretical_loss": 3.5647743557353286, + "tokens_seen": 1286275072 + }, + { + "epoch": 0.39, + "learning_rate": 0.000308234704112337, + "loss": 2.8453, + "theoretical_loss": 3.5647578664751123, + "tokens_seen": 1286340608 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003082246740220662, + "loss": 2.8469, + "theoretical_loss": 3.564741378290175, + "tokens_seen": 1286406144 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003082146439317954, + "loss": 2.6499, + "theoretical_loss": 3.564724891180391, + "tokens_seen": 1286471680 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003082046138415246, + "loss": 2.7469, + "theoretical_loss": 3.564708405145635, + "tokens_seen": 1286537216 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030819458375125374, + "loss": 2.5643, + "theoretical_loss": 3.5646919201857834, + "tokens_seen": 1286602752 + }, + { + "epoch": 0.39, + "learning_rate": 0.000308184553660983, + "loss": 2.4553, + "theoretical_loss": 3.5646754363007105, + "tokens_seen": 1286668288 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003081745235707121, + "loss": 2.6297, + "theoretical_loss": 3.564658953490292, + "tokens_seen": 1286733824 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030816449348044134, + "loss": 2.5938, + "theoretical_loss": 3.5646424717544027, + "tokens_seen": 1286799360 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003081544633901705, + "loss": 2.832, + "theoretical_loss": 3.5646259910929183, + "tokens_seen": 1286864896 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003081444332998997, + "loss": 2.6732, + "theoretical_loss": 3.5646095115057137, + "tokens_seen": 1286930432 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003081344032096289, + "loss": 2.7039, + "theoretical_loss": 3.564593032992664, + "tokens_seen": 1286995968 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003081243731193581, + "loss": 2.7675, + "theoretical_loss": 3.5645765555536446, + "tokens_seen": 1287061504 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030811434302908724, + "loss": 2.8038, + "theoretical_loss": 3.5645600791885315, + "tokens_seen": 1287127040 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003081043129388165, + "loss": 2.7545, + "theoretical_loss": 3.564543603897199, + "tokens_seen": 1287192576 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003080942828485456, + "loss": 2.9194, + "theoretical_loss": 3.564527129679523, + "tokens_seen": 1287258112 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030808425275827484, + "loss": 2.5778, + "theoretical_loss": 3.564510656535379, + "tokens_seen": 1287323648 + }, + { + "epoch": 0.39, + "learning_rate": 0.000308074222668004, + "loss": 2.7615, + "theoretical_loss": 3.5644941844646416, + "tokens_seen": 1287389184 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003080641925777332, + "loss": 2.7191, + "theoretical_loss": 3.5644777134671877, + "tokens_seen": 1287454720 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003080541624874624, + "loss": 2.6188, + "theoretical_loss": 3.5644612435428913, + "tokens_seen": 1287520256 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030804413239719157, + "loss": 2.8617, + "theoretical_loss": 3.564444774691628, + "tokens_seen": 1287585792 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030803410230692075, + "loss": 2.8198, + "theoretical_loss": 3.5644283069132743, + "tokens_seen": 1287651328 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030802407221665, + "loss": 2.9742, + "theoretical_loss": 3.564411840207705, + "tokens_seen": 1287716864 + }, + { + "epoch": 0.39, + "objective/train/docs_used": 730476, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.2410356998443604, + "objective/train/theoretical_loss": 3.564395374574796, + "objective/train/tokens_used": 1308242400, + "theoretical_loss": 3.564395374574796, + "tokens_seen": 1287782400 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003080140421263791, + "loss": 2.8386, + "theoretical_loss": 3.564395374574796, + "tokens_seen": 1287782400 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030800401203610834, + "loss": 2.7999, + "theoretical_loss": 3.564378910014422, + "tokens_seen": 1287847936 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003079939819458375, + "loss": 2.8403, + "theoretical_loss": 3.5643624465264594, + "tokens_seen": 1287913472 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003079839518555667, + "loss": 2.8508, + "theoretical_loss": 3.5643459841107834, + "tokens_seen": 1287979008 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003079739217652959, + "loss": 2.8753, + "theoretical_loss": 3.5643295227672698, + "tokens_seen": 1288044544 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030796389167502507, + "loss": 2.7691, + "theoretical_loss": 3.564313062495794, + "tokens_seen": 1288110080 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030795386158475425, + "loss": 2.8837, + "theoretical_loss": 3.564296603296232, + "tokens_seen": 1288175616 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003079438314944835, + "loss": 2.6574, + "theoretical_loss": 3.564280145168459, + "tokens_seen": 1288241152 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003079338014042126, + "loss": 2.8871, + "theoretical_loss": 3.5642636881123515, + "tokens_seen": 1288306688 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030792377131394185, + "loss": 2.6458, + "theoretical_loss": 3.564247232127784, + "tokens_seen": 1288372224 + }, + { + "epoch": 0.39, + "learning_rate": 0.000307913741223671, + "loss": 2.9371, + "theoretical_loss": 3.5642307772146333, + "tokens_seen": 1288437760 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003079037111334002, + "loss": 2.6266, + "theoretical_loss": 3.5642143233727746, + "tokens_seen": 1288503296 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003078936810431294, + "loss": 2.6399, + "theoretical_loss": 3.5641978706020834, + "tokens_seen": 1288568832 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030788365095285857, + "loss": 2.7763, + "theoretical_loss": 3.5641814189024363, + "tokens_seen": 1288634368 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030787362086258775, + "loss": 2.5499, + "theoretical_loss": 3.5641649682737087, + "tokens_seen": 1288699904 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030786359077231693, + "loss": 2.8277, + "theoretical_loss": 3.5641485187157764, + "tokens_seen": 1288765440 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003078535606820461, + "loss": 2.5792, + "theoretical_loss": 3.5641320702285153, + "tokens_seen": 1288830976 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030784353059177535, + "loss": 2.85, + "theoretical_loss": 3.5641156228118014, + "tokens_seen": 1288896512 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003078335005015045, + "loss": 2.7988, + "theoretical_loss": 3.5640991764655103, + "tokens_seen": 1288962048 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003078234704112337, + "loss": 2.7778, + "theoretical_loss": 3.564082731189518, + "tokens_seen": 1289027584 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003078134403209629, + "loss": 2.627, + "theoretical_loss": 3.5640662869837003, + "tokens_seen": 1289093120 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003078034102306921, + "loss": 2.9919, + "theoretical_loss": 3.5640498438479336, + "tokens_seen": 1289158656 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003077933801404213, + "loss": 2.7435, + "theoretical_loss": 3.564033401782094, + "tokens_seen": 1289224192 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030778335005015044, + "loss": 2.8873, + "theoretical_loss": 3.5640169607860566, + "tokens_seen": 1289289728 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030777331995987967, + "loss": 2.7533, + "theoretical_loss": 3.564000520859698, + "tokens_seen": 1289355264 + }, + { + "epoch": 0.39, + "objective/train/docs_used": 731120, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.0814762115478516, + "objective/train/theoretical_loss": 3.5639840820028947, + "objective/train/tokens_used": 1309880800, + "theoretical_loss": 3.5639840820028947, + "tokens_seen": 1289420800 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030776328986960885, + "loss": 2.8176, + "theoretical_loss": 3.5639840820028947, + "tokens_seen": 1289420800 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030775325977933803, + "loss": 2.4788, + "theoretical_loss": 3.5639676442155217, + "tokens_seen": 1289486336 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003077432296890672, + "loss": 2.5748, + "theoretical_loss": 3.563951207497456, + "tokens_seen": 1289551872 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003077331995987964, + "loss": 2.6725, + "theoretical_loss": 3.563934771848573, + "tokens_seen": 1289617408 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003077231695085256, + "loss": 2.6623, + "theoretical_loss": 3.5639183372687495, + "tokens_seen": 1289682944 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003077131394182548, + "loss": 2.7187, + "theoretical_loss": 3.5639019037578614, + "tokens_seen": 1289748480 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030770310932798394, + "loss": 2.7744, + "theoretical_loss": 3.5638854713157846, + "tokens_seen": 1289814016 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003076930792377132, + "loss": 2.4503, + "theoretical_loss": 3.5638690399423956, + "tokens_seen": 1289879552 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003076830491474423, + "loss": 2.8571, + "theoretical_loss": 3.5638526096375704, + "tokens_seen": 1289945088 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030767301905717154, + "loss": 3.0088, + "theoretical_loss": 3.563836180401185, + "tokens_seen": 1290010624 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003076629889669007, + "loss": 2.9464, + "theoretical_loss": 3.5638197522331163, + "tokens_seen": 1290076160 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003076529588766299, + "loss": 2.7111, + "theoretical_loss": 3.56380332513324, + "tokens_seen": 1290141696 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003076429287863591, + "loss": 2.7756, + "theoretical_loss": 3.5637868991014328, + "tokens_seen": 1290207232 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003076328986960883, + "loss": 2.6807, + "theoretical_loss": 3.563770474137571, + "tokens_seen": 1290272768 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030762286860581744, + "loss": 2.8427, + "theoretical_loss": 3.56375405024153, + "tokens_seen": 1290338304 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003076128385155467, + "loss": 2.5176, + "theoretical_loss": 3.5637376274131873, + "tokens_seen": 1290403840 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003076028084252758, + "loss": 2.7978, + "theoretical_loss": 3.563721205652419, + "tokens_seen": 1290469376 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030759277833500504, + "loss": 2.7106, + "theoretical_loss": 3.563704784959101, + "tokens_seen": 1290534912 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003075827482447342, + "loss": 2.5178, + "theoretical_loss": 3.5636883653331104, + "tokens_seen": 1290600448 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003075727181544634, + "loss": 2.6449, + "theoretical_loss": 3.563671946774323, + "tokens_seen": 1290665984 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003075626880641926, + "loss": 2.7391, + "theoretical_loss": 3.5636555292826158, + "tokens_seen": 1290731520 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030755265797392177, + "loss": 2.8588, + "theoretical_loss": 3.563639112857865, + "tokens_seen": 1290797056 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030754262788365095, + "loss": 2.9643, + "theoretical_loss": 3.563622697499947, + "tokens_seen": 1290862592 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003075325977933802, + "loss": 2.7872, + "theoretical_loss": 3.5636062832087383, + "tokens_seen": 1290928128 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003075225677031093, + "loss": 2.5052, + "theoretical_loss": 3.5635898699841158, + "tokens_seen": 1290993664 + }, + { + "epoch": 0.39, + "objective/train/docs_used": 732356, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.470327854156494, + "objective/train/theoretical_loss": 3.5635734578259557, + "objective/train/tokens_used": 1311519200, + "theoretical_loss": 3.5635734578259557, + "tokens_seen": 1291059200 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030751253761283854, + "loss": 2.5455, + "theoretical_loss": 3.5635734578259557, + "tokens_seen": 1291059200 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003075025075225677, + "loss": 2.6297, + "theoretical_loss": 3.5635570467341346, + "tokens_seen": 1291124736 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003074924774322969, + "loss": 2.7356, + "theoretical_loss": 3.563540636708529, + "tokens_seen": 1291190272 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003074824473420261, + "loss": 2.7078, + "theoretical_loss": 3.5635242277490162, + "tokens_seen": 1291255808 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030747241725175527, + "loss": 2.6395, + "theoretical_loss": 3.563507819855472, + "tokens_seen": 1291321344 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030746238716148445, + "loss": 2.8936, + "theoretical_loss": 3.5634914130277737, + "tokens_seen": 1291386880 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003074523570712137, + "loss": 2.7813, + "theoretical_loss": 3.5634750072657972, + "tokens_seen": 1291452416 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003074423269809428, + "loss": 2.7399, + "theoretical_loss": 3.56345860256942, + "tokens_seen": 1291517952 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030743229689067205, + "loss": 2.6481, + "theoretical_loss": 3.5634421989385183, + "tokens_seen": 1291583488 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003074222668004012, + "loss": 2.8764, + "theoretical_loss": 3.5634257963729694, + "tokens_seen": 1291649024 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003074122367101304, + "loss": 2.6838, + "theoretical_loss": 3.5634093948726493, + "tokens_seen": 1291714560 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003074022066198596, + "loss": 2.8946, + "theoretical_loss": 3.5633929944374354, + "tokens_seen": 1291780096 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030739217652958877, + "loss": 2.7914, + "theoretical_loss": 3.5633765950672043, + "tokens_seen": 1291845632 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030738214643931795, + "loss": 2.5473, + "theoretical_loss": 3.563360196761833, + "tokens_seen": 1291911168 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030737211634904713, + "loss": 2.9781, + "theoretical_loss": 3.5633437995211974, + "tokens_seen": 1291976704 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003073620862587763, + "loss": 2.6867, + "theoretical_loss": 3.5633274033451756, + "tokens_seen": 1292042240 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030735205616850555, + "loss": 2.9298, + "theoretical_loss": 3.563311008233644, + "tokens_seen": 1292107776 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003073420260782347, + "loss": 2.738, + "theoretical_loss": 3.5632946141864794, + "tokens_seen": 1292173312 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003073319959879639, + "loss": 2.7646, + "theoretical_loss": 3.563278221203559, + "tokens_seen": 1292238848 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003073219658976931, + "loss": 2.8032, + "theoretical_loss": 3.5632618292847593, + "tokens_seen": 1292304384 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003073119358074223, + "loss": 2.7591, + "theoretical_loss": 3.5632454384299574, + "tokens_seen": 1292369920 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030730190571715146, + "loss": 2.6215, + "theoretical_loss": 3.563229048639031, + "tokens_seen": 1292435456 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030729187562688064, + "loss": 2.6369, + "theoretical_loss": 3.563212659911856, + "tokens_seen": 1292500992 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003072818455366098, + "loss": 2.6935, + "theoretical_loss": 3.5631962722483097, + "tokens_seen": 1292566528 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030727181544633905, + "loss": 2.765, + "theoretical_loss": 3.56317988564827, + "tokens_seen": 1292632064 + }, + { + "epoch": 0.39, + "objective/train/docs_used": 733109, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.9310474395751953, + "objective/train/theoretical_loss": 3.5631635001116138, + "objective/train/tokens_used": 1313157600, + "theoretical_loss": 3.5631635001116138, + "tokens_seen": 1292697600 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003072617853560682, + "loss": 2.9276, + "theoretical_loss": 3.5631635001116138, + "tokens_seen": 1292697600 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003072517552657974, + "loss": 2.9127, + "theoretical_loss": 3.5631471156382166, + "tokens_seen": 1292763136 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030724172517552654, + "loss": 2.7599, + "theoretical_loss": 3.5631307322279575, + "tokens_seen": 1292828672 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003072316950852558, + "loss": 2.7267, + "theoretical_loss": 3.5631143498807125, + "tokens_seen": 1292894208 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030722166499498496, + "loss": 2.6033, + "theoretical_loss": 3.5630979685963586, + "tokens_seen": 1292959744 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030721163490471414, + "loss": 2.6101, + "theoretical_loss": 3.563081588374774, + "tokens_seen": 1293025280 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003072016048144433, + "loss": 2.725, + "theoretical_loss": 3.563065209215835, + "tokens_seen": 1293090816 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003071915747241725, + "loss": 2.8066, + "theoretical_loss": 3.5630488311194193, + "tokens_seen": 1293156352 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003071815446339017, + "loss": 2.864, + "theoretical_loss": 3.5630324540854037, + "tokens_seen": 1293221888 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003071715145436309, + "loss": 2.6012, + "theoretical_loss": 3.563016078113666, + "tokens_seen": 1293287424 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030716148445336005, + "loss": 2.582, + "theoretical_loss": 3.562999703204083, + "tokens_seen": 1293352960 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003071514543630893, + "loss": 2.7889, + "theoretical_loss": 3.5629833293565323, + "tokens_seen": 1293418496 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030714142427281846, + "loss": 2.7967, + "theoretical_loss": 3.5629669565708904, + "tokens_seen": 1293484032 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030713139418254764, + "loss": 2.8625, + "theoretical_loss": 3.562950584847036, + "tokens_seen": 1293549568 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003071213640922768, + "loss": 2.8094, + "theoretical_loss": 3.5629342141848452, + "tokens_seen": 1293615104 + }, + { + "epoch": 0.39, + "learning_rate": 0.000307111334002006, + "loss": 2.939, + "theoretical_loss": 3.562917844584197, + "tokens_seen": 1293680640 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003071013039117352, + "loss": 3.0824, + "theoretical_loss": 3.5629014760449667, + "tokens_seen": 1293746176 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003070912738214644, + "loss": 2.7515, + "theoretical_loss": 3.562885108567033, + "tokens_seen": 1293811712 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030708124373119355, + "loss": 2.5427, + "theoretical_loss": 3.562868742150273, + "tokens_seen": 1293877248 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003070712136409228, + "loss": 2.7309, + "theoretical_loss": 3.5628523767945643, + "tokens_seen": 1293942784 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003070611835506519, + "loss": 3.0067, + "theoretical_loss": 3.5628360124997847, + "tokens_seen": 1294008320 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030705115346038115, + "loss": 2.7548, + "theoretical_loss": 3.562819649265811, + "tokens_seen": 1294073856 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003070411233701104, + "loss": 2.7441, + "theoretical_loss": 3.562803287092521, + "tokens_seen": 1294139392 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003070310932798395, + "loss": 2.6528, + "theoretical_loss": 3.5627869259797924, + "tokens_seen": 1294204928 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030702106318956874, + "loss": 2.8052, + "theoretical_loss": 3.562770565927502, + "tokens_seen": 1294270464 + }, + { + "epoch": 0.39, + "objective/train/docs_used": 734261, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.1073310375213623, + "objective/train/theoretical_loss": 3.5627542069355282, + "objective/train/tokens_used": 1314796000, + "theoretical_loss": 3.5627542069355282, + "tokens_seen": 1294336000 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003070110330992979, + "loss": 2.6192, + "theoretical_loss": 3.5627542069355282, + "tokens_seen": 1294336000 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003070010030090271, + "loss": 2.8709, + "theoretical_loss": 3.562737849003749, + "tokens_seen": 1294401536 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003069909729187563, + "loss": 2.7808, + "theoretical_loss": 3.5627214921320407, + "tokens_seen": 1294467072 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030698094282848547, + "loss": 2.5081, + "theoretical_loss": 3.5627051363202824, + "tokens_seen": 1294532608 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030697091273821465, + "loss": 2.5161, + "theoretical_loss": 3.56268878156835, + "tokens_seen": 1294598144 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003069608826479439, + "loss": 2.6172, + "theoretical_loss": 3.562672427876123, + "tokens_seen": 1294663680 + }, + { + "epoch": 0.39, + "learning_rate": 0.000306950852557673, + "loss": 2.8294, + "theoretical_loss": 3.5626560752434777, + "tokens_seen": 1294729216 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030694082246740225, + "loss": 2.5879, + "theoretical_loss": 3.5626397236702934, + "tokens_seen": 1294794752 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003069307923771314, + "loss": 2.7859, + "theoretical_loss": 3.5626233731564456, + "tokens_seen": 1294860288 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003069207622868606, + "loss": 2.7186, + "theoretical_loss": 3.562607023701814, + "tokens_seen": 1294925824 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003069107321965898, + "loss": 2.7775, + "theoretical_loss": 3.562590675306276, + "tokens_seen": 1294991360 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030690070210631897, + "loss": 2.9556, + "theoretical_loss": 3.562574327969708, + "tokens_seen": 1295056896 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030689067201604815, + "loss": 2.7647, + "theoretical_loss": 3.56255798169199, + "tokens_seen": 1295122432 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030688064192577733, + "loss": 2.6713, + "theoretical_loss": 3.5625416364729983, + "tokens_seen": 1295187968 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003068706118355065, + "loss": 2.5909, + "theoretical_loss": 3.562525292312611, + "tokens_seen": 1295253504 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030686058174523575, + "loss": 2.5814, + "theoretical_loss": 3.5625089492107063, + "tokens_seen": 1295319040 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003068505516549649, + "loss": 2.5363, + "theoretical_loss": 3.562492607167162, + "tokens_seen": 1295384576 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003068405215646941, + "loss": 2.7949, + "theoretical_loss": 3.5624762661818563, + "tokens_seen": 1295450112 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003068304914744233, + "loss": 2.6938, + "theoretical_loss": 3.5624599262546663, + "tokens_seen": 1295515648 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003068204613841525, + "loss": 2.8126, + "theoretical_loss": 3.562443587385471, + "tokens_seen": 1295581184 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030681043129388166, + "loss": 2.7447, + "theoretical_loss": 3.5624272495741476, + "tokens_seen": 1295646720 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030680040120361084, + "loss": 2.7045, + "theoretical_loss": 3.5624109128205745, + "tokens_seen": 1295712256 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030679037111334, + "loss": 2.8962, + "theoretical_loss": 3.5623945771246297, + "tokens_seen": 1295777792 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030678034102306925, + "loss": 2.8797, + "theoretical_loss": 3.562378242486191, + "tokens_seen": 1295843328 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003067703109327984, + "loss": 2.8716, + "theoretical_loss": 3.562361908905137, + "tokens_seen": 1295908864 + }, + { + "epoch": 0.39, + "objective/train/docs_used": 734748, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.0890066623687744, + "objective/train/theoretical_loss": 3.562345576381345, + "objective/train/tokens_used": 1316434400, + "theoretical_loss": 3.562345576381345, + "tokens_seen": 1295974400 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003067602808425276, + "loss": 3.0154, + "theoretical_loss": 3.562345576381345, + "tokens_seen": 1295974400 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030675025075225674, + "loss": 2.9256, + "theoretical_loss": 3.562329244914694, + "tokens_seen": 1296039936 + }, + { + "epoch": 0.39, + "learning_rate": 0.000306740220661986, + "loss": 2.8377, + "theoretical_loss": 3.5623129145050614, + "tokens_seen": 1296105472 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030673019057171516, + "loss": 2.6489, + "theoretical_loss": 3.5622965851523256, + "tokens_seen": 1296171008 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030672016048144434, + "loss": 3.0688, + "theoretical_loss": 3.5622802568563645, + "tokens_seen": 1296236544 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003067101303911735, + "loss": 2.8964, + "theoretical_loss": 3.562263929617057, + "tokens_seen": 1296302080 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003067001003009027, + "loss": 2.6463, + "theoretical_loss": 3.5622476034342805, + "tokens_seen": 1296367616 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003066900702106319, + "loss": 2.5284, + "theoretical_loss": 3.5622312783079133, + "tokens_seen": 1296433152 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003066800401203611, + "loss": 2.7484, + "theoretical_loss": 3.5622149542378345, + "tokens_seen": 1296498688 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030667001003009025, + "loss": 2.8101, + "theoretical_loss": 3.5621986312239216, + "tokens_seen": 1296564224 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003066599799398195, + "loss": 2.7961, + "theoretical_loss": 3.562182309266053, + "tokens_seen": 1296629760 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030664994984954866, + "loss": 2.8673, + "theoretical_loss": 3.5621659883641072, + "tokens_seen": 1296695296 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030663991975927784, + "loss": 2.7624, + "theoretical_loss": 3.5621496685179626, + "tokens_seen": 1296760832 + }, + { + "epoch": 0.39, + "learning_rate": 0.000306629889669007, + "loss": 2.6998, + "theoretical_loss": 3.562133349727497, + "tokens_seen": 1296826368 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003066198595787362, + "loss": 2.6747, + "theoretical_loss": 3.5621170319925897, + "tokens_seen": 1296891904 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003066098294884654, + "loss": 2.6715, + "theoretical_loss": 3.5621007153131172, + "tokens_seen": 1296957440 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003065997993981946, + "loss": 2.7314, + "theoretical_loss": 3.5620843996889606, + "tokens_seen": 1297022976 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030658976930792375, + "loss": 2.9574, + "theoretical_loss": 3.5620680851199964, + "tokens_seen": 1297088512 + }, + { + "epoch": 0.39, + "learning_rate": 0.000306579739217653, + "loss": 2.8272, + "theoretical_loss": 3.5620517716061038, + "tokens_seen": 1297154048 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003065697091273821, + "loss": 2.7584, + "theoretical_loss": 3.5620354591471606, + "tokens_seen": 1297219584 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030655967903711135, + "loss": 2.674, + "theoretical_loss": 3.5620191477430465, + "tokens_seen": 1297285120 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030654964894684053, + "loss": 2.6473, + "theoretical_loss": 3.5620028373936385, + "tokens_seen": 1297350656 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003065396188565697, + "loss": 2.5738, + "theoretical_loss": 3.5619865280988163, + "tokens_seen": 1297416192 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003065295887662989, + "loss": 2.6113, + "theoretical_loss": 3.5619702198584577, + "tokens_seen": 1297481728 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003065195586760281, + "loss": 2.6486, + "theoretical_loss": 3.561953912672442, + "tokens_seen": 1297547264 + }, + { + "epoch": 0.39, + "objective/train/docs_used": 735602, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.776223659515381, + "objective/train/theoretical_loss": 3.5619376065406474, + "objective/train/tokens_used": 1318072800, + "theoretical_loss": 3.5619376065406474, + "tokens_seen": 1297612800 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030650952858575725, + "loss": 2.6631, + "theoretical_loss": 3.5619376065406474, + "tokens_seen": 1297612800 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003064994984954865, + "loss": 2.8622, + "theoretical_loss": 3.5619213014629523, + "tokens_seen": 1297678336 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003064894684052156, + "loss": 2.9075, + "theoretical_loss": 3.5619049974392354, + "tokens_seen": 1297743872 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030647943831494485, + "loss": 2.7187, + "theoretical_loss": 3.5618886944693755, + "tokens_seen": 1297809408 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030646940822467403, + "loss": 2.7299, + "theoretical_loss": 3.5618723925532514, + "tokens_seen": 1297874944 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003064593781344032, + "loss": 2.5794, + "theoretical_loss": 3.561856091690742, + "tokens_seen": 1297940480 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003064493480441324, + "loss": 2.8512, + "theoretical_loss": 3.5618397918817246, + "tokens_seen": 1298006016 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003064393179538616, + "loss": 2.5381, + "theoretical_loss": 3.56182349312608, + "tokens_seen": 1298071552 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030642928786359076, + "loss": 2.8245, + "theoretical_loss": 3.561807195423685, + "tokens_seen": 1298137088 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030641925777332, + "loss": 2.7599, + "theoretical_loss": 3.5617908987744205, + "tokens_seen": 1298202624 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003064092276830491, + "loss": 2.9704, + "theoretical_loss": 3.561774603178163, + "tokens_seen": 1298268160 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030639919759277835, + "loss": 2.5922, + "theoretical_loss": 3.561758308634793, + "tokens_seen": 1298333696 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003063891675025075, + "loss": 2.9907, + "theoretical_loss": 3.5617420151441888, + "tokens_seen": 1298399232 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003063791374122367, + "loss": 2.8507, + "theoretical_loss": 3.5617257227062287, + "tokens_seen": 1298464768 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003063691073219659, + "loss": 2.877, + "theoretical_loss": 3.561709431320792, + "tokens_seen": 1298530304 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003063590772316951, + "loss": 2.7218, + "theoretical_loss": 3.5616931409877584, + "tokens_seen": 1298595840 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030634904714142426, + "loss": 2.7684, + "theoretical_loss": 3.5616768517070057, + "tokens_seen": 1298661376 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003063390170511535, + "loss": 3.0331, + "theoretical_loss": 3.5616605634784126, + "tokens_seen": 1298726912 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003063289869608826, + "loss": 2.7368, + "theoretical_loss": 3.561644276301859, + "tokens_seen": 1298792448 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030631895687061186, + "loss": 2.7178, + "theoretical_loss": 3.5616279901772234, + "tokens_seen": 1298857984 + }, + { + "epoch": 0.39, + "learning_rate": 0.000306308926780341, + "loss": 3.0054, + "theoretical_loss": 3.5616117051043856, + "tokens_seen": 1298923520 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003062988966900702, + "loss": 2.8699, + "theoretical_loss": 3.561595421083223, + "tokens_seen": 1298989056 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030628886659979945, + "loss": 2.651, + "theoretical_loss": 3.5615791381136157, + "tokens_seen": 1299054592 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003062788365095286, + "loss": 2.7788, + "theoretical_loss": 3.561562856195443, + "tokens_seen": 1299120128 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003062688064192578, + "loss": 2.8185, + "theoretical_loss": 3.5615465753285833, + "tokens_seen": 1299185664 + }, + { + "epoch": 0.39, + "objective/train/docs_used": 736089, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.792696475982666, + "objective/train/theoretical_loss": 3.561530295512916, + "objective/train/tokens_used": 1319711200, + "theoretical_loss": 3.561530295512916, + "tokens_seen": 1299251200 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030625877632898694, + "loss": 2.8616, + "theoretical_loss": 3.561530295512916, + "tokens_seen": 1299251200 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003062487462387162, + "loss": 2.8725, + "theoretical_loss": 3.56151401674832, + "tokens_seen": 1299316736 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030623871614844536, + "loss": 2.867, + "theoretical_loss": 3.5614977390346745, + "tokens_seen": 1299382272 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030622868605817454, + "loss": 3.1027, + "theoretical_loss": 3.5614814623718587, + "tokens_seen": 1299447808 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003062186559679037, + "loss": 2.8571, + "theoretical_loss": 3.5614651867597518, + "tokens_seen": 1299513344 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003062086258776329, + "loss": 2.6892, + "theoretical_loss": 3.5614489121982333, + "tokens_seen": 1299578880 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003061985957873621, + "loss": 2.7532, + "theoretical_loss": 3.5614326386871817, + "tokens_seen": 1299644416 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003061885656970913, + "loss": 2.6665, + "theoretical_loss": 3.561416366226477, + "tokens_seen": 1299709952 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030617853560682045, + "loss": 2.7218, + "theoretical_loss": 3.5614000948159976, + "tokens_seen": 1299775488 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003061685055165497, + "loss": 2.9365, + "theoretical_loss": 3.561383824455624, + "tokens_seen": 1299841024 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030615847542627886, + "loss": 2.7808, + "theoretical_loss": 3.5613675551452344, + "tokens_seen": 1299906560 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030614844533600804, + "loss": 2.9915, + "theoretical_loss": 3.561351286884708, + "tokens_seen": 1299972096 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003061384152457372, + "loss": 2.9194, + "theoretical_loss": 3.5613350196739253, + "tokens_seen": 1300037632 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003061283851554664, + "loss": 2.9055, + "theoretical_loss": 3.561318753512764, + "tokens_seen": 1300103168 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003061183550651956, + "loss": 2.694, + "theoretical_loss": 3.561302488401105, + "tokens_seen": 1300168704 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003061083249749248, + "loss": 2.8235, + "theoretical_loss": 3.5612862243388275, + "tokens_seen": 1300234240 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030609829488465395, + "loss": 2.984, + "theoretical_loss": 3.5612699613258094, + "tokens_seen": 1300299776 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003060882647943832, + "loss": 2.6311, + "theoretical_loss": 3.561253699361932, + "tokens_seen": 1300365312 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003060782347041123, + "loss": 2.8904, + "theoretical_loss": 3.561237438447074, + "tokens_seen": 1300430848 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030606820461384155, + "loss": 2.9138, + "theoretical_loss": 3.561221178581114, + "tokens_seen": 1300496384 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030605817452357073, + "loss": 2.8776, + "theoretical_loss": 3.5612049197639335, + "tokens_seen": 1300561920 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003060481444332999, + "loss": 2.8136, + "theoretical_loss": 3.56118866199541, + "tokens_seen": 1300627456 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003060381143430291, + "loss": 2.8212, + "theoretical_loss": 3.5611724052754243, + "tokens_seen": 1300692992 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003060280842527583, + "loss": 2.9003, + "theoretical_loss": 3.5611561496038555, + "tokens_seen": 1300758528 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030601805416248745, + "loss": 2.5871, + "theoretical_loss": 3.5611398949805833, + "tokens_seen": 1300824064 + }, + { + "epoch": 0.39, + "objective/train/docs_used": 737292, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.2421655654907227, + "objective/train/theoretical_loss": 3.5611236414054868, + "objective/train/tokens_used": 1321349600, + "theoretical_loss": 3.5611236414054868, + "tokens_seen": 1300889600 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003060080240722167, + "loss": 2.6458, + "theoretical_loss": 3.5611236414054868, + "tokens_seen": 1300889600 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003059979939819458, + "loss": 2.9141, + "theoretical_loss": 3.561107388878446, + "tokens_seen": 1300955136 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030598796389167505, + "loss": 2.7988, + "theoretical_loss": 3.561091137399341, + "tokens_seen": 1301020672 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030597793380140423, + "loss": 2.9194, + "theoretical_loss": 3.5610748869680506, + "tokens_seen": 1301086208 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003059679037111334, + "loss": 2.8715, + "theoretical_loss": 3.5610586375844546, + "tokens_seen": 1301151744 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003059578736208626, + "loss": 2.8351, + "theoretical_loss": 3.561042389248433, + "tokens_seen": 1301217280 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003059478435305918, + "loss": 2.8301, + "theoretical_loss": 3.561026141959866, + "tokens_seen": 1301282816 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030593781344032096, + "loss": 2.9549, + "theoretical_loss": 3.5610098957186325, + "tokens_seen": 1301348352 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003059277833500502, + "loss": 2.7135, + "theoretical_loss": 3.5609936505246123, + "tokens_seen": 1301413888 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003059177532597793, + "loss": 2.6076, + "theoretical_loss": 3.5609774063776856, + "tokens_seen": 1301479424 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030590772316950855, + "loss": 3.0576, + "theoretical_loss": 3.5609611632777316, + "tokens_seen": 1301544960 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003058976930792377, + "loss": 2.8576, + "theoretical_loss": 3.560944921224631, + "tokens_seen": 1301610496 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003058876629889669, + "loss": 2.6399, + "theoretical_loss": 3.5609286802182627, + "tokens_seen": 1301676032 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003058776328986961, + "loss": 2.9033, + "theoretical_loss": 3.560912440258507, + "tokens_seen": 1301741568 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003058676028084253, + "loss": 2.8544, + "theoretical_loss": 3.560896201345243, + "tokens_seen": 1301807104 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030585757271815446, + "loss": 2.9476, + "theoretical_loss": 3.5608799634783526, + "tokens_seen": 1301872640 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003058475426278837, + "loss": 2.8149, + "theoretical_loss": 3.560863726657714, + "tokens_seen": 1301938176 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003058375125376128, + "loss": 2.7711, + "theoretical_loss": 3.560847490883207, + "tokens_seen": 1302003712 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030582748244734206, + "loss": 2.5517, + "theoretical_loss": 3.560831256154713, + "tokens_seen": 1302069248 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003058174523570712, + "loss": 2.9455, + "theoretical_loss": 3.5608150224721102, + "tokens_seen": 1302134784 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003058074222668004, + "loss": 2.8634, + "theoretical_loss": 3.5607987898352795, + "tokens_seen": 1302200320 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003057973921765296, + "loss": 2.802, + "theoretical_loss": 3.5607825582441013, + "tokens_seen": 1302265856 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003057873620862588, + "loss": 2.7724, + "theoretical_loss": 3.5607663276984547, + "tokens_seen": 1302331392 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030577733199598796, + "loss": 2.9293, + "theoretical_loss": 3.5607500981982207, + "tokens_seen": 1302396928 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030576730190571714, + "loss": 2.838, + "theoretical_loss": 3.560733869743278, + "tokens_seen": 1302462464 + }, + { + "epoch": 0.39, + "objective/train/docs_used": 737760, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.81878924369812, + "objective/train/theoretical_loss": 3.5607176423335085, + "objective/train/tokens_used": 1322988000, + "theoretical_loss": 3.5607176423335085, + "tokens_seen": 1302528000 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003057572718154463, + "loss": 2.764, + "theoretical_loss": 3.5607176423335085, + "tokens_seen": 1302528000 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030574724172517556, + "loss": 2.8461, + "theoretical_loss": 3.560701415968791, + "tokens_seen": 1302593536 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003057372116349047, + "loss": 2.7095, + "theoretical_loss": 3.560685190649006, + "tokens_seen": 1302659072 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003057271815446339, + "loss": 2.903, + "theoretical_loss": 3.560668966374034, + "tokens_seen": 1302724608 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030571715145436305, + "loss": 2.8707, + "theoretical_loss": 3.560652743143754, + "tokens_seen": 1302790144 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003057071213640923, + "loss": 2.9128, + "theoretical_loss": 3.560636520958047, + "tokens_seen": 1302855680 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030569709127382147, + "loss": 2.8087, + "theoretical_loss": 3.5606202998167937, + "tokens_seen": 1302921216 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030568706118355065, + "loss": 2.8016, + "theoretical_loss": 3.560604079719874, + "tokens_seen": 1302986752 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030567703109327983, + "loss": 2.6508, + "theoretical_loss": 3.5605878606671677, + "tokens_seen": 1303052288 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030566700100300906, + "loss": 2.5794, + "theoretical_loss": 3.560571642658555, + "tokens_seen": 1303117824 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003056569709127382, + "loss": 2.9034, + "theoretical_loss": 3.5605554256939165, + "tokens_seen": 1303183360 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003056469408224674, + "loss": 2.616, + "theoretical_loss": 3.560539209773133, + "tokens_seen": 1303248896 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030563691073219655, + "loss": 2.8869, + "theoretical_loss": 3.560522994896084, + "tokens_seen": 1303314432 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003056268806419258, + "loss": 2.7575, + "theoretical_loss": 3.5605067810626503, + "tokens_seen": 1303379968 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030561685055165497, + "loss": 2.9897, + "theoretical_loss": 3.560490568272712, + "tokens_seen": 1303445504 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030560682046138415, + "loss": 2.6242, + "theoretical_loss": 3.5604743565261496, + "tokens_seen": 1303511040 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030559679037111333, + "loss": 2.5852, + "theoretical_loss": 3.5604581458228437, + "tokens_seen": 1303576576 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003055867602808425, + "loss": 2.8916, + "theoretical_loss": 3.5604419361626745, + "tokens_seen": 1303642112 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003055767301905717, + "loss": 2.696, + "theoretical_loss": 3.560425727545522, + "tokens_seen": 1303707648 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030556670010030093, + "loss": 2.8363, + "theoretical_loss": 3.5604095199712678, + "tokens_seen": 1303773184 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030555667001003006, + "loss": 2.7235, + "theoretical_loss": 3.560393313439791, + "tokens_seen": 1303838720 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003055466399197593, + "loss": 2.6976, + "theoretical_loss": 3.5603771079509734, + "tokens_seen": 1303904256 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003055366098294885, + "loss": 2.98, + "theoretical_loss": 3.5603609035046944, + "tokens_seen": 1303969792 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030552657973921765, + "loss": 3.0041, + "theoretical_loss": 3.5603447001008357, + "tokens_seen": 1304035328 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003055165496489469, + "loss": 2.9218, + "theoretical_loss": 3.5603284977392766, + "tokens_seen": 1304100864 + }, + { + "epoch": 0.4, + "objective/train/docs_used": 739145, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7752797603607178, + "objective/train/theoretical_loss": 3.560312296419899, + "objective/train/tokens_used": 1324626400, + "theoretical_loss": 3.560312296419899, + "tokens_seen": 1304166400 + }, + { + "epoch": 0.4, + "learning_rate": 0.000305506519558676, + "loss": 2.855, + "theoretical_loss": 3.560312296419899, + "tokens_seen": 1304166400 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030549648946840525, + "loss": 2.994, + "theoretical_loss": 3.560296096142582, + "tokens_seen": 1304231936 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030548645937813443, + "loss": 2.5431, + "theoretical_loss": 3.5602798969072076, + "tokens_seen": 1304297472 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003054764292878636, + "loss": 2.8924, + "theoretical_loss": 3.5602636987136558, + "tokens_seen": 1304363008 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003054663991975928, + "loss": 2.724, + "theoretical_loss": 3.5602475015618067, + "tokens_seen": 1304428544 + }, + { + "epoch": 0.4, + "learning_rate": 0.000305456369107322, + "loss": 2.611, + "theoretical_loss": 3.560231305451542, + "tokens_seen": 1304494080 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030544633901705116, + "loss": 2.7322, + "theoretical_loss": 3.5602151103827424, + "tokens_seen": 1304559616 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003054363089267804, + "loss": 2.6499, + "theoretical_loss": 3.5601989163552883, + "tokens_seen": 1304625152 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003054262788365095, + "loss": 2.6927, + "theoretical_loss": 3.56018272336906, + "tokens_seen": 1304690688 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030541624874623875, + "loss": 2.9038, + "theoretical_loss": 3.5601665314239384, + "tokens_seen": 1304756224 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003054062186559679, + "loss": 2.6192, + "theoretical_loss": 3.5601503405198045, + "tokens_seen": 1304821760 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003053961885656971, + "loss": 2.5425, + "theoretical_loss": 3.560134150656539, + "tokens_seen": 1304887296 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003053861584754263, + "loss": 2.9826, + "theoretical_loss": 3.560117961834023, + "tokens_seen": 1304952832 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003053761283851555, + "loss": 2.6977, + "theoretical_loss": 3.5601017740521366, + "tokens_seen": 1305018368 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030536609829488466, + "loss": 2.6911, + "theoretical_loss": 3.5600855873107617, + "tokens_seen": 1305083904 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003053560682046139, + "loss": 2.708, + "theoretical_loss": 3.5600694016097787, + "tokens_seen": 1305149440 + }, + { + "epoch": 0.4, + "learning_rate": 0.000305346038114343, + "loss": 2.6927, + "theoretical_loss": 3.5600532169490684, + "tokens_seen": 1305214976 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030533600802407226, + "loss": 2.4824, + "theoretical_loss": 3.560037033328512, + "tokens_seen": 1305280512 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003053259779338014, + "loss": 2.8791, + "theoretical_loss": 3.560020850747989, + "tokens_seen": 1305346048 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003053159478435306, + "loss": 2.8943, + "theoretical_loss": 3.560004669207383, + "tokens_seen": 1305411584 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003053059177532598, + "loss": 2.5792, + "theoretical_loss": 3.5599884887065727, + "tokens_seen": 1305477120 + }, + { + "epoch": 0.4, + "learning_rate": 0.000305295887662989, + "loss": 2.6738, + "theoretical_loss": 3.55997230924544, + "tokens_seen": 1305542656 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030528585757271816, + "loss": 2.9905, + "theoretical_loss": 3.5599561308238656, + "tokens_seen": 1305608192 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030527582748244734, + "loss": 2.7705, + "theoretical_loss": 3.559939953441731, + "tokens_seen": 1305673728 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003052657973921765, + "loss": 2.5925, + "theoretical_loss": 3.559923777098917, + "tokens_seen": 1305739264 + }, + { + "epoch": 0.4, + "objective/train/docs_used": 739729, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7746076583862305, + "objective/train/theoretical_loss": 3.5599076017953046, + "objective/train/tokens_used": 1326264800, + "theoretical_loss": 3.5599076017953046, + "tokens_seen": 1305804800 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030525576730190576, + "loss": 2.7925, + "theoretical_loss": 3.5599076017953046, + "tokens_seen": 1305804800 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003052457372116349, + "loss": 2.6735, + "theoretical_loss": 3.5598914275307747, + "tokens_seen": 1305870336 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003052357071213641, + "loss": 2.9471, + "theoretical_loss": 3.5598752543052092, + "tokens_seen": 1305935872 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030522567703109325, + "loss": 2.6233, + "theoretical_loss": 3.5598590821184883, + "tokens_seen": 1306001408 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003052156469408225, + "loss": 2.916, + "theoretical_loss": 3.5598429109704934, + "tokens_seen": 1306066944 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030520561685055167, + "loss": 2.7563, + "theoretical_loss": 3.559826740861106, + "tokens_seen": 1306132480 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030519558676028085, + "loss": 2.8432, + "theoretical_loss": 3.5598105717902073, + "tokens_seen": 1306198016 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030518555667001003, + "loss": 2.8032, + "theoretical_loss": 3.559794403757678, + "tokens_seen": 1306263552 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030517552657973926, + "loss": 2.8114, + "theoretical_loss": 3.5597782367633997, + "tokens_seen": 1306329088 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003051654964894684, + "loss": 2.6933, + "theoretical_loss": 3.5597620708072535, + "tokens_seen": 1306394624 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003051554663991976, + "loss": 2.5044, + "theoretical_loss": 3.5597459058891205, + "tokens_seen": 1306460160 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030514543630892675, + "loss": 2.8996, + "theoretical_loss": 3.5597297420088827, + "tokens_seen": 1306525696 + }, + { + "epoch": 0.4, + "learning_rate": 0.000305135406218656, + "loss": 2.7811, + "theoretical_loss": 3.559713579166421, + "tokens_seen": 1306591232 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030512537612838517, + "loss": 2.7376, + "theoretical_loss": 3.5596974173616163, + "tokens_seen": 1306656768 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030511534603811435, + "loss": 2.7578, + "theoretical_loss": 3.55968125659435, + "tokens_seen": 1306722304 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030510531594784353, + "loss": 2.7147, + "theoretical_loss": 3.559665096864504, + "tokens_seen": 1306787840 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003050952858575727, + "loss": 2.7152, + "theoretical_loss": 3.5596489381719594, + "tokens_seen": 1306853376 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003050852557673019, + "loss": 2.8776, + "theoretical_loss": 3.5596327805165977, + "tokens_seen": 1306918912 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030507522567703113, + "loss": 2.8389, + "theoretical_loss": 3.5596166238983002, + "tokens_seen": 1306984448 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030506519558676026, + "loss": 2.7559, + "theoretical_loss": 3.5596004683169484, + "tokens_seen": 1307049984 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003050551654964895, + "loss": 2.7783, + "theoretical_loss": 3.5595843137724237, + "tokens_seen": 1307115520 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003050451354062186, + "loss": 2.7964, + "theoretical_loss": 3.5595681602646074, + "tokens_seen": 1307181056 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030503510531594785, + "loss": 2.9114, + "theoretical_loss": 3.559552007793382, + "tokens_seen": 1307246592 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030502507522567703, + "loss": 2.9301, + "theoretical_loss": 3.5595358563586275, + "tokens_seen": 1307312128 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003050150451354062, + "loss": 2.6285, + "theoretical_loss": 3.559519705960226, + "tokens_seen": 1307377664 + }, + { + "epoch": 0.4, + "objective/train/docs_used": 740313, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.924899101257324, + "objective/train/theoretical_loss": 3.55950355659806, + "objective/train/tokens_used": 1327903200, + "theoretical_loss": 3.55950355659806, + "tokens_seen": 1307443200 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003050050150451354, + "loss": 2.7686, + "theoretical_loss": 3.55950355659806, + "tokens_seen": 1307443200 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030499498495486463, + "loss": 2.8912, + "theoretical_loss": 3.55948740827201, + "tokens_seen": 1307508736 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030498495486459376, + "loss": 2.9552, + "theoretical_loss": 3.5594712609819577, + "tokens_seen": 1307574272 + }, + { + "epoch": 0.4, + "learning_rate": 0.000304974924774323, + "loss": 2.8063, + "theoretical_loss": 3.5594551147277853, + "tokens_seen": 1307639808 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003049648946840521, + "loss": 2.9264, + "theoretical_loss": 3.5594389695093738, + "tokens_seen": 1307705344 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030495486459378136, + "loss": 2.8863, + "theoretical_loss": 3.559422825326605, + "tokens_seen": 1307770880 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030494483450351054, + "loss": 2.7583, + "theoretical_loss": 3.559406682179361, + "tokens_seen": 1307836416 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003049348044132397, + "loss": 2.4771, + "theoretical_loss": 3.5593905400675228, + "tokens_seen": 1307901952 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003049247743229689, + "loss": 2.9733, + "theoretical_loss": 3.5593743989909727, + "tokens_seen": 1307967488 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003049147442326981, + "loss": 2.7771, + "theoretical_loss": 3.559358258949592, + "tokens_seen": 1308033024 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030490471414242726, + "loss": 2.8188, + "theoretical_loss": 3.5593421199432633, + "tokens_seen": 1308098560 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003048946840521565, + "loss": 2.889, + "theoretical_loss": 3.5593259819718677, + "tokens_seen": 1308164096 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003048846539618856, + "loss": 2.918, + "theoretical_loss": 3.5593098450352865, + "tokens_seen": 1308229632 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030487462387161486, + "loss": 2.7817, + "theoretical_loss": 3.559293709133402, + "tokens_seen": 1308295168 + }, + { + "epoch": 0.4, + "learning_rate": 0.000304864593781344, + "loss": 2.8239, + "theoretical_loss": 3.5592775742660967, + "tokens_seen": 1308360704 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003048545636910732, + "loss": 2.706, + "theoretical_loss": 3.5592614404332514, + "tokens_seen": 1308426240 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003048445336008024, + "loss": 2.7558, + "theoretical_loss": 3.5592453076347486, + "tokens_seen": 1308491776 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003048345035105316, + "loss": 2.9029, + "theoretical_loss": 3.55922917587047, + "tokens_seen": 1308557312 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030482447342026076, + "loss": 2.7866, + "theoretical_loss": 3.5592130451402975, + "tokens_seen": 1308622848 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030481444332999, + "loss": 2.7642, + "theoretical_loss": 3.5591969154441125, + "tokens_seen": 1308688384 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003048044132397192, + "loss": 2.9431, + "theoretical_loss": 3.5591807867817984, + "tokens_seen": 1308753920 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030479438314944836, + "loss": 2.7344, + "theoretical_loss": 3.559164659153235, + "tokens_seen": 1308819456 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030478435305917754, + "loss": 2.8924, + "theoretical_loss": 3.5591485325583068, + "tokens_seen": 1308884992 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003047743229689067, + "loss": 2.5976, + "theoretical_loss": 3.559132406996894, + "tokens_seen": 1308950528 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030476429287863596, + "loss": 2.6775, + "theoretical_loss": 3.559116282468879, + "tokens_seen": 1309016064 + }, + { + "epoch": 0.4, + "objective/train/docs_used": 741446, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.0342657566070557, + "objective/train/theoretical_loss": 3.559100158974144, + "objective/train/tokens_used": 1329541600, + "theoretical_loss": 3.559100158974144, + "tokens_seen": 1309081600 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003047542627883651, + "loss": 2.7761, + "theoretical_loss": 3.559100158974144, + "tokens_seen": 1309081600 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003047442326980943, + "loss": 2.667, + "theoretical_loss": 3.559084036512571, + "tokens_seen": 1309147136 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030473420260782345, + "loss": 2.6234, + "theoretical_loss": 3.5590679150840425, + "tokens_seen": 1309212672 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003047241725175527, + "loss": 2.8773, + "theoretical_loss": 3.5590517946884406, + "tokens_seen": 1309278208 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030471414242728187, + "loss": 2.7061, + "theoretical_loss": 3.559035675325646, + "tokens_seen": 1309343744 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030470411233701105, + "loss": 2.9393, + "theoretical_loss": 3.5590195569955423, + "tokens_seen": 1309409280 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030469408224674023, + "loss": 2.8004, + "theoretical_loss": 3.5590034396980115, + "tokens_seen": 1309474816 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030468405215646946, + "loss": 2.6898, + "theoretical_loss": 3.5589873234329357, + "tokens_seen": 1309540352 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003046740220661986, + "loss": 2.7437, + "theoretical_loss": 3.5589712082001963, + "tokens_seen": 1309605888 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003046639919759278, + "loss": 2.9627, + "theoretical_loss": 3.5589550939996766, + "tokens_seen": 1309671424 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030465396188565695, + "loss": 2.8559, + "theoretical_loss": 3.5589389808312584, + "tokens_seen": 1309736960 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003046439317953862, + "loss": 2.6363, + "theoretical_loss": 3.5589228686948235, + "tokens_seen": 1309802496 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030463390170511537, + "loss": 2.7361, + "theoretical_loss": 3.5589067575902553, + "tokens_seen": 1309868032 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030462387161484455, + "loss": 2.8687, + "theoretical_loss": 3.558890647517435, + "tokens_seen": 1309933568 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030461384152457373, + "loss": 2.471, + "theoretical_loss": 3.5588745384762452, + "tokens_seen": 1309999104 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003046038114343029, + "loss": 2.637, + "theoretical_loss": 3.558858430466569, + "tokens_seen": 1310064640 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003045937813440321, + "loss": 2.8307, + "theoretical_loss": 3.558842323488287, + "tokens_seen": 1310130176 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030458375125376133, + "loss": 2.6005, + "theoretical_loss": 3.5588262175412835, + "tokens_seen": 1310195712 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030457372116349046, + "loss": 2.6953, + "theoretical_loss": 3.55881011262544, + "tokens_seen": 1310261248 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003045636910732197, + "loss": 2.7861, + "theoretical_loss": 3.558794008740639, + "tokens_seen": 1310326784 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003045536609829488, + "loss": 2.959, + "theoretical_loss": 3.558777905886763, + "tokens_seen": 1310392320 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030454363089267805, + "loss": 2.8815, + "theoretical_loss": 3.5587618040636944, + "tokens_seen": 1310457856 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030453360080240723, + "loss": 2.7738, + "theoretical_loss": 3.558745703271315, + "tokens_seen": 1310523392 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003045235707121364, + "loss": 2.5894, + "theoretical_loss": 3.5587296035095086, + "tokens_seen": 1310588928 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003045135406218656, + "loss": 2.6306, + "theoretical_loss": 3.558713504778156, + "tokens_seen": 1310654464 + }, + { + "debugging/Self-BLEU-5": 0.2808054379606909, + "debugging/distinct-1-grams": 0.8084398703301545, + "debugging/distinct-2-grams": 0.9774642094229723, + "debugging/entropy-1-grams": 5.141925750104313, + "debugging/entropy-2-grams": 5.814304159018376, + "debugging/length": 488.8, + "debugging/num_segments": 5, + "debugging/score": 0.003229228566653429, + "debugging/score_std": 0.002968572177065524, + "epoch": 0.4, + "objective/train/docs_used": 742066, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.8688571453094482, + "objective/train/theoretical_loss": 3.558697407077142, + "objective/train/tokens_used": 1331180000, + "theoretical_loss": 3.558697407077142, + "tokens_seen": 1310720000 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030450351053159483, + "loss": 2.7947, + "theoretical_loss": 3.558697407077142, + "tokens_seen": 1310720000 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030449348044132396, + "loss": 2.6366, + "theoretical_loss": 3.5586813104063477, + "tokens_seen": 1310785536 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003044834503510532, + "loss": 2.7863, + "theoretical_loss": 3.5586652147656554, + "tokens_seen": 1310851072 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003044734202607823, + "loss": 2.9345, + "theoretical_loss": 3.558649120154948, + "tokens_seen": 1310916608 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030446339017051156, + "loss": 2.6065, + "theoretical_loss": 3.5586330265741086, + "tokens_seen": 1310982144 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030445336008024074, + "loss": 2.7558, + "theoretical_loss": 3.5586169340230196, + "tokens_seen": 1311047680 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003044433299899699, + "loss": 2.8296, + "theoretical_loss": 3.5586008425015634, + "tokens_seen": 1311113216 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003044332998996991, + "loss": 2.8347, + "theoretical_loss": 3.5585847520096228, + "tokens_seen": 1311178752 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003044232698094283, + "loss": 2.8185, + "theoretical_loss": 3.5585686625470805, + "tokens_seen": 1311244288 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030441323971915746, + "loss": 2.6592, + "theoretical_loss": 3.558552574113819, + "tokens_seen": 1311309824 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003044032096288867, + "loss": 2.7851, + "theoretical_loss": 3.558536486709721, + "tokens_seen": 1311375360 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003043931795386158, + "loss": 2.7611, + "theoretical_loss": 3.55852040033467, + "tokens_seen": 1311440896 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030438314944834506, + "loss": 2.9373, + "theoretical_loss": 3.5585043149885474, + "tokens_seen": 1311506432 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003043731193580742, + "loss": 2.8863, + "theoretical_loss": 3.5584882306712373, + "tokens_seen": 1311571968 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003043630892678034, + "loss": 2.7942, + "theoretical_loss": 3.5584721473826217, + "tokens_seen": 1311637504 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003043530591775326, + "loss": 2.4691, + "theoretical_loss": 3.5584560651225834, + "tokens_seen": 1311703040 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003043430290872618, + "loss": 2.6446, + "theoretical_loss": 3.558439983891006, + "tokens_seen": 1311768576 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030433299899699097, + "loss": 2.6019, + "theoretical_loss": 3.558423903687772, + "tokens_seen": 1311834112 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003043229689067202, + "loss": 2.79, + "theoretical_loss": 3.5584078245127637, + "tokens_seen": 1311899648 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030431293881644933, + "loss": 2.8258, + "theoretical_loss": 3.5583917463658645, + "tokens_seen": 1311965184 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030430290872617856, + "loss": 2.7027, + "theoretical_loss": 3.558375669246957, + "tokens_seen": 1312030720 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003042928786359077, + "loss": 2.5016, + "theoretical_loss": 3.5583595931559246, + "tokens_seen": 1312096256 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003042828485456369, + "loss": 3.0614, + "theoretical_loss": 3.5583435180926495, + "tokens_seen": 1312161792 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003042728184553661, + "loss": 2.8557, + "theoretical_loss": 3.558327444057016, + "tokens_seen": 1312227328 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003042627883650953, + "loss": 3.0357, + "theoretical_loss": 3.558311371048906, + "tokens_seen": 1312292864 + }, + { + "epoch": 0.4, + "objective/train/docs_used": 743478, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.808068037033081, + "objective/train/theoretical_loss": 3.5582952990682024, + "objective/train/tokens_used": 1332818400, + "theoretical_loss": 3.5582952990682024, + "tokens_seen": 1312358400 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030425275827482447, + "loss": 2.6421, + "theoretical_loss": 3.5582952990682024, + "tokens_seen": 1312358400 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030424272818455365, + "loss": 2.7857, + "theoretical_loss": 3.5582792281147886, + "tokens_seen": 1312423936 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030423269809428283, + "loss": 2.8439, + "theoretical_loss": 3.5582631581885478, + "tokens_seen": 1312489472 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030422266800401207, + "loss": 2.7866, + "theoretical_loss": 3.5582470892893627, + "tokens_seen": 1312555008 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003042126379137412, + "loss": 2.6876, + "theoretical_loss": 3.558231021417117, + "tokens_seen": 1312620544 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030420260782347043, + "loss": 2.3768, + "theoretical_loss": 3.5582149545716932, + "tokens_seen": 1312686080 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003041925777331996, + "loss": 2.6919, + "theoretical_loss": 3.5581988887529743, + "tokens_seen": 1312751616 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003041825476429288, + "loss": 2.4741, + "theoretical_loss": 3.558182823960844, + "tokens_seen": 1312817152 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030417251755265797, + "loss": 2.7414, + "theoretical_loss": 3.558166760195185, + "tokens_seen": 1312882688 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030416248746238715, + "loss": 2.753, + "theoretical_loss": 3.558150697455881, + "tokens_seen": 1312948224 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030415245737211633, + "loss": 2.795, + "theoretical_loss": 3.5581346357428147, + "tokens_seen": 1313013760 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030414242728184557, + "loss": 2.6958, + "theoretical_loss": 3.5581185750558695, + "tokens_seen": 1313079296 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003041323971915747, + "loss": 2.5995, + "theoretical_loss": 3.5581025153949284, + "tokens_seen": 1313144832 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030412236710130393, + "loss": 2.7241, + "theoretical_loss": 3.558086456759875, + "tokens_seen": 1313210368 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030411233701103306, + "loss": 2.5603, + "theoretical_loss": 3.5580703991505924, + "tokens_seen": 1313275904 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003041023069207623, + "loss": 2.7161, + "theoretical_loss": 3.5580543425669644, + "tokens_seen": 1313341440 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003040922768304915, + "loss": 2.8165, + "theoretical_loss": 3.5580382870088734, + "tokens_seen": 1313406976 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030408224674022066, + "loss": 2.6819, + "theoretical_loss": 3.5580222324762034, + "tokens_seen": 1313472512 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030407221664994984, + "loss": 2.8153, + "theoretical_loss": 3.5580061789688378, + "tokens_seen": 1313538048 + }, + { + "epoch": 0.4, + "learning_rate": 0.000304062186559679, + "loss": 2.6408, + "theoretical_loss": 3.557990126486659, + "tokens_seen": 1313603584 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030405215646940825, + "loss": 2.6724, + "theoretical_loss": 3.5579740750295517, + "tokens_seen": 1313669120 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030404212637913743, + "loss": 2.6963, + "theoretical_loss": 3.5579580245973985, + "tokens_seen": 1313734656 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003040320962888666, + "loss": 2.6941, + "theoretical_loss": 3.557941975190083, + "tokens_seen": 1313800192 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003040220661985958, + "loss": 2.7354, + "theoretical_loss": 3.5579259268074885, + "tokens_seen": 1313865728 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030401203610832503, + "loss": 2.7209, + "theoretical_loss": 3.557909879449499, + "tokens_seen": 1313931264 + }, + { + "epoch": 0.4, + "objective/train/docs_used": 744202, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.225639820098877, + "objective/train/theoretical_loss": 3.5578938331159975, + "objective/train/tokens_used": 1334456800, + "theoretical_loss": 3.5578938331159975, + "tokens_seen": 1313996800 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030400200601805416, + "loss": 2.6785, + "theoretical_loss": 3.5578938331159975, + "tokens_seen": 1313996800 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003039919759277834, + "loss": 2.5031, + "theoretical_loss": 3.5578777878068677, + "tokens_seen": 1314062336 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003039819458375125, + "loss": 2.6897, + "theoretical_loss": 3.5578617435219932, + "tokens_seen": 1314127872 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030397191574724176, + "loss": 2.7109, + "theoretical_loss": 3.557845700261257, + "tokens_seen": 1314193408 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030396188565697094, + "loss": 2.9145, + "theoretical_loss": 3.557829658024543, + "tokens_seen": 1314258944 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003039518555667001, + "loss": 2.8722, + "theoretical_loss": 3.5578136168117354, + "tokens_seen": 1314324480 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003039418254764293, + "loss": 2.5906, + "theoretical_loss": 3.5577975766227166, + "tokens_seen": 1314390016 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003039317953861585, + "loss": 2.9014, + "theoretical_loss": 3.5577815374573714, + "tokens_seen": 1314455552 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030392176529588766, + "loss": 2.6801, + "theoretical_loss": 3.5577654993155825, + "tokens_seen": 1314521088 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003039117352056169, + "loss": 2.7302, + "theoretical_loss": 3.5577494621972345, + "tokens_seen": 1314586624 + }, + { + "epoch": 0.4, + "learning_rate": 0.000303901705115346, + "loss": 2.8587, + "theoretical_loss": 3.55773342610221, + "tokens_seen": 1314652160 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030389167502507526, + "loss": 2.7384, + "theoretical_loss": 3.557717391030393, + "tokens_seen": 1314717696 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003038816449348044, + "loss": 2.6766, + "theoretical_loss": 3.557701356981668, + "tokens_seen": 1314783232 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003038716148445336, + "loss": 2.8675, + "theoretical_loss": 3.5576853239559183, + "tokens_seen": 1314848768 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003038615847542628, + "loss": 3.1361, + "theoretical_loss": 3.557669291953027, + "tokens_seen": 1314914304 + }, + { + "epoch": 0.4, + "learning_rate": 0.000303851554663992, + "loss": 2.7357, + "theoretical_loss": 3.5576532609728786, + "tokens_seen": 1314979840 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030384152457372117, + "loss": 2.8164, + "theoretical_loss": 3.557637231015357, + "tokens_seen": 1315045376 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003038314944834504, + "loss": 2.699, + "theoretical_loss": 3.5576212020803446, + "tokens_seen": 1315110912 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030382146439317953, + "loss": 2.7103, + "theoretical_loss": 3.557605174167727, + "tokens_seen": 1315176448 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030381143430290876, + "loss": 2.5751, + "theoretical_loss": 3.5575891472773877, + "tokens_seen": 1315241984 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003038014042126379, + "loss": 2.6772, + "theoretical_loss": 3.5575731214092094, + "tokens_seen": 1315307520 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003037913741223671, + "loss": 2.9825, + "theoretical_loss": 3.5575570965630776, + "tokens_seen": 1315373056 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003037813440320963, + "loss": 2.7774, + "theoretical_loss": 3.557541072738875, + "tokens_seen": 1315438592 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003037713139418255, + "loss": 2.8429, + "theoretical_loss": 3.557525049936486, + "tokens_seen": 1315504128 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030376128385155467, + "loss": 2.9558, + "theoretical_loss": 3.5575090281557946, + "tokens_seen": 1315569664 + }, + { + "epoch": 0.4, + "objective/train/docs_used": 745555, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.3200926780700684, + "objective/train/theoretical_loss": 3.557493007396684, + "objective/train/tokens_used": 1336095200, + "theoretical_loss": 3.557493007396684, + "tokens_seen": 1315635200 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030375125376128385, + "loss": 2.504, + "theoretical_loss": 3.557493007396684, + "tokens_seen": 1315635200 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030374122367101303, + "loss": 2.7915, + "theoretical_loss": 3.5574769876590397, + "tokens_seen": 1315700736 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030373119358074227, + "loss": 2.9014, + "theoretical_loss": 3.5574609689427446, + "tokens_seen": 1315766272 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003037211634904714, + "loss": 2.7835, + "theoretical_loss": 3.5574449512476822, + "tokens_seen": 1315831808 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030371113340020063, + "loss": 2.7644, + "theoretical_loss": 3.5574289345737378, + "tokens_seen": 1315897344 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003037011033099298, + "loss": 2.7304, + "theoretical_loss": 3.557412918920795, + "tokens_seen": 1315962880 + }, + { + "epoch": 0.4, + "learning_rate": 0.000303691073219659, + "loss": 2.7345, + "theoretical_loss": 3.5573969042887374, + "tokens_seen": 1316028416 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030368104312938817, + "loss": 2.5377, + "theoretical_loss": 3.55738089067745, + "tokens_seen": 1316093952 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030367101303911735, + "loss": 2.78, + "theoretical_loss": 3.557364878086816, + "tokens_seen": 1316159488 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030366098294884653, + "loss": 2.918, + "theoretical_loss": 3.55734886651672, + "tokens_seen": 1316225024 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030365095285857577, + "loss": 2.7614, + "theoretical_loss": 3.5573328559670463, + "tokens_seen": 1316290560 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003036409227683049, + "loss": 2.7329, + "theoretical_loss": 3.5573168464376783, + "tokens_seen": 1316356096 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030363089267803413, + "loss": 2.5911, + "theoretical_loss": 3.557300837928501, + "tokens_seen": 1316421632 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030362086258776326, + "loss": 2.9087, + "theoretical_loss": 3.5572848304393982, + "tokens_seen": 1316487168 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003036108324974925, + "loss": 2.6876, + "theoretical_loss": 3.5572688239702543, + "tokens_seen": 1316552704 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003036008024072217, + "loss": 2.7998, + "theoretical_loss": 3.5572528185209538, + "tokens_seen": 1316618240 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030359077231695086, + "loss": 2.7623, + "theoretical_loss": 3.55723681409138, + "tokens_seen": 1316683776 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030358074222668004, + "loss": 2.7831, + "theoretical_loss": 3.5572208106814185, + "tokens_seen": 1316749312 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003035707121364092, + "loss": 2.8137, + "theoretical_loss": 3.557204808290953, + "tokens_seen": 1316814848 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003035606820461384, + "loss": 2.9662, + "theoretical_loss": 3.557188806919867, + "tokens_seen": 1316880384 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030355065195586763, + "loss": 2.8395, + "theoretical_loss": 3.5571728065680457, + "tokens_seen": 1316945920 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030354062186559676, + "loss": 2.9447, + "theoretical_loss": 3.5571568072353736, + "tokens_seen": 1317011456 + }, + { + "epoch": 0.4, + "learning_rate": 0.000303530591775326, + "loss": 2.7205, + "theoretical_loss": 3.5571408089217353, + "tokens_seen": 1317076992 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003035205616850552, + "loss": 2.7391, + "theoretical_loss": 3.557124811627014, + "tokens_seen": 1317142528 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030351053159478436, + "loss": 2.9419, + "theoretical_loss": 3.5571088153510955, + "tokens_seen": 1317208064 + }, + { + "epoch": 0.4, + "objective/train/docs_used": 746236, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.822262763977051, + "objective/train/theoretical_loss": 3.557092820093863, + "objective/train/tokens_used": 1337733600, + "theoretical_loss": 3.557092820093863, + "tokens_seen": 1317273600 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030350050150451354, + "loss": 2.8262, + "theoretical_loss": 3.557092820093863, + "tokens_seen": 1317273600 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003034904714142427, + "loss": 2.9109, + "theoretical_loss": 3.5570768258552015, + "tokens_seen": 1317339136 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003034804413239719, + "loss": 2.8639, + "theoretical_loss": 3.5570608326349955, + "tokens_seen": 1317404672 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030347041123370114, + "loss": 2.4582, + "theoretical_loss": 3.5570448404331296, + "tokens_seen": 1317470208 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030346038114343026, + "loss": 2.704, + "theoretical_loss": 3.5570288492494884, + "tokens_seen": 1317535744 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003034503510531595, + "loss": 2.677, + "theoretical_loss": 3.5570128590839563, + "tokens_seen": 1317601280 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003034403209628886, + "loss": 2.6345, + "theoretical_loss": 3.5569968699364174, + "tokens_seen": 1317666816 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030343029087261786, + "loss": 2.8018, + "theoretical_loss": 3.5569808818067568, + "tokens_seen": 1317732352 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030342026078234704, + "loss": 2.532, + "theoretical_loss": 3.5569648946948584, + "tokens_seen": 1317797888 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003034102306920762, + "loss": 2.8285, + "theoretical_loss": 3.556948908600608, + "tokens_seen": 1317863424 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003034002006018054, + "loss": 2.6509, + "theoretical_loss": 3.556932923523889, + "tokens_seen": 1317928960 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003033901705115346, + "loss": 2.7614, + "theoretical_loss": 3.556916939464587, + "tokens_seen": 1317994496 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030338014042126377, + "loss": 2.7949, + "theoretical_loss": 3.556900956422586, + "tokens_seen": 1318060032 + }, + { + "epoch": 0.4, + "learning_rate": 0.000303370110330993, + "loss": 2.6685, + "theoretical_loss": 3.5568849743977706, + "tokens_seen": 1318125568 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030336008024072213, + "loss": 3.0776, + "theoretical_loss": 3.5568689933900264, + "tokens_seen": 1318191104 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030335005015045137, + "loss": 2.8651, + "theoretical_loss": 3.5568530133992375, + "tokens_seen": 1318256640 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030334002006018055, + "loss": 2.4253, + "theoretical_loss": 3.556837034425288, + "tokens_seen": 1318322176 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030332998996990973, + "loss": 2.8652, + "theoretical_loss": 3.556821056468064, + "tokens_seen": 1318387712 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003033199598796389, + "loss": 2.8559, + "theoretical_loss": 3.5568050795274493, + "tokens_seen": 1318453248 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003033099297893681, + "loss": 2.7554, + "theoretical_loss": 3.556789103603329, + "tokens_seen": 1318518784 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003032998996990973, + "loss": 2.5539, + "theoretical_loss": 3.5567731286955873, + "tokens_seen": 1318584320 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003032898696088265, + "loss": 2.7626, + "theoretical_loss": 3.55675715480411, + "tokens_seen": 1318649856 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003032798395185557, + "loss": 2.9094, + "theoretical_loss": 3.5567411819287815, + "tokens_seen": 1318715392 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030326980942828487, + "loss": 2.875, + "theoretical_loss": 3.556725210069487, + "tokens_seen": 1318780928 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030325977933801405, + "loss": 2.4969, + "theoretical_loss": 3.5567092392261106, + "tokens_seen": 1318846464 + }, + { + "epoch": 0.4, + "objective/train/docs_used": 747487, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.512132406234741, + "objective/train/theoretical_loss": 3.5566932693985382, + "objective/train/tokens_used": 1339372000, + "theoretical_loss": 3.5566932693985382, + "tokens_seen": 1318912000 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030324974924774323, + "loss": 2.6198, + "theoretical_loss": 3.5566932693985382, + "tokens_seen": 1318912000 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030323971915747247, + "loss": 2.7745, + "theoretical_loss": 3.556677300586654, + "tokens_seen": 1318977536 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003032296890672016, + "loss": 2.7831, + "theoretical_loss": 3.556661332790343, + "tokens_seen": 1319043072 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030321965897693083, + "loss": 2.6368, + "theoretical_loss": 3.55664536600949, + "tokens_seen": 1319108608 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030320962888666, + "loss": 2.6037, + "theoretical_loss": 3.556629400243981, + "tokens_seen": 1319174144 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003031995987963892, + "loss": 2.6194, + "theoretical_loss": 3.5566134354937, + "tokens_seen": 1319239680 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030318956870611837, + "loss": 2.7083, + "theoretical_loss": 3.5565974717585322, + "tokens_seen": 1319305216 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030317953861584755, + "loss": 2.8423, + "theoretical_loss": 3.556581509038363, + "tokens_seen": 1319370752 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030316950852557673, + "loss": 2.5668, + "theoretical_loss": 3.5565655473330766, + "tokens_seen": 1319436288 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030315947843530597, + "loss": 2.5527, + "theoretical_loss": 3.556549586642559, + "tokens_seen": 1319501824 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003031494483450351, + "loss": 2.8386, + "theoretical_loss": 3.5565336269666954, + "tokens_seen": 1319567360 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030313941825476433, + "loss": 2.7792, + "theoretical_loss": 3.5565176683053696, + "tokens_seen": 1319632896 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030312938816449346, + "loss": 2.6571, + "theoretical_loss": 3.5565017106584684, + "tokens_seen": 1319698432 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003031193580742227, + "loss": 2.688, + "theoretical_loss": 3.556485754025876, + "tokens_seen": 1319763968 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003031093279839519, + "loss": 2.7352, + "theoretical_loss": 3.556469798407477, + "tokens_seen": 1319829504 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030309929789368106, + "loss": 2.5138, + "theoretical_loss": 3.556453843803158, + "tokens_seen": 1319895040 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030308926780341024, + "loss": 2.4766, + "theoretical_loss": 3.556437890212803, + "tokens_seen": 1319960576 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003030792377131394, + "loss": 2.8434, + "theoretical_loss": 3.5564219376362978, + "tokens_seen": 1320026112 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003030692076228686, + "loss": 2.6396, + "theoretical_loss": 3.5564059860735275, + "tokens_seen": 1320091648 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030305917753259783, + "loss": 2.5377, + "theoretical_loss": 3.5563900355243776, + "tokens_seen": 1320157184 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030304914744232696, + "loss": 2.8628, + "theoretical_loss": 3.5563740859887334, + "tokens_seen": 1320222720 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003030391173520562, + "loss": 2.6002, + "theoretical_loss": 3.5563581374664794, + "tokens_seen": 1320288256 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003030290872617854, + "loss": 2.6769, + "theoretical_loss": 3.556342189957502, + "tokens_seen": 1320353792 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030301905717151456, + "loss": 2.4929, + "theoretical_loss": 3.556326243461686, + "tokens_seen": 1320419328 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030300902708124374, + "loss": 2.588, + "theoretical_loss": 3.556310297978916, + "tokens_seen": 1320484864 + }, + { + "epoch": 0.4, + "objective/train/docs_used": 748180, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.769991159439087, + "objective/train/theoretical_loss": 3.556294353509079, + "objective/train/tokens_used": 1341010400, + "theoretical_loss": 3.556294353509079, + "tokens_seen": 1320550400 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003029989969909729, + "loss": 2.7585, + "theoretical_loss": 3.556294353509079, + "tokens_seen": 1320550400 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003029889669007021, + "loss": 2.5909, + "theoretical_loss": 3.5562784100520592, + "tokens_seen": 1320615936 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030297893681043134, + "loss": 2.6116, + "theoretical_loss": 3.556262467607742, + "tokens_seen": 1320681472 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030296890672016046, + "loss": 2.9328, + "theoretical_loss": 3.5562465261760137, + "tokens_seen": 1320747008 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003029588766298897, + "loss": 2.6611, + "theoretical_loss": 3.556230585756759, + "tokens_seen": 1320812544 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003029488465396188, + "loss": 2.616, + "theoretical_loss": 3.556214646349864, + "tokens_seen": 1320878080 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030293881644934806, + "loss": 2.5927, + "theoretical_loss": 3.5561987079552133, + "tokens_seen": 1320943616 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030292878635907724, + "loss": 2.6941, + "theoretical_loss": 3.5561827705726925, + "tokens_seen": 1321009152 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003029187562688064, + "loss": 2.7696, + "theoretical_loss": 3.5561668342021884, + "tokens_seen": 1321074688 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003029087261785356, + "loss": 2.8404, + "theoretical_loss": 3.556150898843585, + "tokens_seen": 1321140224 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003028986960882648, + "loss": 2.8874, + "theoretical_loss": 3.556134964496769, + "tokens_seen": 1321205760 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030288866599799397, + "loss": 2.8127, + "theoretical_loss": 3.556119031161624, + "tokens_seen": 1321271296 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003028786359077232, + "loss": 2.7002, + "theoretical_loss": 3.5561030988380384, + "tokens_seen": 1321336832 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030286860581745233, + "loss": 2.5839, + "theoretical_loss": 3.556087167525896, + "tokens_seen": 1321402368 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030285857572718157, + "loss": 2.7066, + "theoretical_loss": 3.5560712372250833, + "tokens_seen": 1321467904 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030284854563691075, + "loss": 2.6829, + "theoretical_loss": 3.556055307935485, + "tokens_seen": 1321533440 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030283851554663993, + "loss": 2.8838, + "theoretical_loss": 3.556039379656988, + "tokens_seen": 1321598976 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003028284854563691, + "loss": 2.753, + "theoretical_loss": 3.5560234523894763, + "tokens_seen": 1321664512 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003028184553660983, + "loss": 2.8894, + "theoretical_loss": 3.556007526132837, + "tokens_seen": 1321730048 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030280842527582747, + "loss": 2.7923, + "theoretical_loss": 3.555991600886956, + "tokens_seen": 1321795584 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003027983951855567, + "loss": 2.843, + "theoretical_loss": 3.555975676651718, + "tokens_seen": 1321861120 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030278836509528583, + "loss": 2.7781, + "theoretical_loss": 3.555959753427009, + "tokens_seen": 1321926656 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030277833500501507, + "loss": 2.7964, + "theoretical_loss": 3.5559438312127147, + "tokens_seen": 1321992192 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003027683049147442, + "loss": 2.5869, + "theoretical_loss": 3.555927910008722, + "tokens_seen": 1322057728 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030275827482447343, + "loss": 2.6843, + "theoretical_loss": 3.5559119898149154, + "tokens_seen": 1322123264 + }, + { + "epoch": 0.4, + "objective/train/docs_used": 749233, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.741284132003784, + "objective/train/theoretical_loss": 3.5558960706311815, + "objective/train/tokens_used": 1342648800, + "theoretical_loss": 3.5558960706311815, + "tokens_seen": 1322188800 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003027482447342026, + "loss": 2.9718, + "theoretical_loss": 3.5558960706311815, + "tokens_seen": 1322188800 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003027382146439318, + "loss": 2.7531, + "theoretical_loss": 3.555880152457406, + "tokens_seen": 1322254336 + }, + { + "epoch": 0.4, + "learning_rate": 0.000302728184553661, + "loss": 2.7012, + "theoretical_loss": 3.555864235293474, + "tokens_seen": 1322319872 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003027181544633902, + "loss": 2.6029, + "theoretical_loss": 3.5558483191392734, + "tokens_seen": 1322385408 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030270812437311934, + "loss": 2.532, + "theoretical_loss": 3.5558324039946876, + "tokens_seen": 1322450944 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030269809428284857, + "loss": 2.9132, + "theoretical_loss": 3.555816489859604, + "tokens_seen": 1322516480 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003026880641925777, + "loss": 2.5737, + "theoretical_loss": 3.5558005767339083, + "tokens_seen": 1322582016 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030267803410230693, + "loss": 2.6383, + "theoretical_loss": 3.5557846646174864, + "tokens_seen": 1322647552 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003026680040120361, + "loss": 2.6886, + "theoretical_loss": 3.5557687535102245, + "tokens_seen": 1322713088 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003026579739217653, + "loss": 2.8626, + "theoretical_loss": 3.5557528434120083, + "tokens_seen": 1322778624 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003026479438314945, + "loss": 2.9123, + "theoretical_loss": 3.5557369343227236, + "tokens_seen": 1322844160 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030263791374122366, + "loss": 2.8616, + "theoretical_loss": 3.5557210262422574, + "tokens_seen": 1322909696 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030262788365095284, + "loss": 2.7586, + "theoretical_loss": 3.5557051191704945, + "tokens_seen": 1322975232 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003026178535606821, + "loss": 2.7487, + "theoretical_loss": 3.555689213107322, + "tokens_seen": 1323040768 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003026078234704112, + "loss": 2.656, + "theoretical_loss": 3.5556733080526257, + "tokens_seen": 1323106304 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030259779338014044, + "loss": 3.0089, + "theoretical_loss": 3.5556574040062916, + "tokens_seen": 1323171840 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030258776328986956, + "loss": 2.8442, + "theoretical_loss": 3.555641500968205, + "tokens_seen": 1323237376 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003025777331995988, + "loss": 2.5696, + "theoretical_loss": 3.5556255989382537, + "tokens_seen": 1323302912 + }, + { + "epoch": 0.4, + "learning_rate": 0.000302567703109328, + "loss": 2.7604, + "theoretical_loss": 3.5556096979163234, + "tokens_seen": 1323368448 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030255767301905716, + "loss": 2.8256, + "theoretical_loss": 3.555593797902299, + "tokens_seen": 1323433984 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003025476429287864, + "loss": 2.7902, + "theoretical_loss": 3.5555778988960682, + "tokens_seen": 1323499520 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003025376128385156, + "loss": 2.7776, + "theoretical_loss": 3.555562000897517, + "tokens_seen": 1323565056 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030252758274824476, + "loss": 2.8276, + "theoretical_loss": 3.5555461039065306, + "tokens_seen": 1323630592 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030251755265797394, + "loss": 2.7027, + "theoretical_loss": 3.5555302079229962, + "tokens_seen": 1323696128 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003025075225677031, + "loss": 2.8487, + "theoretical_loss": 3.5555143129467996, + "tokens_seen": 1323761664 + }, + { + "epoch": 0.4, + "objective/train/docs_used": 749806, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6512277126312256, + "objective/train/theoretical_loss": 3.555498418977828, + "objective/train/tokens_used": 1344287200, + "theoretical_loss": 3.555498418977828, + "tokens_seen": 1323827200 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003024974924774323, + "loss": 2.6681, + "theoretical_loss": 3.555498418977828, + "tokens_seen": 1323827200 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030248746238716154, + "loss": 2.8547, + "theoretical_loss": 3.555482526015967, + "tokens_seen": 1323892736 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030247743229689066, + "loss": 2.8402, + "theoretical_loss": 3.5554666340611023, + "tokens_seen": 1323958272 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003024674022066199, + "loss": 2.5608, + "theoretical_loss": 3.5554507431131217, + "tokens_seen": 1324023808 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030245737211634903, + "loss": 2.556, + "theoretical_loss": 3.555434853171911, + "tokens_seen": 1324089344 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030244734202607826, + "loss": 2.8911, + "theoretical_loss": 3.555418964237356, + "tokens_seen": 1324154880 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030243731193580744, + "loss": 2.7218, + "theoretical_loss": 3.5554030763093434, + "tokens_seen": 1324220416 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003024272818455366, + "loss": 2.6432, + "theoretical_loss": 3.55538718938776, + "tokens_seen": 1324285952 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003024172517552658, + "loss": 2.3284, + "theoretical_loss": 3.555371303472492, + "tokens_seen": 1324351488 + }, + { + "epoch": 0.4, + "learning_rate": 0.000302407221664995, + "loss": 2.739, + "theoretical_loss": 3.555355418563426, + "tokens_seen": 1324417024 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030239719157472417, + "loss": 2.8839, + "theoretical_loss": 3.5553395346604484, + "tokens_seen": 1324482560 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003023871614844534, + "loss": 2.715, + "theoretical_loss": 3.5553236517634454, + "tokens_seen": 1324548096 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030237713139418253, + "loss": 2.9136, + "theoretical_loss": 3.5553077698723037, + "tokens_seen": 1324613632 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030236710130391177, + "loss": 2.8037, + "theoretical_loss": 3.5552918889869103, + "tokens_seen": 1324679168 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030235707121364095, + "loss": 2.8092, + "theoretical_loss": 3.5552760091071507, + "tokens_seen": 1324744704 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030234704112337013, + "loss": 2.7088, + "theoretical_loss": 3.555260130232913, + "tokens_seen": 1324810240 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003023370110330993, + "loss": 2.6225, + "theoretical_loss": 3.5552442523640826, + "tokens_seen": 1324875776 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003023269809428285, + "loss": 2.7675, + "theoretical_loss": 3.5552283755005463, + "tokens_seen": 1324941312 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030231695085255767, + "loss": 2.8563, + "theoretical_loss": 3.5552124996421908, + "tokens_seen": 1325006848 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003023069207622869, + "loss": 2.8391, + "theoretical_loss": 3.5551966247889033, + "tokens_seen": 1325072384 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030229689067201603, + "loss": 2.5565, + "theoretical_loss": 3.5551807509405697, + "tokens_seen": 1325137920 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030228686058174527, + "loss": 2.6991, + "theoretical_loss": 3.5551648780970773, + "tokens_seen": 1325203456 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003022768304914744, + "loss": 2.6445, + "theoretical_loss": 3.555149006258312, + "tokens_seen": 1325268992 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030226680040120363, + "loss": 2.8356, + "theoretical_loss": 3.5551331354241613, + "tokens_seen": 1325334528 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003022567703109328, + "loss": 2.6488, + "theoretical_loss": 3.5551172655945114, + "tokens_seen": 1325400064 + }, + { + "epoch": 0.4, + "objective/train/docs_used": 750658, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.8016040325164795, + "objective/train/theoretical_loss": 3.55510139676925, + "objective/train/tokens_used": 1345925600, + "theoretical_loss": 3.55510139676925, + "tokens_seen": 1325465600 + }, + { + "epoch": 0.4, + "learning_rate": 0.000302246740220662, + "loss": 2.5342, + "theoretical_loss": 3.55510139676925, + "tokens_seen": 1325465600 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003022367101303912, + "loss": 2.7499, + "theoretical_loss": 3.555085528948262, + "tokens_seen": 1325531136 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003022266800401204, + "loss": 2.8614, + "theoretical_loss": 3.5550696621314364, + "tokens_seen": 1325596672 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030221664994984954, + "loss": 2.6206, + "theoretical_loss": 3.555053796318659, + "tokens_seen": 1325662208 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030220661985957877, + "loss": 2.7462, + "theoretical_loss": 3.5550379315098164, + "tokens_seen": 1325727744 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003021965897693079, + "loss": 2.9203, + "theoretical_loss": 3.5550220677047952, + "tokens_seen": 1325793280 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030218655967903713, + "loss": 2.6327, + "theoretical_loss": 3.555006204903483, + "tokens_seen": 1325858816 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003021765295887663, + "loss": 2.7967, + "theoretical_loss": 3.554990343105767, + "tokens_seen": 1325924352 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003021664994984955, + "loss": 2.8361, + "theoretical_loss": 3.554974482311533, + "tokens_seen": 1325989888 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003021564694082247, + "loss": 2.5471, + "theoretical_loss": 3.554958622520669, + "tokens_seen": 1326055424 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030214643931795386, + "loss": 2.8212, + "theoretical_loss": 3.5549427637330604, + "tokens_seen": 1326120960 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030213640922768304, + "loss": 2.5994, + "theoretical_loss": 3.554926905948596, + "tokens_seen": 1326186496 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003021263791374123, + "loss": 2.5763, + "theoretical_loss": 3.5549110491671616, + "tokens_seen": 1326252032 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003021163490471414, + "loss": 2.8415, + "theoretical_loss": 3.554895193388645, + "tokens_seen": 1326317568 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030210631895687064, + "loss": 2.8966, + "theoretical_loss": 3.5548793386129325, + "tokens_seen": 1326383104 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030209628886659976, + "loss": 2.6833, + "theoretical_loss": 3.5548634848399114, + "tokens_seen": 1326448640 + }, + { + "epoch": 0.4, + "learning_rate": 0.000302086258776329, + "loss": 2.7768, + "theoretical_loss": 3.5548476320694684, + "tokens_seen": 1326514176 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003020762286860582, + "loss": 2.6625, + "theoretical_loss": 3.554831780301491, + "tokens_seen": 1326579712 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030206619859578736, + "loss": 2.5274, + "theoretical_loss": 3.554815929535867, + "tokens_seen": 1326645248 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030205616850551654, + "loss": 2.4785, + "theoretical_loss": 3.554800079772482, + "tokens_seen": 1326710784 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003020461384152458, + "loss": 2.694, + "theoretical_loss": 3.554784231011224, + "tokens_seen": 1326776320 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003020361083249749, + "loss": 2.4311, + "theoretical_loss": 3.55476838325198, + "tokens_seen": 1326841856 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030202607823470414, + "loss": 2.7084, + "theoretical_loss": 3.5547525364946373, + "tokens_seen": 1326907392 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030201604814443327, + "loss": 2.9757, + "theoretical_loss": 3.554736690739083, + "tokens_seen": 1326972928 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003020060180541625, + "loss": 2.7145, + "theoretical_loss": 3.5547208459852033, + "tokens_seen": 1327038464 + }, + { + "epoch": 0.4, + "objective/train/docs_used": 751457, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.393244981765747, + "objective/train/theoretical_loss": 3.5547050022328874, + "objective/train/tokens_used": 1347564000, + "theoretical_loss": 3.5547050022328874, + "tokens_seen": 1327104000 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003019959879638917, + "loss": 2.7392, + "theoretical_loss": 3.5547050022328874, + "tokens_seen": 1327104000 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030198595787362087, + "loss": 2.9306, + "theoretical_loss": 3.554689159482021, + "tokens_seen": 1327169536 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030197592778335005, + "loss": 2.758, + "theoretical_loss": 3.554673317732492, + "tokens_seen": 1327235072 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030196589769307923, + "loss": 2.6271, + "theoretical_loss": 3.5546574769841874, + "tokens_seen": 1327300608 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003019558676028084, + "loss": 2.7376, + "theoretical_loss": 3.554641637236995, + "tokens_seen": 1327366144 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030194583751253764, + "loss": 2.7852, + "theoretical_loss": 3.554625798490801, + "tokens_seen": 1327431680 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030193580742226677, + "loss": 2.7784, + "theoretical_loss": 3.554609960745494, + "tokens_seen": 1327497216 + }, + { + "epoch": 0.4, + "learning_rate": 0.000301925777331996, + "loss": 2.7527, + "theoretical_loss": 3.5545941240009604, + "tokens_seen": 1327562752 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030191574724172513, + "loss": 2.3607, + "theoretical_loss": 3.5545782882570878, + "tokens_seen": 1327628288 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030190571715145437, + "loss": 2.5971, + "theoretical_loss": 3.554562453513764, + "tokens_seen": 1327693824 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030189568706118355, + "loss": 2.5812, + "theoretical_loss": 3.554546619770876, + "tokens_seen": 1327759360 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030188565697091273, + "loss": 2.7348, + "theoretical_loss": 3.5545307870283116, + "tokens_seen": 1327824896 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003018756268806419, + "loss": 2.897, + "theoretical_loss": 3.554514955285958, + "tokens_seen": 1327890432 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030186559679037115, + "loss": 2.696, + "theoretical_loss": 3.5544991245437023, + "tokens_seen": 1327955968 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003018555667001003, + "loss": 2.7745, + "theoretical_loss": 3.5544832948014324, + "tokens_seen": 1328021504 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003018455366098295, + "loss": 2.6662, + "theoretical_loss": 3.5544674660590356, + "tokens_seen": 1328087040 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030183550651955864, + "loss": 2.7788, + "theoretical_loss": 3.5544516383164, + "tokens_seen": 1328152576 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030182547642928787, + "loss": 2.5707, + "theoretical_loss": 3.554435811573412, + "tokens_seen": 1328218112 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030181544633901705, + "loss": 2.7995, + "theoretical_loss": 3.55441998582996, + "tokens_seen": 1328283648 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030180541624874623, + "loss": 2.6116, + "theoretical_loss": 3.554404161085931, + "tokens_seen": 1328349184 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030179538615847547, + "loss": 2.3785, + "theoretical_loss": 3.554388337341213, + "tokens_seen": 1328414720 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003017853560682046, + "loss": 2.6847, + "theoretical_loss": 3.554372514595694, + "tokens_seen": 1328480256 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030177532597793383, + "loss": 2.7232, + "theoretical_loss": 3.554356692849261, + "tokens_seen": 1328545792 + }, + { + "epoch": 0.4, + "learning_rate": 0.000301765295887663, + "loss": 2.9167, + "theoretical_loss": 3.5543408721018013, + "tokens_seen": 1328611328 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003017552657973922, + "loss": 2.7022, + "theoretical_loss": 3.5543250523532035, + "tokens_seen": 1328676864 + }, + { + "epoch": 0.4, + "objective/train/docs_used": 752138, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.0572965145111084, + "objective/train/theoretical_loss": 3.5543092336033544, + "objective/train/tokens_used": 1349202400, + "theoretical_loss": 3.5543092336033544, + "tokens_seen": 1328742400 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003017452357071214, + "loss": 2.7774, + "theoretical_loss": 3.5543092336033544, + "tokens_seen": 1328742400 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003017352056168506, + "loss": 2.5149, + "theoretical_loss": 3.5542934158521424, + "tokens_seen": 1328807936 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030172517552657974, + "loss": 2.8782, + "theoretical_loss": 3.554277599099455, + "tokens_seen": 1328873472 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030171514543630897, + "loss": 3.0795, + "theoretical_loss": 3.5542617833451793, + "tokens_seen": 1328939008 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003017051153460381, + "loss": 2.5716, + "theoretical_loss": 3.554245968589204, + "tokens_seen": 1329004544 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030169508525576733, + "loss": 2.6762, + "theoretical_loss": 3.554230154831416, + "tokens_seen": 1329070080 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003016850551654965, + "loss": 2.716, + "theoretical_loss": 3.5542143420717043, + "tokens_seen": 1329135616 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003016750250752257, + "loss": 2.5188, + "theoretical_loss": 3.554198530309955, + "tokens_seen": 1329201152 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003016649949849549, + "loss": 3.0159, + "theoretical_loss": 3.5541827195460574, + "tokens_seen": 1329266688 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030165496489468406, + "loss": 2.4216, + "theoretical_loss": 3.554166909779898, + "tokens_seen": 1329332224 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030164493480441324, + "loss": 2.7091, + "theoretical_loss": 3.5541511010113664, + "tokens_seen": 1329397760 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003016349047141425, + "loss": 2.8578, + "theoretical_loss": 3.554135293240349, + "tokens_seen": 1329463296 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003016248746238716, + "loss": 2.7872, + "theoretical_loss": 3.554119486466734, + "tokens_seen": 1329528832 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030161484453360084, + "loss": 2.8126, + "theoretical_loss": 3.55410368069041, + "tokens_seen": 1329594368 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030160481444332996, + "loss": 2.8519, + "theoretical_loss": 3.554087875911264, + "tokens_seen": 1329659904 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003015947843530592, + "loss": 2.8452, + "theoretical_loss": 3.554072072129184, + "tokens_seen": 1329725440 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003015847542627884, + "loss": 2.9181, + "theoretical_loss": 3.554056269344059, + "tokens_seen": 1329790976 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030157472417251756, + "loss": 2.7795, + "theoretical_loss": 3.554040467555776, + "tokens_seen": 1329856512 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030156469408224674, + "loss": 2.7593, + "theoretical_loss": 3.5540246667642235, + "tokens_seen": 1329922048 + }, + { + "epoch": 0.4, + "learning_rate": 0.000301554663991976, + "loss": 2.856, + "theoretical_loss": 3.554008866969289, + "tokens_seen": 1329987584 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003015446339017051, + "loss": 2.8167, + "theoretical_loss": 3.553993068170861, + "tokens_seen": 1330053120 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030153460381143434, + "loss": 2.5627, + "theoretical_loss": 3.5539772703688275, + "tokens_seen": 1330118656 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030152457372116347, + "loss": 2.7604, + "theoretical_loss": 3.553961473563076, + "tokens_seen": 1330184192 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003015145436308927, + "loss": 2.8246, + "theoretical_loss": 3.5539456777534957, + "tokens_seen": 1330249728 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003015045135406219, + "loss": 2.5076, + "theoretical_loss": 3.5539298829399737, + "tokens_seen": 1330315264 + }, + { + "epoch": 0.4, + "objective/train/docs_used": 753163, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.27243971824646, + "objective/train/theoretical_loss": 3.553914089122399, + "objective/train/tokens_used": 1350840800, + "theoretical_loss": 3.553914089122399, + "tokens_seen": 1330380800 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030149448345035107, + "loss": 2.9, + "theoretical_loss": 3.553914089122399, + "tokens_seen": 1330380800 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030148445336008025, + "loss": 2.742, + "theoretical_loss": 3.5538982963006585, + "tokens_seen": 1330446336 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030147442326980943, + "loss": 2.7952, + "theoretical_loss": 3.5538825044746414, + "tokens_seen": 1330511872 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003014643931795386, + "loss": 2.5422, + "theoretical_loss": 3.5538667136442355, + "tokens_seen": 1330577408 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030145436308926784, + "loss": 2.5313, + "theoretical_loss": 3.553850923809329, + "tokens_seen": 1330642944 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030144433299899697, + "loss": 2.6584, + "theoretical_loss": 3.5538351349698107, + "tokens_seen": 1330708480 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003014343029087262, + "loss": 2.8268, + "theoretical_loss": 3.553819347125568, + "tokens_seen": 1330774016 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030142427281845533, + "loss": 2.5823, + "theoretical_loss": 3.5538035602764895, + "tokens_seen": 1330839552 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030141424272818457, + "loss": 2.8169, + "theoretical_loss": 3.5537877744224633, + "tokens_seen": 1330905088 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030140421263791375, + "loss": 2.7788, + "theoretical_loss": 3.553771989563378, + "tokens_seen": 1330970624 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030139418254764293, + "loss": 2.714, + "theoretical_loss": 3.5537562056991217, + "tokens_seen": 1331036160 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003013841524573721, + "loss": 2.9966, + "theoretical_loss": 3.5537404228295824, + "tokens_seen": 1331101696 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030137412236710135, + "loss": 2.7105, + "theoretical_loss": 3.5537246409546492, + "tokens_seen": 1331167232 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003013640922768305, + "loss": 2.7565, + "theoretical_loss": 3.5537088600742104, + "tokens_seen": 1331232768 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003013540621865597, + "loss": 2.7252, + "theoretical_loss": 3.5536930801881534, + "tokens_seen": 1331298304 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030134403209628884, + "loss": 2.6811, + "theoretical_loss": 3.5536773012963674, + "tokens_seen": 1331363840 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030133400200601807, + "loss": 2.7609, + "theoretical_loss": 3.553661523398741, + "tokens_seen": 1331429376 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030132397191574725, + "loss": 2.6647, + "theoretical_loss": 3.553645746495162, + "tokens_seen": 1331494912 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030131394182547643, + "loss": 2.7151, + "theoretical_loss": 3.553629970585519, + "tokens_seen": 1331560448 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003013039117352056, + "loss": 2.7632, + "theoretical_loss": 3.5536141956697005, + "tokens_seen": 1331625984 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003012938816449348, + "loss": 2.8974, + "theoretical_loss": 3.5535984217475955, + "tokens_seen": 1331691520 + }, + { + "epoch": 0.4, + "learning_rate": 0.000301283851554664, + "loss": 2.6704, + "theoretical_loss": 3.5535826488190922, + "tokens_seen": 1331757056 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003012738214643932, + "loss": 2.7974, + "theoretical_loss": 3.5535668768840782, + "tokens_seen": 1331822592 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030126379137412234, + "loss": 2.5203, + "theoretical_loss": 3.553551105942444, + "tokens_seen": 1331888128 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003012537612838516, + "loss": 2.6648, + "theoretical_loss": 3.5535353359940762, + "tokens_seen": 1331953664 + }, + { + "epoch": 0.4, + "objective/train/docs_used": 753870, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.709228992462158, + "objective/train/theoretical_loss": 3.5535195670388644, + "objective/train/tokens_used": 1352479200, + "theoretical_loss": 3.5535195670388644, + "tokens_seen": 1332019200 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003012437311935807, + "loss": 2.9034, + "theoretical_loss": 3.5535195670388644, + "tokens_seen": 1332019200 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030123370110330994, + "loss": 2.7414, + "theoretical_loss": 3.553503799076697, + "tokens_seen": 1332084736 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003012236710130391, + "loss": 2.7759, + "theoretical_loss": 3.5534880321074627, + "tokens_seen": 1332150272 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003012136409227683, + "loss": 2.9212, + "theoretical_loss": 3.55347226613105, + "tokens_seen": 1332215808 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003012036108324975, + "loss": 2.7421, + "theoretical_loss": 3.5534565011473473, + "tokens_seen": 1332281344 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003011935807422267, + "loss": 2.9612, + "theoretical_loss": 3.5534407371562438, + "tokens_seen": 1332346880 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030118355065195584, + "loss": 2.8173, + "theoretical_loss": 3.553424974157628, + "tokens_seen": 1332412416 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003011735205616851, + "loss": 2.5563, + "theoretical_loss": 3.5534092121513883, + "tokens_seen": 1332477952 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003011634904714142, + "loss": 2.8009, + "theoretical_loss": 3.5533934511374135, + "tokens_seen": 1332543488 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030115346038114344, + "loss": 2.8479, + "theoretical_loss": 3.553377691115593, + "tokens_seen": 1332609024 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003011434302908726, + "loss": 2.448, + "theoretical_loss": 3.5533619320858145, + "tokens_seen": 1332674560 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003011334002006018, + "loss": 2.7808, + "theoretical_loss": 3.5533461740479675, + "tokens_seen": 1332740096 + }, + { + "epoch": 0.4, + "learning_rate": 0.000301123370110331, + "loss": 2.8045, + "theoretical_loss": 3.553330417001941, + "tokens_seen": 1332805632 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030111334002006016, + "loss": 2.738, + "theoretical_loss": 3.553314660947623, + "tokens_seen": 1332871168 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030110330992978935, + "loss": 2.777, + "theoretical_loss": 3.553298905884903, + "tokens_seen": 1332936704 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003010932798395186, + "loss": 2.5566, + "theoretical_loss": 3.5532831518136696, + "tokens_seen": 1333002240 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003010832497492477, + "loss": 2.8453, + "theoretical_loss": 3.5532673987338113, + "tokens_seen": 1333067776 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030107321965897694, + "loss": 2.7015, + "theoretical_loss": 3.5532516466452173, + "tokens_seen": 1333133312 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030106318956870607, + "loss": 2.567, + "theoretical_loss": 3.553235895547777, + "tokens_seen": 1333198848 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003010531594784353, + "loss": 2.7939, + "theoretical_loss": 3.5532201454413785, + "tokens_seen": 1333264384 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030104312938816454, + "loss": 2.6812, + "theoretical_loss": 3.5532043963259112, + "tokens_seen": 1333329920 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030103309929789367, + "loss": 2.6252, + "theoretical_loss": 3.553188648201264, + "tokens_seen": 1333395456 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003010230692076229, + "loss": 2.7426, + "theoretical_loss": 3.5531729010673256, + "tokens_seen": 1333460992 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003010130391173521, + "loss": 2.8594, + "theoretical_loss": 3.5531571549239853, + "tokens_seen": 1333526528 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030100300902708127, + "loss": 2.5893, + "theoretical_loss": 3.5531414097711322, + "tokens_seen": 1333592064 + }, + { + "epoch": 0.4, + "objective/train/docs_used": 755212, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7628026008605957, + "objective/train/theoretical_loss": 3.553125665608655, + "objective/train/tokens_used": 1354117600, + "theoretical_loss": 3.553125665608655, + "tokens_seen": 1333657600 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030099297893681045, + "loss": 2.9157, + "theoretical_loss": 3.553125665608655, + "tokens_seen": 1333657600 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030098294884653963, + "loss": 2.6513, + "theoretical_loss": 3.553109922436443, + "tokens_seen": 1333723136 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003009729187562688, + "loss": 2.9749, + "theoretical_loss": 3.553094180254385, + "tokens_seen": 1333788672 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030096288866599804, + "loss": 2.87, + "theoretical_loss": 3.5530784390623698, + "tokens_seen": 1333854208 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030095285857572717, + "loss": 2.7884, + "theoretical_loss": 3.5530626988602876, + "tokens_seen": 1333919744 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003009428284854564, + "loss": 2.7275, + "theoretical_loss": 3.5530469596480265, + "tokens_seen": 1333985280 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030093279839518553, + "loss": 2.5815, + "theoretical_loss": 3.553031221425476, + "tokens_seen": 1334050816 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030092276830491477, + "loss": 2.6658, + "theoretical_loss": 3.553015484192525, + "tokens_seen": 1334116352 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030091273821464395, + "loss": 2.3775, + "theoretical_loss": 3.5529997479490634, + "tokens_seen": 1334181888 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030090270812437313, + "loss": 2.5213, + "theoretical_loss": 3.552984012694979, + "tokens_seen": 1334247424 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003008926780341023, + "loss": 2.5853, + "theoretical_loss": 3.5529682784301624, + "tokens_seen": 1334312960 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030088264794383155, + "loss": 2.9102, + "theoretical_loss": 3.5529525451545023, + "tokens_seen": 1334378496 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003008726178535607, + "loss": 2.5501, + "theoretical_loss": 3.552936812867888, + "tokens_seen": 1334444032 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003008625877632899, + "loss": 2.7437, + "theoretical_loss": 3.552921081570209, + "tokens_seen": 1334509568 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030085255767301904, + "loss": 2.7785, + "theoretical_loss": 3.5529053512613533, + "tokens_seen": 1334575104 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030084252758274827, + "loss": 2.6937, + "theoretical_loss": 3.552889621941212, + "tokens_seen": 1334640640 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030083249749247745, + "loss": 2.4343, + "theoretical_loss": 3.5528738936096724, + "tokens_seen": 1334706176 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030082246740220663, + "loss": 3.0228, + "theoretical_loss": 3.552858166266626, + "tokens_seen": 1334771712 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003008124373119358, + "loss": 2.6208, + "theoretical_loss": 3.552842439911961, + "tokens_seen": 1334837248 + }, + { + "epoch": 0.4, + "learning_rate": 0.000300802407221665, + "loss": 2.6346, + "theoretical_loss": 3.5528267145455668, + "tokens_seen": 1334902784 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003007923771313942, + "loss": 2.6056, + "theoretical_loss": 3.552810990167333, + "tokens_seen": 1334968320 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003007823470411234, + "loss": 2.5046, + "theoretical_loss": 3.5527952667771485, + "tokens_seen": 1335033856 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030077231695085254, + "loss": 2.5802, + "theoretical_loss": 3.5527795443749035, + "tokens_seen": 1335099392 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003007622868605818, + "loss": 2.5581, + "theoretical_loss": 3.5527638229604865, + "tokens_seen": 1335164928 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003007522567703109, + "loss": 2.6468, + "theoretical_loss": 3.552748102533788, + "tokens_seen": 1335230464 + }, + { + "epoch": 0.4, + "objective/train/docs_used": 755960, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.9571595191955566, + "objective/train/theoretical_loss": 3.552732383094696, + "objective/train/tokens_used": 1355756000, + "theoretical_loss": 3.552732383094696, + "tokens_seen": 1335296000 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030074222668004014, + "loss": 2.8357, + "theoretical_loss": 3.552732383094696, + "tokens_seen": 1335296000 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003007321965897693, + "loss": 2.718, + "theoretical_loss": 3.5527166646431025, + "tokens_seen": 1335361536 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003007221664994985, + "loss": 2.6527, + "theoretical_loss": 3.552700947178894, + "tokens_seen": 1335427072 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003007121364092277, + "loss": 2.8255, + "theoretical_loss": 3.5526852307019623, + "tokens_seen": 1335492608 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003007021063189569, + "loss": 3.031, + "theoretical_loss": 3.5526695152121963, + "tokens_seen": 1335558144 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030069207622868604, + "loss": 2.8235, + "theoretical_loss": 3.5526538007094848, + "tokens_seen": 1335623680 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003006820461384153, + "loss": 2.7895, + "theoretical_loss": 3.552638087193718, + "tokens_seen": 1335689216 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003006720160481444, + "loss": 2.5906, + "theoretical_loss": 3.552622374664786, + "tokens_seen": 1335754752 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030066198595787364, + "loss": 2.7964, + "theoretical_loss": 3.5526066631225772, + "tokens_seen": 1335820288 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003006519558676028, + "loss": 2.766, + "theoretical_loss": 3.5525909525669817, + "tokens_seen": 1335885824 + }, + { + "epoch": 0.4, + "learning_rate": 0.000300641925777332, + "loss": 2.5409, + "theoretical_loss": 3.5525752429978894, + "tokens_seen": 1335951360 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003006318956870612, + "loss": 2.7003, + "theoretical_loss": 3.552559534415191, + "tokens_seen": 1336016896 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030062186559679036, + "loss": 2.6307, + "theoretical_loss": 3.552543826818774, + "tokens_seen": 1336082432 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030061183550651955, + "loss": 2.6505, + "theoretical_loss": 3.552528120208529, + "tokens_seen": 1336147968 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003006018054162488, + "loss": 2.5765, + "theoretical_loss": 3.5525124145843465, + "tokens_seen": 1336213504 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003005917753259779, + "loss": 2.7601, + "theoretical_loss": 3.5524967099461158, + "tokens_seen": 1336279040 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030058174523570714, + "loss": 2.8218, + "theoretical_loss": 3.5524810062937258, + "tokens_seen": 1336344576 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030057171514543627, + "loss": 2.8439, + "theoretical_loss": 3.5524653036270673, + "tokens_seen": 1336410112 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003005616850551655, + "loss": 2.8481, + "theoretical_loss": 3.55244960194603, + "tokens_seen": 1336475648 + }, + { + "epoch": 0.41, + "learning_rate": 0.0003005516549648947, + "loss": 2.4218, + "theoretical_loss": 3.5524339012505033, + "tokens_seen": 1336541184 + }, + { + "epoch": 0.41, + "learning_rate": 0.00030054162487462387, + "loss": 2.7345, + "theoretical_loss": 3.552418201540377, + "tokens_seen": 1336606720 + }, + { + "epoch": 0.41, + "learning_rate": 0.00030053159478435305, + "loss": 2.599, + "theoretical_loss": 3.5524025028155415, + "tokens_seen": 1336672256 + }, + { + "epoch": 0.41, + "learning_rate": 0.0003005215646940823, + "loss": 2.6083, + "theoretical_loss": 3.5523868050758862, + "tokens_seen": 1336737792 + }, + { + "epoch": 0.41, + "learning_rate": 0.0003005115346038114, + "loss": 3.0634, + "theoretical_loss": 3.552371108321301, + "tokens_seen": 1336803328 + }, + { + "epoch": 0.41, + "learning_rate": 0.00030050150451354065, + "loss": 2.674, + "theoretical_loss": 3.552355412551676, + "tokens_seen": 1336868864 + }, + { + "epoch": 0.41, + "objective/train/docs_used": 757325, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.68227481842041, + "objective/train/theoretical_loss": 3.5523397177669005, + "objective/train/tokens_used": 1357394400, + "theoretical_loss": 3.5523397177669005, + "tokens_seen": 1336934400 + }, + { + "epoch": 0.41, + "learning_rate": 0.0003004914744232698, + "loss": 2.6932, + "theoretical_loss": 3.5523397177669005, + "tokens_seen": 1336934400 + }, + { + "epoch": 0.41, + "learning_rate": 0.000300481444332999, + "loss": 2.9058, + "theoretical_loss": 3.5523240239668654, + "tokens_seen": 1336999936 + }, + { + "epoch": 0.41, + "learning_rate": 0.0003004714142427282, + "loss": 2.6379, + "theoretical_loss": 3.55230833115146, + "tokens_seen": 1337065472 + }, + { + "epoch": 0.41, + "learning_rate": 0.00030046138415245737, + "loss": 2.5958, + "theoretical_loss": 3.5522926393205743, + "tokens_seen": 1337131008 + }, + { + "epoch": 0.41, + "learning_rate": 0.00030045135406218655, + "loss": 2.9982, + "theoretical_loss": 3.5522769484740984, + "tokens_seen": 1337196544 + }, + { + "epoch": 0.41, + "learning_rate": 0.00030044132397191573, + "loss": 2.858, + "theoretical_loss": 3.5522612586119227, + "tokens_seen": 1337262080 + }, + { + "epoch": 0.41, + "learning_rate": 0.0003004312938816449, + "loss": 2.6615, + "theoretical_loss": 3.5522455697339366, + "tokens_seen": 1337327616 + }, + { + "epoch": 0.41, + "learning_rate": 0.00030042126379137415, + "loss": 2.6119, + "theoretical_loss": 3.55222988184003, + "tokens_seen": 1337393152 + }, + { + "epoch": 0.41, + "learning_rate": 0.0003004112337011033, + "loss": 2.6299, + "theoretical_loss": 3.552214194930094, + "tokens_seen": 1337458688 + }, + { + "epoch": 0.41, + "learning_rate": 0.0003004012036108325, + "loss": 2.6758, + "theoretical_loss": 3.552198509004018, + "tokens_seen": 1337524224 + }, + { + "epoch": 0.41, + "learning_rate": 0.0003003911735205617, + "loss": 2.667, + "theoretical_loss": 3.5521828240616915, + "tokens_seen": 1337589760 + }, + { + "epoch": 0.41, + "learning_rate": 0.0003003811434302909, + "loss": 2.4177, + "theoretical_loss": 3.5521671401030055, + "tokens_seen": 1337655296 + }, + { + "epoch": 0.41, + "learning_rate": 0.00030037111334002006, + "loss": 2.7302, + "theoretical_loss": 3.5521514571278496, + "tokens_seen": 1337720832 + }, + { + "epoch": 0.41, + "learning_rate": 0.00030036108324974924, + "loss": 2.6653, + "theoretical_loss": 3.552135775136115, + "tokens_seen": 1337786368 + }, + { + "epoch": 0.41, + "learning_rate": 0.0003003510531594784, + "loss": 2.5751, + "theoretical_loss": 3.5521200941276905, + "tokens_seen": 1337851904 + }, + { + "epoch": 0.41, + "learning_rate": 0.00030034102306920765, + "loss": 2.7985, + "theoretical_loss": 3.552104414102467, + "tokens_seen": 1337917440 + }, + { + "epoch": 0.41, + "learning_rate": 0.0003003309929789368, + "loss": 2.7035, + "theoretical_loss": 3.552088735060334, + "tokens_seen": 1337982976 + }, + { + "epoch": 0.41, + "learning_rate": 0.000300320962888666, + "loss": 2.6981, + "theoretical_loss": 3.5520730570011825, + "tokens_seen": 1338048512 + }, + { + "epoch": 0.41, + "learning_rate": 0.00030031093279839514, + "loss": 2.8032, + "theoretical_loss": 3.5520573799249027, + "tokens_seen": 1338114048 + }, + { + "epoch": 0.41, + "learning_rate": 0.0003003009027081244, + "loss": 2.6895, + "theoretical_loss": 3.5520417038313843, + "tokens_seen": 1338179584 + }, + { + "epoch": 0.41, + "learning_rate": 0.0003002908726178536, + "loss": 2.6984, + "theoretical_loss": 3.552026028720518, + "tokens_seen": 1338245120 + }, + { + "epoch": 0.41, + "learning_rate": 0.00030028084252758274, + "loss": 2.4287, + "theoretical_loss": 3.552010354592194, + "tokens_seen": 1338310656 + }, + { + "epoch": 0.41, + "learning_rate": 0.000300270812437312, + "loss": 2.8283, + "theoretical_loss": 3.5519946814463026, + "tokens_seen": 1338376192 + }, + { + "epoch": 0.41, + "learning_rate": 0.0003002607823470411, + "loss": 2.6384, + "theoretical_loss": 3.551979009282734, + "tokens_seen": 1338441728 + }, + { + "epoch": 0.41, + "learning_rate": 0.00030025075225677034, + "loss": 2.6744, + "theoretical_loss": 3.5519633381013787, + "tokens_seen": 1338507264 + }, + { + "epoch": 0.41, + "objective/train/docs_used": 758060, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7282652854919434, + "objective/train/theoretical_loss": 3.5519476679021267, + "objective/train/tokens_used": 1359032800, + "theoretical_loss": 3.5519476679021267, + "tokens_seen": 1338572800 + }, + { + "epoch": 0.41, + "learning_rate": 0.0003002407221664995, + "loss": 2.6094, + "theoretical_loss": 3.5519476679021267, + "tokens_seen": 1338572800 + }, + { + "epoch": 0.41, + "learning_rate": 0.0003002306920762287, + "loss": 2.5453, + "theoretical_loss": 3.551931998684869, + "tokens_seen": 1338638336 + }, + { + "epoch": 0.41, + "learning_rate": 0.0003002206619859579, + "loss": 2.6604, + "theoretical_loss": 3.5519163304494956, + "tokens_seen": 1338703872 + }, + { + "epoch": 0.41, + "learning_rate": 0.0003002106318956871, + "loss": 2.5288, + "theoretical_loss": 3.551900663195897, + "tokens_seen": 1338769408 + }, + { + "epoch": 0.41, + "learning_rate": 0.00030020060180541624, + "loss": 2.6264, + "theoretical_loss": 3.5518849969239636, + "tokens_seen": 1338834944 + }, + { + "epoch": 0.41, + "learning_rate": 0.0003001905717151455, + "loss": 2.6451, + "theoretical_loss": 3.551869331633586, + "tokens_seen": 1338900480 + }, + { + "epoch": 0.41, + "learning_rate": 0.0003001805416248746, + "loss": 2.7688, + "theoretical_loss": 3.5518536673246537, + "tokens_seen": 1338966016 + }, + { + "epoch": 0.41, + "learning_rate": 0.00030017051153460384, + "loss": 2.6557, + "theoretical_loss": 3.5518380039970587, + "tokens_seen": 1339031552 + }, + { + "epoch": 0.41, + "learning_rate": 0.000300160481444333, + "loss": 2.6593, + "theoretical_loss": 3.5518223416506904, + "tokens_seen": 1339097088 + }, + { + "epoch": 0.41, + "learning_rate": 0.0003001504513540622, + "loss": 2.7633, + "theoretical_loss": 3.55180668028544, + "tokens_seen": 1339162624 + }, + { + "epoch": 0.41, + "learning_rate": 0.0003001404212637914, + "loss": 2.739, + "theoretical_loss": 3.5517910199011977, + "tokens_seen": 1339228160 + }, + { + "epoch": 0.41, + "learning_rate": 0.00030013039117352056, + "loss": 2.7241, + "theoretical_loss": 3.551775360497854, + "tokens_seen": 1339293696 + }, + { + "epoch": 0.41, + "learning_rate": 0.00030012036108324975, + "loss": 2.5926, + "theoretical_loss": 3.551759702075299, + "tokens_seen": 1339359232 + }, + { + "epoch": 0.41, + "learning_rate": 0.000300110330992979, + "loss": 2.9531, + "theoretical_loss": 3.5517440446334243, + "tokens_seen": 1339424768 + }, + { + "epoch": 0.41, + "learning_rate": 0.0003001003009027081, + "loss": 2.6521, + "theoretical_loss": 3.55172838817212, + "tokens_seen": 1339490304 + }, + { + "epoch": 0.41, + "learning_rate": 0.00030009027081243734, + "loss": 2.5948, + "theoretical_loss": 3.5517127326912767, + "tokens_seen": 1339555840 + }, + { + "epoch": 0.41, + "learning_rate": 0.00030008024072216647, + "loss": 2.5524, + "theoretical_loss": 3.5516970781907853, + "tokens_seen": 1339621376 + }, + { + "epoch": 0.41, + "learning_rate": 0.0003000702106318957, + "loss": 2.4727, + "theoretical_loss": 3.5516814246705355, + "tokens_seen": 1339686912 + }, + { + "epoch": 0.41, + "learning_rate": 0.0003000601805416249, + "loss": 2.4969, + "theoretical_loss": 3.5516657721304195, + "tokens_seen": 1339752448 + }, + { + "epoch": 0.41, + "learning_rate": 0.00030005015045135407, + "loss": 2.741, + "theoretical_loss": 3.5516501205703266, + "tokens_seen": 1339817984 + }, + { + "epoch": 0.41, + "learning_rate": 0.00030004012036108325, + "loss": 2.5702, + "theoretical_loss": 3.551634469990148, + "tokens_seen": 1339883520 + }, + { + "epoch": 0.41, + "learning_rate": 0.0003000300902708125, + "loss": 2.6369, + "theoretical_loss": 3.551618820389775, + "tokens_seen": 1339949056 + }, + { + "epoch": 0.41, + "learning_rate": 0.0003000200601805416, + "loss": 2.6586, + "theoretical_loss": 3.5516031717690977, + "tokens_seen": 1340014592 + }, + { + "epoch": 0.41, + "learning_rate": 0.00030001003009027085, + "loss": 2.8525, + "theoretical_loss": 3.5515875241280073, + "tokens_seen": 1340080128 + }, + { + "epoch": 0.41, + "learning_rate": 0.0003, + "loss": 2.6664, + "theoretical_loss": 3.551571877466394, + "tokens_seen": 1340145664 + }, + { + "epoch": 0.41, + "objective/train/docs_used": 759300, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.4068970680236816, + "objective/train/theoretical_loss": 3.551556231784149, + "objective/train/tokens_used": 1360671200, + "theoretical_loss": 3.551556231784149, + "tokens_seen": 1340211200 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002999899699097292, + "loss": 2.6676, + "theoretical_loss": 3.551556231784149, + "tokens_seen": 1340211200 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002999799398194584, + "loss": 2.7373, + "theoretical_loss": 3.5515405870811625, + "tokens_seen": 1340276736 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029996990972918757, + "loss": 2.6699, + "theoretical_loss": 3.551524943357326, + "tokens_seen": 1340342272 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029995987963891675, + "loss": 2.7438, + "theoretical_loss": 3.5515093006125307, + "tokens_seen": 1340407808 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029994984954864593, + "loss": 2.7211, + "theoretical_loss": 3.551493658846667, + "tokens_seen": 1340473344 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002999398194583751, + "loss": 2.8479, + "theoretical_loss": 3.5514780180596253, + "tokens_seen": 1340538880 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029992978936810435, + "loss": 2.6864, + "theoretical_loss": 3.551462378251297, + "tokens_seen": 1340604416 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002999197592778335, + "loss": 2.4633, + "theoretical_loss": 3.5514467394215723, + "tokens_seen": 1340669952 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002999097291875627, + "loss": 2.59, + "theoretical_loss": 3.551431101570344, + "tokens_seen": 1340735488 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002998996990972919, + "loss": 2.6975, + "theoretical_loss": 3.551415464697501, + "tokens_seen": 1340801024 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002998896690070211, + "loss": 2.5482, + "theoretical_loss": 3.5513998288029356, + "tokens_seen": 1340866560 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029987963891675026, + "loss": 2.6762, + "theoretical_loss": 3.5513841938865376, + "tokens_seen": 1340932096 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029986960882647944, + "loss": 2.5713, + "theoretical_loss": 3.551368559948199, + "tokens_seen": 1340997632 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002998595787362086, + "loss": 2.5488, + "theoretical_loss": 3.5513529269878106, + "tokens_seen": 1341063168 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029984954864593785, + "loss": 2.8228, + "theoretical_loss": 3.551337295005264, + "tokens_seen": 1341128704 + }, + { + "epoch": 0.41, + "learning_rate": 0.000299839518555667, + "loss": 2.8577, + "theoretical_loss": 3.5513216640004486, + "tokens_seen": 1341194240 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002998294884653962, + "loss": 2.7441, + "theoretical_loss": 3.551306033973257, + "tokens_seen": 1341259776 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029981945837512534, + "loss": 2.3337, + "theoretical_loss": 3.5512904049235794, + "tokens_seen": 1341325312 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002998094282848546, + "loss": 2.6053, + "theoretical_loss": 3.551274776851308, + "tokens_seen": 1341390848 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029979939819458376, + "loss": 2.8362, + "theoretical_loss": 3.5512591497563326, + "tokens_seen": 1341456384 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029978936810431294, + "loss": 2.8117, + "theoretical_loss": 3.551243523638545, + "tokens_seen": 1341521920 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002997793380140421, + "loss": 2.8206, + "theoretical_loss": 3.551227898497836, + "tokens_seen": 1341587456 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002997693079237713, + "loss": 2.7179, + "theoretical_loss": 3.5512122743340977, + "tokens_seen": 1341652992 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002997592778335005, + "loss": 2.6036, + "theoretical_loss": 3.5511966511472206, + "tokens_seen": 1341718528 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002997492477432297, + "loss": 2.6496, + "theoretical_loss": 3.551181028937096, + "tokens_seen": 1341784064 + }, + { + "epoch": 0.41, + "objective/train/docs_used": 759950, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6135246753692627, + "objective/train/theoretical_loss": 3.551165407703615, + "objective/train/tokens_used": 1362309600, + "theoretical_loss": 3.551165407703615, + "tokens_seen": 1341849600 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029973921765295885, + "loss": 2.8968, + "theoretical_loss": 3.551165407703615, + "tokens_seen": 1341849600 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002997291875626881, + "loss": 2.8144, + "theoretical_loss": 3.551149787446669, + "tokens_seen": 1341915136 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029971915747241726, + "loss": 2.9385, + "theoretical_loss": 3.5511341681661492, + "tokens_seen": 1341980672 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029970912738214644, + "loss": 2.611, + "theoretical_loss": 3.551118549861947, + "tokens_seen": 1342046208 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002996990972918756, + "loss": 2.766, + "theoretical_loss": 3.5511029325339534, + "tokens_seen": 1342111744 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002996890672016048, + "loss": 2.5295, + "theoretical_loss": 3.5510873161820604, + "tokens_seen": 1342177280 + }, + { + "epoch": 0.41, + "learning_rate": 0.000299679037111334, + "loss": 2.6992, + "theoretical_loss": 3.5510717008061587, + "tokens_seen": 1342242816 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002996690070210632, + "loss": 2.4816, + "theoretical_loss": 3.55105608640614, + "tokens_seen": 1342308352 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029965897693079235, + "loss": 2.8581, + "theoretical_loss": 3.5510404729818954, + "tokens_seen": 1342373888 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002996489468405216, + "loss": 2.7702, + "theoretical_loss": 3.5510248605333166, + "tokens_seen": 1342439424 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002996389167502507, + "loss": 2.5065, + "theoretical_loss": 3.5510092490602942, + "tokens_seen": 1342504960 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029962888665997995, + "loss": 2.6798, + "theoretical_loss": 3.550993638562721, + "tokens_seen": 1342570496 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029961885656970913, + "loss": 2.6446, + "theoretical_loss": 3.550978029040487, + "tokens_seen": 1342636032 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002996088264794383, + "loss": 2.8831, + "theoretical_loss": 3.5509624204934847, + "tokens_seen": 1342701568 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002995987963891675, + "loss": 2.7048, + "theoretical_loss": 3.5509468129216053, + "tokens_seen": 1342767104 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029958876629889667, + "loss": 2.7739, + "theoretical_loss": 3.55093120632474, + "tokens_seen": 1342832640 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029957873620862585, + "loss": 2.7951, + "theoretical_loss": 3.5509156007027807, + "tokens_seen": 1342898176 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002995687061183551, + "loss": 2.5066, + "theoretical_loss": 3.5508999960556182, + "tokens_seen": 1342963712 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002995586760280842, + "loss": 2.8644, + "theoretical_loss": 3.550884392383145, + "tokens_seen": 1343029248 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029954864593781345, + "loss": 2.558, + "theoretical_loss": 3.550868789685252, + "tokens_seen": 1343094784 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002995386158475427, + "loss": 2.7477, + "theoretical_loss": 3.5508531879618315, + "tokens_seen": 1343160320 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002995285857572718, + "loss": 2.906, + "theoretical_loss": 3.550837587212774, + "tokens_seen": 1343225856 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029951855566700105, + "loss": 2.8014, + "theoretical_loss": 3.5508219874379723, + "tokens_seen": 1343291392 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002995085255767302, + "loss": 2.8204, + "theoretical_loss": 3.5508063886373176, + "tokens_seen": 1343356928 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002994984954864594, + "loss": 2.7041, + "theoretical_loss": 3.5507907908107006, + "tokens_seen": 1343422464 + }, + { + "epoch": 0.41, + "objective/train/docs_used": 761102, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.8779098987579346, + "objective/train/theoretical_loss": 3.5507751939580148, + "objective/train/tokens_used": 1363948000, + "theoretical_loss": 3.5507751939580148, + "tokens_seen": 1343488000 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002994884653961886, + "loss": 2.9535, + "theoretical_loss": 3.5507751939580148, + "tokens_seen": 1343488000 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029947843530591777, + "loss": 2.6919, + "theoretical_loss": 3.55075959807915, + "tokens_seen": 1343553536 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029946840521564695, + "loss": 2.6903, + "theoretical_loss": 3.550744003173999, + "tokens_seen": 1343619072 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029945837512537613, + "loss": 2.529, + "theoretical_loss": 3.5507284092424536, + "tokens_seen": 1343684608 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002994483450351053, + "loss": 2.4831, + "theoretical_loss": 3.550712816284405, + "tokens_seen": 1343750144 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029943831494483455, + "loss": 2.8677, + "theoretical_loss": 3.550697224299745, + "tokens_seen": 1343815680 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002994282848545637, + "loss": 2.6019, + "theoretical_loss": 3.5506816332883657, + "tokens_seen": 1343881216 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002994182547642929, + "loss": 2.707, + "theoretical_loss": 3.5506660432501587, + "tokens_seen": 1343946752 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002994082246740221, + "loss": 2.6753, + "theoretical_loss": 3.550650454185016, + "tokens_seen": 1344012288 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002993981945837513, + "loss": 2.7325, + "theoretical_loss": 3.550634866092829, + "tokens_seen": 1344077824 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029938816449348046, + "loss": 2.6214, + "theoretical_loss": 3.55061927897349, + "tokens_seen": 1344143360 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029937813440320964, + "loss": 2.4051, + "theoretical_loss": 3.5506036928268907, + "tokens_seen": 1344208896 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002993681043129388, + "loss": 2.56, + "theoretical_loss": 3.5505881076529224, + "tokens_seen": 1344274432 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029935807422266805, + "loss": 2.8232, + "theoretical_loss": 3.550572523451478, + "tokens_seen": 1344339968 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002993480441323972, + "loss": 2.4904, + "theoretical_loss": 3.5505569402224486, + "tokens_seen": 1344405504 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002993380140421264, + "loss": 2.3762, + "theoretical_loss": 3.550541357965727, + "tokens_seen": 1344471040 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029932798395185554, + "loss": 2.5362, + "theoretical_loss": 3.550525776681204, + "tokens_seen": 1344536576 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002993179538615848, + "loss": 2.6838, + "theoretical_loss": 3.5505101963687724, + "tokens_seen": 1344602112 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029930792377131396, + "loss": 2.9552, + "theoretical_loss": 3.550494617028324, + "tokens_seen": 1344667648 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029929789368104314, + "loss": 2.5859, + "theoretical_loss": 3.5504790386597507, + "tokens_seen": 1344733184 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002992878635907723, + "loss": 2.6006, + "theoretical_loss": 3.5504634612629444, + "tokens_seen": 1344798720 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002992778335005015, + "loss": 2.8131, + "theoretical_loss": 3.5504478848377974, + "tokens_seen": 1344864256 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002992678034102307, + "loss": 2.725, + "theoretical_loss": 3.5504323093842016, + "tokens_seen": 1344929792 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002992577733199599, + "loss": 2.5522, + "theoretical_loss": 3.550416734902049, + "tokens_seen": 1344995328 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029924774322968905, + "loss": 2.3811, + "theoretical_loss": 3.550401161391232, + "tokens_seen": 1345060864 + }, + { + "epoch": 0.41, + "objective/train/docs_used": 761644, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.2070727348327637, + "objective/train/theoretical_loss": 3.550385588851642, + "objective/train/tokens_used": 1365586400, + "theoretical_loss": 3.550385588851642, + "tokens_seen": 1345126400 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002992377131394183, + "loss": 2.4377, + "theoretical_loss": 3.550385588851642, + "tokens_seen": 1345126400 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029922768304914746, + "loss": 2.6849, + "theoretical_loss": 3.550370017283172, + "tokens_seen": 1345191936 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029921765295887664, + "loss": 2.762, + "theoretical_loss": 3.5503544466857133, + "tokens_seen": 1345257472 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002992076228686058, + "loss": 2.6869, + "theoretical_loss": 3.5503388770591586, + "tokens_seen": 1345323008 + }, + { + "epoch": 0.41, + "learning_rate": 0.000299197592778335, + "loss": 2.7219, + "theoretical_loss": 3.5503233084034003, + "tokens_seen": 1345388544 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002991875626880642, + "loss": 2.7652, + "theoretical_loss": 3.5503077407183294, + "tokens_seen": 1345454080 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002991775325977934, + "loss": 2.5601, + "theoretical_loss": 3.5502921740038396, + "tokens_seen": 1345519616 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029916750250752255, + "loss": 2.7656, + "theoretical_loss": 3.5502766082598223, + "tokens_seen": 1345585152 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002991574724172518, + "loss": 2.6536, + "theoretical_loss": 3.55026104348617, + "tokens_seen": 1345650688 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002991474423269809, + "loss": 2.3709, + "theoretical_loss": 3.5502454796827743, + "tokens_seen": 1345716224 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029913741223671015, + "loss": 2.8504, + "theoretical_loss": 3.5502299168495286, + "tokens_seen": 1345781760 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029912738214643933, + "loss": 2.9137, + "theoretical_loss": 3.550214354986324, + "tokens_seen": 1345847296 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002991173520561685, + "loss": 2.5812, + "theoretical_loss": 3.5501987940930535, + "tokens_seen": 1345912832 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002991073219658977, + "loss": 2.6886, + "theoretical_loss": 3.5501832341696096, + "tokens_seen": 1345978368 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029909729187562687, + "loss": 2.8763, + "theoretical_loss": 3.550167675215884, + "tokens_seen": 1346043904 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029908726178535605, + "loss": 2.8363, + "theoretical_loss": 3.55015211723177, + "tokens_seen": 1346109440 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002990772316950853, + "loss": 2.6432, + "theoretical_loss": 3.5501365602171586, + "tokens_seen": 1346174976 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002990672016048144, + "loss": 2.7864, + "theoretical_loss": 3.5501210041719435, + "tokens_seen": 1346240512 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029905717151454365, + "loss": 2.6761, + "theoretical_loss": 3.550105449096016, + "tokens_seen": 1346306048 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029904714142427283, + "loss": 2.5498, + "theoretical_loss": 3.5500898949892696, + "tokens_seen": 1346371584 + }, + { + "epoch": 0.41, + "learning_rate": 0.000299037111334002, + "loss": 2.7646, + "theoretical_loss": 3.5500743418515963, + "tokens_seen": 1346437120 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002990270812437312, + "loss": 2.9059, + "theoretical_loss": 3.550058789682888, + "tokens_seen": 1346502656 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002990170511534604, + "loss": 2.8066, + "theoretical_loss": 3.5500432384830383, + "tokens_seen": 1346568192 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029900702106318956, + "loss": 2.8549, + "theoretical_loss": 3.550027688251938, + "tokens_seen": 1346633728 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002989969909729188, + "loss": 2.6241, + "theoretical_loss": 3.550012138989482, + "tokens_seen": 1346699264 + }, + { + "epoch": 0.41, + "objective/train/docs_used": 762571, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.717539072036743, + "objective/train/theoretical_loss": 3.5499965906955606, + "objective/train/tokens_used": 1367224800, + "theoretical_loss": 3.5499965906955606, + "tokens_seen": 1346764800 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002989869608826479, + "loss": 2.9187, + "theoretical_loss": 3.5499965906955606, + "tokens_seen": 1346764800 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029897693079237715, + "loss": 2.5232, + "theoretical_loss": 3.549981043370068, + "tokens_seen": 1346830336 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002989669007021063, + "loss": 2.6737, + "theoretical_loss": 3.549965497012895, + "tokens_seen": 1346895872 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002989568706118355, + "loss": 2.5757, + "theoretical_loss": 3.549949951623936, + "tokens_seen": 1346961408 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002989468405215647, + "loss": 2.7354, + "theoretical_loss": 3.5499344072030823, + "tokens_seen": 1347026944 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002989368104312939, + "loss": 2.7589, + "theoretical_loss": 3.5499188637502277, + "tokens_seen": 1347092480 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029892678034102306, + "loss": 2.6747, + "theoretical_loss": 3.5499033212652638, + "tokens_seen": 1347158016 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002989167502507523, + "loss": 2.4844, + "theoretical_loss": 3.549887779748083, + "tokens_seen": 1347223552 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002989067201604814, + "loss": 2.5494, + "theoretical_loss": 3.549872239198579, + "tokens_seen": 1347289088 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029889669007021066, + "loss": 2.7379, + "theoretical_loss": 3.5498566996166443, + "tokens_seen": 1347354624 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002988866599799398, + "loss": 2.7615, + "theoretical_loss": 3.549841161002171, + "tokens_seen": 1347420160 + }, + { + "epoch": 0.41, + "learning_rate": 0.000298876629889669, + "loss": 2.711, + "theoretical_loss": 3.5498256233550523, + "tokens_seen": 1347485696 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002988665997993982, + "loss": 2.9806, + "theoretical_loss": 3.5498100866751807, + "tokens_seen": 1347551232 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002988565697091274, + "loss": 2.4568, + "theoretical_loss": 3.549794550962449, + "tokens_seen": 1347616768 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029884653961885656, + "loss": 3.1012, + "theoretical_loss": 3.5497790162167497, + "tokens_seen": 1347682304 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029883650952858574, + "loss": 2.5441, + "theoretical_loss": 3.5497634824379762, + "tokens_seen": 1347747840 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002988264794383149, + "loss": 2.8105, + "theoretical_loss": 3.5497479496260214, + "tokens_seen": 1347813376 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029881644934804416, + "loss": 2.8267, + "theoretical_loss": 3.549732417780777, + "tokens_seen": 1347878912 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002988064192577733, + "loss": 2.7009, + "theoretical_loss": 3.5497168869021367, + "tokens_seen": 1347944448 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002987963891675025, + "loss": 2.7946, + "theoretical_loss": 3.549701356989994, + "tokens_seen": 1348009984 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002987863590772317, + "loss": 2.7501, + "theoretical_loss": 3.54968582804424, + "tokens_seen": 1348075520 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002987763289869609, + "loss": 2.8205, + "theoretical_loss": 3.549670300064769, + "tokens_seen": 1348141056 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002987662988966901, + "loss": 2.5858, + "theoretical_loss": 3.549654773051473, + "tokens_seen": 1348206592 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029875626880641925, + "loss": 2.6136, + "theoretical_loss": 3.549639247004246, + "tokens_seen": 1348272128 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002987462387161485, + "loss": 2.8202, + "theoretical_loss": 3.54962372192298, + "tokens_seen": 1348337664 + }, + { + "epoch": 0.41, + "objective/train/docs_used": 763291, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.773918628692627, + "objective/train/theoretical_loss": 3.549608197807568, + "objective/train/tokens_used": 1368863200, + "theoretical_loss": 3.549608197807568, + "tokens_seen": 1348403200 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029873620862587766, + "loss": 2.8216, + "theoretical_loss": 3.549608197807568, + "tokens_seen": 1348403200 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029872617853560684, + "loss": 2.7832, + "theoretical_loss": 3.5495926746579034, + "tokens_seen": 1348468736 + }, + { + "epoch": 0.41, + "learning_rate": 0.000298716148445336, + "loss": 2.8492, + "theoretical_loss": 3.5495771524738795, + "tokens_seen": 1348534272 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002987061183550652, + "loss": 2.5956, + "theoretical_loss": 3.5495616312553886, + "tokens_seen": 1348599808 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002986960882647944, + "loss": 2.8332, + "theoretical_loss": 3.549546111002324, + "tokens_seen": 1348665344 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002986860581745236, + "loss": 2.7526, + "theoretical_loss": 3.5495305917145785, + "tokens_seen": 1348730880 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029867602808425275, + "loss": 2.5692, + "theoretical_loss": 3.5495150733920457, + "tokens_seen": 1348796416 + }, + { + "epoch": 0.41, + "learning_rate": 0.000298665997993982, + "loss": 2.7404, + "theoretical_loss": 3.549499556034618, + "tokens_seen": 1348861952 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002986559679037111, + "loss": 2.7728, + "theoretical_loss": 3.5494840396421887, + "tokens_seen": 1348927488 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029864593781344035, + "loss": 2.6708, + "theoretical_loss": 3.5494685242146513, + "tokens_seen": 1348993024 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029863590772316953, + "loss": 2.5288, + "theoretical_loss": 3.549453009751899, + "tokens_seen": 1349058560 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002986258776328987, + "loss": 2.7704, + "theoretical_loss": 3.549437496253824, + "tokens_seen": 1349124096 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002986158475426279, + "loss": 2.5716, + "theoretical_loss": 3.549421983720321, + "tokens_seen": 1349189632 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029860581745235707, + "loss": 2.8059, + "theoretical_loss": 3.549406472151281, + "tokens_seen": 1349255168 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029859578736208625, + "loss": 2.5794, + "theoretical_loss": 3.5493909615465995, + "tokens_seen": 1349320704 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002985857572718155, + "loss": 2.802, + "theoretical_loss": 3.549375451906168, + "tokens_seen": 1349386240 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002985757271815446, + "loss": 3.061, + "theoretical_loss": 3.549359943229881, + "tokens_seen": 1349451776 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029856569709127385, + "loss": 2.5106, + "theoretical_loss": 3.5493444355176305, + "tokens_seen": 1349517312 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029855566700100303, + "loss": 2.6663, + "theoretical_loss": 3.549328928769311, + "tokens_seen": 1349582848 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002985456369107322, + "loss": 2.7399, + "theoretical_loss": 3.549313422984815, + "tokens_seen": 1349648384 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002985356068204614, + "loss": 2.7485, + "theoretical_loss": 3.5492979181640356, + "tokens_seen": 1349713920 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002985255767301906, + "loss": 2.6211, + "theoretical_loss": 3.549282414306867, + "tokens_seen": 1349779456 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029851554663991976, + "loss": 2.8935, + "theoretical_loss": 3.549266911413201, + "tokens_seen": 1349844992 + }, + { + "epoch": 0.41, + "learning_rate": 0.000298505516549649, + "loss": 2.721, + "theoretical_loss": 3.549251409482933, + "tokens_seen": 1349910528 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002984954864593781, + "loss": 2.7419, + "theoretical_loss": 3.5492359085159553, + "tokens_seen": 1349976064 + }, + { + "epoch": 0.41, + "objective/train/docs_used": 764652, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7754642963409424, + "objective/train/theoretical_loss": 3.549220408512161, + "objective/train/tokens_used": 1370501600, + "theoretical_loss": 3.549220408512161, + "tokens_seen": 1350041600 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029848545636910735, + "loss": 2.691, + "theoretical_loss": 3.549220408512161, + "tokens_seen": 1350041600 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002984754262788365, + "loss": 2.508, + "theoretical_loss": 3.549204909471444, + "tokens_seen": 1350107136 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002984653961885657, + "loss": 2.6236, + "theoretical_loss": 3.549189411393697, + "tokens_seen": 1350172672 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002984553660982949, + "loss": 2.8616, + "theoretical_loss": 3.5491739142788146, + "tokens_seen": 1350238208 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002984453360080241, + "loss": 2.4707, + "theoretical_loss": 3.549158418126689, + "tokens_seen": 1350303744 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029843530591775326, + "loss": 2.6285, + "theoretical_loss": 3.5491429229372153, + "tokens_seen": 1350369280 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002984252758274825, + "loss": 2.4846, + "theoretical_loss": 3.549127428710285, + "tokens_seen": 1350434816 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002984152457372116, + "loss": 2.6518, + "theoretical_loss": 3.5491119354457927, + "tokens_seen": 1350500352 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029840521564694086, + "loss": 2.6694, + "theoretical_loss": 3.549096443143632, + "tokens_seen": 1350565888 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029839518555667, + "loss": 2.7667, + "theoretical_loss": 3.5490809518036963, + "tokens_seen": 1350631424 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002983851554663992, + "loss": 2.6059, + "theoretical_loss": 3.5490654614258785, + "tokens_seen": 1350696960 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002983751253761284, + "loss": 2.5888, + "theoretical_loss": 3.5490499720100734, + "tokens_seen": 1350762496 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002983650952858576, + "loss": 2.6247, + "theoretical_loss": 3.5490344835561736, + "tokens_seen": 1350828032 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029835506519558676, + "loss": 2.8307, + "theoretical_loss": 3.5490189960640732, + "tokens_seen": 1350893568 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029834503510531594, + "loss": 2.623, + "theoretical_loss": 3.5490035095336654, + "tokens_seen": 1350959104 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002983350050150451, + "loss": 2.68, + "theoretical_loss": 3.5489880239648444, + "tokens_seen": 1351024640 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029832497492477436, + "loss": 2.89, + "theoretical_loss": 3.5489725393575036, + "tokens_seen": 1351090176 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002983149448345035, + "loss": 2.6409, + "theoretical_loss": 3.548957055711536, + "tokens_seen": 1351155712 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002983049147442327, + "loss": 2.6599, + "theoretical_loss": 3.548941573026836, + "tokens_seen": 1351221248 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029829488465396185, + "loss": 2.7401, + "theoretical_loss": 3.548926091303297, + "tokens_seen": 1351286784 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002982848545636911, + "loss": 2.7235, + "theoretical_loss": 3.5489106105408137, + "tokens_seen": 1351352320 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029827482447342026, + "loss": 2.6908, + "theoretical_loss": 3.548895130739279, + "tokens_seen": 1351417856 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029826479438314945, + "loss": 2.6438, + "theoretical_loss": 3.548879651898586, + "tokens_seen": 1351483392 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002982547642928786, + "loss": 2.8051, + "theoretical_loss": 3.5488641740186293, + "tokens_seen": 1351548928 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029824473420260786, + "loss": 2.8503, + "theoretical_loss": 3.5488486970993027, + "tokens_seen": 1351614464 + }, + { + "epoch": 0.41, + "objective/train/docs_used": 765372, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.4946346282958984, + "objective/train/theoretical_loss": 3.5488332211405, + "objective/train/tokens_used": 1372140000, + "theoretical_loss": 3.5488332211405, + "tokens_seen": 1351680000 + }, + { + "epoch": 0.41, + "learning_rate": 0.000298234704112337, + "loss": 2.559, + "theoretical_loss": 3.5488332211405, + "tokens_seen": 1351680000 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002982246740220662, + "loss": 2.8531, + "theoretical_loss": 3.548817746142115, + "tokens_seen": 1351745536 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029821464393179535, + "loss": 2.7105, + "theoretical_loss": 3.548802272104041, + "tokens_seen": 1351811072 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002982046138415246, + "loss": 2.9003, + "theoretical_loss": 3.548786799026172, + "tokens_seen": 1351876608 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029819458375125377, + "loss": 2.6531, + "theoretical_loss": 3.548771326908403, + "tokens_seen": 1351942144 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029818455366098295, + "loss": 2.7215, + "theoretical_loss": 3.5487558557506267, + "tokens_seen": 1352007680 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029817452357071213, + "loss": 2.5243, + "theoretical_loss": 3.548740385552737, + "tokens_seen": 1352073216 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002981644934804413, + "loss": 2.7883, + "theoretical_loss": 3.5487249163146286, + "tokens_seen": 1352138752 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002981544633901705, + "loss": 2.5462, + "theoretical_loss": 3.5487094480361945, + "tokens_seen": 1352204288 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029814443329989973, + "loss": 2.735, + "theoretical_loss": 3.5486939807173297, + "tokens_seen": 1352269824 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029813440320962885, + "loss": 2.7817, + "theoretical_loss": 3.5486785143579276, + "tokens_seen": 1352335360 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002981243731193581, + "loss": 2.6913, + "theoretical_loss": 3.548663048957882, + "tokens_seen": 1352400896 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002981143430290872, + "loss": 2.5613, + "theoretical_loss": 3.548647584517087, + "tokens_seen": 1352466432 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029810431293881645, + "loss": 2.7385, + "theoretical_loss": 3.5486321210354372, + "tokens_seen": 1352531968 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029809428284854563, + "loss": 2.6671, + "theoretical_loss": 3.548616658512826, + "tokens_seen": 1352597504 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002980842527582748, + "loss": 2.7649, + "theoretical_loss": 3.5486011969491478, + "tokens_seen": 1352663040 + }, + { + "epoch": 0.41, + "learning_rate": 0.000298074222668004, + "loss": 2.6292, + "theoretical_loss": 3.548585736344296, + "tokens_seen": 1352728576 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029806419257773323, + "loss": 2.7041, + "theoretical_loss": 3.5485702766981655, + "tokens_seen": 1352794112 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029805416248746236, + "loss": 2.6375, + "theoretical_loss": 3.5485548180106505, + "tokens_seen": 1352859648 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002980441323971916, + "loss": 2.5962, + "theoretical_loss": 3.5485393602816444, + "tokens_seen": 1352925184 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002980341023069208, + "loss": 2.6058, + "theoretical_loss": 3.548523903511042, + "tokens_seen": 1352990720 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029802407221664996, + "loss": 2.6289, + "theoretical_loss": 3.5485084476987367, + "tokens_seen": 1353056256 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002980140421263792, + "loss": 2.5622, + "theoretical_loss": 3.548492992844623, + "tokens_seen": 1353121792 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002980040120361083, + "loss": 2.7741, + "theoretical_loss": 3.548477538948596, + "tokens_seen": 1353187328 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029799398194583755, + "loss": 2.5927, + "theoretical_loss": 3.5484620860105487, + "tokens_seen": 1353252864 + }, + { + "epoch": 0.41, + "objective/train/docs_used": 766365, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.018037796020508, + "objective/train/theoretical_loss": 3.5484466340303755, + "objective/train/tokens_used": 1373778400, + "theoretical_loss": 3.5484466340303755, + "tokens_seen": 1353318400 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002979839518555667, + "loss": 2.6444, + "theoretical_loss": 3.5484466340303755, + "tokens_seen": 1353318400 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002979739217652959, + "loss": 2.5151, + "theoretical_loss": 3.5484311830079713, + "tokens_seen": 1353383936 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002979638916750251, + "loss": 2.8615, + "theoretical_loss": 3.54841573294323, + "tokens_seen": 1353449472 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002979538615847543, + "loss": 2.7283, + "theoretical_loss": 3.5484002838360453, + "tokens_seen": 1353515008 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029794383149448346, + "loss": 2.7801, + "theoretical_loss": 3.5483848356863126, + "tokens_seen": 1353580544 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002979338014042127, + "loss": 2.6658, + "theoretical_loss": 3.5483693884939256, + "tokens_seen": 1353646080 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002979237713139418, + "loss": 2.6621, + "theoretical_loss": 3.548353942258778, + "tokens_seen": 1353711616 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029791374122367106, + "loss": 2.5207, + "theoretical_loss": 3.5483384969807656, + "tokens_seen": 1353777152 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002979037111334002, + "loss": 2.8063, + "theoretical_loss": 3.548323052659782, + "tokens_seen": 1353842688 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002978936810431294, + "loss": 2.8351, + "theoretical_loss": 3.548307609295721, + "tokens_seen": 1353908224 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002978836509528586, + "loss": 2.612, + "theoretical_loss": 3.548292166888478, + "tokens_seen": 1353973760 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002978736208625878, + "loss": 2.6133, + "theoretical_loss": 3.5482767254379466, + "tokens_seen": 1354039296 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029786359077231696, + "loss": 2.9652, + "theoretical_loss": 3.5482612849440223, + "tokens_seen": 1354104832 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029785356068204614, + "loss": 2.9238, + "theoretical_loss": 3.5482458454065977, + "tokens_seen": 1354170368 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002978435305917753, + "loss": 2.489, + "theoretical_loss": 3.548230406825569, + "tokens_seen": 1354235904 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029783350050150456, + "loss": 2.8582, + "theoretical_loss": 3.54821496920083, + "tokens_seen": 1354301440 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002978234704112337, + "loss": 2.4953, + "theoretical_loss": 3.5481995325322755, + "tokens_seen": 1354366976 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002978134403209629, + "loss": 2.6152, + "theoretical_loss": 3.5481840968197993, + "tokens_seen": 1354432512 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029780341023069205, + "loss": 2.643, + "theoretical_loss": 3.5481686620632966, + "tokens_seen": 1354498048 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002977933801404213, + "loss": 2.6817, + "theoretical_loss": 3.548153228262662, + "tokens_seen": 1354563584 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029778335005015046, + "loss": 2.7658, + "theoretical_loss": 3.5481377954177895, + "tokens_seen": 1354629120 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029777331995987965, + "loss": 2.6047, + "theoretical_loss": 3.548122363528574, + "tokens_seen": 1354694656 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029776328986960883, + "loss": 2.5922, + "theoretical_loss": 3.5481069325949104, + "tokens_seen": 1354760192 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029775325977933806, + "loss": 2.5819, + "theoretical_loss": 3.5480915026166926, + "tokens_seen": 1354825728 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002977432296890672, + "loss": 2.3483, + "theoretical_loss": 3.5480760735938155, + "tokens_seen": 1354891264 + }, + { + "epoch": 0.41, + "objective/train/docs_used": 767078, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.730848550796509, + "objective/train/theoretical_loss": 3.5480606455261743, + "objective/train/tokens_used": 1375416800, + "theoretical_loss": 3.5480606455261743, + "tokens_seen": 1354956800 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002977331995987964, + "loss": 2.6957, + "theoretical_loss": 3.5480606455261743, + "tokens_seen": 1354956800 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029772316950852555, + "loss": 2.6537, + "theoretical_loss": 3.5480452184136633, + "tokens_seen": 1355022336 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002977131394182548, + "loss": 2.4638, + "theoretical_loss": 3.5480297922561768, + "tokens_seen": 1355087872 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029770310932798397, + "loss": 2.8843, + "theoretical_loss": 3.54801436705361, + "tokens_seen": 1355153408 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029769307923771315, + "loss": 2.6635, + "theoretical_loss": 3.547998942805857, + "tokens_seen": 1355218944 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029768304914744233, + "loss": 2.7525, + "theoretical_loss": 3.547983519512813, + "tokens_seen": 1355284480 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002976730190571715, + "loss": 2.7962, + "theoretical_loss": 3.547968097174373, + "tokens_seen": 1355350016 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002976629889669007, + "loss": 2.6456, + "theoretical_loss": 3.547952675790431, + "tokens_seen": 1355415552 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029765295887662993, + "loss": 2.6541, + "theoretical_loss": 3.547937255360883, + "tokens_seen": 1355481088 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029764292878635905, + "loss": 2.449, + "theoretical_loss": 3.5479218358856226, + "tokens_seen": 1355546624 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002976328986960883, + "loss": 2.6241, + "theoretical_loss": 3.547906417364545, + "tokens_seen": 1355612160 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002976228686058174, + "loss": 2.8839, + "theoretical_loss": 3.547890999797545, + "tokens_seen": 1355677696 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029761283851554665, + "loss": 2.7989, + "theoretical_loss": 3.5478755831845175, + "tokens_seen": 1355743232 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029760280842527583, + "loss": 2.5759, + "theoretical_loss": 3.547860167525357, + "tokens_seen": 1355808768 + }, + { + "epoch": 0.41, + "learning_rate": 0.000297592778335005, + "loss": 2.8479, + "theoretical_loss": 3.5478447528199597, + "tokens_seen": 1355874304 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002975827482447342, + "loss": 2.7214, + "theoretical_loss": 3.547829339068219, + "tokens_seen": 1355939840 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029757271815446343, + "loss": 2.6863, + "theoretical_loss": 3.54781392627003, + "tokens_seen": 1356005376 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029756268806419256, + "loss": 2.5133, + "theoretical_loss": 3.5477985144252884, + "tokens_seen": 1356070912 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002975526579739218, + "loss": 2.8081, + "theoretical_loss": 3.547783103533889, + "tokens_seen": 1356136448 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002975426278836509, + "loss": 2.7985, + "theoretical_loss": 3.547767693595726, + "tokens_seen": 1356201984 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029753259779338016, + "loss": 2.6955, + "theoretical_loss": 3.547752284610695, + "tokens_seen": 1356267520 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029752256770310934, + "loss": 2.5433, + "theoretical_loss": 3.547736876578691, + "tokens_seen": 1356333056 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002975125376128385, + "loss": 2.817, + "theoretical_loss": 3.547721469499608, + "tokens_seen": 1356398592 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002975025075225677, + "loss": 2.5982, + "theoretical_loss": 3.547706063373343, + "tokens_seen": 1356464128 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002974924774322969, + "loss": 2.795, + "theoretical_loss": 3.54769065819979, + "tokens_seen": 1356529664 + }, + { + "epoch": 0.41, + "objective/train/docs_used": 768221, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.592787981033325, + "objective/train/theoretical_loss": 3.547675253978843, + "objective/train/tokens_used": 1377055200, + "theoretical_loss": 3.547675253978843, + "tokens_seen": 1356595200 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029748244734202606, + "loss": 2.5759, + "theoretical_loss": 3.547675253978843, + "tokens_seen": 1356595200 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002974724172517553, + "loss": 2.6252, + "theoretical_loss": 3.5476598507103994, + "tokens_seen": 1356660736 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002974623871614844, + "loss": 2.7779, + "theoretical_loss": 3.547644448394352, + "tokens_seen": 1356726272 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029745235707121366, + "loss": 2.7526, + "theoretical_loss": 3.547629047030597, + "tokens_seen": 1356791808 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002974423269809428, + "loss": 2.5468, + "theoretical_loss": 3.5476136466190296, + "tokens_seen": 1356857344 + }, + { + "epoch": 0.41, + "learning_rate": 0.000297432296890672, + "loss": 2.7877, + "theoretical_loss": 3.547598247159545, + "tokens_seen": 1356922880 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002974222668004012, + "loss": 2.5478, + "theoretical_loss": 3.5475828486520378, + "tokens_seen": 1356988416 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002974122367101304, + "loss": 2.619, + "theoretical_loss": 3.5475674510964037, + "tokens_seen": 1357053952 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029740220661985956, + "loss": 2.7616, + "theoretical_loss": 3.547552054492537, + "tokens_seen": 1357119488 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002973921765295888, + "loss": 2.5443, + "theoretical_loss": 3.5475366588403343, + "tokens_seen": 1357185024 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002973821464393179, + "loss": 2.6467, + "theoretical_loss": 3.5475212641396903, + "tokens_seen": 1357250560 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029737211634904716, + "loss": 2.5926, + "theoretical_loss": 3.5475058703905, + "tokens_seen": 1357316096 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002973620862587763, + "loss": 2.8875, + "theoretical_loss": 3.5474904775926586, + "tokens_seen": 1357381632 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002973520561685055, + "loss": 2.6025, + "theoretical_loss": 3.5474750857460613, + "tokens_seen": 1357447168 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002973420260782347, + "loss": 2.9342, + "theoretical_loss": 3.547459694850604, + "tokens_seen": 1357512704 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002973319959879639, + "loss": 2.5825, + "theoretical_loss": 3.5474443049061812, + "tokens_seen": 1357578240 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029732196589769307, + "loss": 2.5542, + "theoretical_loss": 3.5474289159126893, + "tokens_seen": 1357643776 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029731193580742225, + "loss": 2.7559, + "theoretical_loss": 3.5474135278700225, + "tokens_seen": 1357709312 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029730190571715143, + "loss": 2.6472, + "theoretical_loss": 3.5473981407780766, + "tokens_seen": 1357774848 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029729187562688067, + "loss": 2.5685, + "theoretical_loss": 3.5473827546367476, + "tokens_seen": 1357840384 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029728184553660985, + "loss": 2.7824, + "theoretical_loss": 3.54736736944593, + "tokens_seen": 1357905920 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029727181544633903, + "loss": 2.7349, + "theoretical_loss": 3.5473519852055193, + "tokens_seen": 1357971456 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029726178535606826, + "loss": 2.6498, + "theoretical_loss": 3.5473366019154113, + "tokens_seen": 1358036992 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002972517552657974, + "loss": 2.6306, + "theoretical_loss": 3.5473212195755015, + "tokens_seen": 1358102528 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002972417251755266, + "loss": 2.772, + "theoretical_loss": 3.547305838185685, + "tokens_seen": 1358168064 + }, + { + "epoch": 0.41, + "objective/train/docs_used": 768734, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.8365371227264404, + "objective/train/theoretical_loss": 3.5472904577458575, + "objective/train/tokens_used": 1378693600, + "theoretical_loss": 3.5472904577458575, + "tokens_seen": 1358233600 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029723169508525575, + "loss": 2.7758, + "theoretical_loss": 3.5472904577458575, + "tokens_seen": 1358233600 + }, + { + "epoch": 0.41, + "learning_rate": 0.000297221664994985, + "loss": 2.6831, + "theoretical_loss": 3.5472750782559146, + "tokens_seen": 1358299136 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029721163490471417, + "loss": 2.7009, + "theoretical_loss": 3.5472596997157515, + "tokens_seen": 1358364672 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029720160481444335, + "loss": 2.7268, + "theoretical_loss": 3.5472443221252634, + "tokens_seen": 1358430208 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029719157472417253, + "loss": 2.752, + "theoretical_loss": 3.547228945484347, + "tokens_seen": 1358495744 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002971815446339017, + "loss": 2.8133, + "theoretical_loss": 3.5472135697928966, + "tokens_seen": 1358561280 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002971715145436309, + "loss": 2.5795, + "theoretical_loss": 3.547198195050809, + "tokens_seen": 1358626816 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029716148445336013, + "loss": 2.794, + "theoretical_loss": 3.5471828212579792, + "tokens_seen": 1358692352 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029715145436308926, + "loss": 2.5525, + "theoretical_loss": 3.547167448414302, + "tokens_seen": 1358757888 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002971414242728185, + "loss": 2.7141, + "theoretical_loss": 3.5471520765196747, + "tokens_seen": 1358823424 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002971313941825476, + "loss": 2.3996, + "theoretical_loss": 3.5471367055739913, + "tokens_seen": 1358888960 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029712136409227685, + "loss": 2.6931, + "theoretical_loss": 3.547121335577148, + "tokens_seen": 1358954496 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029711133400200603, + "loss": 2.7884, + "theoretical_loss": 3.5471059665290414, + "tokens_seen": 1359020032 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002971013039117352, + "loss": 2.7478, + "theoretical_loss": 3.547090598429566, + "tokens_seen": 1359085568 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002970912738214644, + "loss": 2.6483, + "theoretical_loss": 3.547075231278618, + "tokens_seen": 1359151104 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029708124373119363, + "loss": 2.649, + "theoretical_loss": 3.5470598650760934, + "tokens_seen": 1359216640 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029707121364092276, + "loss": 2.9121, + "theoretical_loss": 3.5470444998218875, + "tokens_seen": 1359282176 + }, + { + "epoch": 0.41, + "learning_rate": 0.000297061183550652, + "loss": 2.8085, + "theoretical_loss": 3.5470291355158956, + "tokens_seen": 1359347712 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002970511534603811, + "loss": 2.5036, + "theoretical_loss": 3.5470137721580146, + "tokens_seen": 1359413248 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029704112337011036, + "loss": 2.628, + "theoretical_loss": 3.5469984097481397, + "tokens_seen": 1359478784 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029703109327983954, + "loss": 2.4137, + "theoretical_loss": 3.546983048286167, + "tokens_seen": 1359544320 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002970210631895687, + "loss": 2.5759, + "theoretical_loss": 3.5469676877719913, + "tokens_seen": 1359609856 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002970110330992979, + "loss": 2.9426, + "theoretical_loss": 3.54695232820551, + "tokens_seen": 1359675392 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002970010030090271, + "loss": 2.7599, + "theoretical_loss": 3.5469369695866177, + "tokens_seen": 1359740928 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029699097291875626, + "loss": 2.7857, + "theoretical_loss": 3.546921611915211, + "tokens_seen": 1359806464 + }, + { + "epoch": 0.41, + "objective/train/docs_used": 769315, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.4634768962860107, + "objective/train/theoretical_loss": 3.5469062551911854, + "objective/train/tokens_used": 1380332000, + "theoretical_loss": 3.5469062551911854, + "tokens_seen": 1359872000 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002969809428284855, + "loss": 2.5293, + "theoretical_loss": 3.5469062551911854, + "tokens_seen": 1359872000 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002969709127382146, + "loss": 2.6636, + "theoretical_loss": 3.5468908994144366, + "tokens_seen": 1359937536 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029696088264794386, + "loss": 2.6849, + "theoretical_loss": 3.546875544584861, + "tokens_seen": 1360003072 + }, + { + "epoch": 0.41, + "learning_rate": 0.000296950852557673, + "loss": 2.8185, + "theoretical_loss": 3.546860190702355, + "tokens_seen": 1360068608 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002969408224674022, + "loss": 2.7172, + "theoretical_loss": 3.5468448377668134, + "tokens_seen": 1360134144 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002969307923771314, + "loss": 2.5614, + "theoretical_loss": 3.5468294857781326, + "tokens_seen": 1360199680 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002969207622868606, + "loss": 2.674, + "theoretical_loss": 3.546814134736209, + "tokens_seen": 1360265216 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029691073219658976, + "loss": 2.6993, + "theoretical_loss": 3.546798784640939, + "tokens_seen": 1360330752 + }, + { + "epoch": 0.41, + "learning_rate": 0.000296900702106319, + "loss": 2.7542, + "theoretical_loss": 3.5467834354922165, + "tokens_seen": 1360396288 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002968906720160481, + "loss": 2.6112, + "theoretical_loss": 3.54676808728994, + "tokens_seen": 1360461824 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029688064192577736, + "loss": 2.7946, + "theoretical_loss": 3.546752740034004, + "tokens_seen": 1360527360 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002968706118355065, + "loss": 2.5583, + "theoretical_loss": 3.546737393724305, + "tokens_seen": 1360592896 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002968605817452357, + "loss": 2.6414, + "theoretical_loss": 3.5467220483607393, + "tokens_seen": 1360658432 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002968505516549649, + "loss": 2.6046, + "theoretical_loss": 3.546706703943203, + "tokens_seen": 1360723968 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002968405215646941, + "loss": 2.4477, + "theoretical_loss": 3.546691360471592, + "tokens_seen": 1360789504 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029683049147442327, + "loss": 2.7042, + "theoretical_loss": 3.546676017945803, + "tokens_seen": 1360855040 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029682046138415245, + "loss": 2.7748, + "theoretical_loss": 3.5466606763657316, + "tokens_seen": 1360920576 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029681043129388163, + "loss": 2.6568, + "theoretical_loss": 3.5466453357312737, + "tokens_seen": 1360986112 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029680040120361087, + "loss": 2.7786, + "theoretical_loss": 3.5466299960423258, + "tokens_seen": 1361051648 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029679037111334, + "loss": 2.6478, + "theoretical_loss": 3.5466146572987842, + "tokens_seen": 1361117184 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029678034102306923, + "loss": 2.5265, + "theoretical_loss": 3.5465993195005456, + "tokens_seen": 1361182720 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002967703109327984, + "loss": 2.9178, + "theoretical_loss": 3.5465839826475047, + "tokens_seen": 1361248256 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002967602808425276, + "loss": 2.563, + "theoretical_loss": 3.5465686467395594, + "tokens_seen": 1361313792 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029675025075225677, + "loss": 2.8381, + "theoretical_loss": 3.5465533117766053, + "tokens_seen": 1361379328 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029674022066198595, + "loss": 2.6964, + "theoretical_loss": 3.5465379777585384, + "tokens_seen": 1361444864 + }, + { + "epoch": 0.41, + "objective/train/docs_used": 770761, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.693655252456665, + "objective/train/theoretical_loss": 3.546522644685256, + "objective/train/tokens_used": 1381970400, + "theoretical_loss": 3.546522644685256, + "tokens_seen": 1361510400 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029673019057171513, + "loss": 2.5735, + "theoretical_loss": 3.546522644685256, + "tokens_seen": 1361510400 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029672016048144437, + "loss": 2.8058, + "theoretical_loss": 3.5465073125566526, + "tokens_seen": 1361575936 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002967101303911735, + "loss": 2.5498, + "theoretical_loss": 3.5464919813726263, + "tokens_seen": 1361641472 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029670010030090273, + "loss": 2.8771, + "theoretical_loss": 3.546476651133073, + "tokens_seen": 1361707008 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029669007021063186, + "loss": 2.8193, + "theoretical_loss": 3.5464613218378886, + "tokens_seen": 1361772544 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002966800401203611, + "loss": 2.6428, + "theoretical_loss": 3.5464459934869694, + "tokens_seen": 1361838080 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002966700100300903, + "loss": 2.7401, + "theoretical_loss": 3.5464306660802123, + "tokens_seen": 1361903616 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029665997993981946, + "loss": 2.5556, + "theoretical_loss": 3.5464153396175138, + "tokens_seen": 1361969152 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029664994984954864, + "loss": 2.7622, + "theoretical_loss": 3.54640001409877, + "tokens_seen": 1362034688 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002966399197592778, + "loss": 2.48, + "theoretical_loss": 3.5463846895238773, + "tokens_seen": 1362100224 + }, + { + "epoch": 0.41, + "learning_rate": 0.000296629889669007, + "loss": 2.7064, + "theoretical_loss": 3.5463693658927324, + "tokens_seen": 1362165760 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029661985957873623, + "loss": 2.6794, + "theoretical_loss": 3.5463540432052314, + "tokens_seen": 1362231296 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029660982948846536, + "loss": 2.6846, + "theoretical_loss": 3.546338721461271, + "tokens_seen": 1362296832 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002965997993981946, + "loss": 2.5242, + "theoretical_loss": 3.5463234006607482, + "tokens_seen": 1362362368 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002965897693079238, + "loss": 2.6567, + "theoretical_loss": 3.546308080803559, + "tokens_seen": 1362427904 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029657973921765296, + "loss": 2.7516, + "theoretical_loss": 3.5462927618895996, + "tokens_seen": 1362493440 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029656970912738214, + "loss": 2.7187, + "theoretical_loss": 3.546277443918768, + "tokens_seen": 1362558976 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002965596790371113, + "loss": 2.7422, + "theoretical_loss": 3.5462621268909587, + "tokens_seen": 1362624512 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002965496489468405, + "loss": 2.6628, + "theoretical_loss": 3.5462468108060703, + "tokens_seen": 1362690048 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029653961885656974, + "loss": 2.5885, + "theoretical_loss": 3.5462314956639975, + "tokens_seen": 1362755584 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002965295887662989, + "loss": 2.6615, + "theoretical_loss": 3.5462161814646387, + "tokens_seen": 1362821120 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002965195586760281, + "loss": 2.7581, + "theoretical_loss": 3.5462008682078894, + "tokens_seen": 1362886656 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002965095285857573, + "loss": 2.6191, + "theoretical_loss": 3.546185555893647, + "tokens_seen": 1362952192 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029649949849548646, + "loss": 2.6532, + "theoretical_loss": 3.5461702445218073, + "tokens_seen": 1363017728 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002964894684052157, + "loss": 2.5259, + "theoretical_loss": 3.5461549340922676, + "tokens_seen": 1363083264 + }, + { + "epoch": 0.41, + "objective/train/docs_used": 771430, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6774184703826904, + "objective/train/theoretical_loss": 3.5461396246049244, + "objective/train/tokens_used": 1383608800, + "theoretical_loss": 3.5461396246049244, + "tokens_seen": 1363148800 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002964794383149448, + "loss": 2.6551, + "theoretical_loss": 3.5461396246049244, + "tokens_seen": 1363148800 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029646940822467406, + "loss": 2.845, + "theoretical_loss": 3.546124316059675, + "tokens_seen": 1363214336 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002964593781344032, + "loss": 2.4919, + "theoretical_loss": 3.546109008456415, + "tokens_seen": 1363279872 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002964493480441324, + "loss": 2.648, + "theoretical_loss": 3.5460937017950425, + "tokens_seen": 1363345408 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002964393179538616, + "loss": 2.6842, + "theoretical_loss": 3.546078396075453, + "tokens_seen": 1363410944 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002964292878635908, + "loss": 2.3725, + "theoretical_loss": 3.546063091297544, + "tokens_seen": 1363476480 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029641925777331996, + "loss": 2.6676, + "theoretical_loss": 3.5460477874612124, + "tokens_seen": 1363542016 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002964092276830492, + "loss": 2.7911, + "theoretical_loss": 3.5460324845663544, + "tokens_seen": 1363607552 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002963991975927783, + "loss": 2.7289, + "theoretical_loss": 3.5460171826128675, + "tokens_seen": 1363673088 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029638916750250756, + "loss": 2.4512, + "theoretical_loss": 3.5460018816006484, + "tokens_seen": 1363738624 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002963791374122367, + "loss": 2.7802, + "theoretical_loss": 3.5459865815295935, + "tokens_seen": 1363804160 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002963691073219659, + "loss": 2.7183, + "theoretical_loss": 3.5459712823996004, + "tokens_seen": 1363869696 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002963590772316951, + "loss": 2.5769, + "theoretical_loss": 3.5459559842105657, + "tokens_seen": 1363935232 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002963490471414243, + "loss": 2.8125, + "theoretical_loss": 3.5459406869623855, + "tokens_seen": 1364000768 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029633901705115347, + "loss": 2.5704, + "theoretical_loss": 3.5459253906549586, + "tokens_seen": 1364066304 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029632898696088265, + "loss": 2.6294, + "theoretical_loss": 3.54591009528818, + "tokens_seen": 1364131840 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029631895687061183, + "loss": 2.4298, + "theoretical_loss": 3.545894800861948, + "tokens_seen": 1364197376 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029630892678034107, + "loss": 2.7947, + "theoretical_loss": 3.545879507376159, + "tokens_seen": 1364262912 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002962988966900702, + "loss": 2.871, + "theoretical_loss": 3.54586421483071, + "tokens_seen": 1364328448 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029628886659979943, + "loss": 2.7425, + "theoretical_loss": 3.545848923225498, + "tokens_seen": 1364393984 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002962788365095286, + "loss": 2.8533, + "theoretical_loss": 3.5458336325604205, + "tokens_seen": 1364459520 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002962688064192578, + "loss": 2.5408, + "theoretical_loss": 3.545818342835374, + "tokens_seen": 1364525056 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029625877632898697, + "loss": 2.821, + "theoretical_loss": 3.545803054050256, + "tokens_seen": 1364590592 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029624874623871615, + "loss": 2.607, + "theoretical_loss": 3.5457877662049633, + "tokens_seen": 1364656128 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029623871614844533, + "loss": 2.8383, + "theoretical_loss": 3.545772479299393, + "tokens_seen": 1364721664 + }, + { + "epoch": 0.41, + "objective/train/docs_used": 772731, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 1.9461311101913452, + "objective/train/theoretical_loss": 3.5457571933334417, + "objective/train/tokens_used": 1385247200, + "theoretical_loss": 3.5457571933334417, + "tokens_seen": 1364787200 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029622868605817457, + "loss": 2.6138, + "theoretical_loss": 3.5457571933334417, + "tokens_seen": 1364787200 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002962186559679037, + "loss": 2.6423, + "theoretical_loss": 3.545741908307008, + "tokens_seen": 1364852736 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029620862587763293, + "loss": 2.713, + "theoretical_loss": 3.545726624219988, + "tokens_seen": 1364918272 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029619859578736206, + "loss": 2.7212, + "theoretical_loss": 3.5457113410722787, + "tokens_seen": 1364983808 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002961885656970913, + "loss": 2.7797, + "theoretical_loss": 3.5456960588637774, + "tokens_seen": 1365049344 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002961785356068205, + "loss": 2.7541, + "theoretical_loss": 3.5456807775943817, + "tokens_seen": 1365114880 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029616850551654966, + "loss": 2.7492, + "theoretical_loss": 3.5456654972639883, + "tokens_seen": 1365180416 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029615847542627884, + "loss": 2.6622, + "theoretical_loss": 3.5456502178724953, + "tokens_seen": 1365245952 + }, + { + "epoch": 0.41, + "learning_rate": 0.000296148445336008, + "loss": 2.6546, + "theoretical_loss": 3.5456349394197986, + "tokens_seen": 1365311488 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002961384152457372, + "loss": 2.7177, + "theoretical_loss": 3.545619661905797, + "tokens_seen": 1365377024 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029612838515546643, + "loss": 2.6437, + "theoretical_loss": 3.5456043853303862, + "tokens_seen": 1365442560 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029611835506519556, + "loss": 2.6537, + "theoretical_loss": 3.545589109693465, + "tokens_seen": 1365508096 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002961083249749248, + "loss": 2.5029, + "theoretical_loss": 3.54557383499493, + "tokens_seen": 1365573632 + }, + { + "epoch": 0.41, + "learning_rate": 0.000296098294884654, + "loss": 2.7198, + "theoretical_loss": 3.545558561234678, + "tokens_seen": 1365639168 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029608826479438316, + "loss": 2.5171, + "theoretical_loss": 3.545543288412607, + "tokens_seen": 1365704704 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029607823470411234, + "loss": 2.7978, + "theoretical_loss": 3.5455280165286145, + "tokens_seen": 1365770240 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002960682046138415, + "loss": 2.64, + "theoretical_loss": 3.545512745582597, + "tokens_seen": 1365835776 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002960581745235707, + "loss": 2.6891, + "theoretical_loss": 3.545497475574453, + "tokens_seen": 1365901312 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029604814443329994, + "loss": 2.7023, + "theoretical_loss": 3.5454822065040794, + "tokens_seen": 1365966848 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029603811434302906, + "loss": 2.8261, + "theoretical_loss": 3.5454669383713737, + "tokens_seen": 1366032384 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002960280842527583, + "loss": 2.7013, + "theoretical_loss": 3.5454516711762327, + "tokens_seen": 1366097920 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002960180541624874, + "loss": 2.6312, + "theoretical_loss": 3.5454364049185547, + "tokens_seen": 1366163456 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029600802407221666, + "loss": 2.7791, + "theoretical_loss": 3.545421139598237, + "tokens_seen": 1366228992 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029599799398194584, + "loss": 2.484, + "theoretical_loss": 3.5454058752151765, + "tokens_seen": 1366294528 + }, + { + "epoch": 0.41, + "learning_rate": 0.000295987963891675, + "loss": 2.7159, + "theoretical_loss": 3.5453906117692715, + "tokens_seen": 1366360064 + }, + { + "epoch": 0.41, + "objective/train/docs_used": 773461, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.5035691261291504, + "objective/train/theoretical_loss": 3.545375349260419, + "objective/train/tokens_used": 1386885600, + "theoretical_loss": 3.545375349260419, + "tokens_seen": 1366425600 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002959779338014042, + "loss": 2.6433, + "theoretical_loss": 3.545375349260419, + "tokens_seen": 1366425600 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002959679037111334, + "loss": 2.735, + "theoretical_loss": 3.545360087688517, + "tokens_seen": 1366491136 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029595787362086257, + "loss": 2.677, + "theoretical_loss": 3.545344827053462, + "tokens_seen": 1366556672 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002959478435305918, + "loss": 2.9056, + "theoretical_loss": 3.5453295673551537, + "tokens_seen": 1366622208 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029593781344032093, + "loss": 2.7194, + "theoretical_loss": 3.545314308593487, + "tokens_seen": 1366687744 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029592778335005016, + "loss": 2.7147, + "theoretical_loss": 3.5452990507683615, + "tokens_seen": 1366753280 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029591775325977935, + "loss": 2.693, + "theoretical_loss": 3.545283793879674, + "tokens_seen": 1366818816 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029590772316950853, + "loss": 2.5053, + "theoretical_loss": 3.545268537927322, + "tokens_seen": 1366884352 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002958976930792377, + "loss": 2.7413, + "theoretical_loss": 3.5452532829112036, + "tokens_seen": 1366949888 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002958876629889669, + "loss": 2.9868, + "theoretical_loss": 3.545238028831216, + "tokens_seen": 1367015424 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029587763289869607, + "loss": 2.5317, + "theoretical_loss": 3.5452227756872574, + "tokens_seen": 1367080960 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002958676028084253, + "loss": 2.8291, + "theoretical_loss": 3.545207523479225, + "tokens_seen": 1367146496 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029585757271815443, + "loss": 2.847, + "theoretical_loss": 3.545192272207017, + "tokens_seen": 1367212032 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029584754262788367, + "loss": 2.6147, + "theoretical_loss": 3.545177021870531, + "tokens_seen": 1367277568 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002958375125376128, + "loss": 2.8907, + "theoretical_loss": 3.5451617724696645, + "tokens_seen": 1367343104 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029582748244734203, + "loss": 2.5669, + "theoretical_loss": 3.5451465240043145, + "tokens_seen": 1367408640 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002958174523570712, + "loss": 2.6977, + "theoretical_loss": 3.5451312764743808, + "tokens_seen": 1367474176 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002958074222668004, + "loss": 2.3962, + "theoretical_loss": 3.5451160298797593, + "tokens_seen": 1367539712 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002957973921765296, + "loss": 2.8379, + "theoretical_loss": 3.545100784220349, + "tokens_seen": 1367605248 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002957873620862588, + "loss": 2.5758, + "theoretical_loss": 3.545085539496047, + "tokens_seen": 1367670784 + }, + { + "epoch": 0.41, + "learning_rate": 0.000295777331995988, + "loss": 2.6891, + "theoretical_loss": 3.5450702957067515, + "tokens_seen": 1367736320 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029576730190571717, + "loss": 2.6433, + "theoretical_loss": 3.5450550528523603, + "tokens_seen": 1367801856 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029575727181544635, + "loss": 2.6515, + "theoretical_loss": 3.545039810932771, + "tokens_seen": 1367867392 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029574724172517553, + "loss": 2.5533, + "theoretical_loss": 3.5450245699478824, + "tokens_seen": 1367932928 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029573721163490477, + "loss": 2.4537, + "theoretical_loss": 3.5450093298975913, + "tokens_seen": 1367998464 + }, + { + "epoch": 0.41, + "objective/train/docs_used": 774789, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.4741592407226562, + "objective/train/theoretical_loss": 3.544994090781796, + "objective/train/tokens_used": 1388524000, + "theoretical_loss": 3.544994090781796, + "tokens_seen": 1368064000 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002957271815446339, + "loss": 2.5356, + "theoretical_loss": 3.544994090781796, + "tokens_seen": 1368064000 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029571715145436313, + "loss": 2.3952, + "theoretical_loss": 3.5449788526003942, + "tokens_seen": 1368129536 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029570712136409226, + "loss": 2.8576, + "theoretical_loss": 3.5449636153532844, + "tokens_seen": 1368195072 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002956970912738215, + "loss": 2.6411, + "theoretical_loss": 3.544948379040364, + "tokens_seen": 1368260608 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002956870611835507, + "loss": 2.5374, + "theoretical_loss": 3.544933143661532, + "tokens_seen": 1368326144 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029567703109327986, + "loss": 2.7693, + "theoretical_loss": 3.5449179092166854, + "tokens_seen": 1368391680 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029566700100300904, + "loss": 2.776, + "theoretical_loss": 3.5449026757057225, + "tokens_seen": 1368457216 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002956569709127382, + "loss": 2.769, + "theoretical_loss": 3.544887443128541, + "tokens_seen": 1368522752 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002956469408224674, + "loss": 2.7925, + "theoretical_loss": 3.5448722114850395, + "tokens_seen": 1368588288 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029563691073219663, + "loss": 2.6775, + "theoretical_loss": 3.544856980775116, + "tokens_seen": 1368653824 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029562688064192576, + "loss": 2.753, + "theoretical_loss": 3.5448417509986685, + "tokens_seen": 1368719360 + }, + { + "epoch": 0.41, + "learning_rate": 0.000295616850551655, + "loss": 2.9956, + "theoretical_loss": 3.544826522155595, + "tokens_seen": 1368784896 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002956068204613842, + "loss": 2.6337, + "theoretical_loss": 3.544811294245793, + "tokens_seen": 1368850432 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029559679037111336, + "loss": 2.7669, + "theoretical_loss": 3.5447960672691616, + "tokens_seen": 1368915968 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029558676028084254, + "loss": 2.7035, + "theoretical_loss": 3.5447808412255988, + "tokens_seen": 1368981504 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002955767301905717, + "loss": 2.5958, + "theoretical_loss": 3.5447656161150025, + "tokens_seen": 1369047040 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002955667001003009, + "loss": 2.7322, + "theoretical_loss": 3.544750391937271, + "tokens_seen": 1369112576 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029555667001003014, + "loss": 2.6119, + "theoretical_loss": 3.5447351686923025, + "tokens_seen": 1369178112 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029554663991975926, + "loss": 2.6333, + "theoretical_loss": 3.544719946379995, + "tokens_seen": 1369243648 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002955366098294885, + "loss": 2.6717, + "theoretical_loss": 3.544704725000247, + "tokens_seen": 1369309184 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002955265797392176, + "loss": 2.5936, + "theoretical_loss": 3.5446895045529567, + "tokens_seen": 1369374720 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029551654964894686, + "loss": 2.71, + "theoretical_loss": 3.5446742850380217, + "tokens_seen": 1369440256 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029550651955867604, + "loss": 2.5491, + "theoretical_loss": 3.544659066455341, + "tokens_seen": 1369505792 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002954964894684052, + "loss": 2.5508, + "theoretical_loss": 3.5446438488048133, + "tokens_seen": 1369571328 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002954864593781344, + "loss": 2.4632, + "theoretical_loss": 3.5446286320863356, + "tokens_seen": 1369636864 + }, + { + "epoch": 0.42, + "objective/train/docs_used": 775516, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.9834017753601074, + "objective/train/theoretical_loss": 3.544613416299808, + "objective/train/tokens_used": 1390162400, + "theoretical_loss": 3.544613416299808, + "tokens_seen": 1369702400 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002954764292878636, + "loss": 2.919, + "theoretical_loss": 3.544613416299808, + "tokens_seen": 1369702400 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029546639919759277, + "loss": 2.5666, + "theoretical_loss": 3.544598201445127, + "tokens_seen": 1369767936 + }, + { + "epoch": 0.42, + "learning_rate": 0.000295456369107322, + "loss": 2.7568, + "theoretical_loss": 3.5445829875221917, + "tokens_seen": 1369833472 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029544633901705113, + "loss": 2.7622, + "theoretical_loss": 3.5445677745309006, + "tokens_seen": 1369899008 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029543630892678037, + "loss": 2.5413, + "theoretical_loss": 3.544552562471152, + "tokens_seen": 1369964544 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029542627883650955, + "loss": 2.7292, + "theoretical_loss": 3.544537351342844, + "tokens_seen": 1370030080 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029541624874623873, + "loss": 2.6185, + "theoretical_loss": 3.544522141145876, + "tokens_seen": 1370095616 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002954062186559679, + "loss": 2.7381, + "theoretical_loss": 3.5445069318801457, + "tokens_seen": 1370161152 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002953961885656971, + "loss": 2.7692, + "theoretical_loss": 3.544491723545551, + "tokens_seen": 1370226688 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029538615847542627, + "loss": 2.9177, + "theoretical_loss": 3.5444765161419913, + "tokens_seen": 1370292224 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002953761283851555, + "loss": 2.7845, + "theoretical_loss": 3.5444613096693645, + "tokens_seen": 1370357760 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029536609829488463, + "loss": 2.7966, + "theoretical_loss": 3.5444461041275694, + "tokens_seen": 1370423296 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029535606820461387, + "loss": 2.6282, + "theoretical_loss": 3.5444308995165046, + "tokens_seen": 1370488832 + }, + { + "epoch": 0.42, + "learning_rate": 0.000295346038114343, + "loss": 2.8776, + "theoretical_loss": 3.544415695836068, + "tokens_seen": 1370554368 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029533600802407223, + "loss": 2.5787, + "theoretical_loss": 3.544400493086159, + "tokens_seen": 1370619904 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002953259779338014, + "loss": 2.6612, + "theoretical_loss": 3.544385291266676, + "tokens_seen": 1370685440 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002953159478435306, + "loss": 2.6639, + "theoretical_loss": 3.544370090377517, + "tokens_seen": 1370750976 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002953059177532598, + "loss": 2.4649, + "theoretical_loss": 3.544354890418581, + "tokens_seen": 1370816512 + }, + { + "epoch": 0.42, + "learning_rate": 0.000295295887662989, + "loss": 2.6367, + "theoretical_loss": 3.5443396913897667, + "tokens_seen": 1370882048 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029528585757271814, + "loss": 2.8307, + "theoretical_loss": 3.544324493290972, + "tokens_seen": 1370947584 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029527582748244737, + "loss": 2.6221, + "theoretical_loss": 3.5443092961220968, + "tokens_seen": 1371013120 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002952657973921765, + "loss": 2.6511, + "theoretical_loss": 3.5442940998830386, + "tokens_seen": 1371078656 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029525576730190573, + "loss": 2.2687, + "theoretical_loss": 3.544278904573697, + "tokens_seen": 1371144192 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002952457372116349, + "loss": 2.7019, + "theoretical_loss": 3.54426371019397, + "tokens_seen": 1371209728 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002952357071213641, + "loss": 2.6321, + "theoretical_loss": 3.544248516743756, + "tokens_seen": 1371275264 + }, + { + "epoch": 0.42, + "objective/train/docs_used": 776338, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.134028196334839, + "objective/train/theoretical_loss": 3.5442333242229545, + "objective/train/tokens_used": 1391800800, + "theoretical_loss": 3.5442333242229545, + "tokens_seen": 1371340800 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002952256770310933, + "loss": 2.4491, + "theoretical_loss": 3.5442333242229545, + "tokens_seen": 1371340800 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029521564694082246, + "loss": 2.6735, + "theoretical_loss": 3.544218132631464, + "tokens_seen": 1371406336 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029520561685055164, + "loss": 2.5032, + "theoretical_loss": 3.5442029419691834, + "tokens_seen": 1371471872 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002951955867602809, + "loss": 2.6468, + "theoretical_loss": 3.5441877522360112, + "tokens_seen": 1371537408 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029518555667001, + "loss": 2.7177, + "theoretical_loss": 3.544172563431846, + "tokens_seen": 1371602944 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029517552657973924, + "loss": 2.5677, + "theoretical_loss": 3.544157375556587, + "tokens_seen": 1371668480 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029516549648946836, + "loss": 2.5501, + "theoretical_loss": 3.5441421886101336, + "tokens_seen": 1371734016 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002951554663991976, + "loss": 2.5802, + "theoretical_loss": 3.544127002592383, + "tokens_seen": 1371799552 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002951454363089268, + "loss": 2.4618, + "theoretical_loss": 3.544111817503235, + "tokens_seen": 1371865088 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029513540621865596, + "loss": 2.6433, + "theoretical_loss": 3.5440966333425887, + "tokens_seen": 1371930624 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029512537612838514, + "loss": 2.6454, + "theoretical_loss": 3.544081450110342, + "tokens_seen": 1371996160 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002951153460381144, + "loss": 2.6617, + "theoretical_loss": 3.5440662678063957, + "tokens_seen": 1372061696 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002951053159478435, + "loss": 2.708, + "theoretical_loss": 3.5440510864306467, + "tokens_seen": 1372127232 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029509528585757274, + "loss": 2.7923, + "theoretical_loss": 3.544035905982995, + "tokens_seen": 1372192768 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029508525576730187, + "loss": 3.0044, + "theoretical_loss": 3.544020726463339, + "tokens_seen": 1372258304 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002950752256770311, + "loss": 2.7399, + "theoretical_loss": 3.5440055478715777, + "tokens_seen": 1372323840 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002950651955867603, + "loss": 2.6246, + "theoretical_loss": 3.5439903702076108, + "tokens_seen": 1372389376 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029505516549648946, + "loss": 2.3524, + "theoretical_loss": 3.5439751934713364, + "tokens_seen": 1372454912 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029504513540621865, + "loss": 2.7339, + "theoretical_loss": 3.543960017662654, + "tokens_seen": 1372520448 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002950351053159478, + "loss": 2.449, + "theoretical_loss": 3.543944842781462, + "tokens_seen": 1372585984 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029502507522567706, + "loss": 2.7151, + "theoretical_loss": 3.5439296688276607, + "tokens_seen": 1372651520 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029501504513540624, + "loss": 2.8035, + "theoretical_loss": 3.5439144958011477, + "tokens_seen": 1372717056 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002950050150451354, + "loss": 2.9308, + "theoretical_loss": 3.5438993237018233, + "tokens_seen": 1372782592 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002949949849548646, + "loss": 2.583, + "theoretical_loss": 3.5438841525295857, + "tokens_seen": 1372848128 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002949849548645938, + "loss": 2.5836, + "theoretical_loss": 3.5438689822843346, + "tokens_seen": 1372913664 + }, + { + "epoch": 0.42, + "objective/train/docs_used": 777606, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.360630512237549, + "objective/train/theoretical_loss": 3.5438538129659687, + "objective/train/tokens_used": 1393439200, + "theoretical_loss": 3.5438538129659687, + "tokens_seen": 1372979200 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029497492477432297, + "loss": 2.7546, + "theoretical_loss": 3.5438538129659687, + "tokens_seen": 1372979200 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002949648946840522, + "loss": 2.8358, + "theoretical_loss": 3.543838644574387, + "tokens_seen": 1373044736 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029495486459378133, + "loss": 2.6468, + "theoretical_loss": 3.543823477109489, + "tokens_seen": 1373110272 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029494483450351057, + "loss": 2.9566, + "theoretical_loss": 3.543808310571174, + "tokens_seen": 1373175808 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029493480441323975, + "loss": 2.8742, + "theoretical_loss": 3.5437931449593405, + "tokens_seen": 1373241344 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029492477432296893, + "loss": 2.3716, + "theoretical_loss": 3.5437779802738887, + "tokens_seen": 1373306880 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002949147442326981, + "loss": 2.7989, + "theoretical_loss": 3.543762816514717, + "tokens_seen": 1373372416 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002949047141424273, + "loss": 2.9004, + "theoretical_loss": 3.5437476536817245, + "tokens_seen": 1373437952 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029489468405215647, + "loss": 2.7237, + "theoretical_loss": 3.5437324917748114, + "tokens_seen": 1373503488 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002948846539618857, + "loss": 2.7089, + "theoretical_loss": 3.543717330793876, + "tokens_seen": 1373569024 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029487462387161483, + "loss": 2.7494, + "theoretical_loss": 3.543702170738818, + "tokens_seen": 1373634560 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029486459378134407, + "loss": 2.6304, + "theoretical_loss": 3.5436870116095367, + "tokens_seen": 1373700096 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002948545636910732, + "loss": 2.6622, + "theoretical_loss": 3.543671853405931, + "tokens_seen": 1373765632 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029484453360080243, + "loss": 2.4505, + "theoretical_loss": 3.543656696127901, + "tokens_seen": 1373831168 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002948345035105316, + "loss": 2.6285, + "theoretical_loss": 3.5436415397753454, + "tokens_seen": 1373896704 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002948244734202608, + "loss": 2.6293, + "theoretical_loss": 3.5436263843481637, + "tokens_seen": 1373962240 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029481444332999, + "loss": 2.7023, + "theoretical_loss": 3.543611229846255, + "tokens_seen": 1374027776 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002948044132397192, + "loss": 2.7307, + "theoretical_loss": 3.5435960762695196, + "tokens_seen": 1374093312 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029479438314944834, + "loss": 2.4951, + "theoretical_loss": 3.543580923617856, + "tokens_seen": 1374158848 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029478435305917757, + "loss": 2.6783, + "theoretical_loss": 3.5435657718911635, + "tokens_seen": 1374224384 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002947743229689067, + "loss": 2.4813, + "theoretical_loss": 3.5435506210893424, + "tokens_seen": 1374289920 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029476429287863593, + "loss": 2.6915, + "theoretical_loss": 3.5435354712122917, + "tokens_seen": 1374355456 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002947542627883651, + "loss": 2.6971, + "theoretical_loss": 3.5435203222599103, + "tokens_seen": 1374420992 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002947442326980943, + "loss": 2.7067, + "theoretical_loss": 3.5435051742320987, + "tokens_seen": 1374486528 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002947342026078235, + "loss": 2.7453, + "theoretical_loss": 3.5434900271287555, + "tokens_seen": 1374552064 + }, + { + "epoch": 0.42, + "objective/train/docs_used": 778164, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7287800312042236, + "objective/train/theoretical_loss": 3.5434748809497814, + "objective/train/tokens_used": 1395077600, + "theoretical_loss": 3.5434748809497814, + "tokens_seen": 1374617600 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029472417251755266, + "loss": 2.8689, + "theoretical_loss": 3.5434748809497814, + "tokens_seen": 1374617600 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029471414242728184, + "loss": 2.6391, + "theoretical_loss": 3.543459735695074, + "tokens_seen": 1374683136 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002947041123370111, + "loss": 2.6542, + "theoretical_loss": 3.543444591364535, + "tokens_seen": 1374748672 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002946940822467402, + "loss": 2.7524, + "theoretical_loss": 3.5434294479580624, + "tokens_seen": 1374814208 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029468405215646944, + "loss": 2.7287, + "theoretical_loss": 3.5434143054755562, + "tokens_seen": 1374879744 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029467402206619856, + "loss": 2.6571, + "theoretical_loss": 3.5433991639169165, + "tokens_seen": 1374945280 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002946639919759278, + "loss": 2.5573, + "theoretical_loss": 3.5433840232820417, + "tokens_seen": 1375010816 + }, + { + "epoch": 0.42, + "learning_rate": 0.000294653961885657, + "loss": 2.7006, + "theoretical_loss": 3.5433688835708326, + "tokens_seen": 1375076352 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029464393179538616, + "loss": 2.5319, + "theoretical_loss": 3.5433537447831887, + "tokens_seen": 1375141888 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029463390170511534, + "loss": 2.9236, + "theoretical_loss": 3.543338606919009, + "tokens_seen": 1375207424 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002946238716148446, + "loss": 2.5282, + "theoretical_loss": 3.543323469978194, + "tokens_seen": 1375272960 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002946138415245737, + "loss": 2.5575, + "theoretical_loss": 3.5433083339606424, + "tokens_seen": 1375338496 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029460381143430294, + "loss": 2.9222, + "theoretical_loss": 3.543293198866255, + "tokens_seen": 1375404032 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029459378134403207, + "loss": 2.4386, + "theoretical_loss": 3.5432780646949302, + "tokens_seen": 1375469568 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002945837512537613, + "loss": 2.8422, + "theoretical_loss": 3.543262931446569, + "tokens_seen": 1375535104 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002945737211634905, + "loss": 2.7512, + "theoretical_loss": 3.5432477991210702, + "tokens_seen": 1375600640 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029456369107321966, + "loss": 2.7479, + "theoretical_loss": 3.5432326677183346, + "tokens_seen": 1375666176 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029455366098294885, + "loss": 2.4851, + "theoretical_loss": 3.5432175372382604, + "tokens_seen": 1375731712 + }, + { + "epoch": 0.42, + "learning_rate": 0.000294543630892678, + "loss": 2.7358, + "theoretical_loss": 3.543202407680749, + "tokens_seen": 1375797248 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002945336008024072, + "loss": 2.5873, + "theoretical_loss": 3.543187279045699, + "tokens_seen": 1375862784 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029452357071213644, + "loss": 2.5648, + "theoretical_loss": 3.543172151333011, + "tokens_seen": 1375928320 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029451354062186557, + "loss": 2.6574, + "theoretical_loss": 3.543157024542585, + "tokens_seen": 1375993856 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002945035105315948, + "loss": 2.7866, + "theoretical_loss": 3.54314189867432, + "tokens_seen": 1376059392 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029449348044132393, + "loss": 2.8504, + "theoretical_loss": 3.5431267737281167, + "tokens_seen": 1376124928 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029448345035105317, + "loss": 2.6212, + "theoretical_loss": 3.5431116497038744, + "tokens_seen": 1376190464 + }, + { + "epoch": 0.42, + "objective/train/docs_used": 779255, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.917086601257324, + "objective/train/theoretical_loss": 3.5430965266014933, + "objective/train/tokens_used": 1396716000, + "theoretical_loss": 3.5430965266014933, + "tokens_seen": 1376256000 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029447342026078235, + "loss": 2.5503, + "theoretical_loss": 3.5430965266014933, + "tokens_seen": 1376256000 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029446339017051153, + "loss": 2.643, + "theoretical_loss": 3.5430814044208727, + "tokens_seen": 1376321536 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002944533600802407, + "loss": 2.7949, + "theoretical_loss": 3.5430662831619135, + "tokens_seen": 1376387072 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029444332998996995, + "loss": 2.6542, + "theoretical_loss": 3.543051162824515, + "tokens_seen": 1376452608 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002944332998996991, + "loss": 2.6859, + "theoretical_loss": 3.5430360434085775, + "tokens_seen": 1376518144 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002944232698094283, + "loss": 2.7596, + "theoretical_loss": 3.543020924914001, + "tokens_seen": 1376583680 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029441323971915744, + "loss": 2.6152, + "theoretical_loss": 3.543005807340685, + "tokens_seen": 1376649216 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029440320962888667, + "loss": 2.613, + "theoretical_loss": 3.542990690688529, + "tokens_seen": 1376714752 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029439317953861585, + "loss": 2.7143, + "theoretical_loss": 3.542975574957435, + "tokens_seen": 1376780288 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029438314944834503, + "loss": 2.4603, + "theoretical_loss": 3.5429604601473015, + "tokens_seen": 1376845824 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002943731193580742, + "loss": 2.4991, + "theoretical_loss": 3.5429453462580294, + "tokens_seen": 1376911360 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002943630892678034, + "loss": 2.3327, + "theoretical_loss": 3.542930233289518, + "tokens_seen": 1376976896 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002943530591775326, + "loss": 2.9971, + "theoretical_loss": 3.5429151212416676, + "tokens_seen": 1377042432 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002943430290872618, + "loss": 2.5762, + "theoretical_loss": 3.542900010114378, + "tokens_seen": 1377107968 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029433299899699094, + "loss": 2.626, + "theoretical_loss": 3.5428848999075506, + "tokens_seen": 1377173504 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002943229689067202, + "loss": 2.5622, + "theoretical_loss": 3.542869790621084, + "tokens_seen": 1377239040 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002943129388164493, + "loss": 2.785, + "theoretical_loss": 3.5428546822548794, + "tokens_seen": 1377304576 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029430290872617854, + "loss": 2.5257, + "theoretical_loss": 3.542839574808837, + "tokens_seen": 1377370112 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029429287863590777, + "loss": 2.6811, + "theoretical_loss": 3.5428244682828556, + "tokens_seen": 1377435648 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002942828485456369, + "loss": 2.8046, + "theoretical_loss": 3.5428093626768367, + "tokens_seen": 1377501184 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029427281845536613, + "loss": 2.8273, + "theoretical_loss": 3.54279425799068, + "tokens_seen": 1377566720 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002942627883650953, + "loss": 2.9532, + "theoretical_loss": 3.542779154224286, + "tokens_seen": 1377632256 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002942527582748245, + "loss": 2.7221, + "theoretical_loss": 3.5427640513775556, + "tokens_seen": 1377697792 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002942427281845537, + "loss": 2.5298, + "theoretical_loss": 3.542748949450387, + "tokens_seen": 1377763328 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029423269809428286, + "loss": 2.6546, + "theoretical_loss": 3.5427338484426825, + "tokens_seen": 1377828864 + }, + { + "epoch": 0.42, + "objective/train/docs_used": 780014, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.73660945892334, + "objective/train/theoretical_loss": 3.542718748354342, + "objective/train/tokens_used": 1398354400, + "theoretical_loss": 3.542718748354342, + "tokens_seen": 1377894400 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029422266800401204, + "loss": 2.6366, + "theoretical_loss": 3.542718748354342, + "tokens_seen": 1377894400 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002942126379137413, + "loss": 2.8346, + "theoretical_loss": 3.542703649185265, + "tokens_seen": 1377959936 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002942026078234704, + "loss": 2.5871, + "theoretical_loss": 3.542688550935352, + "tokens_seen": 1378025472 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029419257773319964, + "loss": 2.8856, + "theoretical_loss": 3.542673453604504, + "tokens_seen": 1378091008 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029418254764292876, + "loss": 2.7507, + "theoretical_loss": 3.5426583571926207, + "tokens_seen": 1378156544 + }, + { + "epoch": 0.42, + "learning_rate": 0.000294172517552658, + "loss": 2.6982, + "theoretical_loss": 3.5426432616996033, + "tokens_seen": 1378222080 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002941624874623872, + "loss": 2.835, + "theoretical_loss": 3.5426281671253506, + "tokens_seen": 1378287616 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029415245737211636, + "loss": 2.8998, + "theoretical_loss": 3.542613073469765, + "tokens_seen": 1378353152 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029414242728184554, + "loss": 2.8075, + "theoretical_loss": 3.5425979807327455, + "tokens_seen": 1378418688 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002941323971915748, + "loss": 2.5066, + "theoretical_loss": 3.5425828889141933, + "tokens_seen": 1378484224 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002941223671013039, + "loss": 2.7969, + "theoretical_loss": 3.542567798014008, + "tokens_seen": 1378549760 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029411233701103314, + "loss": 2.9403, + "theoretical_loss": 3.542552708032091, + "tokens_seen": 1378615296 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029410230692076227, + "loss": 2.6784, + "theoretical_loss": 3.5425376189683417, + "tokens_seen": 1378680832 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002940922768304915, + "loss": 2.6447, + "theoretical_loss": 3.542522530822662, + "tokens_seen": 1378746368 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002940822467402207, + "loss": 2.5443, + "theoretical_loss": 3.5425074435949515, + "tokens_seen": 1378811904 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029407221664994986, + "loss": 2.683, + "theoretical_loss": 3.54249235728511, + "tokens_seen": 1378877440 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029406218655967905, + "loss": 2.5131, + "theoretical_loss": 3.54247727189304, + "tokens_seen": 1378942976 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002940521564694082, + "loss": 2.6648, + "theoretical_loss": 3.5424621874186406, + "tokens_seen": 1379008512 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002940421263791374, + "loss": 3.02, + "theoretical_loss": 3.5424471038618126, + "tokens_seen": 1379074048 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029403209628886664, + "loss": 2.7911, + "theoretical_loss": 3.5424320212224565, + "tokens_seen": 1379139584 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029402206619859577, + "loss": 2.9922, + "theoretical_loss": 3.5424169395004736, + "tokens_seen": 1379205120 + }, + { + "epoch": 0.42, + "learning_rate": 0.000294012036108325, + "loss": 2.6719, + "theoretical_loss": 3.542401858695764, + "tokens_seen": 1379270656 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029400200601805413, + "loss": 2.6389, + "theoretical_loss": 3.542386778808228, + "tokens_seen": 1379336192 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029399197592778337, + "loss": 2.6955, + "theoretical_loss": 3.5423716998377666, + "tokens_seen": 1379401728 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029398194583751255, + "loss": 2.6948, + "theoretical_loss": 3.542356621784281, + "tokens_seen": 1379467264 + }, + { + "epoch": 0.42, + "objective/train/docs_used": 781246, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.57826566696167, + "objective/train/theoretical_loss": 3.5423415446476705, + "objective/train/tokens_used": 1399992800, + "theoretical_loss": 3.5423415446476705, + "tokens_seen": 1379532800 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029397191574724173, + "loss": 2.5344, + "theoretical_loss": 3.5423415446476705, + "tokens_seen": 1379532800 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002939618856569709, + "loss": 2.6346, + "theoretical_loss": 3.542326468427837, + "tokens_seen": 1379598336 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029395185556670015, + "loss": 2.7439, + "theoretical_loss": 3.542311393124681, + "tokens_seen": 1379663872 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002939418254764293, + "loss": 2.4321, + "theoretical_loss": 3.5422963187381034, + "tokens_seen": 1379729408 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002939317953861585, + "loss": 2.7164, + "theoretical_loss": 3.542281245268004, + "tokens_seen": 1379794944 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029392176529588764, + "loss": 2.6644, + "theoretical_loss": 3.5422661727142843, + "tokens_seen": 1379860480 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029391173520561687, + "loss": 2.6293, + "theoretical_loss": 3.542251101076845, + "tokens_seen": 1379926016 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029390170511534605, + "loss": 2.8498, + "theoretical_loss": 3.542236030355587, + "tokens_seen": 1379991552 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029389167502507523, + "loss": 2.559, + "theoretical_loss": 3.5422209605504107, + "tokens_seen": 1380057088 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002938816449348044, + "loss": 2.8711, + "theoretical_loss": 3.542205891661217, + "tokens_seen": 1380122624 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002938716148445336, + "loss": 2.7591, + "theoretical_loss": 3.542190823687907, + "tokens_seen": 1380188160 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002938615847542628, + "loss": 2.6481, + "theoretical_loss": 3.5421757566303818, + "tokens_seen": 1380253696 + }, + { + "epoch": 0.42, + "learning_rate": 0.000293851554663992, + "loss": 2.4278, + "theoretical_loss": 3.542160690488541, + "tokens_seen": 1380319232 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029384152457372114, + "loss": 2.4424, + "theoretical_loss": 3.542145625262287, + "tokens_seen": 1380384768 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002938314944834504, + "loss": 2.6028, + "theoretical_loss": 3.54213056095152, + "tokens_seen": 1380450304 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002938214643931795, + "loss": 2.7837, + "theoretical_loss": 3.542115497556141, + "tokens_seen": 1380515840 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029381143430290874, + "loss": 2.5627, + "theoretical_loss": 3.5421004350760508, + "tokens_seen": 1380581376 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002938014042126379, + "loss": 2.9382, + "theoretical_loss": 3.54208537351115, + "tokens_seen": 1380646912 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002937913741223671, + "loss": 2.5331, + "theoretical_loss": 3.54207031286134, + "tokens_seen": 1380712448 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002937813440320963, + "loss": 2.8005, + "theoretical_loss": 3.542055253126523, + "tokens_seen": 1380777984 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002937713139418255, + "loss": 2.7297, + "theoretical_loss": 3.5420401943065976, + "tokens_seen": 1380843520 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029376128385155464, + "loss": 2.8479, + "theoretical_loss": 3.542025136401466, + "tokens_seen": 1380909056 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002937512537612839, + "loss": 2.7568, + "theoretical_loss": 3.542010079411029, + "tokens_seen": 1380974592 + }, + { + "epoch": 0.42, + "learning_rate": 0.000293741223671013, + "loss": 2.7572, + "theoretical_loss": 3.5419950233351885, + "tokens_seen": 1381040128 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029373119358074224, + "loss": 2.6858, + "theoretical_loss": 3.5419799681738446, + "tokens_seen": 1381105664 + }, + { + "epoch": 0.42, + "objective/train/docs_used": 781914, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.076110363006592, + "objective/train/theoretical_loss": 3.5419649139268987, + "objective/train/tokens_used": 1401631200, + "theoretical_loss": 3.5419649139268987, + "tokens_seen": 1381171200 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002937211634904714, + "loss": 2.8151, + "theoretical_loss": 3.5419649139268987, + "tokens_seen": 1381171200 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002937111334002006, + "loss": 2.6209, + "theoretical_loss": 3.541949860594251, + "tokens_seen": 1381236736 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002937011033099298, + "loss": 2.6552, + "theoretical_loss": 3.541934808175804, + "tokens_seen": 1381302272 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029369107321965896, + "loss": 2.5337, + "theoretical_loss": 3.541919756671458, + "tokens_seen": 1381367808 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029368104312938815, + "loss": 2.5981, + "theoretical_loss": 3.541904706081114, + "tokens_seen": 1381433344 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002936710130391174, + "loss": 2.7709, + "theoretical_loss": 3.541889656404674, + "tokens_seen": 1381498880 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002936609829488465, + "loss": 2.5221, + "theoretical_loss": 3.541874607642039, + "tokens_seen": 1381564416 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029365095285857574, + "loss": 2.7877, + "theoretical_loss": 3.541859559793109, + "tokens_seen": 1381629952 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029364092276830487, + "loss": 2.7678, + "theoretical_loss": 3.5418445128577862, + "tokens_seen": 1381695488 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002936308926780341, + "loss": 2.7324, + "theoretical_loss": 3.5418294668359716, + "tokens_seen": 1381761024 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002936208625877633, + "loss": 2.8444, + "theoretical_loss": 3.541814421727566, + "tokens_seen": 1381826560 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029361083249749247, + "loss": 2.6223, + "theoretical_loss": 3.5417993775324716, + "tokens_seen": 1381892096 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029360080240722165, + "loss": 2.6202, + "theoretical_loss": 3.541784334250589, + "tokens_seen": 1381957632 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002935907723169509, + "loss": 2.6006, + "theoretical_loss": 3.5417692918818187, + "tokens_seen": 1382023168 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029358074222668, + "loss": 2.7653, + "theoretical_loss": 3.541754250426064, + "tokens_seen": 1382088704 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029357071213640925, + "loss": 2.9863, + "theoretical_loss": 3.5417392098832243, + "tokens_seen": 1382154240 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002935606820461384, + "loss": 2.5254, + "theoretical_loss": 3.541724170253201, + "tokens_seen": 1382219776 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002935506519558676, + "loss": 2.4626, + "theoretical_loss": 3.5417091315358973, + "tokens_seen": 1382285312 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029354062186559684, + "loss": 2.431, + "theoretical_loss": 3.5416940937312127, + "tokens_seen": 1382350848 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029353059177532597, + "loss": 2.6108, + "theoretical_loss": 3.5416790568390493, + "tokens_seen": 1382416384 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002935205616850552, + "loss": 2.636, + "theoretical_loss": 3.541664020859308, + "tokens_seen": 1382481920 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029351053159478433, + "loss": 2.831, + "theoretical_loss": 3.5416489857918902, + "tokens_seen": 1382547456 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029350050150451357, + "loss": 2.7371, + "theoretical_loss": 3.5416339516366984, + "tokens_seen": 1382612992 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029349047141424275, + "loss": 2.8879, + "theoretical_loss": 3.541618918393633, + "tokens_seen": 1382678528 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029348044132397193, + "loss": 2.4292, + "theoretical_loss": 3.541603886062595, + "tokens_seen": 1382744064 + }, + { + "epoch": 0.42, + "objective/train/docs_used": 783265, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.92041277885437, + "objective/train/theoretical_loss": 3.541588854643487, + "objective/train/tokens_used": 1403269600, + "theoretical_loss": 3.541588854643487, + "tokens_seen": 1382809600 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002934704112337011, + "loss": 2.8049, + "theoretical_loss": 3.541588854643487, + "tokens_seen": 1382809600 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029346038114343035, + "loss": 2.9001, + "theoretical_loss": 3.54157382413621, + "tokens_seen": 1382875136 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002934503510531595, + "loss": 2.5338, + "theoretical_loss": 3.541558794540665, + "tokens_seen": 1382940672 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002934403209628887, + "loss": 2.6146, + "theoretical_loss": 3.5415437658567543, + "tokens_seen": 1383006208 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029343029087261784, + "loss": 2.8176, + "theoretical_loss": 3.541528738084379, + "tokens_seen": 1383071744 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029342026078234707, + "loss": 2.739, + "theoretical_loss": 3.5415137112234403, + "tokens_seen": 1383137280 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029341023069207625, + "loss": 2.8763, + "theoretical_loss": 3.54149868527384, + "tokens_seen": 1383202816 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029340020060180543, + "loss": 2.888, + "theoretical_loss": 3.541483660235481, + "tokens_seen": 1383268352 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002933901705115346, + "loss": 2.563, + "theoretical_loss": 3.541468636108262, + "tokens_seen": 1383333888 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002933801404212638, + "loss": 2.6678, + "theoretical_loss": 3.541453612892087, + "tokens_seen": 1383399424 + }, + { + "epoch": 0.42, + "learning_rate": 0.000293370110330993, + "loss": 2.7013, + "theoretical_loss": 3.541438590586857, + "tokens_seen": 1383464960 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002933600802407222, + "loss": 2.5872, + "theoretical_loss": 3.5414235691924727, + "tokens_seen": 1383530496 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029335005015045134, + "loss": 2.9281, + "theoretical_loss": 3.541408548708837, + "tokens_seen": 1383596032 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002933400200601806, + "loss": 2.7881, + "theoretical_loss": 3.541393529135851, + "tokens_seen": 1383661568 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002933299899699097, + "loss": 2.6125, + "theoretical_loss": 3.5413785104734163, + "tokens_seen": 1383727104 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029331995987963894, + "loss": 2.6799, + "theoretical_loss": 3.5413634927214344, + "tokens_seen": 1383792640 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002933099297893681, + "loss": 2.744, + "theoretical_loss": 3.5413484758798073, + "tokens_seen": 1383858176 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002932998996990973, + "loss": 2.7611, + "theoretical_loss": 3.5413334599484365, + "tokens_seen": 1383923712 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002932898696088265, + "loss": 2.6971, + "theoretical_loss": 3.5413184449272244, + "tokens_seen": 1383989248 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002932798395185557, + "loss": 2.5649, + "theoretical_loss": 3.541303430816072, + "tokens_seen": 1384054784 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029326980942828484, + "loss": 2.5075, + "theoretical_loss": 3.5412884176148807, + "tokens_seen": 1384120320 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002932597793380141, + "loss": 2.5946, + "theoretical_loss": 3.541273405323553, + "tokens_seen": 1384185856 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002932497492477432, + "loss": 2.6995, + "theoretical_loss": 3.5412583939419906, + "tokens_seen": 1384251392 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029323971915747244, + "loss": 2.4576, + "theoretical_loss": 3.5412433834700954, + "tokens_seen": 1384316928 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002932296890672016, + "loss": 2.6193, + "theoretical_loss": 3.541228373907769, + "tokens_seen": 1384382464 + }, + { + "epoch": 0.42, + "objective/train/docs_used": 783958, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.74815034866333, + "objective/train/theoretical_loss": 3.541213365254913, + "objective/train/tokens_used": 1404908000, + "theoretical_loss": 3.541213365254913, + "tokens_seen": 1384448000 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002932196589769308, + "loss": 2.8452, + "theoretical_loss": 3.541213365254913, + "tokens_seen": 1384448000 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029320962888666, + "loss": 2.9722, + "theoretical_loss": 3.5411983575114294, + "tokens_seen": 1384513536 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029319959879638916, + "loss": 2.7468, + "theoretical_loss": 3.5411833506772203, + "tokens_seen": 1384579072 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029318956870611835, + "loss": 2.5473, + "theoretical_loss": 3.5411683447521876, + "tokens_seen": 1384644608 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002931795386158476, + "loss": 2.4899, + "theoretical_loss": 3.5411533397362325, + "tokens_seen": 1384710144 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002931695085255767, + "loss": 2.711, + "theoretical_loss": 3.541138335629258, + "tokens_seen": 1384775680 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029315947843530594, + "loss": 2.6024, + "theoretical_loss": 3.541123332431165, + "tokens_seen": 1384841216 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029314944834503507, + "loss": 2.6262, + "theoretical_loss": 3.541108330141856, + "tokens_seen": 1384906752 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002931394182547643, + "loss": 2.6652, + "theoretical_loss": 3.5410933287612325, + "tokens_seen": 1384972288 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002931293881644935, + "loss": 2.6327, + "theoretical_loss": 3.5410783282891973, + "tokens_seen": 1385037824 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029311935807422267, + "loss": 2.5344, + "theoretical_loss": 3.541063328725652, + "tokens_seen": 1385103360 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029310932798395185, + "loss": 2.5935, + "theoretical_loss": 3.541048330070498, + "tokens_seen": 1385168896 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002930992978936811, + "loss": 2.7137, + "theoretical_loss": 3.541033332323638, + "tokens_seen": 1385234432 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002930892678034102, + "loss": 2.6031, + "theoretical_loss": 3.541018335484974, + "tokens_seen": 1385299968 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029307923771313945, + "loss": 2.5355, + "theoretical_loss": 3.5410033395544076, + "tokens_seen": 1385365504 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002930692076228686, + "loss": 2.6847, + "theoretical_loss": 3.540988344531841, + "tokens_seen": 1385431040 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002930591775325978, + "loss": 2.7504, + "theoretical_loss": 3.5409733504171763, + "tokens_seen": 1385496576 + }, + { + "epoch": 0.42, + "learning_rate": 0.000293049147442327, + "loss": 2.7242, + "theoretical_loss": 3.540958357210316, + "tokens_seen": 1385562112 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029303911735205617, + "loss": 2.4053, + "theoretical_loss": 3.5409433649111617, + "tokens_seen": 1385627648 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029302908726178535, + "loss": 2.9008, + "theoretical_loss": 3.5409283735196153, + "tokens_seen": 1385693184 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029301905717151453, + "loss": 2.7889, + "theoretical_loss": 3.54091338303558, + "tokens_seen": 1385758720 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002930090270812437, + "loss": 2.7431, + "theoretical_loss": 3.5408983934589573, + "tokens_seen": 1385824256 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029299899699097295, + "loss": 2.6809, + "theoretical_loss": 3.540883404789649, + "tokens_seen": 1385889792 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002929889669007021, + "loss": 2.5918, + "theoretical_loss": 3.5408684170275575, + "tokens_seen": 1385955328 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002929789368104313, + "loss": 2.4599, + "theoretical_loss": 3.5408534301725854, + "tokens_seen": 1386020864 + }, + { + "epoch": 0.42, + "objective/train/docs_used": 785103, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.8873586654663086, + "objective/train/theoretical_loss": 3.5408384442246343, + "objective/train/tokens_used": 1406546400, + "theoretical_loss": 3.5408384442246343, + "tokens_seen": 1386086400 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002929689067201605, + "loss": 2.8346, + "theoretical_loss": 3.5408384442246343, + "tokens_seen": 1386086400 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002929588766298897, + "loss": 2.6898, + "theoretical_loss": 3.540823459183607, + "tokens_seen": 1386151936 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029294884653961885, + "loss": 2.8109, + "theoretical_loss": 3.5408084750494053, + "tokens_seen": 1386217472 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029293881644934804, + "loss": 2.7269, + "theoretical_loss": 3.540793491821932, + "tokens_seen": 1386283008 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002929287863590772, + "loss": 2.7502, + "theoretical_loss": 3.540778509501089, + "tokens_seen": 1386348544 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029291875626880645, + "loss": 2.6837, + "theoretical_loss": 3.540763528086778, + "tokens_seen": 1386414080 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002929087261785356, + "loss": 2.538, + "theoretical_loss": 3.540748547578902, + "tokens_seen": 1386479616 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002928986960882648, + "loss": 2.6287, + "theoretical_loss": 3.540733567977364, + "tokens_seen": 1386545152 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029288866599799394, + "loss": 2.9144, + "theoretical_loss": 3.540718589282065, + "tokens_seen": 1386610688 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002928786359077232, + "loss": 2.6486, + "theoretical_loss": 3.5407036114929076, + "tokens_seen": 1386676224 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029286860581745236, + "loss": 2.6647, + "theoretical_loss": 3.540688634609795, + "tokens_seen": 1386741760 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029285857572718154, + "loss": 2.7514, + "theoretical_loss": 3.5406736586326284, + "tokens_seen": 1386807296 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002928485456369107, + "loss": 2.687, + "theoretical_loss": 3.540658683561311, + "tokens_seen": 1386872832 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002928385155466399, + "loss": 2.6101, + "theoretical_loss": 3.5406437093957455, + "tokens_seen": 1386938368 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002928284854563691, + "loss": 2.7266, + "theoretical_loss": 3.540628736135834, + "tokens_seen": 1387003904 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002928184553660983, + "loss": 2.7933, + "theoretical_loss": 3.540613763781478, + "tokens_seen": 1387069440 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029280842527582744, + "loss": 2.9202, + "theoretical_loss": 3.540598792332581, + "tokens_seen": 1387134976 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002927983951855567, + "loss": 2.6864, + "theoretical_loss": 3.5405838217890455, + "tokens_seen": 1387200512 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002927883650952859, + "loss": 2.7334, + "theoretical_loss": 3.5405688521507734, + "tokens_seen": 1387266048 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029277833500501504, + "loss": 2.8457, + "theoretical_loss": 3.5405538834176675, + "tokens_seen": 1387331584 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002927683049147443, + "loss": 2.6398, + "theoretical_loss": 3.5405389155896305, + "tokens_seen": 1387397120 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002927582748244734, + "loss": 2.8485, + "theoretical_loss": 3.5405239486665643, + "tokens_seen": 1387462656 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029274824473420264, + "loss": 2.5451, + "theoretical_loss": 3.540508982648372, + "tokens_seen": 1387528192 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002927382146439318, + "loss": 2.6681, + "theoretical_loss": 3.5404940175349564, + "tokens_seen": 1387593728 + }, + { + "epoch": 0.42, + "learning_rate": 0.000292728184553661, + "loss": 2.8184, + "theoretical_loss": 3.5404790533262194, + "tokens_seen": 1387659264 + }, + { + "epoch": 0.42, + "objective/train/docs_used": 785794, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.4556312561035156, + "objective/train/theoretical_loss": 3.5404640900220636, + "objective/train/tokens_used": 1408184800, + "theoretical_loss": 3.5404640900220636, + "tokens_seen": 1387724800 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002927181544633902, + "loss": 2.5046, + "theoretical_loss": 3.5404640900220636, + "tokens_seen": 1387724800 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029270812437311936, + "loss": 2.6187, + "theoretical_loss": 3.5404491276223924, + "tokens_seen": 1387790336 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029269809428284855, + "loss": 2.6185, + "theoretical_loss": 3.540434166127108, + "tokens_seen": 1387855872 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002926880641925778, + "loss": 2.6793, + "theoretical_loss": 3.5404192055361126, + "tokens_seen": 1387921408 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002926780341023069, + "loss": 2.6766, + "theoretical_loss": 3.5404042458493086, + "tokens_seen": 1387986944 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029266800401203614, + "loss": 2.9252, + "theoretical_loss": 3.5403892870666, + "tokens_seen": 1388052480 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029265797392176527, + "loss": 2.7765, + "theoretical_loss": 3.5403743291878884, + "tokens_seen": 1388118016 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002926479438314945, + "loss": 2.7599, + "theoretical_loss": 3.5403593722130773, + "tokens_seen": 1388183552 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002926379137412237, + "loss": 2.7427, + "theoretical_loss": 3.5403444161420685, + "tokens_seen": 1388249088 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029262788365095287, + "loss": 2.7058, + "theoretical_loss": 3.5403294609747653, + "tokens_seen": 1388314624 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029261785356068205, + "loss": 2.7555, + "theoretical_loss": 3.54031450671107, + "tokens_seen": 1388380160 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002926078234704113, + "loss": 2.7576, + "theoretical_loss": 3.540299553350886, + "tokens_seen": 1388445696 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002925977933801404, + "loss": 2.6775, + "theoretical_loss": 3.5402846008941156, + "tokens_seen": 1388511232 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029258776328986965, + "loss": 2.6241, + "theoretical_loss": 3.5402696493406616, + "tokens_seen": 1388576768 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002925777331995988, + "loss": 2.4819, + "theoretical_loss": 3.5402546986904273, + "tokens_seen": 1388642304 + }, + { + "epoch": 0.42, + "learning_rate": 0.000292567703109328, + "loss": 2.4903, + "theoretical_loss": 3.5402397489433146, + "tokens_seen": 1388707840 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002925576730190572, + "loss": 2.7399, + "theoretical_loss": 3.540224800099227, + "tokens_seen": 1388773376 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029254764292878637, + "loss": 2.7132, + "theoretical_loss": 3.5402098521580676, + "tokens_seen": 1388838912 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029253761283851555, + "loss": 2.7736, + "theoretical_loss": 3.540194905119738, + "tokens_seen": 1388904448 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029252758274824473, + "loss": 2.7155, + "theoretical_loss": 3.5401799589841425, + "tokens_seen": 1388969984 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002925175526579739, + "loss": 2.7678, + "theoretical_loss": 3.5401650137511833, + "tokens_seen": 1389035520 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029250752256770315, + "loss": 2.6109, + "theoretical_loss": 3.5401500694207635, + "tokens_seen": 1389101056 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002924974924774323, + "loss": 2.399, + "theoretical_loss": 3.5401351259927853, + "tokens_seen": 1389166592 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002924874623871615, + "loss": 2.6577, + "theoretical_loss": 3.540120183467153, + "tokens_seen": 1389232128 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002924774322968907, + "loss": 2.6486, + "theoretical_loss": 3.5401052418437686, + "tokens_seen": 1389297664 + }, + { + "epoch": 0.42, + "objective/train/docs_used": 786491, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6160640716552734, + "objective/train/theoretical_loss": 3.540090301122535, + "objective/train/tokens_used": 1409823200, + "theoretical_loss": 3.540090301122535, + "tokens_seen": 1389363200 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002924674022066199, + "loss": 2.5536, + "theoretical_loss": 3.540090301122535, + "tokens_seen": 1389363200 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029245737211634906, + "loss": 2.8319, + "theoretical_loss": 3.540075361303356, + "tokens_seen": 1389428736 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029244734202607824, + "loss": 2.6688, + "theoretical_loss": 3.5400604223861336, + "tokens_seen": 1389494272 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002924373119358074, + "loss": 2.849, + "theoretical_loss": 3.540045484370771, + "tokens_seen": 1389559808 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029242728184553665, + "loss": 2.8512, + "theoretical_loss": 3.5400305472571723, + "tokens_seen": 1389625344 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002924172517552658, + "loss": 2.7389, + "theoretical_loss": 3.540015611045239, + "tokens_seen": 1389690880 + }, + { + "epoch": 0.42, + "learning_rate": 0.000292407221664995, + "loss": 2.4401, + "theoretical_loss": 3.540000675734875, + "tokens_seen": 1389756416 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029239719157472414, + "loss": 2.8601, + "theoretical_loss": 3.5399857413259834, + "tokens_seen": 1389821952 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002923871614844534, + "loss": 2.5225, + "theoretical_loss": 3.5399708078184675, + "tokens_seen": 1389887488 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029237713139418256, + "loss": 2.7083, + "theoretical_loss": 3.5399558752122293, + "tokens_seen": 1389953024 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029236710130391174, + "loss": 2.7147, + "theoretical_loss": 3.5399409435071734, + "tokens_seen": 1390018560 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002923570712136409, + "loss": 2.5643, + "theoretical_loss": 3.5399260127032015, + "tokens_seen": 1390084096 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002923470411233701, + "loss": 2.7923, + "theoretical_loss": 3.5399110828002174, + "tokens_seen": 1390149632 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002923370110330993, + "loss": 2.7726, + "theoretical_loss": 3.5398961537981246, + "tokens_seen": 1390215168 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002923269809428285, + "loss": 2.7182, + "theoretical_loss": 3.539881225696826, + "tokens_seen": 1390280704 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029231695085255765, + "loss": 2.7654, + "theoretical_loss": 3.539866298496224, + "tokens_seen": 1390346240 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002923069207622869, + "loss": 2.9584, + "theoretical_loss": 3.5398513721962237, + "tokens_seen": 1390411776 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029229689067201606, + "loss": 2.6725, + "theoretical_loss": 3.5398364467967265, + "tokens_seen": 1390477312 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029228686058174524, + "loss": 2.6968, + "theoretical_loss": 3.539821522297636, + "tokens_seen": 1390542848 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002922768304914744, + "loss": 2.8632, + "theoretical_loss": 3.5398065986988563, + "tokens_seen": 1390608384 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002922668004012036, + "loss": 2.5506, + "theoretical_loss": 3.53979167600029, + "tokens_seen": 1390673920 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002922567703109328, + "loss": 2.8354, + "theoretical_loss": 3.5397767542018403, + "tokens_seen": 1390739456 + }, + { + "epoch": 0.42, + "learning_rate": 0.000292246740220662, + "loss": 2.7653, + "theoretical_loss": 3.5397618333034107, + "tokens_seen": 1390804992 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029223671013039115, + "loss": 2.774, + "theoretical_loss": 3.5397469133049047, + "tokens_seen": 1390870528 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002922266800401204, + "loss": 2.8137, + "theoretical_loss": 3.5397319942062255, + "tokens_seen": 1390936064 + }, + { + "epoch": 0.42, + "objective/train/docs_used": 787676, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.9149885177612305, + "objective/train/theoretical_loss": 3.539717076007276, + "objective/train/tokens_used": 1411461600, + "theoretical_loss": 3.539717076007276, + "tokens_seen": 1391001600 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002922166499498495, + "loss": 2.7172, + "theoretical_loss": 3.539717076007276, + "tokens_seen": 1391001600 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029220661985957875, + "loss": 2.5102, + "theoretical_loss": 3.53970215870796, + "tokens_seen": 1391067136 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002921965897693079, + "loss": 2.7085, + "theoretical_loss": 3.539687242308181, + "tokens_seen": 1391132672 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002921865596790371, + "loss": 2.7334, + "theoretical_loss": 3.5396723268078416, + "tokens_seen": 1391198208 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002921765295887663, + "loss": 2.7365, + "theoretical_loss": 3.539657412206846, + "tokens_seen": 1391263744 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029216649949849547, + "loss": 2.6805, + "theoretical_loss": 3.5396424985050974, + "tokens_seen": 1391329280 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029215646940822465, + "loss": 2.3154, + "theoretical_loss": 3.5396275857024992, + "tokens_seen": 1391394816 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002921464393179539, + "loss": 2.7529, + "theoretical_loss": 3.5396126737989553, + "tokens_seen": 1391460352 + }, + { + "epoch": 0.42, + "learning_rate": 0.000292136409227683, + "loss": 2.5227, + "theoretical_loss": 3.5395977627943678, + "tokens_seen": 1391525888 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029212637913741225, + "loss": 2.6371, + "theoretical_loss": 3.5395828526886417, + "tokens_seen": 1391591424 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029211634904714143, + "loss": 2.7816, + "theoretical_loss": 3.5395679434816794, + "tokens_seen": 1391656960 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002921063189568706, + "loss": 2.8741, + "theoretical_loss": 3.5395530351733853, + "tokens_seen": 1391722496 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002920962888665998, + "loss": 2.7861, + "theoretical_loss": 3.539538127763662, + "tokens_seen": 1391788032 + }, + { + "epoch": 0.42, + "learning_rate": 0.000292086258776329, + "loss": 2.7084, + "theoretical_loss": 3.539523221252414, + "tokens_seen": 1391853568 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029207622868605815, + "loss": 2.565, + "theoretical_loss": 3.5395083156395444, + "tokens_seen": 1391919104 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002920661985957874, + "loss": 2.8309, + "theoretical_loss": 3.5394934109249565, + "tokens_seen": 1391984640 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002920561685055165, + "loss": 2.7357, + "theoretical_loss": 3.5394785071085537, + "tokens_seen": 1392050176 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029204613841524575, + "loss": 2.6079, + "theoretical_loss": 3.5394636041902405, + "tokens_seen": 1392115712 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029203610832497493, + "loss": 2.6042, + "theoretical_loss": 3.5394487021699197, + "tokens_seen": 1392181248 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002920260782347041, + "loss": 2.7643, + "theoretical_loss": 3.5394338010474957, + "tokens_seen": 1392246784 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029201604814443335, + "loss": 2.7545, + "theoretical_loss": 3.539418900822871, + "tokens_seen": 1392312320 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002920060180541625, + "loss": 2.4634, + "theoretical_loss": 3.5394040014959502, + "tokens_seen": 1392377856 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002919959879638917, + "loss": 2.8252, + "theoretical_loss": 3.539389103066637, + "tokens_seen": 1392443392 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002919859578736209, + "loss": 2.7419, + "theoretical_loss": 3.539374205534834, + "tokens_seen": 1392508928 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002919759277833501, + "loss": 2.6291, + "theoretical_loss": 3.539359308900446, + "tokens_seen": 1392574464 + }, + { + "epoch": 0.42, + "objective/train/docs_used": 788318, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.3744328022003174, + "objective/train/theoretical_loss": 3.5393444131633762, + "objective/train/tokens_used": 1413100000, + "theoretical_loss": 3.5393444131633762, + "tokens_seen": 1392640000 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029196589769307926, + "loss": 2.6069, + "theoretical_loss": 3.5393444131633762, + "tokens_seen": 1392640000 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029195586760280844, + "loss": 2.8778, + "theoretical_loss": 3.5393295183235294, + "tokens_seen": 1392705536 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002919458375125376, + "loss": 2.4805, + "theoretical_loss": 3.5393146243808076, + "tokens_seen": 1392771072 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029193580742226685, + "loss": 2.5169, + "theoretical_loss": 3.539299731335115, + "tokens_seen": 1392836608 + }, + { + "epoch": 0.42, + "learning_rate": 0.000291925777331996, + "loss": 2.5708, + "theoretical_loss": 3.5392848391863563, + "tokens_seen": 1392902144 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002919157472417252, + "loss": 2.7143, + "theoretical_loss": 3.539269947934435, + "tokens_seen": 1392967680 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029190571715145434, + "loss": 2.7454, + "theoretical_loss": 3.539255057579254, + "tokens_seen": 1393033216 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002918956870611836, + "loss": 2.6531, + "theoretical_loss": 3.539240168120718, + "tokens_seen": 1393098752 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029188565697091276, + "loss": 2.9833, + "theoretical_loss": 3.539225279558731, + "tokens_seen": 1393164288 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029187562688064194, + "loss": 2.667, + "theoretical_loss": 3.539210391893196, + "tokens_seen": 1393229824 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002918655967903711, + "loss": 2.9396, + "theoretical_loss": 3.539195505124017, + "tokens_seen": 1393295360 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002918555667001003, + "loss": 2.632, + "theoretical_loss": 3.5391806192510984, + "tokens_seen": 1393360896 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002918455366098295, + "loss": 2.7077, + "theoretical_loss": 3.539165734274344, + "tokens_seen": 1393426432 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002918355065195587, + "loss": 2.8032, + "theoretical_loss": 3.539150850193658, + "tokens_seen": 1393491968 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029182547642928785, + "loss": 2.7129, + "theoretical_loss": 3.539135967008943, + "tokens_seen": 1393557504 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002918154463390171, + "loss": 2.6775, + "theoretical_loss": 3.5391210847201044, + "tokens_seen": 1393623040 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029180541624874626, + "loss": 3.0375, + "theoretical_loss": 3.5391062033270453, + "tokens_seen": 1393688576 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029179538615847544, + "loss": 2.5136, + "theoretical_loss": 3.5390913228296697, + "tokens_seen": 1393754112 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002917853560682046, + "loss": 2.7903, + "theoretical_loss": 3.539076443227882, + "tokens_seen": 1393819648 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002917753259779338, + "loss": 2.8006, + "theoretical_loss": 3.539061564521586, + "tokens_seen": 1393885184 + }, + { + "epoch": 0.42, + "learning_rate": 0.000291765295887663, + "loss": 2.8193, + "theoretical_loss": 3.5390466867106856, + "tokens_seen": 1393950720 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002917552657973922, + "loss": 2.6122, + "theoretical_loss": 3.539031809795085, + "tokens_seen": 1394016256 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029174523570712135, + "loss": 2.6196, + "theoretical_loss": 3.5390169337746884, + "tokens_seen": 1394081792 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002917352056168506, + "loss": 2.7951, + "theoretical_loss": 3.539002058649399, + "tokens_seen": 1394147328 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002917251755265797, + "loss": 2.7399, + "theoretical_loss": 3.5389871844191223, + "tokens_seen": 1394212864 + }, + { + "epoch": 0.42, + "objective/train/docs_used": 789787, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.568711996078491, + "objective/train/theoretical_loss": 3.5389723110837608, + "objective/train/tokens_used": 1414738400, + "theoretical_loss": 3.5389723110837608, + "tokens_seen": 1394278400 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029171514543630895, + "loss": 2.6704, + "theoretical_loss": 3.5389723110837608, + "tokens_seen": 1394278400 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002917051153460381, + "loss": 2.5324, + "theoretical_loss": 3.53895743864322, + "tokens_seen": 1394343936 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002916950852557673, + "loss": 2.8043, + "theoretical_loss": 3.538942567097403, + "tokens_seen": 1394409472 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002916850551654965, + "loss": 2.9299, + "theoretical_loss": 3.538927696446214, + "tokens_seen": 1394475008 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029167502507522567, + "loss": 2.8274, + "theoretical_loss": 3.538912826689558, + "tokens_seen": 1394540544 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029166499498495485, + "loss": 2.5667, + "theoretical_loss": 3.5388979578273383, + "tokens_seen": 1394606080 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002916549648946841, + "loss": 2.7376, + "theoretical_loss": 3.538883089859459, + "tokens_seen": 1394671616 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002916449348044132, + "loss": 2.7333, + "theoretical_loss": 3.5388682227858252, + "tokens_seen": 1394737152 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029163490471414245, + "loss": 2.8075, + "theoretical_loss": 3.5388533566063405, + "tokens_seen": 1394802688 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029162487462387163, + "loss": 2.831, + "theoretical_loss": 3.538838491320909, + "tokens_seen": 1394868224 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002916148445336008, + "loss": 2.6397, + "theoretical_loss": 3.5388236269294353, + "tokens_seen": 1394933760 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029160481444333, + "loss": 2.6267, + "theoretical_loss": 3.538808763431823, + "tokens_seen": 1394999296 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002915947843530592, + "loss": 2.7757, + "theoretical_loss": 3.5387939008279767, + "tokens_seen": 1395064832 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029158475426278835, + "loss": 2.5981, + "theoretical_loss": 3.5387790391178013, + "tokens_seen": 1395130368 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002915747241725176, + "loss": 2.8588, + "theoretical_loss": 3.5387641783012, + "tokens_seen": 1395195904 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002915646940822467, + "loss": 2.7057, + "theoretical_loss": 3.5387493183780783, + "tokens_seen": 1395261440 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029155466399197595, + "loss": 2.8257, + "theoretical_loss": 3.5387344593483396, + "tokens_seen": 1395326976 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002915446339017051, + "loss": 2.6196, + "theoretical_loss": 3.5387196012118887, + "tokens_seen": 1395392512 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002915346038114343, + "loss": 2.7181, + "theoretical_loss": 3.538704743968629, + "tokens_seen": 1395458048 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002915245737211635, + "loss": 2.7396, + "theoretical_loss": 3.5386898876184665, + "tokens_seen": 1395523584 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002915145436308927, + "loss": 2.7794, + "theoretical_loss": 3.5386750321613043, + "tokens_seen": 1395589120 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029150451354062186, + "loss": 2.5786, + "theoretical_loss": 3.538660177597047, + "tokens_seen": 1395654656 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002914944834503511, + "loss": 2.8728, + "theoretical_loss": 3.5386453239255995, + "tokens_seen": 1395720192 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002914844533600802, + "loss": 2.5933, + "theoretical_loss": 3.538630471146866, + "tokens_seen": 1395785728 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029147442326980946, + "loss": 2.8822, + "theoretical_loss": 3.5386156192607503, + "tokens_seen": 1395851264 + }, + { + "epoch": 0.42, + "objective/train/docs_used": 790377, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.0071396827697754, + "objective/train/theoretical_loss": 3.5386007682671576, + "objective/train/tokens_used": 1416376800, + "theoretical_loss": 3.5386007682671576, + "tokens_seen": 1395916800 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002914643931795386, + "loss": 2.803, + "theoretical_loss": 3.5386007682671576, + "tokens_seen": 1395916800 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002914543630892678, + "loss": 2.7203, + "theoretical_loss": 3.5385859181659924, + "tokens_seen": 1395982336 + }, + { + "epoch": 0.42, + "learning_rate": 0.000291444332998997, + "loss": 2.8441, + "theoretical_loss": 3.538571068957159, + "tokens_seen": 1396047872 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002914343029087262, + "loss": 2.5507, + "theoretical_loss": 3.5385562206405616, + "tokens_seen": 1396113408 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029142427281845536, + "loss": 2.683, + "theoretical_loss": 3.5385413732161046, + "tokens_seen": 1396178944 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029141424272818454, + "loss": 2.8457, + "theoretical_loss": 3.5385265266836936, + "tokens_seen": 1396244480 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002914042126379137, + "loss": 2.6324, + "theoretical_loss": 3.538511681043232, + "tokens_seen": 1396310016 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029139418254764296, + "loss": 2.6382, + "theoretical_loss": 3.538496836294625, + "tokens_seen": 1396375552 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002913841524573721, + "loss": 2.5292, + "theoretical_loss": 3.5384819924377764, + "tokens_seen": 1396441088 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002913741223671013, + "loss": 2.6696, + "theoretical_loss": 3.538467149472592, + "tokens_seen": 1396506624 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029136409227683045, + "loss": 2.5142, + "theoretical_loss": 3.538452307398975, + "tokens_seen": 1396572160 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002913540621865597, + "loss": 2.6617, + "theoretical_loss": 3.5384374662168314, + "tokens_seen": 1396637696 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029134403209628886, + "loss": 2.5527, + "theoretical_loss": 3.5384226259260645, + "tokens_seen": 1396703232 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029133400200601805, + "loss": 2.9413, + "theoretical_loss": 3.53840778652658, + "tokens_seen": 1396768768 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002913239719157472, + "loss": 2.4189, + "theoretical_loss": 3.5383929480182825, + "tokens_seen": 1396834304 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029131394182547646, + "loss": 2.7026, + "theoretical_loss": 3.5383781104010756, + "tokens_seen": 1396899840 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002913039117352056, + "loss": 2.6234, + "theoretical_loss": 3.5383632736748654, + "tokens_seen": 1396965376 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002912938816449348, + "loss": 2.5333, + "theoretical_loss": 3.5383484378395553, + "tokens_seen": 1397030912 + }, + { + "epoch": 0.42, + "learning_rate": 0.000291283851554664, + "loss": 2.6544, + "theoretical_loss": 3.538333602895051, + "tokens_seen": 1397096448 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002912738214643932, + "loss": 2.7261, + "theoretical_loss": 3.5383187688412567, + "tokens_seen": 1397161984 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002912637913741224, + "loss": 2.8062, + "theoretical_loss": 3.538303935678077, + "tokens_seen": 1397227520 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029125376128385155, + "loss": 2.5107, + "theoretical_loss": 3.5382891034054174, + "tokens_seen": 1397293056 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002912437311935808, + "loss": 2.9902, + "theoretical_loss": 3.538274272023182, + "tokens_seen": 1397358592 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002912337011033099, + "loss": 2.834, + "theoretical_loss": 3.538259441531276, + "tokens_seen": 1397424128 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029122367101303915, + "loss": 2.5364, + "theoretical_loss": 3.5382446119296036, + "tokens_seen": 1397489664 + }, + { + "epoch": 0.42, + "objective/train/docs_used": 791674, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.246107578277588, + "objective/train/theoretical_loss": 3.5382297832180702, + "objective/train/tokens_used": 1418015200, + "theoretical_loss": 3.5382297832180702, + "tokens_seen": 1397555200 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029121364092276833, + "loss": 2.6555, + "theoretical_loss": 3.5382297832180702, + "tokens_seen": 1397555200 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002912036108324975, + "loss": 2.8216, + "theoretical_loss": 3.5382149553965805, + "tokens_seen": 1397620736 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002911935807422267, + "loss": 2.7497, + "theoretical_loss": 3.5382001284650397, + "tokens_seen": 1397686272 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029118355065195587, + "loss": 2.7157, + "theoretical_loss": 3.538185302423352, + "tokens_seen": 1397751808 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029117352056168505, + "loss": 2.7896, + "theoretical_loss": 3.5381704772714224, + "tokens_seen": 1397817344 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002911634904714143, + "loss": 2.5904, + "theoretical_loss": 3.5381556530091562, + "tokens_seen": 1397882880 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002911534603811434, + "loss": 2.516, + "theoretical_loss": 3.5381408296364576, + "tokens_seen": 1397948416 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029114343029087265, + "loss": 2.7466, + "theoretical_loss": 3.5381260071532328, + "tokens_seen": 1398013952 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029113340020060183, + "loss": 2.5202, + "theoretical_loss": 3.538111185559385, + "tokens_seen": 1398079488 + }, + { + "epoch": 0.42, + "learning_rate": 0.000291123370110331, + "loss": 2.6806, + "theoretical_loss": 3.5380963648548205, + "tokens_seen": 1398145024 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002911133400200602, + "loss": 2.9612, + "theoretical_loss": 3.538081545039444, + "tokens_seen": 1398210560 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002911033099297894, + "loss": 2.2884, + "theoretical_loss": 3.5380667261131595, + "tokens_seen": 1398276096 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029109327983951855, + "loss": 2.5974, + "theoretical_loss": 3.5380519080758734, + "tokens_seen": 1398341632 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002910832497492478, + "loss": 2.6868, + "theoretical_loss": 3.5380370909274896, + "tokens_seen": 1398407168 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002910732196589769, + "loss": 2.5405, + "theoretical_loss": 3.538022274667914, + "tokens_seen": 1398472704 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029106318956870615, + "loss": 2.9272, + "theoretical_loss": 3.5380074592970514, + "tokens_seen": 1398538240 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002910531594784353, + "loss": 2.5379, + "theoretical_loss": 3.537992644814807, + "tokens_seen": 1398603776 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002910431293881645, + "loss": 2.65, + "theoretical_loss": 3.5379778312210846, + "tokens_seen": 1398669312 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002910330992978937, + "loss": 2.5511, + "theoretical_loss": 3.537963018515791, + "tokens_seen": 1398734848 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002910230692076229, + "loss": 2.6981, + "theoretical_loss": 3.53794820669883, + "tokens_seen": 1398800384 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029101303911735206, + "loss": 2.8514, + "theoretical_loss": 3.537933395770108, + "tokens_seen": 1398865920 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002910030090270813, + "loss": 2.6274, + "theoretical_loss": 3.5379185857295283, + "tokens_seen": 1398931456 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002909929789368104, + "loss": 2.7271, + "theoretical_loss": 3.5379037765769983, + "tokens_seen": 1398996992 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029098294884653966, + "loss": 2.7589, + "theoretical_loss": 3.537888968312421, + "tokens_seen": 1399062528 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002909729187562688, + "loss": 2.8668, + "theoretical_loss": 3.5378741609357034, + "tokens_seen": 1399128064 + }, + { + "epoch": 0.42, + "objective/train/docs_used": 792315, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7772862911224365, + "objective/train/theoretical_loss": 3.5378593544467494, + "objective/train/tokens_used": 1419653600, + "theoretical_loss": 3.5378593544467494, + "tokens_seen": 1399193600 + }, + { + "epoch": 0.42, + "learning_rate": 0.000290962888665998, + "loss": 2.7528, + "theoretical_loss": 3.5378593544467494, + "tokens_seen": 1399193600 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002909528585757272, + "loss": 2.7564, + "theoretical_loss": 3.537844548845464, + "tokens_seen": 1399259136 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002909428284854564, + "loss": 2.6026, + "theoretical_loss": 3.537829744131754, + "tokens_seen": 1399324672 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029093279839518556, + "loss": 2.8379, + "theoretical_loss": 3.5378149403055232, + "tokens_seen": 1399390208 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029092276830491474, + "loss": 2.6144, + "theoretical_loss": 3.537800137366677, + "tokens_seen": 1399455744 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002909127382146439, + "loss": 2.8726, + "theoretical_loss": 3.5377853353151214, + "tokens_seen": 1399521280 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029090270812437316, + "loss": 2.7744, + "theoretical_loss": 3.537770534150761, + "tokens_seen": 1399586816 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002908926780341023, + "loss": 2.9124, + "theoretical_loss": 3.5377557338735017, + "tokens_seen": 1399652352 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002908826479438315, + "loss": 2.8343, + "theoretical_loss": 3.537740934483248, + "tokens_seen": 1399717888 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029087261785356065, + "loss": 2.3503, + "theoretical_loss": 3.5377261359799057, + "tokens_seen": 1399783424 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002908625877632899, + "loss": 2.6967, + "theoretical_loss": 3.5377113383633803, + "tokens_seen": 1399848960 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029085255767301906, + "loss": 2.6423, + "theoretical_loss": 3.5376965416335766, + "tokens_seen": 1399914496 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029084252758274825, + "loss": 2.6614, + "theoretical_loss": 3.5376817457904, + "tokens_seen": 1399980032 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002908324974924774, + "loss": 2.611, + "theoretical_loss": 3.5376669508337564, + "tokens_seen": 1400045568 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029082246740220666, + "loss": 2.5502, + "theoretical_loss": 3.537652156763551, + "tokens_seen": 1400111104 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002908124373119358, + "loss": 2.5838, + "theoretical_loss": 3.537637363579689, + "tokens_seen": 1400176640 + }, + { + "epoch": 0.42, + "learning_rate": 0.000290802407221665, + "loss": 2.5755, + "theoretical_loss": 3.537622571282076, + "tokens_seen": 1400242176 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029079237713139415, + "loss": 2.6407, + "theoretical_loss": 3.5376077798706174, + "tokens_seen": 1400307712 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002907823470411234, + "loss": 2.6103, + "theoretical_loss": 3.5375929893452187, + "tokens_seen": 1400373248 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029077231695085257, + "loss": 2.7672, + "theoretical_loss": 3.537578199705785, + "tokens_seen": 1400438784 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029076228686058175, + "loss": 2.5821, + "theoretical_loss": 3.537563410952222, + "tokens_seen": 1400504320 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029075225677031093, + "loss": 2.4568, + "theoretical_loss": 3.5375486230844353, + "tokens_seen": 1400569856 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002907422266800401, + "loss": 2.6626, + "theoretical_loss": 3.5375338361023303, + "tokens_seen": 1400635392 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002907321965897693, + "loss": 2.805, + "theoretical_loss": 3.5375190500058125, + "tokens_seen": 1400700928 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029072216649949853, + "loss": 2.827, + "theoretical_loss": 3.5375042647947876, + "tokens_seen": 1400766464 + }, + { + "epoch": 0.42, + "objective/train/docs_used": 793814, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.0992543697357178, + "objective/train/theoretical_loss": 3.537489480469161, + "objective/train/tokens_used": 1421292000, + "theoretical_loss": 3.537489480469161, + "tokens_seen": 1400832000 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029071213640922765, + "loss": 2.8767, + "theoretical_loss": 3.537489480469161, + "tokens_seen": 1400832000 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002907021063189569, + "loss": 2.636, + "theoretical_loss": 3.5374746970288378, + "tokens_seen": 1400897536 + }, + { + "epoch": 0.42, + "learning_rate": 0.000290692076228686, + "loss": 2.7563, + "theoretical_loss": 3.5374599144737244, + "tokens_seen": 1400963072 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029068204613841525, + "loss": 2.7482, + "theoretical_loss": 3.537445132803726, + "tokens_seen": 1401028608 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029067201604814443, + "loss": 2.7705, + "theoretical_loss": 3.537430352018749, + "tokens_seen": 1401094144 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002906619859578736, + "loss": 2.726, + "theoretical_loss": 3.537415572118697, + "tokens_seen": 1401159680 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002906519558676028, + "loss": 2.8213, + "theoretical_loss": 3.5374007931034774, + "tokens_seen": 1401225216 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029064192577733203, + "loss": 2.6376, + "theoretical_loss": 3.5373860149729954, + "tokens_seen": 1401290752 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029063189568706116, + "loss": 2.6821, + "theoretical_loss": 3.5373712377271564, + "tokens_seen": 1401356288 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002906218655967904, + "loss": 2.5741, + "theoretical_loss": 3.537356461365867, + "tokens_seen": 1401421824 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002906118355065195, + "loss": 2.6717, + "theoretical_loss": 3.5373416858890314, + "tokens_seen": 1401487360 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029060180541624876, + "loss": 2.4888, + "theoretical_loss": 3.5373269112965566, + "tokens_seen": 1401552896 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029059177532597794, + "loss": 2.6431, + "theoretical_loss": 3.537312137588347, + "tokens_seen": 1401618432 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002905817452357071, + "loss": 2.7676, + "theoretical_loss": 3.53729736476431, + "tokens_seen": 1401683968 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002905717151454363, + "loss": 2.5916, + "theoretical_loss": 3.53728259282435, + "tokens_seen": 1401749504 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002905616850551655, + "loss": 2.8283, + "theoretical_loss": 3.5372678217683733, + "tokens_seen": 1401815040 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029055165496489466, + "loss": 2.6634, + "theoretical_loss": 3.5372530515962852, + "tokens_seen": 1401880576 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002905416248746239, + "loss": 2.7816, + "theoretical_loss": 3.5372382823079924, + "tokens_seen": 1401946112 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002905315947843531, + "loss": 2.6508, + "theoretical_loss": 3.5372235139034, + "tokens_seen": 1402011648 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029052156469408226, + "loss": 2.5887, + "theoretical_loss": 3.5372087463824142, + "tokens_seen": 1402077184 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002905115346038115, + "loss": 2.9463, + "theoretical_loss": 3.5371939797449405, + "tokens_seen": 1402142720 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002905015045135406, + "loss": 2.5881, + "theoretical_loss": 3.5371792139908855, + "tokens_seen": 1402208256 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029049147442326986, + "loss": 2.6401, + "theoretical_loss": 3.5371644491201537, + "tokens_seen": 1402273792 + }, + { + "epoch": 0.42, + "learning_rate": 0.000290481444332999, + "loss": 2.7061, + "theoretical_loss": 3.5371496851326523, + "tokens_seen": 1402339328 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002904714142427282, + "loss": 2.8337, + "theoretical_loss": 3.5371349220282866, + "tokens_seen": 1402404864 + }, + { + "epoch": 0.42, + "objective/train/docs_used": 794381, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.609896659851074, + "objective/train/theoretical_loss": 3.537120159806962, + "objective/train/tokens_used": 1422930400, + "theoretical_loss": 3.537120159806962, + "tokens_seen": 1402470400 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002904613841524574, + "loss": 2.5713, + "theoretical_loss": 3.537120159806962, + "tokens_seen": 1402470400 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002904513540621866, + "loss": 2.6322, + "theoretical_loss": 3.5371053984685856, + "tokens_seen": 1402535936 + }, + { + "epoch": 0.43, + "learning_rate": 0.00029044132397191576, + "loss": 2.717, + "theoretical_loss": 3.5370906380130624, + "tokens_seen": 1402601472 + }, + { + "epoch": 0.43, + "learning_rate": 0.00029043129388164494, + "loss": 2.4921, + "theoretical_loss": 3.537075878440299, + "tokens_seen": 1402667008 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002904212637913741, + "loss": 2.8781, + "theoretical_loss": 3.5370611197502004, + "tokens_seen": 1402732544 + }, + { + "epoch": 0.43, + "learning_rate": 0.00029041123370110336, + "loss": 2.8191, + "theoretical_loss": 3.5370463619426733, + "tokens_seen": 1402798080 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002904012036108325, + "loss": 2.7562, + "theoretical_loss": 3.5370316050176243, + "tokens_seen": 1402863616 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002903911735205617, + "loss": 2.6787, + "theoretical_loss": 3.5370168489749583, + "tokens_seen": 1402929152 + }, + { + "epoch": 0.43, + "learning_rate": 0.00029038114343029085, + "loss": 2.2378, + "theoretical_loss": 3.5370020938145816, + "tokens_seen": 1402994688 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002903711133400201, + "loss": 2.5001, + "theoretical_loss": 3.5369873395364007, + "tokens_seen": 1403060224 + }, + { + "epoch": 0.43, + "learning_rate": 0.00029036108324974926, + "loss": 2.5045, + "theoretical_loss": 3.5369725861403207, + "tokens_seen": 1403125760 + }, + { + "epoch": 0.43, + "learning_rate": 0.00029035105315947845, + "loss": 2.5549, + "theoretical_loss": 3.536957833626249, + "tokens_seen": 1403191296 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002903410230692076, + "loss": 2.6757, + "theoretical_loss": 3.5369430819940906, + "tokens_seen": 1403256832 + }, + { + "epoch": 0.43, + "learning_rate": 0.00029033099297893686, + "loss": 2.6569, + "theoretical_loss": 3.536928331243752, + "tokens_seen": 1403322368 + }, + { + "epoch": 0.43, + "learning_rate": 0.000290320962888666, + "loss": 2.843, + "theoretical_loss": 3.5369135813751393, + "tokens_seen": 1403387904 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002903109327983952, + "loss": 2.5453, + "theoretical_loss": 3.536898832388159, + "tokens_seen": 1403453440 + }, + { + "epoch": 0.43, + "learning_rate": 0.00029030090270812435, + "loss": 2.5624, + "theoretical_loss": 3.5368840842827165, + "tokens_seen": 1403518976 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002902908726178536, + "loss": 2.6611, + "theoretical_loss": 3.536869337058718, + "tokens_seen": 1403584512 + }, + { + "epoch": 0.43, + "learning_rate": 0.00029028084252758277, + "loss": 2.4407, + "theoretical_loss": 3.5368545907160707, + "tokens_seen": 1403650048 + }, + { + "epoch": 0.43, + "learning_rate": 0.00029027081243731195, + "loss": 2.7488, + "theoretical_loss": 3.536839845254679, + "tokens_seen": 1403715584 + }, + { + "epoch": 0.43, + "learning_rate": 0.00029026078234704113, + "loss": 2.7578, + "theoretical_loss": 3.536825100674451, + "tokens_seen": 1403781120 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002902507522567703, + "loss": 2.5392, + "theoretical_loss": 3.5368103569752916, + "tokens_seen": 1403846656 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002902407221664995, + "loss": 2.7465, + "theoretical_loss": 3.5367956141571075, + "tokens_seen": 1403912192 + }, + { + "epoch": 0.43, + "learning_rate": 0.00029023069207622873, + "loss": 2.7643, + "theoretical_loss": 3.536780872219805, + "tokens_seen": 1403977728 + }, + { + "epoch": 0.43, + "learning_rate": 0.00029022066198595785, + "loss": 2.4454, + "theoretical_loss": 3.53676613116329, + "tokens_seen": 1404043264 + }, + { + "epoch": 0.43, + "objective/train/docs_used": 795506, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.984286308288574, + "objective/train/theoretical_loss": 3.5367513909874697, + "objective/train/tokens_used": 1424568800, + "theoretical_loss": 3.5367513909874697, + "tokens_seen": 1404108800 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002902106318956871, + "loss": 2.8666, + "theoretical_loss": 3.5367513909874697, + "tokens_seen": 1404108800 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002902006018054162, + "loss": 2.6748, + "theoretical_loss": 3.536736651692249, + "tokens_seen": 1404174336 + }, + { + "epoch": 0.43, + "learning_rate": 0.00029019057171514545, + "loss": 2.9133, + "theoretical_loss": 3.5367219132775354, + "tokens_seen": 1404239872 + }, + { + "epoch": 0.43, + "learning_rate": 0.00029018054162487463, + "loss": 2.6633, + "theoretical_loss": 3.5367071757432345, + "tokens_seen": 1404305408 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002901705115346038, + "loss": 2.6337, + "theoretical_loss": 3.5366924390892525, + "tokens_seen": 1404370944 + }, + { + "epoch": 0.43, + "learning_rate": 0.000290160481444333, + "loss": 2.6926, + "theoretical_loss": 3.5366777033154966, + "tokens_seen": 1404436480 + }, + { + "epoch": 0.43, + "learning_rate": 0.00029015045135406223, + "loss": 2.7725, + "theoretical_loss": 3.536662968421872, + "tokens_seen": 1404502016 + }, + { + "epoch": 0.43, + "learning_rate": 0.00029014042126379136, + "loss": 2.5553, + "theoretical_loss": 3.536648234408286, + "tokens_seen": 1404567552 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002901303911735206, + "loss": 2.6814, + "theoretical_loss": 3.536633501274645, + "tokens_seen": 1404633088 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002901203610832497, + "loss": 2.7467, + "theoretical_loss": 3.536618769020855, + "tokens_seen": 1404698624 + }, + { + "epoch": 0.43, + "learning_rate": 0.00029011033099297896, + "loss": 2.6019, + "theoretical_loss": 3.536604037646822, + "tokens_seen": 1404764160 + }, + { + "epoch": 0.43, + "learning_rate": 0.00029010030090270814, + "loss": 2.685, + "theoretical_loss": 3.5365893071524535, + "tokens_seen": 1404829696 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002900902708124373, + "loss": 2.6238, + "theoretical_loss": 3.536574577537655, + "tokens_seen": 1404895232 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002900802407221665, + "loss": 2.531, + "theoretical_loss": 3.5365598488023333, + "tokens_seen": 1404960768 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002900702106318957, + "loss": 2.6452, + "theoretical_loss": 3.536545120946395, + "tokens_seen": 1405026304 + }, + { + "epoch": 0.43, + "learning_rate": 0.00029006018054162486, + "loss": 2.4475, + "theoretical_loss": 3.5365303939697466, + "tokens_seen": 1405091840 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002900501504513541, + "loss": 2.5954, + "theoretical_loss": 3.536515667872294, + "tokens_seen": 1405157376 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002900401203610832, + "loss": 2.5647, + "theoretical_loss": 3.5365009426539444, + "tokens_seen": 1405222912 + }, + { + "epoch": 0.43, + "learning_rate": 0.00029003009027081246, + "loss": 2.6156, + "theoretical_loss": 3.5364862183146037, + "tokens_seen": 1405288448 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002900200601805416, + "loss": 2.6938, + "theoretical_loss": 3.5364714948541796, + "tokens_seen": 1405353984 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002900100300902708, + "loss": 2.8169, + "theoretical_loss": 3.536456772272577, + "tokens_seen": 1405419520 + }, + { + "epoch": 0.43, + "learning_rate": 0.00029, + "loss": 2.7436, + "theoretical_loss": 3.536442050569704, + "tokens_seen": 1405485056 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002899899699097292, + "loss": 2.7093, + "theoretical_loss": 3.5364273297454663, + "tokens_seen": 1405550592 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028997993981945836, + "loss": 2.8382, + "theoretical_loss": 3.5364126097997706, + "tokens_seen": 1405616128 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002899699097291876, + "loss": 2.9042, + "theoretical_loss": 3.5363978907325233, + "tokens_seen": 1405681664 + }, + { + "epoch": 0.43, + "objective/train/docs_used": 796159, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.8775532245635986, + "objective/train/theoretical_loss": 3.5363831725436317, + "objective/train/tokens_used": 1426207200, + "theoretical_loss": 3.5363831725436317, + "tokens_seen": 1405747200 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002899598796389167, + "loss": 2.6255, + "theoretical_loss": 3.5363831725436317, + "tokens_seen": 1405747200 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028994984954864596, + "loss": 2.4707, + "theoretical_loss": 3.536368455233003, + "tokens_seen": 1405812736 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002899398194583751, + "loss": 2.8992, + "theoretical_loss": 3.5363537388005417, + "tokens_seen": 1405878272 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002899297893681043, + "loss": 2.5213, + "theoretical_loss": 3.536339023246156, + "tokens_seen": 1405943808 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002899197592778335, + "loss": 2.8246, + "theoretical_loss": 3.5363243085697524, + "tokens_seen": 1406009344 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002899097291875627, + "loss": 2.5559, + "theoretical_loss": 3.5363095947712377, + "tokens_seen": 1406074880 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028989969909729187, + "loss": 2.5866, + "theoretical_loss": 3.5362948818505178, + "tokens_seen": 1406140416 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028988966900702105, + "loss": 2.6793, + "theoretical_loss": 3.5362801698075006, + "tokens_seen": 1406205952 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028987963891675023, + "loss": 2.6136, + "theoretical_loss": 3.536265458642092, + "tokens_seen": 1406271488 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028986960882647946, + "loss": 2.5962, + "theoretical_loss": 3.536250748354199, + "tokens_seen": 1406337024 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002898595787362086, + "loss": 2.759, + "theoretical_loss": 3.536236038943729, + "tokens_seen": 1406402560 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002898495486459378, + "loss": 2.8791, + "theoretical_loss": 3.536221330410587, + "tokens_seen": 1406468096 + }, + { + "epoch": 0.43, + "learning_rate": 0.000289839518555667, + "loss": 2.8834, + "theoretical_loss": 3.5362066227546816, + "tokens_seen": 1406533632 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002898294884653962, + "loss": 2.8258, + "theoretical_loss": 3.5361919159759196, + "tokens_seen": 1406599168 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028981945837512537, + "loss": 2.8201, + "theoretical_loss": 3.5361772100742064, + "tokens_seen": 1406664704 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028980942828485455, + "loss": 2.8255, + "theoretical_loss": 3.5361625050494494, + "tokens_seen": 1406730240 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028979939819458373, + "loss": 2.8586, + "theoretical_loss": 3.5361478009015563, + "tokens_seen": 1406795776 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028978936810431297, + "loss": 2.721, + "theoretical_loss": 3.536133097630433, + "tokens_seen": 1406861312 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028977933801404215, + "loss": 2.744, + "theoretical_loss": 3.536118395235987, + "tokens_seen": 1406926848 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028976930792377133, + "loss": 2.6076, + "theoretical_loss": 3.5361036937181254, + "tokens_seen": 1406992384 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002897592778335005, + "loss": 2.7001, + "theoretical_loss": 3.5360889930767536, + "tokens_seen": 1407057920 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002897492477432297, + "loss": 2.7609, + "theoretical_loss": 3.5360742933117804, + "tokens_seen": 1407123456 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028973921765295893, + "loss": 2.667, + "theoretical_loss": 3.5360595944231115, + "tokens_seen": 1407188992 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028972918756268805, + "loss": 2.6166, + "theoretical_loss": 3.536044896410654, + "tokens_seen": 1407254528 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002897191574724173, + "loss": 2.7681, + "theoretical_loss": 3.5360301992743155, + "tokens_seen": 1407320064 + }, + { + "epoch": 0.43, + "objective/train/docs_used": 797613, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.113302230834961, + "objective/train/theoretical_loss": 3.536015503014003, + "objective/train/tokens_used": 1427845600, + "theoretical_loss": 3.536015503014003, + "tokens_seen": 1407385600 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002897091273821464, + "loss": 2.791, + "theoretical_loss": 3.536015503014003, + "tokens_seen": 1407385600 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028969909729187565, + "loss": 2.8303, + "theoretical_loss": 3.5360008076296223, + "tokens_seen": 1407451136 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028968906720160483, + "loss": 2.6961, + "theoretical_loss": 3.5359861131210817, + "tokens_seen": 1407516672 + }, + { + "epoch": 0.43, + "learning_rate": 0.000289679037111334, + "loss": 2.7221, + "theoretical_loss": 3.5359714194882876, + "tokens_seen": 1407582208 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002896690070210632, + "loss": 2.6187, + "theoretical_loss": 3.535956726731147, + "tokens_seen": 1407647744 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028965897693079243, + "loss": 2.4869, + "theoretical_loss": 3.535942034849567, + "tokens_seen": 1407713280 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028964894684052156, + "loss": 2.6782, + "theoretical_loss": 3.535927343843455, + "tokens_seen": 1407778816 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002896389167502508, + "loss": 2.6461, + "theoretical_loss": 3.5359126537127183, + "tokens_seen": 1407844352 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002896288866599799, + "loss": 2.6029, + "theoretical_loss": 3.535897964457263, + "tokens_seen": 1407909888 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028961885656970916, + "loss": 2.5564, + "theoretical_loss": 3.535883276076997, + "tokens_seen": 1407975424 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028960882647943834, + "loss": 2.589, + "theoretical_loss": 3.5358685885718275, + "tokens_seen": 1408040960 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002895987963891675, + "loss": 2.5306, + "theoretical_loss": 3.535853901941661, + "tokens_seen": 1408106496 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002895887662988967, + "loss": 2.6547, + "theoretical_loss": 3.535839216186405, + "tokens_seen": 1408172032 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002895787362086259, + "loss": 2.6197, + "theoretical_loss": 3.535824531305966, + "tokens_seen": 1408237568 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028956870611835506, + "loss": 2.5992, + "theoretical_loss": 3.5358098473002526, + "tokens_seen": 1408303104 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002895586760280843, + "loss": 2.6947, + "theoretical_loss": 3.5357951641691714, + "tokens_seen": 1408368640 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002895486459378134, + "loss": 2.7242, + "theoretical_loss": 3.5357804819126293, + "tokens_seen": 1408434176 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028953861584754266, + "loss": 2.8451, + "theoretical_loss": 3.5357658005305335, + "tokens_seen": 1408499712 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002895285857572718, + "loss": 2.7235, + "theoretical_loss": 3.535751120022791, + "tokens_seen": 1408565248 + }, + { + "epoch": 0.43, + "learning_rate": 0.000289518555667001, + "loss": 2.5221, + "theoretical_loss": 3.53573644038931, + "tokens_seen": 1408630784 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002895085255767302, + "loss": 2.7303, + "theoretical_loss": 3.535721761629997, + "tokens_seen": 1408696320 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002894984954864594, + "loss": 2.5658, + "theoretical_loss": 3.53570708374476, + "tokens_seen": 1408761856 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028948846539618856, + "loss": 2.6725, + "theoretical_loss": 3.5356924067335047, + "tokens_seen": 1408827392 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002894784353059178, + "loss": 2.987, + "theoretical_loss": 3.53567773059614, + "tokens_seen": 1408892928 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002894684052156469, + "loss": 2.7758, + "theoretical_loss": 3.5356630553325727, + "tokens_seen": 1408958464 + }, + { + "epoch": 0.43, + "objective/train/docs_used": 798401, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6743273735046387, + "objective/train/theoretical_loss": 3.5356483809427104, + "objective/train/tokens_used": 1429484000, + "theoretical_loss": 3.5356483809427104, + "tokens_seen": 1409024000 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028945837512537616, + "loss": 2.4886, + "theoretical_loss": 3.5356483809427104, + "tokens_seen": 1409024000 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002894483450351053, + "loss": 2.5378, + "theoretical_loss": 3.5356337074264603, + "tokens_seen": 1409089536 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002894383149448345, + "loss": 2.5901, + "theoretical_loss": 3.5356190347837293, + "tokens_seen": 1409155072 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002894282848545637, + "loss": 2.7235, + "theoretical_loss": 3.535604363014425, + "tokens_seen": 1409220608 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002894182547642929, + "loss": 2.9367, + "theoretical_loss": 3.535589692118455, + "tokens_seen": 1409286144 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028940822467402207, + "loss": 2.9514, + "theoretical_loss": 3.5355750220957267, + "tokens_seen": 1409351680 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028939819458375125, + "loss": 2.8851, + "theoretical_loss": 3.5355603529461477, + "tokens_seen": 1409417216 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028938816449348043, + "loss": 2.7575, + "theoretical_loss": 3.535545684669625, + "tokens_seen": 1409482752 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028937813440320966, + "loss": 2.7067, + "theoretical_loss": 3.535531017266066, + "tokens_seen": 1409548288 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002893681043129388, + "loss": 2.746, + "theoretical_loss": 3.535516350735379, + "tokens_seen": 1409613824 + }, + { + "epoch": 0.43, + "learning_rate": 0.000289358074222668, + "loss": 2.7201, + "theoretical_loss": 3.5355016850774703, + "tokens_seen": 1409679360 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002893480441323972, + "loss": 2.883, + "theoretical_loss": 3.535487020292248, + "tokens_seen": 1409744896 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002893380140421264, + "loss": 2.9311, + "theoretical_loss": 3.5354723563796195, + "tokens_seen": 1409810432 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028932798395185557, + "loss": 2.8922, + "theoretical_loss": 3.5354576933394926, + "tokens_seen": 1409875968 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028931795386158475, + "loss": 2.8043, + "theoretical_loss": 3.5354430311717744, + "tokens_seen": 1409941504 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028930792377131393, + "loss": 2.6917, + "theoretical_loss": 3.535428369876373, + "tokens_seen": 1410007040 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028929789368104317, + "loss": 2.6382, + "theoretical_loss": 3.535413709453196, + "tokens_seen": 1410072576 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002892878635907723, + "loss": 2.5945, + "theoretical_loss": 3.53539904990215, + "tokens_seen": 1410138112 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028927783350050153, + "loss": 2.6537, + "theoretical_loss": 3.5353843912231433, + "tokens_seen": 1410203648 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028926780341023066, + "loss": 2.818, + "theoretical_loss": 3.535369733416083, + "tokens_seen": 1410269184 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002892577733199599, + "loss": 2.942, + "theoretical_loss": 3.5353550764808777, + "tokens_seen": 1410334720 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002892477432296891, + "loss": 2.7801, + "theoretical_loss": 3.535340420417434, + "tokens_seen": 1410400256 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028923771313941825, + "loss": 2.6822, + "theoretical_loss": 3.53532576522566, + "tokens_seen": 1410465792 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028922768304914744, + "loss": 2.5695, + "theoretical_loss": 3.5353111109054636, + "tokens_seen": 1410531328 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002892176529588766, + "loss": 2.7838, + "theoretical_loss": 3.5352964574567523, + "tokens_seen": 1410596864 + }, + { + "epoch": 0.43, + "objective/train/docs_used": 799700, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.46821928024292, + "objective/train/theoretical_loss": 3.5352818048794337, + "objective/train/tokens_used": 1431122400, + "theoretical_loss": 3.5352818048794337, + "tokens_seen": 1410662400 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002892076228686058, + "loss": 2.7264, + "theoretical_loss": 3.5352818048794337, + "tokens_seen": 1410662400 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028919759277833503, + "loss": 2.418, + "theoretical_loss": 3.5352671531734154, + "tokens_seen": 1410727936 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028918756268806416, + "loss": 2.6467, + "theoretical_loss": 3.535252502338605, + "tokens_seen": 1410793472 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002891775325977934, + "loss": 2.591, + "theoretical_loss": 3.5352378523749106, + "tokens_seen": 1410859008 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002891675025075226, + "loss": 2.9828, + "theoretical_loss": 3.53522320328224, + "tokens_seen": 1410924544 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028915747241725176, + "loss": 2.8149, + "theoretical_loss": 3.5352085550605006, + "tokens_seen": 1410990080 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028914744232698094, + "loss": 2.6566, + "theoretical_loss": 3.5351939077096004, + "tokens_seen": 1411055616 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002891374122367101, + "loss": 2.5621, + "theoretical_loss": 3.5351792612294473, + "tokens_seen": 1411121152 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002891273821464393, + "loss": 2.6836, + "theoretical_loss": 3.5351646156199488, + "tokens_seen": 1411186688 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028911735205616854, + "loss": 2.6648, + "theoretical_loss": 3.5351499708810126, + "tokens_seen": 1411252224 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028910732196589766, + "loss": 2.6763, + "theoretical_loss": 3.5351353270125463, + "tokens_seen": 1411317760 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002890972918756269, + "loss": 2.578, + "theoretical_loss": 3.5351206840144593, + "tokens_seen": 1411383296 + }, + { + "epoch": 0.43, + "learning_rate": 0.000289087261785356, + "loss": 2.8947, + "theoretical_loss": 3.5351060418866576, + "tokens_seen": 1411448832 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028907723169508526, + "loss": 2.5744, + "theoretical_loss": 3.5350914006290504, + "tokens_seen": 1411514368 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028906720160481444, + "loss": 2.742, + "theoretical_loss": 3.5350767602415445, + "tokens_seen": 1411579904 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002890571715145436, + "loss": 2.7346, + "theoretical_loss": 3.5350621207240485, + "tokens_seen": 1411645440 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002890471414242728, + "loss": 2.6266, + "theoretical_loss": 3.53504748207647, + "tokens_seen": 1411710976 + }, + { + "epoch": 0.43, + "learning_rate": 0.000289037111334002, + "loss": 2.6786, + "theoretical_loss": 3.5350328442987173, + "tokens_seen": 1411776512 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002890270812437312, + "loss": 2.6583, + "theoretical_loss": 3.535018207390698, + "tokens_seen": 1411842048 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002890170511534604, + "loss": 2.669, + "theoretical_loss": 3.53500357135232, + "tokens_seen": 1411907584 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002890070210631896, + "loss": 2.8737, + "theoretical_loss": 3.534988936183492, + "tokens_seen": 1411973120 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028899699097291876, + "loss": 2.7265, + "theoretical_loss": 3.5349743018841204, + "tokens_seen": 1412038656 + }, + { + "epoch": 0.43, + "learning_rate": 0.000288986960882648, + "loss": 2.7878, + "theoretical_loss": 3.534959668454115, + "tokens_seen": 1412104192 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002889769307923771, + "loss": 3.0749, + "theoretical_loss": 3.5349450358933825, + "tokens_seen": 1412169728 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028896690070210636, + "loss": 2.5464, + "theoretical_loss": 3.5349304042018317, + "tokens_seen": 1412235264 + }, + { + "epoch": 0.43, + "objective/train/docs_used": 800754, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.4482576847076416, + "objective/train/theoretical_loss": 3.53491577337937, + "objective/train/tokens_used": 1432760800, + "theoretical_loss": 3.53491577337937, + "tokens_seen": 1412300800 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002889568706118355, + "loss": 2.527, + "theoretical_loss": 3.53491577337937, + "tokens_seen": 1412300800 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002889568706118355, + "loss": 2.641, + "theoretical_loss": 3.5349011434259063, + "tokens_seen": 1412366336 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002889468405215647, + "loss": 2.5088, + "theoretical_loss": 3.5348865143413484, + "tokens_seen": 1412431872 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002889368104312939, + "loss": 2.73, + "theoretical_loss": 3.534871886125604, + "tokens_seen": 1412497408 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002889267803410231, + "loss": 2.6985, + "theoretical_loss": 3.534857258778581, + "tokens_seen": 1412562944 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028891675025075227, + "loss": 2.8569, + "theoretical_loss": 3.5348426323001876, + "tokens_seen": 1412628480 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028890672016048145, + "loss": 2.7264, + "theoretical_loss": 3.534828006690333, + "tokens_seen": 1412694016 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028889669007021063, + "loss": 2.5771, + "theoretical_loss": 3.534813381948924, + "tokens_seen": 1412759552 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028888665997993986, + "loss": 2.7552, + "theoretical_loss": 3.53479875807587, + "tokens_seen": 1412825088 + }, + { + "epoch": 0.43, + "learning_rate": 0.000288876629889669, + "loss": 2.822, + "theoretical_loss": 3.534784135071077, + "tokens_seen": 1412890624 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028886659979939823, + "loss": 2.7651, + "theoretical_loss": 3.5347695129344556, + "tokens_seen": 1412956160 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002888565697091274, + "loss": 2.8196, + "theoretical_loss": 3.534754891665913, + "tokens_seen": 1413021696 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002888465396188566, + "loss": 2.7344, + "theoretical_loss": 3.534740271265358, + "tokens_seen": 1413087232 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028883650952858577, + "loss": 2.6102, + "theoretical_loss": 3.5347256517326975, + "tokens_seen": 1413152768 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028882647943831495, + "loss": 2.7657, + "theoretical_loss": 3.5347110330678406, + "tokens_seen": 1413218304 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028881644934804413, + "loss": 2.5193, + "theoretical_loss": 3.5346964152706954, + "tokens_seen": 1413283840 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028880641925777337, + "loss": 2.5648, + "theoretical_loss": 3.53468179834117, + "tokens_seen": 1413349376 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002887963891675025, + "loss": 2.5952, + "theoretical_loss": 3.534667182279173, + "tokens_seen": 1413414912 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028878635907723173, + "loss": 2.5292, + "theoretical_loss": 3.5346525670846125, + "tokens_seen": 1413480448 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028877632898696086, + "loss": 2.8043, + "theoretical_loss": 3.5346379527573974, + "tokens_seen": 1413545984 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002887662988966901, + "loss": 2.8449, + "theoretical_loss": 3.534623339297435, + "tokens_seen": 1413611520 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002887562688064193, + "loss": 2.7133, + "theoretical_loss": 3.534608726704634, + "tokens_seen": 1413677056 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028874623871614845, + "loss": 2.593, + "theoretical_loss": 3.534594114978903, + "tokens_seen": 1413742592 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028873620862587764, + "loss": 2.9476, + "theoretical_loss": 3.5345795041201504, + "tokens_seen": 1413808128 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002887261785356068, + "loss": 2.7316, + "theoretical_loss": 3.5345648941282843, + "tokens_seen": 1413873664 + }, + { + "epoch": 0.43, + "objective/train/docs_used": 801653, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.762237071990967, + "objective/train/theoretical_loss": 3.5345502850032133, + "objective/train/tokens_used": 1434399200, + "theoretical_loss": 3.5345502850032133, + "tokens_seen": 1413939200 + }, + { + "epoch": 0.43, + "learning_rate": 0.000288716148445336, + "loss": 2.8215, + "theoretical_loss": 3.5345502850032133, + "tokens_seen": 1413939200 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028870611835506523, + "loss": 2.6413, + "theoretical_loss": 3.534535676744845, + "tokens_seen": 1414004736 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028869608826479436, + "loss": 2.8426, + "theoretical_loss": 3.5345210693530893, + "tokens_seen": 1414070272 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002886860581745236, + "loss": 2.6531, + "theoretical_loss": 3.5345064628278537, + "tokens_seen": 1414135808 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002886760280842528, + "loss": 2.5972, + "theoretical_loss": 3.5344918571690465, + "tokens_seen": 1414201344 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028866599799398196, + "loss": 2.8286, + "theoretical_loss": 3.5344772523765764, + "tokens_seen": 1414266880 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028865596790371114, + "loss": 2.5543, + "theoretical_loss": 3.534462648450352, + "tokens_seen": 1414332416 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002886459378134403, + "loss": 2.8259, + "theoretical_loss": 3.534448045390282, + "tokens_seen": 1414397952 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002886359077231695, + "loss": 2.7597, + "theoretical_loss": 3.5344334431962743, + "tokens_seen": 1414463488 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028862587763289874, + "loss": 2.6695, + "theoretical_loss": 3.5344188418682374, + "tokens_seen": 1414529024 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028861584754262786, + "loss": 2.64, + "theoretical_loss": 3.5344042414060803, + "tokens_seen": 1414594560 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002886058174523571, + "loss": 2.6012, + "theoretical_loss": 3.5343896418097116, + "tokens_seen": 1414660096 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002885957873620862, + "loss": 2.6704, + "theoretical_loss": 3.5343750430790397, + "tokens_seen": 1414725632 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028858575727181546, + "loss": 2.6927, + "theoretical_loss": 3.5343604452139727, + "tokens_seen": 1414791168 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028857572718154464, + "loss": 2.7709, + "theoretical_loss": 3.5343458482144197, + "tokens_seen": 1414856704 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002885656970912738, + "loss": 2.6046, + "theoretical_loss": 3.534331252080289, + "tokens_seen": 1414922240 + }, + { + "epoch": 0.43, + "learning_rate": 0.000288555667001003, + "loss": 2.7526, + "theoretical_loss": 3.5343166568114897, + "tokens_seen": 1414987776 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002885456369107322, + "loss": 2.7896, + "theoretical_loss": 3.5343020624079298, + "tokens_seen": 1415053312 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028853560682046137, + "loss": 2.7516, + "theoretical_loss": 3.5342874688695183, + "tokens_seen": 1415118848 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002885255767301906, + "loss": 2.6705, + "theoretical_loss": 3.534272876196164, + "tokens_seen": 1415184384 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028851554663991973, + "loss": 2.9614, + "theoretical_loss": 3.534258284387775, + "tokens_seen": 1415249920 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028850551654964896, + "loss": 2.6896, + "theoretical_loss": 3.5342436934442607, + "tokens_seen": 1415315456 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028849548645937815, + "loss": 2.6404, + "theoretical_loss": 3.5342291033655293, + "tokens_seen": 1415380992 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002884854563691073, + "loss": 2.7416, + "theoretical_loss": 3.53421451415149, + "tokens_seen": 1415446528 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002884754262788365, + "loss": 2.86, + "theoretical_loss": 3.5341999258020502, + "tokens_seen": 1415512064 + }, + { + "epoch": 0.43, + "objective/train/docs_used": 802505, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.880215883255005, + "objective/train/theoretical_loss": 3.5341853383171205, + "objective/train/tokens_used": 1436037600, + "theoretical_loss": 3.5341853383171205, + "tokens_seen": 1415577600 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002884653961885657, + "loss": 2.8495, + "theoretical_loss": 3.5341853383171205, + "tokens_seen": 1415577600 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028845536609829487, + "loss": 2.7051, + "theoretical_loss": 3.5341707516966085, + "tokens_seen": 1415643136 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002884453360080241, + "loss": 2.6769, + "theoretical_loss": 3.534156165940423, + "tokens_seen": 1415708672 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028843530591775323, + "loss": 2.6987, + "theoretical_loss": 3.5341415810484733, + "tokens_seen": 1415774208 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028842527582748247, + "loss": 2.6753, + "theoretical_loss": 3.5341269970206675, + "tokens_seen": 1415839744 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002884152457372116, + "loss": 2.6776, + "theoretical_loss": 3.534112413856915, + "tokens_seen": 1415905280 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028840521564694083, + "loss": 2.9284, + "theoretical_loss": 3.5340978315571245, + "tokens_seen": 1415970816 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028839518555667, + "loss": 2.5508, + "theoretical_loss": 3.534083250121204, + "tokens_seen": 1416036352 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002883851554663992, + "loss": 2.5504, + "theoretical_loss": 3.5340686695490637, + "tokens_seen": 1416101888 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002883751253761284, + "loss": 2.6287, + "theoretical_loss": 3.5340540898406116, + "tokens_seen": 1416167424 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002883650952858576, + "loss": 2.9257, + "theoretical_loss": 3.534039510995757, + "tokens_seen": 1416232960 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028835506519558674, + "loss": 2.9053, + "theoretical_loss": 3.5340249330144085, + "tokens_seen": 1416298496 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028834503510531597, + "loss": 2.4895, + "theoretical_loss": 3.5340103558964753, + "tokens_seen": 1416364032 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002883350050150451, + "loss": 2.6563, + "theoretical_loss": 3.5339957796418657, + "tokens_seen": 1416429568 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028832497492477433, + "loss": 2.6396, + "theoretical_loss": 3.5339812042504897, + "tokens_seen": 1416495104 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002883149448345035, + "loss": 2.8738, + "theoretical_loss": 3.533966629722255, + "tokens_seen": 1416560640 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002883049147442327, + "loss": 2.7723, + "theoretical_loss": 3.533952056057071, + "tokens_seen": 1416626176 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002882948846539619, + "loss": 2.7235, + "theoretical_loss": 3.5339374832548476, + "tokens_seen": 1416691712 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028828485456369106, + "loss": 2.6116, + "theoretical_loss": 3.5339229113154924, + "tokens_seen": 1416757248 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002882748244734203, + "loss": 2.7077, + "theoretical_loss": 3.5339083402389146, + "tokens_seen": 1416822784 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002882647943831495, + "loss": 2.9025, + "theoretical_loss": 3.5338937700250246, + "tokens_seen": 1416888320 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028825476429287866, + "loss": 2.6936, + "theoretical_loss": 3.53387920067373, + "tokens_seen": 1416953856 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028824473420260784, + "loss": 2.6703, + "theoretical_loss": 3.53386463218494, + "tokens_seen": 1417019392 + }, + { + "epoch": 0.43, + "learning_rate": 0.000288234704112337, + "loss": 2.668, + "theoretical_loss": 3.533850064558564, + "tokens_seen": 1417084928 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002882246740220662, + "loss": 2.6865, + "theoretical_loss": 3.5338354977945112, + "tokens_seen": 1417150464 + }, + { + "epoch": 0.43, + "objective/train/docs_used": 803171, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.926589012145996, + "objective/train/theoretical_loss": 3.5338209318926905, + "objective/train/tokens_used": 1437676000, + "theoretical_loss": 3.5338209318926905, + "tokens_seen": 1417216000 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028821464393179543, + "loss": 2.8291, + "theoretical_loss": 3.5338209318926905, + "tokens_seen": 1417216000 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028820461384152456, + "loss": 2.8008, + "theoretical_loss": 3.5338063668530104, + "tokens_seen": 1417281536 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002881945837512538, + "loss": 2.7066, + "theoretical_loss": 3.533791802675381, + "tokens_seen": 1417347072 + }, + { + "epoch": 0.43, + "learning_rate": 0.000288184553660983, + "loss": 2.4674, + "theoretical_loss": 3.533777239359711, + "tokens_seen": 1417412608 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028817452357071216, + "loss": 2.8786, + "theoretical_loss": 3.5337626769059094, + "tokens_seen": 1417478144 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028816449348044134, + "loss": 2.7606, + "theoretical_loss": 3.5337481153138857, + "tokens_seen": 1417543680 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002881544633901705, + "loss": 2.7969, + "theoretical_loss": 3.5337335545835487, + "tokens_seen": 1417609216 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002881444332998997, + "loss": 2.5008, + "theoretical_loss": 3.5337189947148078, + "tokens_seen": 1417674752 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028813440320962894, + "loss": 2.5741, + "theoretical_loss": 3.5337044357075715, + "tokens_seen": 1417740288 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028812437311935806, + "loss": 3.0092, + "theoretical_loss": 3.5336898775617502, + "tokens_seen": 1417805824 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002881143430290873, + "loss": 2.8034, + "theoretical_loss": 3.5336753202772524, + "tokens_seen": 1417871360 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002881043129388164, + "loss": 2.6878, + "theoretical_loss": 3.5336607638539874, + "tokens_seen": 1417936896 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028809428284854566, + "loss": 2.7739, + "theoretical_loss": 3.5336462082918647, + "tokens_seen": 1418002432 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028808425275827484, + "loss": 2.3597, + "theoretical_loss": 3.5336316535907937, + "tokens_seen": 1418067968 + }, + { + "epoch": 0.43, + "learning_rate": 0.000288074222668004, + "loss": 2.8305, + "theoretical_loss": 3.533617099750683, + "tokens_seen": 1418133504 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002880641925777332, + "loss": 2.5673, + "theoretical_loss": 3.5336025467714416, + "tokens_seen": 1418199040 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002880541624874624, + "loss": 2.8973, + "theoretical_loss": 3.5335879946529802, + "tokens_seen": 1418264576 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028804413239719157, + "loss": 2.9297, + "theoretical_loss": 3.5335734433952073, + "tokens_seen": 1418330112 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002880341023069208, + "loss": 2.8764, + "theoretical_loss": 3.5335588929980317, + "tokens_seen": 1418395648 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028802407221664993, + "loss": 2.7885, + "theoretical_loss": 3.5335443434613643, + "tokens_seen": 1418461184 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028801404212637916, + "loss": 2.6061, + "theoretical_loss": 3.5335297947851125, + "tokens_seen": 1418526720 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028800401203610835, + "loss": 2.566, + "theoretical_loss": 3.533515246969187, + "tokens_seen": 1418592256 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002879939819458375, + "loss": 2.8807, + "theoretical_loss": 3.533500700013497, + "tokens_seen": 1418657792 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002879839518555667, + "loss": 2.7758, + "theoretical_loss": 3.533486153917951, + "tokens_seen": 1418723328 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002879739217652959, + "loss": 2.613, + "theoretical_loss": 3.5334716086824605, + "tokens_seen": 1418788864 + }, + { + "epoch": 0.43, + "objective/train/docs_used": 804514, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 1.9776077270507812, + "objective/train/theoretical_loss": 3.5334570643069325, + "objective/train/tokens_used": 1439314400, + "theoretical_loss": 3.5334570643069325, + "tokens_seen": 1418854400 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028796389167502507, + "loss": 2.6007, + "theoretical_loss": 3.5334570643069325, + "tokens_seen": 1418854400 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002879538615847543, + "loss": 2.7904, + "theoretical_loss": 3.5334425207912776, + "tokens_seen": 1418919936 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028794383149448343, + "loss": 2.7082, + "theoretical_loss": 3.533427978135405, + "tokens_seen": 1418985472 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028793380140421267, + "loss": 2.462, + "theoretical_loss": 3.5334134363392247, + "tokens_seen": 1419051008 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002879237713139418, + "loss": 2.4578, + "theoretical_loss": 3.533398895402646, + "tokens_seen": 1419116544 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028791374122367103, + "loss": 2.7236, + "theoretical_loss": 3.5333843553255777, + "tokens_seen": 1419182080 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002879037111334002, + "loss": 2.9282, + "theoretical_loss": 3.53336981610793, + "tokens_seen": 1419247616 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002878936810431294, + "loss": 2.7505, + "theoretical_loss": 3.5333552777496124, + "tokens_seen": 1419313152 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002878836509528586, + "loss": 2.6151, + "theoretical_loss": 3.5333407402505337, + "tokens_seen": 1419378688 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002878736208625878, + "loss": 2.7688, + "theoretical_loss": 3.5333262036106046, + "tokens_seen": 1419444224 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028786359077231694, + "loss": 2.8991, + "theoretical_loss": 3.533311667829734, + "tokens_seen": 1419509760 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028785356068204617, + "loss": 2.6357, + "theoretical_loss": 3.5332971329078315, + "tokens_seen": 1419575296 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002878435305917753, + "loss": 2.8846, + "theoretical_loss": 3.5332825988448064, + "tokens_seen": 1419640832 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028783350050150453, + "loss": 2.7733, + "theoretical_loss": 3.533268065640569, + "tokens_seen": 1419706368 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002878234704112337, + "loss": 2.7916, + "theoretical_loss": 3.533253533295028, + "tokens_seen": 1419771904 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002878134403209629, + "loss": 2.7407, + "theoretical_loss": 3.533239001808094, + "tokens_seen": 1419837440 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002878034102306921, + "loss": 2.6002, + "theoretical_loss": 3.5332244711796763, + "tokens_seen": 1419902976 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028779338014042126, + "loss": 2.8125, + "theoretical_loss": 3.533209941409684, + "tokens_seen": 1419968512 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028778335005015044, + "loss": 2.5466, + "theoretical_loss": 3.533195412498028, + "tokens_seen": 1420034048 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002877733199598797, + "loss": 3.0395, + "theoretical_loss": 3.5331808844446164, + "tokens_seen": 1420099584 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002877632898696088, + "loss": 2.2661, + "theoretical_loss": 3.5331663572493603, + "tokens_seen": 1420165120 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028775325977933804, + "loss": 2.5033, + "theoretical_loss": 3.5331518309121686, + "tokens_seen": 1420230656 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028774322968906716, + "loss": 2.7992, + "theoretical_loss": 3.533137305432951, + "tokens_seen": 1420296192 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002877331995987964, + "loss": 2.6175, + "theoretical_loss": 3.5331227808116177, + "tokens_seen": 1420361728 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002877231695085256, + "loss": 2.6846, + "theoretical_loss": 3.5331082570480787, + "tokens_seen": 1420427264 + }, + { + "epoch": 0.43, + "objective/train/docs_used": 804990, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.716604709625244, + "objective/train/theoretical_loss": 3.533093734142243, + "objective/train/tokens_used": 1440952800, + "theoretical_loss": 3.533093734142243, + "tokens_seen": 1420492800 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028771313941825476, + "loss": 2.489, + "theoretical_loss": 3.533093734142243, + "tokens_seen": 1420492800 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028770310932798394, + "loss": 2.7268, + "theoretical_loss": 3.5330792120940204, + "tokens_seen": 1420558336 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002876930792377132, + "loss": 2.6474, + "theoretical_loss": 3.5330646909033216, + "tokens_seen": 1420623872 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002876830491474423, + "loss": 2.3482, + "theoretical_loss": 3.533050170570055, + "tokens_seen": 1420689408 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028767301905717154, + "loss": 2.5633, + "theoretical_loss": 3.5330356510941323, + "tokens_seen": 1420754944 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028766298896690067, + "loss": 2.7887, + "theoretical_loss": 3.5330211324754615, + "tokens_seen": 1420820480 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002876529588766299, + "loss": 2.7168, + "theoretical_loss": 3.533006614713954, + "tokens_seen": 1420886016 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002876429287863591, + "loss": 2.9791, + "theoretical_loss": 3.532992097809518, + "tokens_seen": 1420951552 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028763289869608826, + "loss": 2.7153, + "theoretical_loss": 3.5329775817620646, + "tokens_seen": 1421017088 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028762286860581745, + "loss": 2.6688, + "theoretical_loss": 3.532963066571503, + "tokens_seen": 1421082624 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002876128385155466, + "loss": 2.544, + "theoretical_loss": 3.5329485522377437, + "tokens_seen": 1421148160 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002876028084252758, + "loss": 2.6524, + "theoretical_loss": 3.532934038760697, + "tokens_seen": 1421213696 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028759277833500504, + "loss": 2.818, + "theoretical_loss": 3.5329195261402715, + "tokens_seen": 1421279232 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028758274824473417, + "loss": 2.7451, + "theoretical_loss": 3.532905014376378, + "tokens_seen": 1421344768 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002875727181544634, + "loss": 2.7492, + "theoretical_loss": 3.532890503468926, + "tokens_seen": 1421410304 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028756268806419253, + "loss": 2.6762, + "theoretical_loss": 3.532875993417826, + "tokens_seen": 1421475840 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028755265797392177, + "loss": 2.9395, + "theoretical_loss": 3.5328614842229875, + "tokens_seen": 1421541376 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028754262788365095, + "loss": 2.6369, + "theoretical_loss": 3.532846975884321, + "tokens_seen": 1421606912 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028753259779338013, + "loss": 2.779, + "theoretical_loss": 3.5328324684017365, + "tokens_seen": 1421672448 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028752256770310936, + "loss": 2.6777, + "theoretical_loss": 3.5328179617751436, + "tokens_seen": 1421737984 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028751253761283855, + "loss": 2.6714, + "theoretical_loss": 3.5328034560044523, + "tokens_seen": 1421803520 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002875025075225677, + "loss": 2.788, + "theoretical_loss": 3.5327889510895734, + "tokens_seen": 1421869056 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002874924774322969, + "loss": 2.749, + "theoretical_loss": 3.532774447030416, + "tokens_seen": 1421934592 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002874824473420261, + "loss": 2.5994, + "theoretical_loss": 3.532759943826891, + "tokens_seen": 1422000128 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028747241725175527, + "loss": 2.7461, + "theoretical_loss": 3.532745441478908, + "tokens_seen": 1422065664 + }, + { + "epoch": 0.43, + "objective/train/docs_used": 806295, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7887356281280518, + "objective/train/theoretical_loss": 3.5327309399863767, + "objective/train/tokens_used": 1442591200, + "theoretical_loss": 3.5327309399863767, + "tokens_seen": 1422131200 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002874623871614845, + "loss": 2.7315, + "theoretical_loss": 3.5327309399863767, + "tokens_seen": 1422131200 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028745235707121363, + "loss": 2.6923, + "theoretical_loss": 3.5327164393492083, + "tokens_seen": 1422196736 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028744232698094287, + "loss": 2.6882, + "theoretical_loss": 3.5327019395673123, + "tokens_seen": 1422262272 + }, + { + "epoch": 0.43, + "learning_rate": 0.000287432296890672, + "loss": 2.5807, + "theoretical_loss": 3.532687440640599, + "tokens_seen": 1422327808 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028742226680040123, + "loss": 2.6118, + "theoretical_loss": 3.5326729425689782, + "tokens_seen": 1422393344 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002874122367101304, + "loss": 2.8524, + "theoretical_loss": 3.5326584453523604, + "tokens_seen": 1422458880 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002874022066198596, + "loss": 2.7094, + "theoretical_loss": 3.532643948990656, + "tokens_seen": 1422524416 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002873921765295888, + "loss": 2.7577, + "theoretical_loss": 3.532629453483775, + "tokens_seen": 1422589952 + }, + { + "epoch": 0.43, + "learning_rate": 0.000287382146439318, + "loss": 2.8415, + "theoretical_loss": 3.532614958831627, + "tokens_seen": 1422655488 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028737211634904714, + "loss": 2.6467, + "theoretical_loss": 3.5326004650341236, + "tokens_seen": 1422721024 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028736208625877637, + "loss": 2.8961, + "theoretical_loss": 3.532585972091174, + "tokens_seen": 1422786560 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002873520561685055, + "loss": 2.7115, + "theoretical_loss": 3.532571480002688, + "tokens_seen": 1422852096 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028734202607823473, + "loss": 2.6321, + "theoretical_loss": 3.5325569887685777, + "tokens_seen": 1422917632 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002873319959879639, + "loss": 2.7971, + "theoretical_loss": 3.532542498388752, + "tokens_seen": 1422983168 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002873219658976931, + "loss": 2.64, + "theoretical_loss": 3.532528008863121, + "tokens_seen": 1423048704 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002873119358074223, + "loss": 2.6911, + "theoretical_loss": 3.5325135201915954, + "tokens_seen": 1423114240 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028730190571715146, + "loss": 2.6882, + "theoretical_loss": 3.532499032374086, + "tokens_seen": 1423179776 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028729187562688064, + "loss": 2.6651, + "theoretical_loss": 3.5324845454105027, + "tokens_seen": 1423245312 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002872818455366099, + "loss": 2.6121, + "theoretical_loss": 3.5324700593007554, + "tokens_seen": 1423310848 + }, + { + "epoch": 0.43, + "learning_rate": 0.000287271815446339, + "loss": 2.7149, + "theoretical_loss": 3.5324555740447554, + "tokens_seen": 1423376384 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028726178535606824, + "loss": 2.686, + "theoretical_loss": 3.532441089642413, + "tokens_seen": 1423441920 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028725175526579736, + "loss": 2.684, + "theoretical_loss": 3.5324266060936376, + "tokens_seen": 1423507456 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002872417251755266, + "loss": 2.6065, + "theoretical_loss": 3.5324121233983403, + "tokens_seen": 1423572992 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002872316950852558, + "loss": 2.751, + "theoretical_loss": 3.5323976415564315, + "tokens_seen": 1423638528 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028722166499498496, + "loss": 2.6398, + "theoretical_loss": 3.5323831605678215, + "tokens_seen": 1423704064 + }, + { + "epoch": 0.43, + "objective/train/docs_used": 806909, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.9317681789398193, + "objective/train/theoretical_loss": 3.5323686804324215, + "objective/train/tokens_used": 1444229600, + "theoretical_loss": 3.5323686804324215, + "tokens_seen": 1423769600 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028721163490471414, + "loss": 2.7101, + "theoretical_loss": 3.5323686804324215, + "tokens_seen": 1423769600 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002872016048144434, + "loss": 2.6476, + "theoretical_loss": 3.5323542011501408, + "tokens_seen": 1423835136 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002871915747241725, + "loss": 2.5503, + "theoretical_loss": 3.53233972272089, + "tokens_seen": 1423900672 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028718154463390174, + "loss": 2.7035, + "theoretical_loss": 3.5323252451445804, + "tokens_seen": 1423966208 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028717151454363087, + "loss": 2.8828, + "theoretical_loss": 3.5323107684211217, + "tokens_seen": 1424031744 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002871614844533601, + "loss": 2.4619, + "theoretical_loss": 3.5322962925504253, + "tokens_seen": 1424097280 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002871514543630893, + "loss": 2.7092, + "theoretical_loss": 3.5322818175324007, + "tokens_seen": 1424162816 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028714142427281846, + "loss": 2.4629, + "theoretical_loss": 3.532267343366959, + "tokens_seen": 1424228352 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028713139418254765, + "loss": 2.6778, + "theoretical_loss": 3.532252870054011, + "tokens_seen": 1424293888 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002871213640922768, + "loss": 2.6975, + "theoretical_loss": 3.532238397593466, + "tokens_seen": 1424359424 + }, + { + "epoch": 0.43, + "learning_rate": 0.000287111334002006, + "loss": 2.7124, + "theoretical_loss": 3.532223925985237, + "tokens_seen": 1424424960 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028710130391173524, + "loss": 3.0112, + "theoretical_loss": 3.532209455229232, + "tokens_seen": 1424490496 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028709127382146437, + "loss": 2.7326, + "theoretical_loss": 3.532194985325363, + "tokens_seen": 1424556032 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002870812437311936, + "loss": 2.8026, + "theoretical_loss": 3.5321805162735407, + "tokens_seen": 1424621568 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028707121364092273, + "loss": 2.6926, + "theoretical_loss": 3.532166048073675, + "tokens_seen": 1424687104 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028706118355065197, + "loss": 2.8657, + "theoretical_loss": 3.532151580725677, + "tokens_seen": 1424752640 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028705115346038115, + "loss": 2.5078, + "theoretical_loss": 3.5321371142294575, + "tokens_seen": 1424818176 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028704112337011033, + "loss": 2.8154, + "theoretical_loss": 3.532122648584927, + "tokens_seen": 1424883712 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002870310932798395, + "loss": 2.9962, + "theoretical_loss": 3.532108183791996, + "tokens_seen": 1424949248 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028702106318956875, + "loss": 2.8988, + "theoretical_loss": 3.5320937198505753, + "tokens_seen": 1425014784 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002870110330992979, + "loss": 2.6136, + "theoretical_loss": 3.532079256760576, + "tokens_seen": 1425080320 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002870010030090271, + "loss": 2.771, + "theoretical_loss": 3.532064794521908, + "tokens_seen": 1425145856 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028699097291875624, + "loss": 2.8835, + "theoretical_loss": 3.5320503331344826, + "tokens_seen": 1425211392 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028698094282848547, + "loss": 2.6819, + "theoretical_loss": 3.532035872598211, + "tokens_seen": 1425276928 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028697091273821465, + "loss": 2.6696, + "theoretical_loss": 3.5320214129130036, + "tokens_seen": 1425342464 + }, + { + "epoch": 0.43, + "objective/train/docs_used": 808391, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.72674298286438, + "objective/train/theoretical_loss": 3.5320069540787706, + "objective/train/tokens_used": 1445868000, + "theoretical_loss": 3.5320069540787706, + "tokens_seen": 1425408000 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028696088264794383, + "loss": 2.4637, + "theoretical_loss": 3.5320069540787706, + "tokens_seen": 1425408000 + }, + { + "epoch": 0.43, + "learning_rate": 0.000286950852557673, + "loss": 2.7103, + "theoretical_loss": 3.531992496095423, + "tokens_seen": 1425473536 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002869408224674022, + "loss": 2.5886, + "theoretical_loss": 3.5319780389628725, + "tokens_seen": 1425539072 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002869307923771314, + "loss": 2.6923, + "theoretical_loss": 3.531963582681029, + "tokens_seen": 1425604608 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002869207622868606, + "loss": 2.735, + "theoretical_loss": 3.531949127249804, + "tokens_seen": 1425670144 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028691073219658974, + "loss": 2.7114, + "theoretical_loss": 3.5319346726691077, + "tokens_seen": 1425735680 + }, + { + "epoch": 0.43, + "learning_rate": 0.000286900702106319, + "loss": 2.7972, + "theoretical_loss": 3.5319202189388514, + "tokens_seen": 1425801216 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002868906720160481, + "loss": 2.712, + "theoretical_loss": 3.5319057660589452, + "tokens_seen": 1425866752 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028688064192577734, + "loss": 2.5612, + "theoretical_loss": 3.531891314029301, + "tokens_seen": 1425932288 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002868706118355065, + "loss": 2.7228, + "theoretical_loss": 3.53187686284983, + "tokens_seen": 1425997824 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002868605817452357, + "loss": 2.7657, + "theoretical_loss": 3.531862412520442, + "tokens_seen": 1426063360 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002868505516549649, + "loss": 2.6842, + "theoretical_loss": 3.531847963041048, + "tokens_seen": 1426128896 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002868405215646941, + "loss": 2.6189, + "theoretical_loss": 3.53183351441156, + "tokens_seen": 1426194432 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028683049147442324, + "loss": 2.6796, + "theoretical_loss": 3.531819066631887, + "tokens_seen": 1426259968 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002868204613841525, + "loss": 2.6831, + "theoretical_loss": 3.5318046197019424, + "tokens_seen": 1426325504 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002868104312938816, + "loss": 2.5167, + "theoretical_loss": 3.531790173621636, + "tokens_seen": 1426391040 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028680040120361084, + "loss": 2.6071, + "theoretical_loss": 3.531775728390879, + "tokens_seen": 1426456576 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028679037111334, + "loss": 2.8452, + "theoretical_loss": 3.5317612840095816, + "tokens_seen": 1426522112 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002867803410230692, + "loss": 2.5919, + "theoretical_loss": 3.5317468404776555, + "tokens_seen": 1426587648 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028677031093279844, + "loss": 2.7364, + "theoretical_loss": 3.531732397795012, + "tokens_seen": 1426653184 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028676028084252756, + "loss": 2.5215, + "theoretical_loss": 3.531717955961562, + "tokens_seen": 1426718720 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002867502507522568, + "loss": 2.5934, + "theoretical_loss": 3.531703514977216, + "tokens_seen": 1426784256 + }, + { + "epoch": 0.43, + "learning_rate": 0.000286740220661986, + "loss": 2.7107, + "theoretical_loss": 3.5316890748418857, + "tokens_seen": 1426849792 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028673019057171516, + "loss": 2.7643, + "theoretical_loss": 3.531674635555482, + "tokens_seen": 1426915328 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028672016048144434, + "loss": 2.6445, + "theoretical_loss": 3.5316601971179162, + "tokens_seen": 1426980864 + }, + { + "epoch": 0.43, + "objective/train/docs_used": 809012, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.594926357269287, + "objective/train/theoretical_loss": 3.531645759529099, + "objective/train/tokens_used": 1447506400, + "theoretical_loss": 3.531645759529099, + "tokens_seen": 1427046400 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002867101303911736, + "loss": 2.6491, + "theoretical_loss": 3.531645759529099, + "tokens_seen": 1427046400 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002867001003009027, + "loss": 2.8329, + "theoretical_loss": 3.5316313227889413, + "tokens_seen": 1427111936 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028669007021063194, + "loss": 2.6368, + "theoretical_loss": 3.5316168868973556, + "tokens_seen": 1427177472 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028668004012036107, + "loss": 2.7562, + "theoretical_loss": 3.5316024518542517, + "tokens_seen": 1427243008 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002866700100300903, + "loss": 2.8861, + "theoretical_loss": 3.5315880176595416, + "tokens_seen": 1427308544 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002866599799398195, + "loss": 2.6876, + "theoretical_loss": 3.5315735843131355, + "tokens_seen": 1427374080 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028664994984954866, + "loss": 2.7623, + "theoretical_loss": 3.5315591518149456, + "tokens_seen": 1427439616 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028663991975927785, + "loss": 2.6039, + "theoretical_loss": 3.5315447201648826, + "tokens_seen": 1427505152 + }, + { + "epoch": 0.43, + "learning_rate": 0.000286629889669007, + "loss": 2.7417, + "theoretical_loss": 3.531530289362858, + "tokens_seen": 1427570688 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002866198595787362, + "loss": 2.8051, + "theoretical_loss": 3.531515859408783, + "tokens_seen": 1427636224 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028660982948846544, + "loss": 2.7145, + "theoretical_loss": 3.5315014303025682, + "tokens_seen": 1427701760 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028659979939819457, + "loss": 2.6313, + "theoretical_loss": 3.531487002044126, + "tokens_seen": 1427767296 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002865897693079238, + "loss": 2.8151, + "theoretical_loss": 3.5314725746333666, + "tokens_seen": 1427832832 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028657973921765293, + "loss": 2.6333, + "theoretical_loss": 3.531458148070202, + "tokens_seen": 1427898368 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028656970912738217, + "loss": 2.506, + "theoretical_loss": 3.5314437223545436, + "tokens_seen": 1427963904 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028655967903711135, + "loss": 2.7595, + "theoretical_loss": 3.5314292974863024, + "tokens_seen": 1428029440 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028654964894684053, + "loss": 2.6501, + "theoretical_loss": 3.5314148734653896, + "tokens_seen": 1428094976 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002865396188565697, + "loss": 2.3816, + "theoretical_loss": 3.5314004502917165, + "tokens_seen": 1428160512 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028652958876629895, + "loss": 2.7017, + "theoretical_loss": 3.531386027965195, + "tokens_seen": 1428226048 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002865195586760281, + "loss": 2.7418, + "theoretical_loss": 3.5313716064857354, + "tokens_seen": 1428291584 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002865095285857573, + "loss": 2.6968, + "theoretical_loss": 3.531357185853251, + "tokens_seen": 1428357120 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028649949849548644, + "loss": 2.8504, + "theoretical_loss": 3.5313427660676506, + "tokens_seen": 1428422656 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028648946840521567, + "loss": 2.5126, + "theoretical_loss": 3.5313283471288477, + "tokens_seen": 1428488192 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028647943831494485, + "loss": 2.8047, + "theoretical_loss": 3.5313139290367532, + "tokens_seen": 1428553728 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028646940822467403, + "loss": 2.8398, + "theoretical_loss": 3.5312995117912784, + "tokens_seen": 1428619264 + }, + { + "epoch": 0.43, + "objective/train/docs_used": 809683, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.873274803161621, + "objective/train/theoretical_loss": 3.531285095392335, + "objective/train/tokens_used": 1449144800, + "theoretical_loss": 3.531285095392335, + "tokens_seen": 1428684800 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002864593781344032, + "loss": 2.601, + "theoretical_loss": 3.531285095392335, + "tokens_seen": 1428684800 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002864493480441324, + "loss": 2.6431, + "theoretical_loss": 3.531270679839834, + "tokens_seen": 1428750336 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002864393179538616, + "loss": 2.6668, + "theoretical_loss": 3.5312562651336865, + "tokens_seen": 1428815872 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002864292878635908, + "loss": 2.759, + "theoretical_loss": 3.5312418512738053, + "tokens_seen": 1428881408 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028641925777331994, + "loss": 2.6262, + "theoretical_loss": 3.5312274382601005, + "tokens_seen": 1428946944 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002864092276830492, + "loss": 2.6075, + "theoretical_loss": 3.5312130260924848, + "tokens_seen": 1429012480 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002863991975927783, + "loss": 2.8011, + "theoretical_loss": 3.5311986147708696, + "tokens_seen": 1429078016 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028638916750250754, + "loss": 2.8363, + "theoretical_loss": 3.5311842042951653, + "tokens_seen": 1429143552 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002863791374122367, + "loss": 2.5668, + "theoretical_loss": 3.531169794665285, + "tokens_seen": 1429209088 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002863691073219659, + "loss": 2.3217, + "theoretical_loss": 3.531155385881139, + "tokens_seen": 1429274624 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002863590772316951, + "loss": 2.5678, + "theoretical_loss": 3.5311409779426395, + "tokens_seen": 1429340160 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002863490471414243, + "loss": 2.5338, + "theoretical_loss": 3.531126570849698, + "tokens_seen": 1429405696 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028633901705115344, + "loss": 2.5985, + "theoretical_loss": 3.5311121646022263, + "tokens_seen": 1429471232 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002863289869608827, + "loss": 2.7991, + "theoretical_loss": 3.5310977592001356, + "tokens_seen": 1429536768 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002863189568706118, + "loss": 2.7926, + "theoretical_loss": 3.531083354643338, + "tokens_seen": 1429602304 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028630892678034104, + "loss": 2.52, + "theoretical_loss": 3.5310689509317448, + "tokens_seen": 1429667840 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002862988966900702, + "loss": 2.6422, + "theoretical_loss": 3.531054548065268, + "tokens_seen": 1429733376 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002862888665997994, + "loss": 2.7252, + "theoretical_loss": 3.5310401460438188, + "tokens_seen": 1429798912 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002862788365095286, + "loss": 2.8201, + "theoretical_loss": 3.5310257448673092, + "tokens_seen": 1429864448 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028626880641925776, + "loss": 2.7326, + "theoretical_loss": 3.531011344535651, + "tokens_seen": 1429929984 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028625877632898694, + "loss": 2.5019, + "theoretical_loss": 3.5309969450487557, + "tokens_seen": 1429995520 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002862487462387162, + "loss": 2.5531, + "theoretical_loss": 3.5309825464065354, + "tokens_seen": 1430061056 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002862387161484453, + "loss": 2.8508, + "theoretical_loss": 3.5309681486089013, + "tokens_seen": 1430126592 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028622868605817454, + "loss": 2.7157, + "theoretical_loss": 3.5309537516557654, + "tokens_seen": 1430192128 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028621865596790367, + "loss": 2.3609, + "theoretical_loss": 3.53093935554704, + "tokens_seen": 1430257664 + }, + { + "epoch": 0.43, + "objective/train/docs_used": 810937, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.279047966003418, + "objective/train/theoretical_loss": 3.530924960282636, + "objective/train/tokens_used": 1450783200, + "theoretical_loss": 3.530924960282636, + "tokens_seen": 1430323200 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002862086258776329, + "loss": 2.3905, + "theoretical_loss": 3.530924960282636, + "tokens_seen": 1430323200 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002861985957873621, + "loss": 2.5656, + "theoretical_loss": 3.5309105658624658, + "tokens_seen": 1430388736 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028618856569709127, + "loss": 2.918, + "theoretical_loss": 3.5308961722864405, + "tokens_seen": 1430454272 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028617853560682045, + "loss": 2.6966, + "theoretical_loss": 3.530881779554473, + "tokens_seen": 1430519808 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002861685055165497, + "loss": 2.5693, + "theoretical_loss": 3.5308673876664742, + "tokens_seen": 1430585344 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002861584754262788, + "loss": 2.6172, + "theoretical_loss": 3.5308529966223565, + "tokens_seen": 1430650880 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028614844533600805, + "loss": 2.7527, + "theoretical_loss": 3.5308386064220314, + "tokens_seen": 1430716416 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028613841524573717, + "loss": 2.9601, + "theoretical_loss": 3.530824217065411, + "tokens_seen": 1430781952 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002861283851554664, + "loss": 2.9833, + "theoretical_loss": 3.530809828552407, + "tokens_seen": 1430847488 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002861183550651956, + "loss": 2.6763, + "theoretical_loss": 3.530795440882932, + "tokens_seen": 1430913024 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028610832497492477, + "loss": 2.6554, + "theoretical_loss": 3.530781054056897, + "tokens_seen": 1430978560 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028609829488465395, + "loss": 2.6272, + "theoretical_loss": 3.5307666680742145, + "tokens_seen": 1431044096 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028608826479438313, + "loss": 2.5109, + "theoretical_loss": 3.530752282934796, + "tokens_seen": 1431109632 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002860782347041123, + "loss": 2.6605, + "theoretical_loss": 3.530737898638554, + "tokens_seen": 1431175168 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028606820461384155, + "loss": 2.6758, + "theoretical_loss": 3.5307235151854, + "tokens_seen": 1431240704 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002860581745235707, + "loss": 2.8341, + "theoretical_loss": 3.5307091325752458, + "tokens_seen": 1431306240 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002860481444332999, + "loss": 2.556, + "theoretical_loss": 3.5306947508080047, + "tokens_seen": 1431371776 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002860381143430291, + "loss": 2.7449, + "theoretical_loss": 3.530680369883587, + "tokens_seen": 1431437312 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002860280842527583, + "loss": 2.8057, + "theoretical_loss": 3.5306659898019053, + "tokens_seen": 1431502848 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002860180541624875, + "loss": 2.6575, + "theoretical_loss": 3.530651610562872, + "tokens_seen": 1431568384 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028600802407221664, + "loss": 2.7273, + "theoretical_loss": 3.5306372321663995, + "tokens_seen": 1431633920 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028599799398194587, + "loss": 2.3612, + "theoretical_loss": 3.530622854612399, + "tokens_seen": 1431699456 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028598796389167505, + "loss": 2.6662, + "theoretical_loss": 3.530608477900783, + "tokens_seen": 1431764992 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028597793380140423, + "loss": 2.8994, + "theoretical_loss": 3.5305941020314635, + "tokens_seen": 1431830528 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002859679037111334, + "loss": 2.7261, + "theoretical_loss": 3.5305797270043526, + "tokens_seen": 1431896064 + }, + { + "epoch": 0.43, + "objective/train/docs_used": 811574, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.5372531414031982, + "objective/train/theoretical_loss": 3.5305653528193623, + "objective/train/tokens_used": 1452421600, + "theoretical_loss": 3.5305653528193623, + "tokens_seen": 1431961600 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002859578736208626, + "loss": 2.596, + "theoretical_loss": 3.5305653528193623, + "tokens_seen": 1431961600 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002859478435305918, + "loss": 2.753, + "theoretical_loss": 3.530550979476405, + "tokens_seen": 1432027136 + }, + { + "epoch": 0.43, + "learning_rate": 0.000285937813440321, + "loss": 2.7245, + "theoretical_loss": 3.5305366069753927, + "tokens_seen": 1432092672 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028592778335005014, + "loss": 2.5895, + "theoretical_loss": 3.530522235316237, + "tokens_seen": 1432158208 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002859177532597794, + "loss": 2.5021, + "theoretical_loss": 3.5305078644988512, + "tokens_seen": 1432223744 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002859077231695085, + "loss": 2.8294, + "theoretical_loss": 3.530493494523147, + "tokens_seen": 1432289280 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028589769307923774, + "loss": 2.6894, + "theoretical_loss": 3.5304791253890366, + "tokens_seen": 1432354816 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002858876629889669, + "loss": 2.7532, + "theoretical_loss": 3.530464757096432, + "tokens_seen": 1432420352 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002858776328986961, + "loss": 2.7632, + "theoretical_loss": 3.530450389645245, + "tokens_seen": 1432485888 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002858676028084253, + "loss": 2.5186, + "theoretical_loss": 3.5304360230353886, + "tokens_seen": 1432551424 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002858575727181545, + "loss": 2.6441, + "theoretical_loss": 3.530421657266775, + "tokens_seen": 1432616960 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028584754262788364, + "loss": 2.6832, + "theoretical_loss": 3.5304072923393166, + "tokens_seen": 1432682496 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002858375125376129, + "loss": 3.0046, + "theoretical_loss": 3.530392928252925, + "tokens_seen": 1432748032 + }, + { + "epoch": 0.43, + "learning_rate": 0.000285827482447342, + "loss": 2.5749, + "theoretical_loss": 3.5303785650075126, + "tokens_seen": 1432813568 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028581745235707124, + "loss": 2.5933, + "theoretical_loss": 3.5303642026029918, + "tokens_seen": 1432879104 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002858074222668004, + "loss": 2.7199, + "theoretical_loss": 3.5303498410392757, + "tokens_seen": 1432944640 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002857973921765296, + "loss": 2.5655, + "theoretical_loss": 3.5303354803162756, + "tokens_seen": 1433010176 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002857873620862588, + "loss": 2.6288, + "theoretical_loss": 3.5303211204339044, + "tokens_seen": 1433075712 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028577733199598796, + "loss": 2.7839, + "theoretical_loss": 3.530306761392074, + "tokens_seen": 1433141248 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028576730190571714, + "loss": 2.6439, + "theoretical_loss": 3.530292403190697, + "tokens_seen": 1433206784 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002857572718154464, + "loss": 2.5106, + "theoretical_loss": 3.530278045829686, + "tokens_seen": 1433272320 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002857472417251755, + "loss": 2.5469, + "theoretical_loss": 3.530263689308953, + "tokens_seen": 1433337856 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028573721163490474, + "loss": 2.7134, + "theoretical_loss": 3.5302493336284106, + "tokens_seen": 1433403392 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028572718154463387, + "loss": 2.7114, + "theoretical_loss": 3.5302349787879717, + "tokens_seen": 1433468928 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002857171514543631, + "loss": 2.6924, + "theoretical_loss": 3.530220624787548, + "tokens_seen": 1433534464 + }, + { + "epoch": 0.43, + "objective/train/docs_used": 812859, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6041934490203857, + "objective/train/theoretical_loss": 3.530206271627052, + "objective/train/tokens_used": 1454060000, + "theoretical_loss": 3.530206271627052, + "tokens_seen": 1433600000 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002857071213640923, + "loss": 2.7103, + "theoretical_loss": 3.530206271627052, + "tokens_seen": 1433600000 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028569709127382147, + "loss": 2.5745, + "theoretical_loss": 3.5301919193063966, + "tokens_seen": 1433665536 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028568706118355065, + "loss": 2.5899, + "theoretical_loss": 3.5301775678254943, + "tokens_seen": 1433731072 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002856770310932799, + "loss": 2.8353, + "theoretical_loss": 3.5301632171842567, + "tokens_seen": 1433796608 + }, + { + "epoch": 0.43, + "learning_rate": 0.000285667001003009, + "loss": 2.8744, + "theoretical_loss": 3.5301488673825974, + "tokens_seen": 1433862144 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028565697091273825, + "loss": 2.6901, + "theoretical_loss": 3.5301345184204282, + "tokens_seen": 1433927680 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028564694082246737, + "loss": 2.5848, + "theoretical_loss": 3.5301201702976615, + "tokens_seen": 1433993216 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002856369107321966, + "loss": 2.6775, + "theoretical_loss": 3.530105823014211, + "tokens_seen": 1434058752 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002856268806419258, + "loss": 2.7262, + "theoretical_loss": 3.5300914765699876, + "tokens_seen": 1434124288 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028561685055165497, + "loss": 2.7241, + "theoretical_loss": 3.5300771309649055, + "tokens_seen": 1434189824 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028560682046138415, + "loss": 2.51, + "theoretical_loss": 3.530062786198876, + "tokens_seen": 1434255360 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028559679037111333, + "loss": 2.666, + "theoretical_loss": 3.530048442271812, + "tokens_seen": 1434320896 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002855867602808425, + "loss": 2.5929, + "theoretical_loss": 3.5300340991836268, + "tokens_seen": 1434386432 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028557673019057175, + "loss": 2.5367, + "theoretical_loss": 3.530019756934232, + "tokens_seen": 1434451968 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002855667001003009, + "loss": 2.5494, + "theoretical_loss": 3.530005415523541, + "tokens_seen": 1434517504 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002855566700100301, + "loss": 2.9005, + "theoretical_loss": 3.5299910749514662, + "tokens_seen": 1434583040 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002855466399197593, + "loss": 2.4912, + "theoretical_loss": 3.52997673521792, + "tokens_seen": 1434648576 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002855366098294885, + "loss": 2.6381, + "theoretical_loss": 3.529962396322815, + "tokens_seen": 1434714112 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028552657973921765, + "loss": 2.5958, + "theoretical_loss": 3.529948058266065, + "tokens_seen": 1434779648 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028551654964894684, + "loss": 2.6402, + "theoretical_loss": 3.529933721047581, + "tokens_seen": 1434845184 + }, + { + "epoch": 0.43, + "learning_rate": 0.000285506519558676, + "loss": 2.5987, + "theoretical_loss": 3.529919384667277, + "tokens_seen": 1434910720 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028549648946840525, + "loss": 2.6864, + "theoretical_loss": 3.529905049125065, + "tokens_seen": 1434976256 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002854864593781344, + "loss": 2.6201, + "theoretical_loss": 3.5298907144208584, + "tokens_seen": 1435041792 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002854764292878636, + "loss": 2.7303, + "theoretical_loss": 3.529876380554569, + "tokens_seen": 1435107328 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028546639919759274, + "loss": 2.5187, + "theoretical_loss": 3.5298620475261107, + "tokens_seen": 1435172864 + }, + { + "epoch": 0.43, + "objective/train/docs_used": 813625, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7453997135162354, + "objective/train/theoretical_loss": 3.5298477153353955, + "objective/train/tokens_used": 1455698400, + "theoretical_loss": 3.5298477153353955, + "tokens_seen": 1435238400 + }, + { + "epoch": 0.43, + "learning_rate": 0.000285456369107322, + "loss": 2.7429, + "theoretical_loss": 3.5298477153353955, + "tokens_seen": 1435238400 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028544633901705116, + "loss": 2.6856, + "theoretical_loss": 3.5298333839823366, + "tokens_seen": 1435303936 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028543630892678034, + "loss": 2.5348, + "theoretical_loss": 3.529819053466846, + "tokens_seen": 1435369472 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002854262788365095, + "loss": 2.6413, + "theoretical_loss": 3.5298047237888373, + "tokens_seen": 1435435008 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002854162487462387, + "loss": 2.8222, + "theoretical_loss": 3.529790394948223, + "tokens_seen": 1435500544 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002854062186559679, + "loss": 2.7331, + "theoretical_loss": 3.5297760669449163, + "tokens_seen": 1435566080 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002853961885656971, + "loss": 2.7046, + "theoretical_loss": 3.52976173977883, + "tokens_seen": 1435631616 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028538615847542624, + "loss": 2.8609, + "theoretical_loss": 3.5297474134498765, + "tokens_seen": 1435697152 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002853761283851555, + "loss": 2.8432, + "theoretical_loss": 3.529733087957969, + "tokens_seen": 1435762688 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028536609829488466, + "loss": 2.6629, + "theoretical_loss": 3.5297187633030203, + "tokens_seen": 1435828224 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028535606820461384, + "loss": 2.7653, + "theoretical_loss": 3.5297044394849433, + "tokens_seen": 1435893760 + }, + { + "epoch": 0.44, + "learning_rate": 0.000285346038114343, + "loss": 2.6821, + "theoretical_loss": 3.5296901165036507, + "tokens_seen": 1435959296 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002853360080240722, + "loss": 2.493, + "theoretical_loss": 3.529675794359056, + "tokens_seen": 1436024832 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002853259779338014, + "loss": 2.7298, + "theoretical_loss": 3.529661473051072, + "tokens_seen": 1436090368 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002853159478435306, + "loss": 2.5171, + "theoretical_loss": 3.529647152579611, + "tokens_seen": 1436155904 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028530591775325975, + "loss": 2.7021, + "theoretical_loss": 3.5296328329445865, + "tokens_seen": 1436221440 + }, + { + "epoch": 0.44, + "learning_rate": 0.000285295887662989, + "loss": 2.5292, + "theoretical_loss": 3.5296185141459118, + "tokens_seen": 1436286976 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002852858575727181, + "loss": 2.6627, + "theoretical_loss": 3.529604196183499, + "tokens_seen": 1436352512 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028527582748244735, + "loss": 2.6408, + "theoretical_loss": 3.529589879057262, + "tokens_seen": 1436418048 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002852657973921766, + "loss": 2.7842, + "theoretical_loss": 3.5295755627671133, + "tokens_seen": 1436483584 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002852557673019057, + "loss": 2.4794, + "theoretical_loss": 3.5295612473129663, + "tokens_seen": 1436549120 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028524573721163494, + "loss": 2.7194, + "theoretical_loss": 3.5295469326947337, + "tokens_seen": 1436614656 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028523570712136407, + "loss": 2.5288, + "theoretical_loss": 3.529532618912328, + "tokens_seen": 1436680192 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002852256770310933, + "loss": 2.4727, + "theoretical_loss": 3.529518305965664, + "tokens_seen": 1436745728 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002852156469408225, + "loss": 2.7643, + "theoretical_loss": 3.5295039938546537, + "tokens_seen": 1436811264 + }, + { + "epoch": 0.44, + "objective/train/docs_used": 814744, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.809866428375244, + "objective/train/theoretical_loss": 3.52948968257921, + "objective/train/tokens_used": 1457336800, + "theoretical_loss": 3.52948968257921, + "tokens_seen": 1436876800 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028520561685055167, + "loss": 2.7177, + "theoretical_loss": 3.52948968257921, + "tokens_seen": 1436876800 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028519558676028085, + "loss": 2.5842, + "theoretical_loss": 3.5294753721392453, + "tokens_seen": 1436942336 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002851855566700101, + "loss": 2.8052, + "theoretical_loss": 3.5294610625346747, + "tokens_seen": 1437007872 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002851755265797392, + "loss": 2.5863, + "theoretical_loss": 3.52944675376541, + "tokens_seen": 1437073408 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028516549648946845, + "loss": 2.6355, + "theoretical_loss": 3.5294324458313646, + "tokens_seen": 1437138944 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028515546639919757, + "loss": 2.9434, + "theoretical_loss": 3.529418138732452, + "tokens_seen": 1437204480 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002851454363089268, + "loss": 2.8048, + "theoretical_loss": 3.529403832468585, + "tokens_seen": 1437270016 + }, + { + "epoch": 0.44, + "learning_rate": 0.000285135406218656, + "loss": 2.7889, + "theoretical_loss": 3.529389527039677, + "tokens_seen": 1437335552 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028512537612838517, + "loss": 2.6638, + "theoretical_loss": 3.529375222445641, + "tokens_seen": 1437401088 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028511534603811435, + "loss": 2.6986, + "theoretical_loss": 3.5293609186863906, + "tokens_seen": 1437466624 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028510531594784353, + "loss": 2.5535, + "theoretical_loss": 3.5293466157618383, + "tokens_seen": 1437532160 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002850952858575727, + "loss": 2.8235, + "theoretical_loss": 3.5293323136718975, + "tokens_seen": 1437597696 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028508525576730195, + "loss": 2.849, + "theoretical_loss": 3.529318012416482, + "tokens_seen": 1437663232 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002850752256770311, + "loss": 2.6257, + "theoretical_loss": 3.529303711995505, + "tokens_seen": 1437728768 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002850651955867603, + "loss": 2.5204, + "theoretical_loss": 3.52928941240888, + "tokens_seen": 1437794304 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002850551654964895, + "loss": 2.7333, + "theoretical_loss": 3.529275113656519, + "tokens_seen": 1437859840 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002850451354062187, + "loss": 2.816, + "theoretical_loss": 3.5292608157383363, + "tokens_seen": 1437925376 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028503510531594785, + "loss": 2.6051, + "theoretical_loss": 3.5292465186542454, + "tokens_seen": 1437990912 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028502507522567704, + "loss": 2.9118, + "theoretical_loss": 3.529232222404159, + "tokens_seen": 1438056448 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002850150451354062, + "loss": 2.7888, + "theoretical_loss": 3.5292179269879913, + "tokens_seen": 1438121984 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028500501504513545, + "loss": 2.5529, + "theoretical_loss": 3.529203632405655, + "tokens_seen": 1438187520 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002849949849548646, + "loss": 2.6793, + "theoretical_loss": 3.529189338657063, + "tokens_seen": 1438253056 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002849849548645938, + "loss": 2.5794, + "theoretical_loss": 3.5291750457421296, + "tokens_seen": 1438318592 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028497492477432294, + "loss": 2.6467, + "theoretical_loss": 3.5291607536607685, + "tokens_seen": 1438384128 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002849648946840522, + "loss": 2.7756, + "theoretical_loss": 3.529146462412892, + "tokens_seen": 1438449664 + }, + { + "epoch": 0.44, + "objective/train/docs_used": 815309, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.530625104904175, + "objective/train/theoretical_loss": 3.5291321719984134, + "objective/train/tokens_used": 1458975200, + "theoretical_loss": 3.5291321719984134, + "tokens_seen": 1438515200 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028495486459378136, + "loss": 2.5775, + "theoretical_loss": 3.5291321719984134, + "tokens_seen": 1438515200 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028494483450351054, + "loss": 2.7396, + "theoretical_loss": 3.5291178824172476, + "tokens_seen": 1438580736 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002849348044132397, + "loss": 2.4951, + "theoretical_loss": 3.5291035936693067, + "tokens_seen": 1438646272 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002849247743229689, + "loss": 2.8526, + "theoretical_loss": 3.529089305754505, + "tokens_seen": 1438711808 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002849147442326981, + "loss": 2.464, + "theoretical_loss": 3.5290750186727555, + "tokens_seen": 1438777344 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002849047141424273, + "loss": 2.6597, + "theoretical_loss": 3.529060732423972, + "tokens_seen": 1438842880 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028489468405215644, + "loss": 2.4165, + "theoretical_loss": 3.5290464470080676, + "tokens_seen": 1438908416 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002848846539618857, + "loss": 2.7833, + "theoretical_loss": 3.529032162424956, + "tokens_seen": 1438973952 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028487462387161486, + "loss": 2.6966, + "theoretical_loss": 3.529017878674551, + "tokens_seen": 1439039488 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028486459378134404, + "loss": 2.6768, + "theoretical_loss": 3.5290035957567656, + "tokens_seen": 1439105024 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002848545636910732, + "loss": 2.6737, + "theoretical_loss": 3.528989313671514, + "tokens_seen": 1439170560 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002848445336008024, + "loss": 2.7154, + "theoretical_loss": 3.5289750324187086, + "tokens_seen": 1439236096 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002848345035105316, + "loss": 2.5969, + "theoretical_loss": 3.5289607519982646, + "tokens_seen": 1439301632 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002848244734202608, + "loss": 2.7224, + "theoretical_loss": 3.5289464724100945, + "tokens_seen": 1439367168 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028481444332998995, + "loss": 2.6522, + "theoretical_loss": 3.5289321936541125, + "tokens_seen": 1439432704 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002848044132397192, + "loss": 2.8746, + "theoretical_loss": 3.5289179157302315, + "tokens_seen": 1439498240 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002847943831494483, + "loss": 2.6718, + "theoretical_loss": 3.528903638638366, + "tokens_seen": 1439563776 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028478435305917755, + "loss": 2.7382, + "theoretical_loss": 3.5288893623784285, + "tokens_seen": 1439629312 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002847743229689067, + "loss": 2.6455, + "theoretical_loss": 3.5288750869503334, + "tokens_seen": 1439694848 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002847642928786359, + "loss": 2.5324, + "theoretical_loss": 3.5288608123539946, + "tokens_seen": 1439760384 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002847542627883651, + "loss": 2.733, + "theoretical_loss": 3.5288465385893257, + "tokens_seen": 1439825920 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028474423269809427, + "loss": 2.6636, + "theoretical_loss": 3.5288322656562396, + "tokens_seen": 1439891456 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028473420260782345, + "loss": 2.7352, + "theoretical_loss": 3.528817993554651, + "tokens_seen": 1439956992 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002847241725175527, + "loss": 2.4791, + "theoretical_loss": 3.528803722284473, + "tokens_seen": 1440022528 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002847141424272818, + "loss": 2.5866, + "theoretical_loss": 3.5287894518456193, + "tokens_seen": 1440088064 + }, + { + "epoch": 0.44, + "objective/train/docs_used": 816515, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.823932409286499, + "objective/train/theoretical_loss": 3.5287751822380047, + "objective/train/tokens_used": 1460613600, + "theoretical_loss": 3.5287751822380047, + "tokens_seen": 1440153600 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028470411233701105, + "loss": 2.5665, + "theoretical_loss": 3.5287751822380047, + "tokens_seen": 1440153600 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028469408224674023, + "loss": 2.8361, + "theoretical_loss": 3.528760913461541, + "tokens_seen": 1440219136 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002846840521564694, + "loss": 2.8039, + "theoretical_loss": 3.528746645516144, + "tokens_seen": 1440284672 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002846740220661986, + "loss": 2.7947, + "theoretical_loss": 3.5287323784017266, + "tokens_seen": 1440350208 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002846639919759278, + "loss": 2.4342, + "theoretical_loss": 3.528718112118202, + "tokens_seen": 1440415744 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028465396188565695, + "loss": 2.5138, + "theoretical_loss": 3.5287038466654845, + "tokens_seen": 1440481280 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002846439317953862, + "loss": 2.6579, + "theoretical_loss": 3.5286895820434885, + "tokens_seen": 1440546816 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002846339017051153, + "loss": 2.6093, + "theoretical_loss": 3.5286753182521275, + "tokens_seen": 1440612352 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028462387161484455, + "loss": 2.7997, + "theoretical_loss": 3.5286610552913147, + "tokens_seen": 1440677888 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002846138415245737, + "loss": 2.5719, + "theoretical_loss": 3.5286467931609646, + "tokens_seen": 1440743424 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002846038114343029, + "loss": 2.7789, + "theoretical_loss": 3.528632531860991, + "tokens_seen": 1440808960 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002845937813440321, + "loss": 2.8064, + "theoretical_loss": 3.5286182713913075, + "tokens_seen": 1440874496 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002845837512537613, + "loss": 2.9147, + "theoretical_loss": 3.5286040117518285, + "tokens_seen": 1440940032 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028457372116349046, + "loss": 2.822, + "theoretical_loss": 3.5285897529424677, + "tokens_seen": 1441005568 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002845636910732197, + "loss": 2.723, + "theoretical_loss": 3.5285754949631385, + "tokens_seen": 1441071104 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002845536609829488, + "loss": 2.7802, + "theoretical_loss": 3.5285612378137556, + "tokens_seen": 1441136640 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028454363089267805, + "loss": 2.4715, + "theoretical_loss": 3.528546981494233, + "tokens_seen": 1441202176 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028453360080240724, + "loss": 2.8832, + "theoretical_loss": 3.5285327260044834, + "tokens_seen": 1441267712 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002845235707121364, + "loss": 2.537, + "theoretical_loss": 3.528518471344422, + "tokens_seen": 1441333248 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028451354062186565, + "loss": 2.9509, + "theoretical_loss": 3.528504217513963, + "tokens_seen": 1441398784 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002845035105315948, + "loss": 2.605, + "theoretical_loss": 3.528489964513019, + "tokens_seen": 1441464320 + }, + { + "epoch": 0.44, + "learning_rate": 0.000284493480441324, + "loss": 2.5348, + "theoretical_loss": 3.528475712341506, + "tokens_seen": 1441529856 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028448345035105314, + "loss": 2.6461, + "theoretical_loss": 3.528461460999336, + "tokens_seen": 1441595392 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002844734202607824, + "loss": 2.6134, + "theoretical_loss": 3.5284472104864246, + "tokens_seen": 1441660928 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028446339017051156, + "loss": 2.534, + "theoretical_loss": 3.528432960802685, + "tokens_seen": 1441726464 + }, + { + "epoch": 0.44, + "objective/train/docs_used": 817059, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6012871265411377, + "objective/train/theoretical_loss": 3.5284187119480315, + "objective/train/tokens_used": 1462252000, + "theoretical_loss": 3.5284187119480315, + "tokens_seen": 1441792000 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028445336008024074, + "loss": 2.6351, + "theoretical_loss": 3.5284187119480315, + "tokens_seen": 1441792000 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002844433299899699, + "loss": 2.5977, + "theoretical_loss": 3.528404463922378, + "tokens_seen": 1441857536 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002844332998996991, + "loss": 2.7772, + "theoretical_loss": 3.5283902167256382, + "tokens_seen": 1441923072 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002844232698094283, + "loss": 2.7188, + "theoretical_loss": 3.528375970357728, + "tokens_seen": 1441988608 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002844132397191575, + "loss": 2.5445, + "theoretical_loss": 3.5283617248185593, + "tokens_seen": 1442054144 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028440320962888664, + "loss": 2.54, + "theoretical_loss": 3.5283474801080477, + "tokens_seen": 1442119680 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002843931795386159, + "loss": 2.9519, + "theoretical_loss": 3.528333236226106, + "tokens_seen": 1442185216 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028438314944834506, + "loss": 2.9077, + "theoretical_loss": 3.5283189931726504, + "tokens_seen": 1442250752 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028437311935807424, + "loss": 2.5333, + "theoretical_loss": 3.5283047509475933, + "tokens_seen": 1442316288 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002843630892678034, + "loss": 2.7241, + "theoretical_loss": 3.528290509550849, + "tokens_seen": 1442381824 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002843530591775326, + "loss": 2.9595, + "theoretical_loss": 3.528276268982333, + "tokens_seen": 1442447360 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002843430290872618, + "loss": 2.6703, + "theoretical_loss": 3.528262029241958, + "tokens_seen": 1442512896 + }, + { + "epoch": 0.44, + "learning_rate": 0.000284332998996991, + "loss": 2.8144, + "theoretical_loss": 3.5282477903296394, + "tokens_seen": 1442578432 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028432296890672015, + "loss": 2.5717, + "theoretical_loss": 3.5282335522452906, + "tokens_seen": 1442643968 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002843129388164494, + "loss": 2.6533, + "theoretical_loss": 3.528219314988826, + "tokens_seen": 1442709504 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002843029087261785, + "loss": 2.6734, + "theoretical_loss": 3.52820507856016, + "tokens_seen": 1442775040 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028429287863590775, + "loss": 2.5422, + "theoretical_loss": 3.528190842959207, + "tokens_seen": 1442840576 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002842828485456369, + "loss": 2.7416, + "theoretical_loss": 3.5281766081858814, + "tokens_seen": 1442906112 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002842728184553661, + "loss": 2.8339, + "theoretical_loss": 3.5281623742400967, + "tokens_seen": 1442971648 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002842627883650953, + "loss": 2.7706, + "theoretical_loss": 3.5281481411217683, + "tokens_seen": 1443037184 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028425275827482447, + "loss": 2.932, + "theoretical_loss": 3.52813390883081, + "tokens_seen": 1443102720 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028424272818455365, + "loss": 2.5467, + "theoretical_loss": 3.528119677367136, + "tokens_seen": 1443168256 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002842326980942829, + "loss": 2.5544, + "theoretical_loss": 3.5281054467306605, + "tokens_seen": 1443233792 + }, + { + "epoch": 0.44, + "learning_rate": 0.000284222668004012, + "loss": 2.5863, + "theoretical_loss": 3.5280912169212986, + "tokens_seen": 1443299328 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028421263791374125, + "loss": 2.7362, + "theoretical_loss": 3.528076987938964, + "tokens_seen": 1443364864 + }, + { + "epoch": 0.44, + "objective/train/docs_used": 818508, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6614174842834473, + "objective/train/theoretical_loss": 3.5280627597835705, + "objective/train/tokens_used": 1463890400, + "theoretical_loss": 3.5280627597835705, + "tokens_seen": 1443430400 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028420260782347043, + "loss": 2.6241, + "theoretical_loss": 3.5280627597835705, + "tokens_seen": 1443430400 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002841925777331996, + "loss": 2.6064, + "theoretical_loss": 3.528048532455034, + "tokens_seen": 1443495936 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002841825476429288, + "loss": 2.5895, + "theoretical_loss": 3.5280343059532684, + "tokens_seen": 1443561472 + }, + { + "epoch": 0.44, + "learning_rate": 0.000284172517552658, + "loss": 2.7986, + "theoretical_loss": 3.528020080278188, + "tokens_seen": 1443627008 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028416248746238715, + "loss": 2.8935, + "theoretical_loss": 3.528005855429707, + "tokens_seen": 1443692544 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002841524573721164, + "loss": 2.7718, + "theoretical_loss": 3.5279916314077395, + "tokens_seen": 1443758080 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002841424272818455, + "loss": 2.5576, + "theoretical_loss": 3.527977408212201, + "tokens_seen": 1443823616 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028413239719157475, + "loss": 2.8829, + "theoretical_loss": 3.527963185843005, + "tokens_seen": 1443889152 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002841223671013039, + "loss": 2.7617, + "theoretical_loss": 3.5279489643000668, + "tokens_seen": 1443954688 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002841123370110331, + "loss": 2.7803, + "theoretical_loss": 3.527934743583301, + "tokens_seen": 1444020224 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002841023069207623, + "loss": 2.6027, + "theoretical_loss": 3.5279205236926208, + "tokens_seen": 1444085760 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002840922768304915, + "loss": 2.6894, + "theoretical_loss": 3.5279063046279417, + "tokens_seen": 1444151296 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028408224674022066, + "loss": 2.6994, + "theoretical_loss": 3.5278920863891785, + "tokens_seen": 1444216832 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002840722166499499, + "loss": 2.7663, + "theoretical_loss": 3.527877868976245, + "tokens_seen": 1444282368 + }, + { + "epoch": 0.44, + "learning_rate": 0.000284062186559679, + "loss": 2.7233, + "theoretical_loss": 3.5278636523890565, + "tokens_seen": 1444347904 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028405215646940825, + "loss": 2.8227, + "theoretical_loss": 3.527849436627527, + "tokens_seen": 1444413440 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002840421263791374, + "loss": 2.6867, + "theoretical_loss": 3.5278352216915714, + "tokens_seen": 1444478976 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002840320962888666, + "loss": 2.6912, + "theoretical_loss": 3.527821007581104, + "tokens_seen": 1444544512 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002840220661985958, + "loss": 2.8248, + "theoretical_loss": 3.5278067942960396, + "tokens_seen": 1444610048 + }, + { + "epoch": 0.44, + "learning_rate": 0.000284012036108325, + "loss": 2.7708, + "theoretical_loss": 3.5277925818362927, + "tokens_seen": 1444675584 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028400200601805416, + "loss": 2.6526, + "theoretical_loss": 3.527778370201778, + "tokens_seen": 1444741120 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028399197592778334, + "loss": 2.9498, + "theoretical_loss": 3.527764159392411, + "tokens_seen": 1444806656 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002839819458375125, + "loss": 2.5629, + "theoretical_loss": 3.527749949408105, + "tokens_seen": 1444872192 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028397191574724176, + "loss": 2.6956, + "theoretical_loss": 3.5277357402487755, + "tokens_seen": 1444937728 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002839618856569709, + "loss": 2.6336, + "theoretical_loss": 3.527721531914337, + "tokens_seen": 1445003264 + }, + { + "epoch": 0.44, + "objective/train/docs_used": 819330, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.721529722213745, + "objective/train/theoretical_loss": 3.527707324404704, + "objective/train/tokens_used": 1465528800, + "theoretical_loss": 3.527707324404704, + "tokens_seen": 1445068800 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002839518555667001, + "loss": 2.8314, + "theoretical_loss": 3.527707324404704, + "tokens_seen": 1445068800 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028394182547642925, + "loss": 2.7685, + "theoretical_loss": 3.5276931177197914, + "tokens_seen": 1445134336 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002839317953861585, + "loss": 2.6947, + "theoretical_loss": 3.527678911859514, + "tokens_seen": 1445199872 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028392176529588766, + "loss": 2.9891, + "theoretical_loss": 3.5276647068237867, + "tokens_seen": 1445265408 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028391173520561684, + "loss": 2.9808, + "theoretical_loss": 3.5276505026125236, + "tokens_seen": 1445330944 + }, + { + "epoch": 0.44, + "learning_rate": 0.000283901705115346, + "loss": 2.6941, + "theoretical_loss": 3.52763629922564, + "tokens_seen": 1445396480 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028389167502507526, + "loss": 2.6094, + "theoretical_loss": 3.5276220966630505, + "tokens_seen": 1445462016 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002838816449348044, + "loss": 2.7066, + "theoretical_loss": 3.52760789492467, + "tokens_seen": 1445527552 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002838716148445336, + "loss": 2.7622, + "theoretical_loss": 3.527593694010413, + "tokens_seen": 1445593088 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028386158475426275, + "loss": 2.8053, + "theoretical_loss": 3.5275794939201948, + "tokens_seen": 1445658624 + }, + { + "epoch": 0.44, + "learning_rate": 0.000283851554663992, + "loss": 2.7633, + "theoretical_loss": 3.52756529465393, + "tokens_seen": 1445724160 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028384152457372117, + "loss": 2.672, + "theoretical_loss": 3.5275510962115337, + "tokens_seen": 1445789696 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028383149448345035, + "loss": 2.7019, + "theoretical_loss": 3.5275368985929205, + "tokens_seen": 1445855232 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028382146439317953, + "loss": 2.8034, + "theoretical_loss": 3.527522701798005, + "tokens_seen": 1445920768 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002838114343029087, + "loss": 2.5845, + "theoretical_loss": 3.5275085058267024, + "tokens_seen": 1445986304 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002838014042126379, + "loss": 2.7324, + "theoretical_loss": 3.5274943106789274, + "tokens_seen": 1446051840 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002837913741223671, + "loss": 2.3596, + "theoretical_loss": 3.527480116354595, + "tokens_seen": 1446117376 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002837813440320963, + "loss": 2.8119, + "theoretical_loss": 3.5274659228536205, + "tokens_seen": 1446182912 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002837713139418255, + "loss": 2.6295, + "theoretical_loss": 3.5274517301759185, + "tokens_seen": 1446248448 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028376128385155467, + "loss": 2.6169, + "theoretical_loss": 3.527437538321404, + "tokens_seen": 1446313984 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028375125376128385, + "loss": 2.552, + "theoretical_loss": 3.5274233472899916, + "tokens_seen": 1446379520 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002837412236710131, + "loss": 2.885, + "theoretical_loss": 3.527409157081597, + "tokens_seen": 1446445056 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002837311935807422, + "loss": 2.8021, + "theoretical_loss": 3.527394967696134, + "tokens_seen": 1446510592 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028372116349047145, + "loss": 2.7222, + "theoretical_loss": 3.5273807791335194, + "tokens_seen": 1446576128 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028371113340020063, + "loss": 2.8239, + "theoretical_loss": 3.5273665913936663, + "tokens_seen": 1446641664 + }, + { + "epoch": 0.44, + "objective/train/docs_used": 820636, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.4957058429718018, + "objective/train/theoretical_loss": 3.527352404476491, + "objective/train/tokens_used": 1467167200, + "theoretical_loss": 3.527352404476491, + "tokens_seen": 1446707200 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002837011033099298, + "loss": 2.6335, + "theoretical_loss": 3.527352404476491, + "tokens_seen": 1446707200 + }, + { + "epoch": 0.44, + "learning_rate": 0.000283691073219659, + "loss": 2.6218, + "theoretical_loss": 3.527338218381908, + "tokens_seen": 1446772736 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002836810431293882, + "loss": 2.6958, + "theoretical_loss": 3.5273240331098323, + "tokens_seen": 1446838272 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028367101303911735, + "loss": 2.8662, + "theoretical_loss": 3.5273098486601793, + "tokens_seen": 1446903808 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002836609829488466, + "loss": 2.8132, + "theoretical_loss": 3.527295665032864, + "tokens_seen": 1446969344 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002836509528585757, + "loss": 2.4651, + "theoretical_loss": 3.527281482227801, + "tokens_seen": 1447034880 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028364092276830495, + "loss": 2.6914, + "theoretical_loss": 3.527267300244906, + "tokens_seen": 1447100416 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002836308926780341, + "loss": 2.5294, + "theoretical_loss": 3.5272531190840937, + "tokens_seen": 1447165952 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002836208625877633, + "loss": 2.656, + "theoretical_loss": 3.52723893874528, + "tokens_seen": 1447231488 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002836108324974925, + "loss": 2.4776, + "theoretical_loss": 3.5272247592283787, + "tokens_seen": 1447297024 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002836008024072217, + "loss": 2.5675, + "theoretical_loss": 3.527210580533306, + "tokens_seen": 1447362560 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028359077231695086, + "loss": 2.7692, + "theoretical_loss": 3.527196402659977, + "tokens_seen": 1447428096 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002835807422266801, + "loss": 2.7417, + "theoretical_loss": 3.527182225608306, + "tokens_seen": 1447493632 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002835707121364092, + "loss": 2.6763, + "theoretical_loss": 3.527168049378209, + "tokens_seen": 1447559168 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028356068204613846, + "loss": 3.0316, + "theoretical_loss": 3.5271538739696013, + "tokens_seen": 1447624704 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002835506519558676, + "loss": 2.7627, + "theoretical_loss": 3.5271396993823974, + "tokens_seen": 1447690240 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002835406218655968, + "loss": 2.8959, + "theoretical_loss": 3.5271255256165133, + "tokens_seen": 1447755776 + }, + { + "epoch": 0.44, + "learning_rate": 0.000283530591775326, + "loss": 2.6064, + "theoretical_loss": 3.527111352671864, + "tokens_seen": 1447821312 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002835205616850552, + "loss": 2.6553, + "theoretical_loss": 3.527097180548364, + "tokens_seen": 1447886848 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028351053159478436, + "loss": 2.6128, + "theoretical_loss": 3.5270830092459295, + "tokens_seen": 1447952384 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028350050150451354, + "loss": 2.485, + "theoretical_loss": 3.527068838764475, + "tokens_seen": 1448017920 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002834904714142427, + "loss": 2.7765, + "theoretical_loss": 3.527054669103917, + "tokens_seen": 1448083456 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028348044132397196, + "loss": 2.6285, + "theoretical_loss": 3.527040500264169, + "tokens_seen": 1448148992 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002834704112337011, + "loss": 2.6512, + "theoretical_loss": 3.527026332245148, + "tokens_seen": 1448214528 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002834603811434303, + "loss": 2.7611, + "theoretical_loss": 3.527012165046769, + "tokens_seen": 1448280064 + }, + { + "epoch": 0.44, + "objective/train/docs_used": 821297, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.9668633937835693, + "objective/train/theoretical_loss": 3.5269979986689464, + "objective/train/tokens_used": 1468805600, + "theoretical_loss": 3.5269979986689464, + "tokens_seen": 1448345600 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028345035105315945, + "loss": 2.6533, + "theoretical_loss": 3.5269979986689464, + "tokens_seen": 1448345600 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002834403209628887, + "loss": 2.5279, + "theoretical_loss": 3.526983833111596, + "tokens_seen": 1448411136 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028343029087261786, + "loss": 2.715, + "theoretical_loss": 3.526969668374634, + "tokens_seen": 1448476672 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028342026078234705, + "loss": 2.8302, + "theoretical_loss": 3.5269555044579746, + "tokens_seen": 1448542208 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002834102306920762, + "loss": 2.6919, + "theoretical_loss": 3.5269413413615336, + "tokens_seen": 1448607744 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028340020060180546, + "loss": 2.8131, + "theoretical_loss": 3.5269271790852263, + "tokens_seen": 1448673280 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002833901705115346, + "loss": 2.6199, + "theoretical_loss": 3.526913017628969, + "tokens_seen": 1448738816 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002833801404212638, + "loss": 2.654, + "theoretical_loss": 3.5268988569926756, + "tokens_seen": 1448804352 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028337011033099295, + "loss": 2.539, + "theoretical_loss": 3.5268846971762624, + "tokens_seen": 1448869888 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002833600802407222, + "loss": 2.379, + "theoretical_loss": 3.5268705381796455, + "tokens_seen": 1448935424 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028335005015045137, + "loss": 2.5523, + "theoretical_loss": 3.526856380002739, + "tokens_seen": 1449000960 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028334002006018055, + "loss": 2.5992, + "theoretical_loss": 3.5268422226454588, + "tokens_seen": 1449066496 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028332998996990973, + "loss": 2.8011, + "theoretical_loss": 3.526828066107721, + "tokens_seen": 1449132032 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002833199598796389, + "loss": 2.6276, + "theoretical_loss": 3.5268139103894405, + "tokens_seen": 1449197568 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002833099297893681, + "loss": 2.6553, + "theoretical_loss": 3.526799755490533, + "tokens_seen": 1449263104 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002832998996990973, + "loss": 2.6693, + "theoretical_loss": 3.5267856014109142, + "tokens_seen": 1449328640 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028328986960882645, + "loss": 2.7742, + "theoretical_loss": 3.526771448150499, + "tokens_seen": 1449394176 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002832798395185557, + "loss": 2.7112, + "theoretical_loss": 3.5267572957092037, + "tokens_seen": 1449459712 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002832698094282848, + "loss": 2.7484, + "theoretical_loss": 3.526743144086944, + "tokens_seen": 1449525248 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028325977933801405, + "loss": 2.6917, + "theoretical_loss": 3.5267289932836343, + "tokens_seen": 1449590784 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028324974924774323, + "loss": 2.882, + "theoretical_loss": 3.5267148432991915, + "tokens_seen": 1449656320 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002832397191574724, + "loss": 2.8373, + "theoretical_loss": 3.52670069413353, + "tokens_seen": 1449721856 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002832296890672016, + "loss": 2.8674, + "theoretical_loss": 3.526686545786567, + "tokens_seen": 1449787392 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028321965897693083, + "loss": 2.6696, + "theoretical_loss": 3.526672398258216, + "tokens_seen": 1449852928 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028320962888665996, + "loss": 2.7011, + "theoretical_loss": 3.5266582515483944, + "tokens_seen": 1449918464 + }, + { + "epoch": 0.44, + "objective/train/docs_used": 822517, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6085617542266846, + "objective/train/theoretical_loss": 3.5266441056570175, + "objective/train/tokens_used": 1470444000, + "theoretical_loss": 3.5266441056570175, + "tokens_seen": 1449984000 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002831995987963892, + "loss": 2.6089, + "theoretical_loss": 3.5266441056570175, + "tokens_seen": 1449984000 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002831895687061183, + "loss": 2.696, + "theoretical_loss": 3.526629960584, + "tokens_seen": 1450049536 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028317953861584755, + "loss": 2.6792, + "theoretical_loss": 3.526615816329259, + "tokens_seen": 1450115072 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028316950852557674, + "loss": 2.5487, + "theoretical_loss": 3.526601672892709, + "tokens_seen": 1450180608 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002831594784353059, + "loss": 2.6217, + "theoretical_loss": 3.526587530274266, + "tokens_seen": 1450246144 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002831494483450351, + "loss": 2.7544, + "theoretical_loss": 3.5265733884738464, + "tokens_seen": 1450311680 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002831394182547643, + "loss": 2.6912, + "theoretical_loss": 3.526559247491365, + "tokens_seen": 1450377216 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028312938816449346, + "loss": 2.6236, + "theoretical_loss": 3.526545107326738, + "tokens_seen": 1450442752 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002831193580742227, + "loss": 2.7444, + "theoretical_loss": 3.5265309679798813, + "tokens_seen": 1450508288 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002831093279839518, + "loss": 2.8458, + "theoretical_loss": 3.52651682945071, + "tokens_seen": 1450573824 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028309929789368106, + "loss": 2.7377, + "theoretical_loss": 3.526502691739141, + "tokens_seen": 1450639360 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002830892678034102, + "loss": 2.6282, + "theoretical_loss": 3.5264885548450886, + "tokens_seen": 1450704896 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002830792377131394, + "loss": 2.8682, + "theoretical_loss": 3.52647441876847, + "tokens_seen": 1450770432 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002830692076228686, + "loss": 2.7349, + "theoretical_loss": 3.5264602835092003, + "tokens_seen": 1450835968 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002830591775325978, + "loss": 2.6422, + "theoretical_loss": 3.526446149067195, + "tokens_seen": 1450901504 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028304914744232696, + "loss": 2.6574, + "theoretical_loss": 3.5264320154423707, + "tokens_seen": 1450967040 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002830391173520562, + "loss": 2.7041, + "theoretical_loss": 3.526417882634643, + "tokens_seen": 1451032576 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002830290872617854, + "loss": 2.6528, + "theoretical_loss": 3.5264037506439276, + "tokens_seen": 1451098112 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028301905717151456, + "loss": 2.8009, + "theoretical_loss": 3.52638961947014, + "tokens_seen": 1451163648 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028300902708124374, + "loss": 2.5589, + "theoretical_loss": 3.5263754891131973, + "tokens_seen": 1451229184 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002829989969909729, + "loss": 2.9879, + "theoretical_loss": 3.5263613595730146, + "tokens_seen": 1451294720 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028298896690070216, + "loss": 2.6231, + "theoretical_loss": 3.526347230849507, + "tokens_seen": 1451360256 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002829789368104313, + "loss": 2.7041, + "theoretical_loss": 3.526333102942592, + "tokens_seen": 1451425792 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002829689067201605, + "loss": 2.7777, + "theoretical_loss": 3.5263189758521847, + "tokens_seen": 1451491328 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028295887662988965, + "loss": 2.5811, + "theoretical_loss": 3.5263048495782012, + "tokens_seen": 1451556864 + }, + { + "epoch": 0.44, + "objective/train/docs_used": 823297, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.061185359954834, + "objective/train/theoretical_loss": 3.5262907241205568, + "objective/train/tokens_used": 1472082400, + "theoretical_loss": 3.5262907241205568, + "tokens_seen": 1451622400 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002829488465396189, + "loss": 2.6823, + "theoretical_loss": 3.5262907241205568, + "tokens_seen": 1451622400 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028293881644934806, + "loss": 2.6582, + "theoretical_loss": 3.5262765994791687, + "tokens_seen": 1451687936 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028292878635907725, + "loss": 2.6943, + "theoretical_loss": 3.5262624756539522, + "tokens_seen": 1451753472 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002829187562688064, + "loss": 2.9404, + "theoretical_loss": 3.5262483526448234, + "tokens_seen": 1451819008 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028290872617853566, + "loss": 2.5835, + "theoretical_loss": 3.526234230451698, + "tokens_seen": 1451884544 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002828986960882648, + "loss": 2.7227, + "theoretical_loss": 3.5262201090744925, + "tokens_seen": 1451950080 + }, + { + "epoch": 0.44, + "learning_rate": 0.000282888665997994, + "loss": 2.5475, + "theoretical_loss": 3.526205988513123, + "tokens_seen": 1452015616 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028287863590772315, + "loss": 2.6567, + "theoretical_loss": 3.5261918687675045, + "tokens_seen": 1452081152 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002828686058174524, + "loss": 2.9099, + "theoretical_loss": 3.5261777498375544, + "tokens_seen": 1452146688 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028285857572718157, + "loss": 2.6315, + "theoretical_loss": 3.526163631723188, + "tokens_seen": 1452212224 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028284854563691075, + "loss": 2.7956, + "theoretical_loss": 3.526149514424322, + "tokens_seen": 1452277760 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028283851554663993, + "loss": 2.7972, + "theoretical_loss": 3.5261353979408714, + "tokens_seen": 1452343296 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002828284854563691, + "loss": 2.4808, + "theoretical_loss": 3.5261212822727535, + "tokens_seen": 1452408832 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002828184553660983, + "loss": 2.6515, + "theoretical_loss": 3.526107167419884, + "tokens_seen": 1452474368 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002828084252758275, + "loss": 2.5691, + "theoretical_loss": 3.526093053382179, + "tokens_seen": 1452539904 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028279839518555665, + "loss": 2.7988, + "theoretical_loss": 3.5260789401595543, + "tokens_seen": 1452605440 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002827883650952859, + "loss": 2.5993, + "theoretical_loss": 3.5260648277519264, + "tokens_seen": 1452670976 + }, + { + "epoch": 0.44, + "learning_rate": 0.000282778335005015, + "loss": 2.8906, + "theoretical_loss": 3.5260507161592116, + "tokens_seen": 1452736512 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028276830491474425, + "loss": 2.6707, + "theoretical_loss": 3.526036605381326, + "tokens_seen": 1452802048 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028275827482447343, + "loss": 2.4031, + "theoretical_loss": 3.5260224954181854, + "tokens_seen": 1452867584 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002827482447342026, + "loss": 2.6554, + "theoretical_loss": 3.5260083862697065, + "tokens_seen": 1452933120 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002827382146439318, + "loss": 2.423, + "theoretical_loss": 3.5259942779358053, + "tokens_seen": 1452998656 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028272818455366103, + "loss": 2.8823, + "theoretical_loss": 3.525980170416398, + "tokens_seen": 1453064192 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028271815446339016, + "loss": 2.7797, + "theoretical_loss": 3.525966063711401, + "tokens_seen": 1453129728 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002827081243731194, + "loss": 2.7432, + "theoretical_loss": 3.525951957820731, + "tokens_seen": 1453195264 + }, + { + "epoch": 0.44, + "objective/train/docs_used": 824661, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6765828132629395, + "objective/train/theoretical_loss": 3.5259378527443035, + "objective/train/tokens_used": 1473720800, + "theoretical_loss": 3.5259378527443035, + "tokens_seen": 1453260800 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002826980942828485, + "loss": 2.8734, + "theoretical_loss": 3.5259378527443035, + "tokens_seen": 1453260800 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028268806419257775, + "loss": 2.7489, + "theoretical_loss": 3.5259237484820347, + "tokens_seen": 1453326336 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028267803410230694, + "loss": 2.7029, + "theoretical_loss": 3.525909645033841, + "tokens_seen": 1453391872 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002826680040120361, + "loss": 2.5712, + "theoretical_loss": 3.5258955423996396, + "tokens_seen": 1453457408 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002826579739217653, + "loss": 2.7158, + "theoretical_loss": 3.525881440579346, + "tokens_seen": 1453522944 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002826479438314945, + "loss": 2.7784, + "theoretical_loss": 3.5258673395728763, + "tokens_seen": 1453588480 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028263791374122366, + "loss": 2.7213, + "theoretical_loss": 3.5258532393801474, + "tokens_seen": 1453654016 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002826278836509529, + "loss": 2.7112, + "theoretical_loss": 3.525839140001076, + "tokens_seen": 1453719552 + }, + { + "epoch": 0.44, + "learning_rate": 0.000282617853560682, + "loss": 2.6118, + "theoretical_loss": 3.525825041435578, + "tokens_seen": 1453785088 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028260782347041126, + "loss": 2.599, + "theoretical_loss": 3.525810943683569, + "tokens_seen": 1453850624 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002825977933801404, + "loss": 2.7429, + "theoretical_loss": 3.5257968467449663, + "tokens_seen": 1453916160 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002825877632898696, + "loss": 2.6931, + "theoretical_loss": 3.525782750619687, + "tokens_seen": 1453981696 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002825777331995988, + "loss": 2.7472, + "theoretical_loss": 3.5257686553076457, + "tokens_seen": 1454047232 + }, + { + "epoch": 0.44, + "learning_rate": 0.000282567703109328, + "loss": 2.6985, + "theoretical_loss": 3.5257545608087604, + "tokens_seen": 1454112768 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028255767301905716, + "loss": 2.7411, + "theoretical_loss": 3.5257404671229464, + "tokens_seen": 1454178304 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002825476429287864, + "loss": 2.7563, + "theoretical_loss": 3.5257263742501213, + "tokens_seen": 1454243840 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002825376128385155, + "loss": 2.9927, + "theoretical_loss": 3.5257122821902005, + "tokens_seen": 1454309376 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028252758274824476, + "loss": 2.8397, + "theoretical_loss": 3.525698190943101, + "tokens_seen": 1454374912 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002825175526579739, + "loss": 2.4947, + "theoretical_loss": 3.5256841005087396, + "tokens_seen": 1454440448 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002825075225677031, + "loss": 2.4798, + "theoretical_loss": 3.5256700108870325, + "tokens_seen": 1454505984 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002824974924774323, + "loss": 2.7615, + "theoretical_loss": 3.5256559220778954, + "tokens_seen": 1454571520 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002824874623871615, + "loss": 2.5818, + "theoretical_loss": 3.525641834081246, + "tokens_seen": 1454637056 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028247743229689067, + "loss": 2.691, + "theoretical_loss": 3.525627746897001, + "tokens_seen": 1454702592 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028246740220661985, + "loss": 2.6427, + "theoretical_loss": 3.5256136605250763, + "tokens_seen": 1454768128 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028245737211634903, + "loss": 2.9393, + "theoretical_loss": 3.525599574965388, + "tokens_seen": 1454833664 + }, + { + "epoch": 0.44, + "objective/train/docs_used": 824875, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.514620065689087, + "objective/train/theoretical_loss": 3.5255854902178534, + "objective/train/tokens_used": 1475359200, + "theoretical_loss": 3.5255854902178534, + "tokens_seen": 1454899200 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028244734202607826, + "loss": 2.7567, + "theoretical_loss": 3.5255854902178534, + "tokens_seen": 1454899200 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002824373119358074, + "loss": 2.8022, + "theoretical_loss": 3.5255714062823893, + "tokens_seen": 1454964736 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002824272818455366, + "loss": 2.8816, + "theoretical_loss": 3.5255573231589117, + "tokens_seen": 1455030272 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002824172517552658, + "loss": 2.9316, + "theoretical_loss": 3.5255432408473375, + "tokens_seen": 1455095808 + }, + { + "epoch": 0.44, + "learning_rate": 0.000282407221664995, + "loss": 2.7389, + "theoretical_loss": 3.525529159347583, + "tokens_seen": 1455161344 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028239719157472417, + "loss": 2.8412, + "theoretical_loss": 3.5255150786595655, + "tokens_seen": 1455226880 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028238716148445335, + "loss": 2.6305, + "theoretical_loss": 3.525500998783201, + "tokens_seen": 1455292416 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028237713139418253, + "loss": 2.7918, + "theoretical_loss": 3.5254869197184067, + "tokens_seen": 1455357952 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028236710130391177, + "loss": 2.6841, + "theoretical_loss": 3.525472841465099, + "tokens_seen": 1455423488 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002823570712136409, + "loss": 2.7606, + "theoretical_loss": 3.5254587640231945, + "tokens_seen": 1455489024 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028234704112337013, + "loss": 2.8253, + "theoretical_loss": 3.52544468739261, + "tokens_seen": 1455554560 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028233701103309926, + "loss": 2.8922, + "theoretical_loss": 3.5254306115732623, + "tokens_seen": 1455620096 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002823269809428285, + "loss": 3.0537, + "theoretical_loss": 3.525416536565068, + "tokens_seen": 1455685632 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002823169508525577, + "loss": 2.7664, + "theoretical_loss": 3.525402462367944, + "tokens_seen": 1455751168 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028230692076228685, + "loss": 2.9412, + "theoretical_loss": 3.525388388981807, + "tokens_seen": 1455816704 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028229689067201604, + "loss": 2.616, + "theoretical_loss": 3.525374316406574, + "tokens_seen": 1455882240 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002822868605817452, + "loss": 2.5525, + "theoretical_loss": 3.525360244642161, + "tokens_seen": 1455947776 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028227683049147445, + "loss": 2.9115, + "theoretical_loss": 3.5253461736884852, + "tokens_seen": 1456013312 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028226680040120363, + "loss": 2.7514, + "theoretical_loss": 3.525332103545464, + "tokens_seen": 1456078848 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002822567703109328, + "loss": 2.6955, + "theoretical_loss": 3.5253180342130133, + "tokens_seen": 1456144384 + }, + { + "epoch": 0.44, + "learning_rate": 0.000282246740220662, + "loss": 2.7167, + "theoretical_loss": 3.5253039656910508, + "tokens_seen": 1456209920 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028223671013039123, + "loss": 2.6703, + "theoretical_loss": 3.5252898979794924, + "tokens_seen": 1456275456 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028222668004012036, + "loss": 2.7853, + "theoretical_loss": 3.5252758310782557, + "tokens_seen": 1456340992 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002822166499498496, + "loss": 2.6779, + "theoretical_loss": 3.525261764987257, + "tokens_seen": 1456406528 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002822066198595787, + "loss": 2.7038, + "theoretical_loss": 3.5252476997064135, + "tokens_seen": 1456472064 + }, + { + "epoch": 0.44, + "objective/train/docs_used": 826174, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.547358274459839, + "objective/train/theoretical_loss": 3.525233635235642, + "objective/train/tokens_used": 1476997600, + "theoretical_loss": 3.525233635235642, + "tokens_seen": 1456537600 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028219658976930795, + "loss": 2.6206, + "theoretical_loss": 3.525233635235642, + "tokens_seen": 1456537600 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028218655967903714, + "loss": 2.8112, + "theoretical_loss": 3.5252195715748598, + "tokens_seen": 1456603136 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002821765295887663, + "loss": 2.6877, + "theoretical_loss": 3.5252055087239835, + "tokens_seen": 1456668672 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002821664994984955, + "loss": 2.8055, + "theoretical_loss": 3.5251914466829293, + "tokens_seen": 1456734208 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002821564694082247, + "loss": 2.5488, + "theoretical_loss": 3.5251773854516153, + "tokens_seen": 1456799744 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028214643931795386, + "loss": 2.8156, + "theoretical_loss": 3.5251633250299577, + "tokens_seen": 1456865280 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002821364092276831, + "loss": 2.7732, + "theoretical_loss": 3.525149265417874, + "tokens_seen": 1456930816 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002821263791374122, + "loss": 2.8711, + "theoretical_loss": 3.525135206615281, + "tokens_seen": 1456996352 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028211634904714146, + "loss": 2.6597, + "theoretical_loss": 3.5251211486220955, + "tokens_seen": 1457061888 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002821063189568706, + "loss": 2.5461, + "theoretical_loss": 3.525107091438234, + "tokens_seen": 1457127424 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002820962888665998, + "loss": 2.5836, + "theoretical_loss": 3.5250930350636147, + "tokens_seen": 1457192960 + }, + { + "epoch": 0.44, + "learning_rate": 0.000282086258776329, + "loss": 2.6777, + "theoretical_loss": 3.525078979498154, + "tokens_seen": 1457258496 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002820762286860582, + "loss": 2.977, + "theoretical_loss": 3.525064924741769, + "tokens_seen": 1457324032 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028206619859578736, + "loss": 2.5611, + "theoretical_loss": 3.5250508707943764, + "tokens_seen": 1457389568 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002820561685055166, + "loss": 2.7629, + "theoretical_loss": 3.5250368176558933, + "tokens_seen": 1457455104 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002820461384152457, + "loss": 2.9055, + "theoretical_loss": 3.5250227653262374, + "tokens_seen": 1457520640 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028203610832497496, + "loss": 2.7436, + "theoretical_loss": 3.5250087138053257, + "tokens_seen": 1457586176 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002820260782347041, + "loss": 2.6902, + "theoretical_loss": 3.524994663093074, + "tokens_seen": 1457651712 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002820160481444333, + "loss": 2.8227, + "theoretical_loss": 3.5249806131894013, + "tokens_seen": 1457717248 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002820060180541625, + "loss": 2.8615, + "theoretical_loss": 3.524966564094224, + "tokens_seen": 1457782784 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002819959879638917, + "loss": 2.8763, + "theoretical_loss": 3.524952515807458, + "tokens_seen": 1457848320 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028198595787362087, + "loss": 2.7489, + "theoretical_loss": 3.5249384683290224, + "tokens_seen": 1457913856 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028197592778335005, + "loss": 2.8313, + "theoretical_loss": 3.524924421658833, + "tokens_seen": 1457979392 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028196589769307923, + "loss": 2.9171, + "theoretical_loss": 3.5249103757968077, + "tokens_seen": 1458044928 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028195586760280846, + "loss": 2.8347, + "theoretical_loss": 3.5248963307428633, + "tokens_seen": 1458110464 + }, + { + "epoch": 0.44, + "objective/train/docs_used": 826810, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.8243699073791504, + "objective/train/theoretical_loss": 3.524882286496917, + "objective/train/tokens_used": 1478636000, + "theoretical_loss": 3.524882286496917, + "tokens_seen": 1458176000 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002819458375125376, + "loss": 2.8355, + "theoretical_loss": 3.524882286496917, + "tokens_seen": 1458176000 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002819358074222668, + "loss": 2.8321, + "theoretical_loss": 3.5248682430588865, + "tokens_seen": 1458241536 + }, + { + "epoch": 0.44, + "learning_rate": 0.000281925777331996, + "loss": 2.8182, + "theoretical_loss": 3.5248542004286882, + "tokens_seen": 1458307072 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002819157472417252, + "loss": 2.7743, + "theoretical_loss": 3.52484015860624, + "tokens_seen": 1458372608 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028190571715145437, + "loss": 2.6999, + "theoretical_loss": 3.5248261175914584, + "tokens_seen": 1458438144 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028189568706118355, + "loss": 2.8775, + "theoretical_loss": 3.5248120773842615, + "tokens_seen": 1458503680 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028188565697091273, + "loss": 2.67, + "theoretical_loss": 3.5247980379845663, + "tokens_seen": 1458569216 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028187562688064197, + "loss": 2.7309, + "theoretical_loss": 3.52478399939229, + "tokens_seen": 1458634752 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002818655967903711, + "loss": 2.8851, + "theoretical_loss": 3.5247699616073502, + "tokens_seen": 1458700288 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028185556670010033, + "loss": 2.8312, + "theoretical_loss": 3.5247559246296634, + "tokens_seen": 1458765824 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028184553660982946, + "loss": 2.6098, + "theoretical_loss": 3.5247418884591473, + "tokens_seen": 1458831360 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002818355065195587, + "loss": 2.7806, + "theoretical_loss": 3.52472785309572, + "tokens_seen": 1458896896 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002818254764292879, + "loss": 2.7408, + "theoretical_loss": 3.524713818539297, + "tokens_seen": 1458962432 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028181544633901705, + "loss": 2.6884, + "theoretical_loss": 3.5246997847897976, + "tokens_seen": 1459027968 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028180541624874624, + "loss": 2.6622, + "theoretical_loss": 3.5246857518471386, + "tokens_seen": 1459093504 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002817953861584754, + "loss": 2.773, + "theoretical_loss": 3.524671719711237, + "tokens_seen": 1459159040 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002817853560682046, + "loss": 2.8274, + "theoretical_loss": 3.52465768838201, + "tokens_seen": 1459224576 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028177532597793383, + "loss": 2.6145, + "theoretical_loss": 3.5246436578593756, + "tokens_seen": 1459290112 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028176529588766296, + "loss": 2.4478, + "theoretical_loss": 3.5246296281432508, + "tokens_seen": 1459355648 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002817552657973922, + "loss": 2.4065, + "theoretical_loss": 3.5246155992335533, + "tokens_seen": 1459421184 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002817452357071214, + "loss": 2.7446, + "theoretical_loss": 3.5246015711302006, + "tokens_seen": 1459486720 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028173520561685056, + "loss": 2.8509, + "theoretical_loss": 3.5245875438331096, + "tokens_seen": 1459552256 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028172517552657974, + "loss": 2.565, + "theoretical_loss": 3.5245735173421986, + "tokens_seen": 1459617792 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002817151454363089, + "loss": 2.834, + "theoretical_loss": 3.524559491657384, + "tokens_seen": 1459683328 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002817051153460381, + "loss": 2.8835, + "theoretical_loss": 3.5245454667785845, + "tokens_seen": 1459748864 + }, + { + "epoch": 0.44, + "objective/train/docs_used": 827996, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.1048243045806885, + "objective/train/theoretical_loss": 3.524531442705716, + "objective/train/tokens_used": 1480274400, + "theoretical_loss": 3.524531442705716, + "tokens_seen": 1459814400 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028169508525576734, + "loss": 3.0502, + "theoretical_loss": 3.524531442705716, + "tokens_seen": 1459814400 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028168505516549646, + "loss": 2.7425, + "theoretical_loss": 3.5245174194386983, + "tokens_seen": 1459879936 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002816750250752257, + "loss": 2.7638, + "theoretical_loss": 3.5245033969774466, + "tokens_seen": 1459945472 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002816649949849548, + "loss": 2.7313, + "theoretical_loss": 3.5244893753218802, + "tokens_seen": 1460011008 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028165496489468406, + "loss": 2.8461, + "theoretical_loss": 3.5244753544719156, + "tokens_seen": 1460076544 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028164493480441324, + "loss": 2.6744, + "theoretical_loss": 3.5244613344274707, + "tokens_seen": 1460142080 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002816349047141424, + "loss": 2.9318, + "theoretical_loss": 3.5244473151884628, + "tokens_seen": 1460207616 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002816248746238716, + "loss": 2.5638, + "theoretical_loss": 3.52443329675481, + "tokens_seen": 1460273152 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002816148445336008, + "loss": 2.6304, + "theoretical_loss": 3.5244192791264295, + "tokens_seen": 1460338688 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028160481444332997, + "loss": 2.7429, + "theoretical_loss": 3.524405262303239, + "tokens_seen": 1460404224 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002815947843530592, + "loss": 2.7197, + "theoretical_loss": 3.524391246285156, + "tokens_seen": 1460469760 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028158475426278833, + "loss": 2.7523, + "theoretical_loss": 3.524377231072098, + "tokens_seen": 1460535296 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028157472417251756, + "loss": 2.5455, + "theoretical_loss": 3.5243632166639838, + "tokens_seen": 1460600832 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028156469408224674, + "loss": 2.5973, + "theoretical_loss": 3.5243492030607295, + "tokens_seen": 1460666368 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002815546639919759, + "loss": 2.6874, + "theoretical_loss": 3.5243351902622537, + "tokens_seen": 1460731904 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002815446339017051, + "loss": 2.8898, + "theoretical_loss": 3.5243211782684734, + "tokens_seen": 1460797440 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002815346038114343, + "loss": 2.5696, + "theoretical_loss": 3.524307167079307, + "tokens_seen": 1460862976 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002815245737211635, + "loss": 2.4849, + "theoretical_loss": 3.5242931566946716, + "tokens_seen": 1460928512 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002815145436308927, + "loss": 2.7806, + "theoretical_loss": 3.524279147114486, + "tokens_seen": 1460994048 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002815045135406219, + "loss": 2.6386, + "theoretical_loss": 3.5242651383386665, + "tokens_seen": 1461059584 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028149448345035107, + "loss": 2.9799, + "theoretical_loss": 3.5242511303671313, + "tokens_seen": 1461125120 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028148445336008025, + "loss": 2.6795, + "theoretical_loss": 3.5242371231997986, + "tokens_seen": 1461190656 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028147442326980943, + "loss": 2.4912, + "theoretical_loss": 3.524223116836586, + "tokens_seen": 1461256192 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028146439317953866, + "loss": 2.8644, + "theoretical_loss": 3.524209111277411, + "tokens_seen": 1461321728 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002814543630892678, + "loss": 2.8116, + "theoretical_loss": 3.5241951065221917, + "tokens_seen": 1461387264 + }, + { + "epoch": 0.44, + "objective/train/docs_used": 828576, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.956451892852783, + "objective/train/theoretical_loss": 3.524181102570846, + "objective/train/tokens_used": 1481912800, + "theoretical_loss": 3.524181102570846, + "tokens_seen": 1461452800 + }, + { + "epoch": 0.44, + "learning_rate": 0.000281444332998997, + "loss": 2.8707, + "theoretical_loss": 3.524181102570846, + "tokens_seen": 1461452800 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002814343029087262, + "loss": 2.8568, + "theoretical_loss": 3.524167099423291, + "tokens_seen": 1461518336 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002814242728184554, + "loss": 2.8624, + "theoretical_loss": 3.5241530970794455, + "tokens_seen": 1461583872 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028141424272818457, + "loss": 2.741, + "theoretical_loss": 3.524139095539226, + "tokens_seen": 1461649408 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028140421263791375, + "loss": 2.6104, + "theoretical_loss": 3.524125094802552, + "tokens_seen": 1461714944 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028139418254764293, + "loss": 2.7743, + "theoretical_loss": 3.5241110948693404, + "tokens_seen": 1461780480 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028138415245737217, + "loss": 2.8001, + "theoretical_loss": 3.524097095739509, + "tokens_seen": 1461846016 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002813741223671013, + "loss": 2.8375, + "theoretical_loss": 3.5240830974129764, + "tokens_seen": 1461911552 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028136409227683053, + "loss": 2.6554, + "theoretical_loss": 3.5240690998896596, + "tokens_seen": 1461977088 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028135406218655966, + "loss": 2.6156, + "theoretical_loss": 3.524055103169477, + "tokens_seen": 1462042624 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002813440320962889, + "loss": 2.9524, + "theoretical_loss": 3.524041107252346, + "tokens_seen": 1462108160 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002813340020060181, + "loss": 2.6897, + "theoretical_loss": 3.524027112138186, + "tokens_seen": 1462173696 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028132397191574725, + "loss": 2.5745, + "theoretical_loss": 3.524013117826913, + "tokens_seen": 1462239232 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028131394182547644, + "loss": 2.8154, + "theoretical_loss": 3.523999124318446, + "tokens_seen": 1462304768 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002813039117352056, + "loss": 2.7216, + "theoretical_loss": 3.5239851316127035, + "tokens_seen": 1462370304 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002812938816449348, + "loss": 2.7985, + "theoretical_loss": 3.5239711397096025, + "tokens_seen": 1462435840 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028128385155466403, + "loss": 2.9623, + "theoretical_loss": 3.523957148609061, + "tokens_seen": 1462501376 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028127382146439316, + "loss": 2.7305, + "theoretical_loss": 3.5239431583109972, + "tokens_seen": 1462566912 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002812637913741224, + "loss": 2.752, + "theoretical_loss": 3.5239291688153296, + "tokens_seen": 1462632448 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002812537612838516, + "loss": 2.5356, + "theoretical_loss": 3.5239151801219757, + "tokens_seen": 1462697984 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028124373119358076, + "loss": 2.8385, + "theoretical_loss": 3.523901192230854, + "tokens_seen": 1462763520 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028123370110330994, + "loss": 2.7429, + "theoretical_loss": 3.523887205141882, + "tokens_seen": 1462829056 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002812236710130391, + "loss": 2.6958, + "theoretical_loss": 3.5238732188549777, + "tokens_seen": 1462894592 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002812136409227683, + "loss": 2.7464, + "theoretical_loss": 3.52385923337006, + "tokens_seen": 1462960128 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028120361083249754, + "loss": 2.721, + "theoretical_loss": 3.5238452486870466, + "tokens_seen": 1463025664 + }, + { + "epoch": 0.44, + "objective/train/docs_used": 830024, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.449328899383545, + "objective/train/theoretical_loss": 3.5238312648058554, + "objective/train/tokens_used": 1483551200, + "theoretical_loss": 3.5238312648058554, + "tokens_seen": 1463091200 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028119358074222666, + "loss": 2.666, + "theoretical_loss": 3.5238312648058554, + "tokens_seen": 1463091200 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002811835506519559, + "loss": 2.8344, + "theoretical_loss": 3.5238172817264046, + "tokens_seen": 1463156736 + }, + { + "epoch": 0.44, + "learning_rate": 0.000281173520561685, + "loss": 2.8676, + "theoretical_loss": 3.523803299448612, + "tokens_seen": 1463222272 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028116349047141426, + "loss": 2.7044, + "theoretical_loss": 3.5237893179723967, + "tokens_seen": 1463287808 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028115346038114344, + "loss": 2.557, + "theoretical_loss": 3.5237753372976757, + "tokens_seen": 1463353344 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002811434302908726, + "loss": 2.7228, + "theoretical_loss": 3.523761357424368, + "tokens_seen": 1463418880 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002811334002006018, + "loss": 2.7976, + "theoretical_loss": 3.5237473783523914, + "tokens_seen": 1463484416 + }, + { + "epoch": 0.44, + "learning_rate": 0.000281123370110331, + "loss": 2.642, + "theoretical_loss": 3.523733400081664, + "tokens_seen": 1463549952 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028111334002006017, + "loss": 2.6614, + "theoretical_loss": 3.5237194226121042, + "tokens_seen": 1463615488 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002811033099297894, + "loss": 2.462, + "theoretical_loss": 3.52370544594363, + "tokens_seen": 1463681024 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028109327983951853, + "loss": 2.8253, + "theoretical_loss": 3.5236914700761606, + "tokens_seen": 1463746560 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028108324974924776, + "loss": 3.0595, + "theoretical_loss": 3.5236774950096126, + "tokens_seen": 1463812096 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028107321965897695, + "loss": 2.4429, + "theoretical_loss": 3.523663520743905, + "tokens_seen": 1463877632 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002810631895687061, + "loss": 2.5085, + "theoretical_loss": 3.5236495472789566, + "tokens_seen": 1463943168 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002810531594784353, + "loss": 2.4993, + "theoretical_loss": 3.5236355746146852, + "tokens_seen": 1464008704 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002810431293881645, + "loss": 2.6386, + "theoretical_loss": 3.5236216027510086, + "tokens_seen": 1464074240 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028103309929789367, + "loss": 2.6573, + "theoretical_loss": 3.523607631687846, + "tokens_seen": 1464139776 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002810230692076229, + "loss": 2.7306, + "theoretical_loss": 3.523593661425115, + "tokens_seen": 1464205312 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028101303911735203, + "loss": 2.3777, + "theoretical_loss": 3.5235796919627345, + "tokens_seen": 1464270848 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028100300902708127, + "loss": 2.7411, + "theoretical_loss": 3.5235657233006226, + "tokens_seen": 1464336384 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002809929789368104, + "loss": 2.7239, + "theoretical_loss": 3.523551755438697, + "tokens_seen": 1464401920 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028098294884653963, + "loss": 2.8697, + "theoretical_loss": 3.523537788376877, + "tokens_seen": 1464467456 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002809729187562688, + "loss": 2.7116, + "theoretical_loss": 3.523523822115081, + "tokens_seen": 1464532992 + }, + { + "epoch": 0.44, + "learning_rate": 0.000280962888665998, + "loss": 2.689, + "theoretical_loss": 3.5235098566532264, + "tokens_seen": 1464598528 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028095285857572717, + "loss": 2.5199, + "theoretical_loss": 3.5234958919912316, + "tokens_seen": 1464664064 + }, + { + "epoch": 0.44, + "objective/train/docs_used": 830651, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7020297050476074, + "objective/train/theoretical_loss": 3.5234819281290166, + "objective/train/tokens_used": 1485189600, + "theoretical_loss": 3.5234819281290166, + "tokens_seen": 1464729600 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028094282848545635, + "loss": 2.5748, + "theoretical_loss": 3.5234819281290166, + "tokens_seen": 1464729600 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028093279839518553, + "loss": 2.8379, + "theoretical_loss": 3.5234679650664984, + "tokens_seen": 1464795136 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028092276830491477, + "loss": 2.8653, + "theoretical_loss": 3.5234540028035957, + "tokens_seen": 1464860672 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002809127382146439, + "loss": 2.8992, + "theoretical_loss": 3.523440041340227, + "tokens_seen": 1464926208 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028090270812437313, + "loss": 2.7805, + "theoretical_loss": 3.523426080676311, + "tokens_seen": 1464991744 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002808926780341023, + "loss": 2.8153, + "theoretical_loss": 3.523412120811766, + "tokens_seen": 1465057280 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002808826479438315, + "loss": 2.6767, + "theoretical_loss": 3.5233981617465107, + "tokens_seen": 1465122816 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002808726178535607, + "loss": 2.8201, + "theoretical_loss": 3.5233842034804628, + "tokens_seen": 1465188352 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028086258776328986, + "loss": 2.712, + "theoretical_loss": 3.523370246013541, + "tokens_seen": 1465253888 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028085255767301904, + "loss": 2.6495, + "theoretical_loss": 3.5233562893456645, + "tokens_seen": 1465319424 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002808425275827483, + "loss": 2.6656, + "theoretical_loss": 3.5233423334767515, + "tokens_seen": 1465384960 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002808324974924774, + "loss": 2.4941, + "theoretical_loss": 3.5233283784067204, + "tokens_seen": 1465450496 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028082246740220664, + "loss": 2.6508, + "theoretical_loss": 3.5233144241354903, + "tokens_seen": 1465516032 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028081243731193576, + "loss": 2.4237, + "theoretical_loss": 3.5233004706629787, + "tokens_seen": 1465581568 + }, + { + "epoch": 0.44, + "learning_rate": 0.000280802407221665, + "loss": 2.8495, + "theoretical_loss": 3.5232865179891046, + "tokens_seen": 1465647104 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002807923771313942, + "loss": 2.8494, + "theoretical_loss": 3.5232725661137874, + "tokens_seen": 1465712640 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028078234704112336, + "loss": 2.7551, + "theoretical_loss": 3.5232586150369443, + "tokens_seen": 1465778176 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002807723169508526, + "loss": 2.7722, + "theoretical_loss": 3.5232446647584954, + "tokens_seen": 1465843712 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002807622868605818, + "loss": 2.7523, + "theoretical_loss": 3.5232307152783577, + "tokens_seen": 1465909248 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028075225677031096, + "loss": 2.634, + "theoretical_loss": 3.523216766596451, + "tokens_seen": 1465974784 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028074222668004014, + "loss": 2.5855, + "theoretical_loss": 3.523202818712694, + "tokens_seen": 1466040320 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002807321965897693, + "loss": 2.8502, + "theoretical_loss": 3.5231888716270046, + "tokens_seen": 1466105856 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002807221664994985, + "loss": 2.7994, + "theoretical_loss": 3.523174925339302, + "tokens_seen": 1466171392 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028071213640922774, + "loss": 2.8441, + "theoretical_loss": 3.5231609798495045, + "tokens_seen": 1466236928 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028070210631895686, + "loss": 2.8296, + "theoretical_loss": 3.523147035157531, + "tokens_seen": 1466302464 + }, + { + "epoch": 0.44, + "objective/train/docs_used": 831840, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.9130706787109375, + "objective/train/theoretical_loss": 3.5231330912633005, + "objective/train/tokens_used": 1486828000, + "theoretical_loss": 3.5231330912633005, + "tokens_seen": 1466368000 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002806920762286861, + "loss": 2.7256, + "theoretical_loss": 3.5231330912633005, + "tokens_seen": 1466368000 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002806820461384152, + "loss": 2.8226, + "theoretical_loss": 3.523119148166731, + "tokens_seen": 1466433536 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028067201604814446, + "loss": 2.4687, + "theoretical_loss": 3.5231052058677417, + "tokens_seen": 1466499072 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028066198595787364, + "loss": 2.9567, + "theoretical_loss": 3.523091264366252, + "tokens_seen": 1466564608 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002806519558676028, + "loss": 2.4661, + "theoretical_loss": 3.523077323662179, + "tokens_seen": 1466630144 + }, + { + "epoch": 0.44, + "learning_rate": 0.000280641925777332, + "loss": 2.5857, + "theoretical_loss": 3.523063383755443, + "tokens_seen": 1466695680 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002806318956870612, + "loss": 2.7417, + "theoretical_loss": 3.523049444645962, + "tokens_seen": 1466761216 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028062186559679037, + "loss": 2.877, + "theoretical_loss": 3.5230355063336547, + "tokens_seen": 1466826752 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002806118355065196, + "loss": 2.5773, + "theoretical_loss": 3.5230215688184403, + "tokens_seen": 1466892288 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028060180541624873, + "loss": 2.6114, + "theoretical_loss": 3.5230076321002373, + "tokens_seen": 1466957824 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028059177532597796, + "loss": 2.7221, + "theoretical_loss": 3.5229936961789647, + "tokens_seen": 1467023360 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028058174523570715, + "loss": 2.4619, + "theoretical_loss": 3.5229797610545415, + "tokens_seen": 1467088896 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002805717151454363, + "loss": 2.7895, + "theoretical_loss": 3.522965826726886, + "tokens_seen": 1467154432 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002805616850551655, + "loss": 2.547, + "theoretical_loss": 3.522951893195918, + "tokens_seen": 1467219968 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002805516549648947, + "loss": 2.7961, + "theoretical_loss": 3.5229379604615554, + "tokens_seen": 1467285504 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028054162487462387, + "loss": 2.7092, + "theoretical_loss": 3.5229240285237173, + "tokens_seen": 1467351040 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002805315947843531, + "loss": 2.6086, + "theoretical_loss": 3.522910097382323, + "tokens_seen": 1467416576 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028052156469408223, + "loss": 2.7956, + "theoretical_loss": 3.522896167037291, + "tokens_seen": 1467482112 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028051153460381147, + "loss": 2.8708, + "theoretical_loss": 3.52288223748854, + "tokens_seen": 1467547648 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002805015045135406, + "loss": 2.8736, + "theoretical_loss": 3.5228683087359896, + "tokens_seen": 1467613184 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028049147442326983, + "loss": 2.6352, + "theoretical_loss": 3.5228543807795587, + "tokens_seen": 1467678720 + }, + { + "epoch": 0.44, + "learning_rate": 0.000280481444332999, + "loss": 2.7676, + "theoretical_loss": 3.5228404536191658, + "tokens_seen": 1467744256 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002804714142427282, + "loss": 2.6589, + "theoretical_loss": 3.52282652725473, + "tokens_seen": 1467809792 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028046138415245737, + "loss": 2.7382, + "theoretical_loss": 3.5228126016861703, + "tokens_seen": 1467875328 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028045135406218655, + "loss": 2.7901, + "theoretical_loss": 3.522798676913405, + "tokens_seen": 1467940864 + }, + { + "epoch": 0.44, + "objective/train/docs_used": 832610, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.66243052482605, + "objective/train/theoretical_loss": 3.5227847529363547, + "objective/train/tokens_used": 1488466400, + "theoretical_loss": 3.5227847529363547, + "tokens_seen": 1468006400 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028044132397191574, + "loss": 2.6096, + "theoretical_loss": 3.5227847529363547, + "tokens_seen": 1468006400 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028043129388164497, + "loss": 2.5883, + "theoretical_loss": 3.522770829754937, + "tokens_seen": 1468071936 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002804212637913741, + "loss": 2.7276, + "theoretical_loss": 3.5227569073690717, + "tokens_seen": 1468137472 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028041123370110333, + "loss": 2.9286, + "theoretical_loss": 3.522742985778678, + "tokens_seen": 1468203008 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002804012036108325, + "loss": 2.7836, + "theoretical_loss": 3.5227290649836736, + "tokens_seen": 1468268544 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002803911735205617, + "loss": 2.6527, + "theoretical_loss": 3.5227151449839793, + "tokens_seen": 1468334080 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002803811434302909, + "loss": 2.7256, + "theoretical_loss": 3.5227012257795125, + "tokens_seen": 1468399616 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028037111334002006, + "loss": 2.5399, + "theoretical_loss": 3.5226873073701936, + "tokens_seen": 1468465152 + }, + { + "epoch": 0.45, + "learning_rate": 0.00028036108324974924, + "loss": 2.3848, + "theoretical_loss": 3.522673389755941, + "tokens_seen": 1468530688 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002803510531594785, + "loss": 2.7388, + "theoretical_loss": 3.522659472936674, + "tokens_seen": 1468596224 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002803410230692076, + "loss": 2.6396, + "theoretical_loss": 3.522645556912312, + "tokens_seen": 1468661760 + }, + { + "epoch": 0.45, + "learning_rate": 0.00028033099297893684, + "loss": 2.7159, + "theoretical_loss": 3.5226316416827737, + "tokens_seen": 1468727296 + }, + { + "epoch": 0.45, + "learning_rate": 0.00028032096288866596, + "loss": 2.6288, + "theoretical_loss": 3.5226177272479786, + "tokens_seen": 1468792832 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002803109327983952, + "loss": 2.4776, + "theoretical_loss": 3.5226038136078452, + "tokens_seen": 1468858368 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002803009027081244, + "loss": 2.4762, + "theoretical_loss": 3.522589900762293, + "tokens_seen": 1468923904 + }, + { + "epoch": 0.45, + "learning_rate": 0.00028029087261785356, + "loss": 2.9656, + "theoretical_loss": 3.522575988711242, + "tokens_seen": 1468989440 + }, + { + "epoch": 0.45, + "learning_rate": 0.00028028084252758274, + "loss": 2.7465, + "theoretical_loss": 3.5225620774546105, + "tokens_seen": 1469054976 + }, + { + "epoch": 0.45, + "learning_rate": 0.000280270812437312, + "loss": 2.4558, + "theoretical_loss": 3.5225481669923173, + "tokens_seen": 1469120512 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002802607823470411, + "loss": 2.4784, + "theoretical_loss": 3.522534257324282, + "tokens_seen": 1469186048 + }, + { + "epoch": 0.45, + "learning_rate": 0.00028025075225677034, + "loss": 2.7622, + "theoretical_loss": 3.522520348450425, + "tokens_seen": 1469251584 + }, + { + "epoch": 0.45, + "learning_rate": 0.00028024072216649947, + "loss": 2.4648, + "theoretical_loss": 3.5225064403706643, + "tokens_seen": 1469317120 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002802306920762287, + "loss": 2.8224, + "theoretical_loss": 3.522492533084919, + "tokens_seen": 1469382656 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002802206619859579, + "loss": 2.5337, + "theoretical_loss": 3.5224786265931085, + "tokens_seen": 1469448192 + }, + { + "epoch": 0.45, + "learning_rate": 0.00028021063189568706, + "loss": 2.9252, + "theoretical_loss": 3.5224647208951527, + "tokens_seen": 1469513728 + }, + { + "epoch": 0.45, + "learning_rate": 0.00028020060180541624, + "loss": 2.7846, + "theoretical_loss": 3.522450815990971, + "tokens_seen": 1469579264 + }, + { + "epoch": 0.45, + "objective/train/docs_used": 834187, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.8911235332489014, + "objective/train/theoretical_loss": 3.5224369118804817, + "objective/train/tokens_used": 1490104800, + "theoretical_loss": 3.5224369118804817, + "tokens_seen": 1469644800 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002801905717151454, + "loss": 2.749, + "theoretical_loss": 3.5224369118804817, + "tokens_seen": 1469644800 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002801805416248746, + "loss": 2.539, + "theoretical_loss": 3.5224230085636044, + "tokens_seen": 1469710336 + }, + { + "epoch": 0.45, + "learning_rate": 0.00028017051153460384, + "loss": 2.6418, + "theoretical_loss": 3.522409106040259, + "tokens_seen": 1469775872 + }, + { + "epoch": 0.45, + "learning_rate": 0.00028016048144433297, + "loss": 2.64, + "theoretical_loss": 3.522395204310364, + "tokens_seen": 1469841408 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002801504513540622, + "loss": 2.8457, + "theoretical_loss": 3.5223813033738396, + "tokens_seen": 1469906944 + }, + { + "epoch": 0.45, + "learning_rate": 0.00028014042126379133, + "loss": 2.8484, + "theoretical_loss": 3.5223674032306045, + "tokens_seen": 1469972480 + }, + { + "epoch": 0.45, + "learning_rate": 0.00028013039117352057, + "loss": 2.7832, + "theoretical_loss": 3.5223535038805784, + "tokens_seen": 1470038016 + }, + { + "epoch": 0.45, + "learning_rate": 0.00028012036108324975, + "loss": 2.7384, + "theoretical_loss": 3.5223396053236806, + "tokens_seen": 1470103552 + }, + { + "epoch": 0.45, + "learning_rate": 0.00028011033099297893, + "loss": 2.666, + "theoretical_loss": 3.5223257075598307, + "tokens_seen": 1470169088 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002801003009027081, + "loss": 2.7595, + "theoretical_loss": 3.5223118105889473, + "tokens_seen": 1470234624 + }, + { + "epoch": 0.45, + "learning_rate": 0.00028009027081243735, + "loss": 2.5983, + "theoretical_loss": 3.5222979144109505, + "tokens_seen": 1470300160 + }, + { + "epoch": 0.45, + "learning_rate": 0.00028008024072216647, + "loss": 2.4271, + "theoretical_loss": 3.52228401902576, + "tokens_seen": 1470365696 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002800702106318957, + "loss": 2.9254, + "theoretical_loss": 3.5222701244332946, + "tokens_seen": 1470431232 + }, + { + "epoch": 0.45, + "learning_rate": 0.00028006018054162483, + "loss": 2.6574, + "theoretical_loss": 3.5222562306334737, + "tokens_seen": 1470496768 + }, + { + "epoch": 0.45, + "learning_rate": 0.00028005015045135407, + "loss": 2.8678, + "theoretical_loss": 3.522242337626218, + "tokens_seen": 1470562304 + }, + { + "epoch": 0.45, + "learning_rate": 0.00028004012036108325, + "loss": 2.2476, + "theoretical_loss": 3.5222284454114448, + "tokens_seen": 1470627840 + }, + { + "epoch": 0.45, + "learning_rate": 0.00028003009027081243, + "loss": 2.6056, + "theoretical_loss": 3.5222145539890755, + "tokens_seen": 1470693376 + }, + { + "epoch": 0.45, + "learning_rate": 0.00028002006018054167, + "loss": 2.8812, + "theoretical_loss": 3.522200663359029, + "tokens_seen": 1470758912 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002800100300902708, + "loss": 2.5519, + "theoretical_loss": 3.5221867735212244, + "tokens_seen": 1470824448 + }, + { + "epoch": 0.45, + "learning_rate": 0.00028000000000000003, + "loss": 2.9593, + "theoretical_loss": 3.5221728844755815, + "tokens_seen": 1470889984 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002799899699097292, + "loss": 2.7201, + "theoretical_loss": 3.5221589962220197, + "tokens_seen": 1470955520 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002799799398194584, + "loss": 2.9024, + "theoretical_loss": 3.5221451087604594, + "tokens_seen": 1471021056 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002799699097291876, + "loss": 2.6428, + "theoretical_loss": 3.5221312220908185, + "tokens_seen": 1471086592 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027995987963891675, + "loss": 2.7465, + "theoretical_loss": 3.522117336213018, + "tokens_seen": 1471152128 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027994984954864594, + "loss": 2.5503, + "theoretical_loss": 3.522103451126977, + "tokens_seen": 1471217664 + }, + { + "epoch": 0.45, + "objective/train/docs_used": 834912, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.569326400756836, + "objective/train/theoretical_loss": 3.522089566832615, + "objective/train/tokens_used": 1491743200, + "theoretical_loss": 3.522089566832615, + "tokens_seen": 1471283200 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027993981945837517, + "loss": 2.6246, + "theoretical_loss": 3.522089566832615, + "tokens_seen": 1471283200 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002799297893681043, + "loss": 2.6421, + "theoretical_loss": 3.5220756833298514, + "tokens_seen": 1471348736 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027991975927783353, + "loss": 2.5252, + "theoretical_loss": 3.522061800618607, + "tokens_seen": 1471414272 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002799097291875627, + "loss": 2.7266, + "theoretical_loss": 3.5220479186987994, + "tokens_seen": 1471479808 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002798996990972919, + "loss": 2.787, + "theoretical_loss": 3.52203403757035, + "tokens_seen": 1471545344 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002798896690070211, + "loss": 2.6756, + "theoretical_loss": 3.5220201572331775, + "tokens_seen": 1471610880 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027987963891675026, + "loss": 2.7065, + "theoretical_loss": 3.522006277687202, + "tokens_seen": 1471676416 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027986960882647944, + "loss": 2.6174, + "theoretical_loss": 3.521992398932343, + "tokens_seen": 1471741952 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002798595787362087, + "loss": 2.8024, + "theoretical_loss": 3.52197852096852, + "tokens_seen": 1471807488 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002798495486459378, + "loss": 2.8304, + "theoretical_loss": 3.521964643795653, + "tokens_seen": 1471873024 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027983951855566704, + "loss": 2.6994, + "theoretical_loss": 3.521950767413662, + "tokens_seen": 1471938560 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027982948846539616, + "loss": 2.9409, + "theoretical_loss": 3.521936891822466, + "tokens_seen": 1472004096 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002798194583751254, + "loss": 2.5199, + "theoretical_loss": 3.521923017021985, + "tokens_seen": 1472069632 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002798094282848546, + "loss": 2.7888, + "theoretical_loss": 3.521909143012139, + "tokens_seen": 1472135168 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027979939819458376, + "loss": 2.7292, + "theoretical_loss": 3.5218952697928474, + "tokens_seen": 1472200704 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027978936810431294, + "loss": 2.7845, + "theoretical_loss": 3.5218813973640297, + "tokens_seen": 1472266240 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002797793380140422, + "loss": 2.6577, + "theoretical_loss": 3.5218675257256065, + "tokens_seen": 1472331776 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002797693079237713, + "loss": 2.8385, + "theoretical_loss": 3.5218536548774972, + "tokens_seen": 1472397312 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027975927783350054, + "loss": 2.633, + "theoretical_loss": 3.5218397848196217, + "tokens_seen": 1472462848 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027974924774322967, + "loss": 2.6025, + "theoretical_loss": 3.521825915551899, + "tokens_seen": 1472528384 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002797392176529589, + "loss": 2.824, + "theoretical_loss": 3.52181204707425, + "tokens_seen": 1472593920 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002797291875626881, + "loss": 2.7336, + "theoretical_loss": 3.5217981793865945, + "tokens_seen": 1472659456 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027971915747241726, + "loss": 2.7379, + "theoretical_loss": 3.5217843124888515, + "tokens_seen": 1472724992 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027970912738214644, + "loss": 2.6069, + "theoretical_loss": 3.5217704463809416, + "tokens_seen": 1472790528 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002796990972918756, + "loss": 2.5572, + "theoretical_loss": 3.5217565810627844, + "tokens_seen": 1472856064 + }, + { + "epoch": 0.45, + "objective/train/docs_used": 835528, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.5229275226593018, + "objective/train/theoretical_loss": 3.5217427165343, + "objective/train/tokens_used": 1493381600, + "theoretical_loss": 3.5217427165343, + "tokens_seen": 1472921600 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002796890672016048, + "loss": 2.7587, + "theoretical_loss": 3.5217427165343, + "tokens_seen": 1472921600 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027967903711133404, + "loss": 2.7241, + "theoretical_loss": 3.521728852795407, + "tokens_seen": 1472987136 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027966900702106317, + "loss": 2.6459, + "theoretical_loss": 3.5217149898460276, + "tokens_seen": 1473052672 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002796589769307924, + "loss": 2.7775, + "theoretical_loss": 3.52170112768608, + "tokens_seen": 1473118208 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027964894684052153, + "loss": 2.5067, + "theoretical_loss": 3.5216872663154852, + "tokens_seen": 1473183744 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027963891675025077, + "loss": 2.5222, + "theoretical_loss": 3.5216734057341617, + "tokens_seen": 1473249280 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027962888665997995, + "loss": 2.5791, + "theoretical_loss": 3.521659545942031, + "tokens_seen": 1473314816 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027961885656970913, + "loss": 2.6725, + "theoretical_loss": 3.5216456869390123, + "tokens_seen": 1473380352 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002796088264794383, + "loss": 2.5475, + "theoretical_loss": 3.5216318287250257, + "tokens_seen": 1473445888 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027959879638916755, + "loss": 2.9087, + "theoretical_loss": 3.5216179712999907, + "tokens_seen": 1473511424 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027958876629889667, + "loss": 2.553, + "theoretical_loss": 3.5216041146638277, + "tokens_seen": 1473576960 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002795787362086259, + "loss": 2.7327, + "theoretical_loss": 3.521590258816457, + "tokens_seen": 1473642496 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027956870611835503, + "loss": 2.6918, + "theoretical_loss": 3.5215764037577983, + "tokens_seen": 1473708032 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027955867602808427, + "loss": 2.4869, + "theoretical_loss": 3.521562549487772, + "tokens_seen": 1473773568 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027954864593781345, + "loss": 2.5998, + "theoretical_loss": 3.521548696006298, + "tokens_seen": 1473839104 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027953861584754263, + "loss": 2.6093, + "theoretical_loss": 3.521534843313296, + "tokens_seen": 1473904640 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002795285857572718, + "loss": 2.6285, + "theoretical_loss": 3.5215209914086865, + "tokens_seen": 1473970176 + }, + { + "epoch": 0.45, + "learning_rate": 0.000279518555667001, + "loss": 2.7486, + "theoretical_loss": 3.521507140292389, + "tokens_seen": 1474035712 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002795085255767302, + "loss": 2.6534, + "theoretical_loss": 3.521493289964324, + "tokens_seen": 1474101248 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002794984954864594, + "loss": 2.7689, + "theoretical_loss": 3.521479440424412, + "tokens_seen": 1474166784 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027948846539618854, + "loss": 2.8692, + "theoretical_loss": 3.5214655916725714, + "tokens_seen": 1474232320 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002794784353059178, + "loss": 2.5907, + "theoretical_loss": 3.5214517437087247, + "tokens_seen": 1474297856 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002794684052156469, + "loss": 2.604, + "theoretical_loss": 3.521437896532791, + "tokens_seen": 1474363392 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027945837512537614, + "loss": 2.6847, + "theoretical_loss": 3.52142405014469, + "tokens_seen": 1474428928 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002794483450351053, + "loss": 2.5085, + "theoretical_loss": 3.5214102045443423, + "tokens_seen": 1474494464 + }, + { + "debugging/Self-BLEU-5": 0.1491206753141767, + "debugging/distinct-1-grams": 0.7939110070257612, + "debugging/distinct-2-grams": 0.9656565656565657, + "debugging/entropy-1-grams": 4.718800747001991, + "debugging/entropy-2-grams": 5.1093341994787185, + "debugging/length": 431.6666666666667, + "debugging/num_segments": 3, + "debugging/score": 0.005702674830344072, + "debugging/score_std": 0.004506248112125528, + "epoch": 0.45, + "objective/train/docs_used": 836610, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.230942964553833, + "objective/train/theoretical_loss": 3.521396359731668, + "objective/train/tokens_used": 1495020000, + "theoretical_loss": 3.521396359731668, + "tokens_seen": 1474560000 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002794383149448345, + "loss": 2.4984, + "theoretical_loss": 3.521396359731668, + "tokens_seen": 1474560000 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002794282848545637, + "loss": 2.5988, + "theoretical_loss": 3.5213825157065877, + "tokens_seen": 1474625536 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002794182547642929, + "loss": 2.8186, + "theoretical_loss": 3.5213686724690207, + "tokens_seen": 1474691072 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027940822467402204, + "loss": 2.761, + "theoretical_loss": 3.521354830018888, + "tokens_seen": 1474756608 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002793981945837513, + "loss": 2.5733, + "theoretical_loss": 3.5213409883561093, + "tokens_seen": 1474822144 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002793881644934804, + "loss": 2.6023, + "theoretical_loss": 3.521327147480605, + "tokens_seen": 1474887680 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027937813440320964, + "loss": 2.5179, + "theoretical_loss": 3.5213133073922958, + "tokens_seen": 1474953216 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002793681043129388, + "loss": 2.6206, + "theoretical_loss": 3.5212994680911014, + "tokens_seen": 1475018752 + }, + { + "epoch": 0.45, + "learning_rate": 0.000279358074222668, + "loss": 2.8195, + "theoretical_loss": 3.5212856295769424, + "tokens_seen": 1475084288 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002793480441323972, + "loss": 2.7936, + "theoretical_loss": 3.5212717918497383, + "tokens_seen": 1475149824 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027933801404212636, + "loss": 2.9768, + "theoretical_loss": 3.5212579549094105, + "tokens_seen": 1475215360 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027932798395185554, + "loss": 2.7005, + "theoretical_loss": 3.5212441187558787, + "tokens_seen": 1475280896 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002793179538615848, + "loss": 2.5674, + "theoretical_loss": 3.5212302833890634, + "tokens_seen": 1475346432 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002793079237713139, + "loss": 2.7426, + "theoretical_loss": 3.5212164488088846, + "tokens_seen": 1475411968 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027929789368104314, + "loss": 2.6325, + "theoretical_loss": 3.5212026150152633, + "tokens_seen": 1475477504 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027928786359077227, + "loss": 2.5163, + "theoretical_loss": 3.521188782008119, + "tokens_seen": 1475543040 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002792778335005015, + "loss": 2.6282, + "theoretical_loss": 3.5211749497873726, + "tokens_seen": 1475608576 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027926780341023074, + "loss": 2.7768, + "theoretical_loss": 3.521161118352944, + "tokens_seen": 1475674112 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027925777331995987, + "loss": 2.5721, + "theoretical_loss": 3.5211472877047543, + "tokens_seen": 1475739648 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002792477432296891, + "loss": 2.4633, + "theoretical_loss": 3.5211334578427236, + "tokens_seen": 1475805184 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002792377131394183, + "loss": 2.8251, + "theoretical_loss": 3.5211196287667716, + "tokens_seen": 1475870720 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027922768304914746, + "loss": 2.6161, + "theoretical_loss": 3.5211058004768203, + "tokens_seen": 1475936256 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027921765295887664, + "loss": 2.6934, + "theoretical_loss": 3.5210919729727888, + "tokens_seen": 1476001792 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002792076228686058, + "loss": 2.747, + "theoretical_loss": 3.521078146254598, + "tokens_seen": 1476067328 + }, + { + "epoch": 0.45, + "learning_rate": 0.000279197592778335, + "loss": 2.5382, + "theoretical_loss": 3.521064320322168, + "tokens_seen": 1476132864 + }, + { + "epoch": 0.45, + "objective/train/docs_used": 837052, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6828176975250244, + "objective/train/theoretical_loss": 3.5210504951754196, + "objective/train/tokens_used": 1496658400, + "theoretical_loss": 3.5210504951754196, + "tokens_seen": 1476198400 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027918756268806424, + "loss": 2.4638, + "theoretical_loss": 3.5210504951754196, + "tokens_seen": 1476198400 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027917753259779337, + "loss": 2.7816, + "theoretical_loss": 3.521036670814273, + "tokens_seen": 1476263936 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002791675025075226, + "loss": 2.6253, + "theoretical_loss": 3.5210228472386493, + "tokens_seen": 1476329472 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027915747241725173, + "loss": 2.5664, + "theoretical_loss": 3.521009024448468, + "tokens_seen": 1476395008 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027914744232698097, + "loss": 2.6042, + "theoretical_loss": 3.5209952024436504, + "tokens_seen": 1476460544 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027913741223671015, + "loss": 2.879, + "theoretical_loss": 3.520981381224117, + "tokens_seen": 1476526080 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027912738214643933, + "loss": 2.4008, + "theoretical_loss": 3.520967560789788, + "tokens_seen": 1476591616 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002791173520561685, + "loss": 2.8119, + "theoretical_loss": 3.5209537411405845, + "tokens_seen": 1476657152 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027910732196589775, + "loss": 2.5794, + "theoretical_loss": 3.520939922276426, + "tokens_seen": 1476722688 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027909729187562687, + "loss": 2.5422, + "theoretical_loss": 3.520926104197234, + "tokens_seen": 1476788224 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002790872617853561, + "loss": 2.8681, + "theoretical_loss": 3.520912286902929, + "tokens_seen": 1476853760 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027907723169508523, + "loss": 2.6222, + "theoretical_loss": 3.5208984703934316, + "tokens_seen": 1476919296 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027906720160481447, + "loss": 2.6332, + "theoretical_loss": 3.520884654668662, + "tokens_seen": 1476984832 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027905717151454365, + "loss": 2.9233, + "theoretical_loss": 3.5208708397285404, + "tokens_seen": 1477050368 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027904714142427283, + "loss": 2.8201, + "theoretical_loss": 3.5208570255729885, + "tokens_seen": 1477115904 + }, + { + "epoch": 0.45, + "learning_rate": 0.000279037111334002, + "loss": 2.7718, + "theoretical_loss": 3.5208432122019264, + "tokens_seen": 1477181440 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002790270812437312, + "loss": 2.7719, + "theoretical_loss": 3.520829399615275, + "tokens_seen": 1477246976 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002790170511534604, + "loss": 2.7933, + "theoretical_loss": 3.5208155878129546, + "tokens_seen": 1477312512 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002790070210631896, + "loss": 2.6974, + "theoretical_loss": 3.5208017767948863, + "tokens_seen": 1477378048 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027899699097291874, + "loss": 2.6619, + "theoretical_loss": 3.5207879665609902, + "tokens_seen": 1477443584 + }, + { + "epoch": 0.45, + "learning_rate": 0.000278986960882648, + "loss": 2.5814, + "theoretical_loss": 3.5207741571111875, + "tokens_seen": 1477509120 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002789769307923771, + "loss": 2.7302, + "theoretical_loss": 3.520760348445399, + "tokens_seen": 1477574656 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027896690070210634, + "loss": 2.6, + "theoretical_loss": 3.520746540563544, + "tokens_seen": 1477640192 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002789568706118355, + "loss": 2.7216, + "theoretical_loss": 3.520732733465546, + "tokens_seen": 1477705728 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002789468405215647, + "loss": 2.5873, + "theoretical_loss": 3.520718927151323, + "tokens_seen": 1477771264 + }, + { + "epoch": 0.45, + "objective/train/docs_used": 837925, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6238677501678467, + "objective/train/theoretical_loss": 3.520705121620797, + "objective/train/tokens_used": 1498296800, + "theoretical_loss": 3.520705121620797, + "tokens_seen": 1477836800 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002789368104312939, + "loss": 2.8242, + "theoretical_loss": 3.520705121620797, + "tokens_seen": 1477836800 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002789267803410231, + "loss": 2.6353, + "theoretical_loss": 3.5206913168738887, + "tokens_seen": 1477902336 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027891675025075224, + "loss": 2.8546, + "theoretical_loss": 3.520677512910519, + "tokens_seen": 1477967872 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002789067201604815, + "loss": 2.6426, + "theoretical_loss": 3.5206637097306084, + "tokens_seen": 1478033408 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002788966900702106, + "loss": 2.5846, + "theoretical_loss": 3.520649907334078, + "tokens_seen": 1478098944 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027888665997993984, + "loss": 2.5639, + "theoretical_loss": 3.5206361057208477, + "tokens_seen": 1478164480 + }, + { + "epoch": 0.45, + "learning_rate": 0.000278876629889669, + "loss": 2.5756, + "theoretical_loss": 3.5206223048908396, + "tokens_seen": 1478230016 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002788665997993982, + "loss": 2.8072, + "theoretical_loss": 3.520608504843974, + "tokens_seen": 1478295552 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002788565697091274, + "loss": 2.5982, + "theoretical_loss": 3.520594705580171, + "tokens_seen": 1478361088 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027884653961885656, + "loss": 2.7661, + "theoretical_loss": 3.520580907099353, + "tokens_seen": 1478426624 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027883650952858574, + "loss": 2.3985, + "theoretical_loss": 3.5205671094014397, + "tokens_seen": 1478492160 + }, + { + "epoch": 0.45, + "learning_rate": 0.000278826479438315, + "loss": 2.5743, + "theoretical_loss": 3.5205533124863524, + "tokens_seen": 1478557696 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002788164493480441, + "loss": 3.0573, + "theoretical_loss": 3.520539516354012, + "tokens_seen": 1478623232 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027880641925777334, + "loss": 2.5768, + "theoretical_loss": 3.520525721004339, + "tokens_seen": 1478688768 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027879638916750247, + "loss": 2.5012, + "theoretical_loss": 3.520511926437255, + "tokens_seen": 1478754304 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002787863590772317, + "loss": 2.5787, + "theoretical_loss": 3.52049813265268, + "tokens_seen": 1478819840 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002787763289869609, + "loss": 2.7102, + "theoretical_loss": 3.520484339650536, + "tokens_seen": 1478885376 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027876629889669007, + "loss": 2.6868, + "theoretical_loss": 3.520470547430743, + "tokens_seen": 1478950912 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027875626880641925, + "loss": 2.8525, + "theoretical_loss": 3.520456755993222, + "tokens_seen": 1479016448 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002787462387161485, + "loss": 2.76, + "theoretical_loss": 3.520442965337895, + "tokens_seen": 1479081984 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002787362086258776, + "loss": 2.6903, + "theoretical_loss": 3.520429175464682, + "tokens_seen": 1479147520 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027872617853560685, + "loss": 2.5357, + "theoretical_loss": 3.5204153863735046, + "tokens_seen": 1479213056 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027871614844533597, + "loss": 2.5798, + "theoretical_loss": 3.5204015980642835, + "tokens_seen": 1479278592 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002787061183550652, + "loss": 3.0257, + "theoretical_loss": 3.5203878105369397, + "tokens_seen": 1479344128 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002786960882647944, + "loss": 2.8914, + "theoretical_loss": 3.520374023791394, + "tokens_seen": 1479409664 + }, + { + "epoch": 0.45, + "objective/train/docs_used": 838738, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.653764486312866, + "objective/train/theoretical_loss": 3.520360237827568, + "objective/train/tokens_used": 1499935200, + "theoretical_loss": 3.520360237827568, + "tokens_seen": 1479475200 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027868605817452357, + "loss": 2.6139, + "theoretical_loss": 3.520360237827568, + "tokens_seen": 1479475200 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027867602808425275, + "loss": 2.7147, + "theoretical_loss": 3.5203464526453825, + "tokens_seen": 1479540736 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027866599799398193, + "loss": 2.8952, + "theoretical_loss": 3.5203326682447584, + "tokens_seen": 1479606272 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002786559679037111, + "loss": 2.6036, + "theoretical_loss": 3.520318884625617, + "tokens_seen": 1479671808 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027864593781344035, + "loss": 2.7428, + "theoretical_loss": 3.520305101787879, + "tokens_seen": 1479737344 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002786359077231695, + "loss": 2.7516, + "theoretical_loss": 3.5202913197314665, + "tokens_seen": 1479802880 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002786258776328987, + "loss": 2.7804, + "theoretical_loss": 3.520277538456299, + "tokens_seen": 1479868416 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002786158475426279, + "loss": 2.6136, + "theoretical_loss": 3.520263757962299, + "tokens_seen": 1479933952 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027860581745235707, + "loss": 2.7875, + "theoretical_loss": 3.520249978249387, + "tokens_seen": 1479999488 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027859578736208625, + "loss": 2.6069, + "theoretical_loss": 3.5202361993174844, + "tokens_seen": 1480065024 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027858575727181544, + "loss": 2.4373, + "theoretical_loss": 3.5202224211665123, + "tokens_seen": 1480130560 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002785757271815446, + "loss": 2.7815, + "theoretical_loss": 3.5202086437963915, + "tokens_seen": 1480196096 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027856569709127385, + "loss": 2.9926, + "theoretical_loss": 3.520194867207044, + "tokens_seen": 1480261632 + }, + { + "epoch": 0.45, + "learning_rate": 0.000278555667001003, + "loss": 2.5868, + "theoretical_loss": 3.52018109139839, + "tokens_seen": 1480327168 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002785456369107322, + "loss": 2.6405, + "theoretical_loss": 3.520167316370351, + "tokens_seen": 1480392704 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027853560682046134, + "loss": 2.9723, + "theoretical_loss": 3.520153542122849, + "tokens_seen": 1480458240 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002785255767301906, + "loss": 2.9442, + "theoretical_loss": 3.5201397686558042, + "tokens_seen": 1480523776 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002785155466399198, + "loss": 2.8925, + "theoretical_loss": 3.5201259959691384, + "tokens_seen": 1480589312 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027850551654964894, + "loss": 2.8234, + "theoretical_loss": 3.5201122240627725, + "tokens_seen": 1480654848 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002784954864593782, + "loss": 2.7389, + "theoretical_loss": 3.520098452936628, + "tokens_seen": 1480720384 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002784854563691073, + "loss": 2.6942, + "theoretical_loss": 3.5200846825906265, + "tokens_seen": 1480785920 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027847542627883654, + "loss": 2.7445, + "theoretical_loss": 3.5200709130246883, + "tokens_seen": 1480851456 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002784653961885657, + "loss": 2.8849, + "theoretical_loss": 3.5200571442387356, + "tokens_seen": 1480916992 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002784553660982949, + "loss": 2.6905, + "theoretical_loss": 3.5200433762326897, + "tokens_seen": 1480982528 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002784453360080241, + "loss": 2.5213, + "theoretical_loss": 3.520029609006471, + "tokens_seen": 1481048064 + }, + { + "epoch": 0.45, + "objective/train/docs_used": 839976, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6811907291412354, + "objective/train/theoretical_loss": 3.520015842560001, + "objective/train/tokens_used": 1501573600, + "theoretical_loss": 3.520015842560001, + "tokens_seen": 1481113600 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002784353059177533, + "loss": 2.5716, + "theoretical_loss": 3.520015842560001, + "tokens_seen": 1481113600 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027842527582748244, + "loss": 2.7463, + "theoretical_loss": 3.5200020768932028, + "tokens_seen": 1481179136 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002784152457372117, + "loss": 2.7665, + "theoretical_loss": 3.519988312005996, + "tokens_seen": 1481244672 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002784052156469408, + "loss": 2.7836, + "theoretical_loss": 3.5199745478983013, + "tokens_seen": 1481310208 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027839518555667004, + "loss": 2.803, + "theoretical_loss": 3.519960784570042, + "tokens_seen": 1481375744 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002783851554663992, + "loss": 2.6752, + "theoretical_loss": 3.519947022021139, + "tokens_seen": 1481441280 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002783751253761284, + "loss": 2.7718, + "theoretical_loss": 3.5199332602515128, + "tokens_seen": 1481506816 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002783650952858576, + "loss": 2.9027, + "theoretical_loss": 3.5199194992610856, + "tokens_seen": 1481572352 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027835506519558676, + "loss": 2.5081, + "theoretical_loss": 3.5199057390497783, + "tokens_seen": 1481637888 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027834503510531594, + "loss": 2.7769, + "theoretical_loss": 3.5198919796175128, + "tokens_seen": 1481703424 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002783350050150452, + "loss": 2.7453, + "theoretical_loss": 3.51987822096421, + "tokens_seen": 1481768960 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002783249749247743, + "loss": 2.8503, + "theoretical_loss": 3.5198644630897915, + "tokens_seen": 1481834496 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027831494483450354, + "loss": 2.6473, + "theoretical_loss": 3.5198507059941795, + "tokens_seen": 1481900032 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027830491474423267, + "loss": 2.4544, + "theoretical_loss": 3.5198369496772948, + "tokens_seen": 1481965568 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002782948846539619, + "loss": 2.4796, + "theoretical_loss": 3.5198231941390588, + "tokens_seen": 1482031104 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002782848545636911, + "loss": 2.5411, + "theoretical_loss": 3.5198094393793933, + "tokens_seen": 1482096640 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027827482447342027, + "loss": 2.7739, + "theoretical_loss": 3.5197956853982193, + "tokens_seen": 1482162176 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027826479438314945, + "loss": 2.7628, + "theoretical_loss": 3.5197819321954595, + "tokens_seen": 1482227712 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002782547642928787, + "loss": 2.6959, + "theoretical_loss": 3.5197681797710336, + "tokens_seen": 1482293248 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002782447342026078, + "loss": 2.7964, + "theoretical_loss": 3.519754428124865, + "tokens_seen": 1482358784 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027823470411233705, + "loss": 2.8914, + "theoretical_loss": 3.5197406772568742, + "tokens_seen": 1482424320 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027822467402206617, + "loss": 2.7016, + "theoretical_loss": 3.519726927166983, + "tokens_seen": 1482489856 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002782146439317954, + "loss": 2.7924, + "theoretical_loss": 3.519713177855113, + "tokens_seen": 1482555392 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002782046138415246, + "loss": 2.733, + "theoretical_loss": 3.519699429321186, + "tokens_seen": 1482620928 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027819458375125377, + "loss": 2.5954, + "theoretical_loss": 3.519685681565123, + "tokens_seen": 1482686464 + }, + { + "epoch": 0.45, + "objective/train/docs_used": 840589, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.8037850856781006, + "objective/train/theoretical_loss": 3.519671934586846, + "objective/train/tokens_used": 1503212000, + "theoretical_loss": 3.519671934586846, + "tokens_seen": 1482752000 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027818455366098295, + "loss": 2.6653, + "theoretical_loss": 3.519671934586846, + "tokens_seen": 1482752000 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027817452357071213, + "loss": 2.831, + "theoretical_loss": 3.519658188386277, + "tokens_seen": 1482817536 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002781644934804413, + "loss": 2.8794, + "theoretical_loss": 3.5196444429633367, + "tokens_seen": 1482883072 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027815446339017055, + "loss": 2.6329, + "theoretical_loss": 3.5196306983179477, + "tokens_seen": 1482948608 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002781444332998997, + "loss": 2.629, + "theoretical_loss": 3.519616954450031, + "tokens_seen": 1483014144 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002781344032096289, + "loss": 2.8906, + "theoretical_loss": 3.5196032113595086, + "tokens_seen": 1483079680 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002781243731193581, + "loss": 2.6507, + "theoretical_loss": 3.519589469046302, + "tokens_seen": 1483145216 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002781143430290873, + "loss": 2.854, + "theoretical_loss": 3.519575727510333, + "tokens_seen": 1483210752 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027810431293881645, + "loss": 2.7315, + "theoretical_loss": 3.5195619867515235, + "tokens_seen": 1483276288 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027809428284854564, + "loss": 2.7497, + "theoretical_loss": 3.5195482467697943, + "tokens_seen": 1483341824 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002780842527582748, + "loss": 2.7572, + "theoretical_loss": 3.5195345075650684, + "tokens_seen": 1483407360 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027807422266800405, + "loss": 2.736, + "theoretical_loss": 3.5195207691372667, + "tokens_seen": 1483472896 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002780641925777332, + "loss": 2.6463, + "theoretical_loss": 3.5195070314863113, + "tokens_seen": 1483538432 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002780541624874624, + "loss": 2.4682, + "theoretical_loss": 3.519493294612124, + "tokens_seen": 1483603968 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027804413239719154, + "loss": 2.6499, + "theoretical_loss": 3.519479558514626, + "tokens_seen": 1483669504 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002780341023069208, + "loss": 2.7896, + "theoretical_loss": 3.51946582319374, + "tokens_seen": 1483735040 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027802407221664996, + "loss": 2.6566, + "theoretical_loss": 3.519452088649387, + "tokens_seen": 1483800576 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027801404212637914, + "loss": 2.6473, + "theoretical_loss": 3.5194383548814896, + "tokens_seen": 1483866112 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002780040120361083, + "loss": 2.4884, + "theoretical_loss": 3.5194246218899683, + "tokens_seen": 1483931648 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002779939819458375, + "loss": 2.7638, + "theoretical_loss": 3.519410889674746, + "tokens_seen": 1483997184 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002779839518555667, + "loss": 2.7682, + "theoretical_loss": 3.519397158235745, + "tokens_seen": 1484062720 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002779739217652959, + "loss": 2.8429, + "theoretical_loss": 3.5193834275728855, + "tokens_seen": 1484128256 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027796389167502504, + "loss": 2.636, + "theoretical_loss": 3.5193696976860904, + "tokens_seen": 1484193792 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002779538615847543, + "loss": 2.6659, + "theoretical_loss": 3.519355968575282, + "tokens_seen": 1484259328 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027794383149448346, + "loss": 2.4421, + "theoretical_loss": 3.5193422402403813, + "tokens_seen": 1484324864 + }, + { + "epoch": 0.45, + "objective/train/docs_used": 841910, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6618845462799072, + "objective/train/theoretical_loss": 3.5193285126813105, + "objective/train/tokens_used": 1504850400, + "theoretical_loss": 3.5193285126813105, + "tokens_seen": 1484390400 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027793380140421264, + "loss": 2.728, + "theoretical_loss": 3.5193285126813105, + "tokens_seen": 1484390400 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002779237713139418, + "loss": 2.6565, + "theoretical_loss": 3.5193147858979916, + "tokens_seen": 1484455936 + }, + { + "epoch": 0.45, + "learning_rate": 0.000277913741223671, + "loss": 2.646, + "theoretical_loss": 3.5193010598903465, + "tokens_seen": 1484521472 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002779037111334002, + "loss": 2.5919, + "theoretical_loss": 3.519287334658297, + "tokens_seen": 1484587008 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002778936810431294, + "loss": 2.6531, + "theoretical_loss": 3.5192736102017648, + "tokens_seen": 1484652544 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027788365095285855, + "loss": 2.661, + "theoretical_loss": 3.519259886520673, + "tokens_seen": 1484718080 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002778736208625878, + "loss": 2.764, + "theoretical_loss": 3.519246163614942, + "tokens_seen": 1484783616 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002778635907723169, + "loss": 2.68, + "theoretical_loss": 3.519232441484495, + "tokens_seen": 1484849152 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027785356068204614, + "loss": 2.7573, + "theoretical_loss": 3.519218720129253, + "tokens_seen": 1484914688 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002778435305917753, + "loss": 2.5998, + "theoretical_loss": 3.519204999549139, + "tokens_seen": 1484980224 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002778335005015045, + "loss": 2.5989, + "theoretical_loss": 3.519191279744074, + "tokens_seen": 1485045760 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002778234704112337, + "loss": 2.6599, + "theoretical_loss": 3.519177560713981, + "tokens_seen": 1485111296 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027781344032096287, + "loss": 2.4882, + "theoretical_loss": 3.5191638424587808, + "tokens_seen": 1485176832 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027780341023069205, + "loss": 2.4336, + "theoretical_loss": 3.519150124978397, + "tokens_seen": 1485242368 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002777933801404213, + "loss": 2.6234, + "theoretical_loss": 3.5191364082727503, + "tokens_seen": 1485307904 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002777833500501504, + "loss": 2.7044, + "theoretical_loss": 3.519122692341764, + "tokens_seen": 1485373440 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027777331995987965, + "loss": 2.9693, + "theoretical_loss": 3.5191089771853585, + "tokens_seen": 1485438976 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002777632898696089, + "loss": 2.7428, + "theoretical_loss": 3.5190952628034573, + "tokens_seen": 1485504512 + }, + { + "epoch": 0.45, + "learning_rate": 0.000277753259779338, + "loss": 2.5534, + "theoretical_loss": 3.519081549195982, + "tokens_seen": 1485570048 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027774322968906725, + "loss": 2.9384, + "theoretical_loss": 3.519067836362855, + "tokens_seen": 1485635584 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027773319959879637, + "loss": 2.7474, + "theoretical_loss": 3.519054124303998, + "tokens_seen": 1485701120 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002777231695085256, + "loss": 2.7464, + "theoretical_loss": 3.519040413019334, + "tokens_seen": 1485766656 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002777131394182548, + "loss": 2.7428, + "theoretical_loss": 3.519026702508783, + "tokens_seen": 1485832192 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027770310932798397, + "loss": 2.5267, + "theoretical_loss": 3.5190129927722698, + "tokens_seen": 1485897728 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027769307923771315, + "loss": 2.8613, + "theoretical_loss": 3.518999283809715, + "tokens_seen": 1485963264 + }, + { + "epoch": 0.45, + "objective/train/docs_used": 842517, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.8399932384490967, + "objective/train/theoretical_loss": 3.5189855756210413, + "objective/train/tokens_used": 1506488800, + "theoretical_loss": 3.5189855756210413, + "tokens_seen": 1486028800 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027768304914744233, + "loss": 2.6981, + "theoretical_loss": 3.5189855756210413, + "tokens_seen": 1486028800 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002776730190571715, + "loss": 2.862, + "theoretical_loss": 3.5189718682061706, + "tokens_seen": 1486094336 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027766298896690075, + "loss": 2.7635, + "theoretical_loss": 3.518958161565025, + "tokens_seen": 1486159872 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002776529588766299, + "loss": 2.5662, + "theoretical_loss": 3.518944455697527, + "tokens_seen": 1486225408 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002776429287863591, + "loss": 2.6077, + "theoretical_loss": 3.5189307506035994, + "tokens_seen": 1486290944 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002776328986960883, + "loss": 2.6199, + "theoretical_loss": 3.518917046283163, + "tokens_seen": 1486356480 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002776228686058175, + "loss": 2.6772, + "theoretical_loss": 3.5189033427361416, + "tokens_seen": 1486422016 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027761283851554665, + "loss": 2.7694, + "theoretical_loss": 3.518889639962456, + "tokens_seen": 1486487552 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027760280842527584, + "loss": 2.6747, + "theoretical_loss": 3.51887593796203, + "tokens_seen": 1486553088 + }, + { + "epoch": 0.45, + "learning_rate": 0.000277592778335005, + "loss": 2.8542, + "theoretical_loss": 3.5188622367347846, + "tokens_seen": 1486618624 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027758274824473425, + "loss": 2.692, + "theoretical_loss": 3.5188485362806423, + "tokens_seen": 1486684160 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002775727181544634, + "loss": 2.6838, + "theoretical_loss": 3.5188348365995257, + "tokens_seen": 1486749696 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002775626880641926, + "loss": 2.738, + "theoretical_loss": 3.5188211376913574, + "tokens_seen": 1486815232 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027755265797392174, + "loss": 2.5795, + "theoretical_loss": 3.518807439556059, + "tokens_seen": 1486880768 + }, + { + "epoch": 0.45, + "learning_rate": 0.000277542627883651, + "loss": 2.6148, + "theoretical_loss": 3.5187937421935533, + "tokens_seen": 1486946304 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027753259779338016, + "loss": 2.8262, + "theoretical_loss": 3.518780045603763, + "tokens_seen": 1487011840 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027752256770310934, + "loss": 2.7024, + "theoretical_loss": 3.5187663497866097, + "tokens_seen": 1487077376 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002775125376128385, + "loss": 2.7371, + "theoretical_loss": 3.518752654742016, + "tokens_seen": 1487142912 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002775025075225677, + "loss": 2.6951, + "theoretical_loss": 3.5187389604699044, + "tokens_seen": 1487208448 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002774924774322969, + "loss": 2.5398, + "theoretical_loss": 3.518725266970197, + "tokens_seen": 1487273984 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002774824473420261, + "loss": 2.6907, + "theoretical_loss": 3.5187115742428166, + "tokens_seen": 1487339520 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027747241725175524, + "loss": 2.5485, + "theoretical_loss": 3.5186978822876855, + "tokens_seen": 1487405056 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002774623871614845, + "loss": 2.822, + "theoretical_loss": 3.518684191104726, + "tokens_seen": 1487470592 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027745235707121366, + "loss": 2.8993, + "theoretical_loss": 3.518670500693861, + "tokens_seen": 1487536128 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027744232698094284, + "loss": 2.7392, + "theoretical_loss": 3.518656811055012, + "tokens_seen": 1487601664 + }, + { + "epoch": 0.45, + "objective/train/docs_used": 843832, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.814084768295288, + "objective/train/theoretical_loss": 3.5186431221881023, + "objective/train/tokens_used": 1508127200, + "theoretical_loss": 3.5186431221881023, + "tokens_seen": 1487667200 + }, + { + "epoch": 0.45, + "learning_rate": 0.000277432296890672, + "loss": 2.8174, + "theoretical_loss": 3.5186431221881023, + "tokens_seen": 1487667200 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002774222668004012, + "loss": 2.302, + "theoretical_loss": 3.5186294340930537, + "tokens_seen": 1487732736 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002774122367101304, + "loss": 2.533, + "theoretical_loss": 3.5186157467697896, + "tokens_seen": 1487798272 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002774022066198596, + "loss": 2.7901, + "theoretical_loss": 3.5186020602182317, + "tokens_seen": 1487863808 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027739217652958875, + "loss": 2.8078, + "theoretical_loss": 3.518588374438303, + "tokens_seen": 1487929344 + }, + { + "epoch": 0.45, + "learning_rate": 0.000277382146439318, + "loss": 2.5966, + "theoretical_loss": 3.5185746894299257, + "tokens_seen": 1487994880 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002773721163490471, + "loss": 2.6267, + "theoretical_loss": 3.518561005193022, + "tokens_seen": 1488060416 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027736208625877634, + "loss": 2.5304, + "theoretical_loss": 3.5185473217275147, + "tokens_seen": 1488125952 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002773520561685055, + "loss": 2.6197, + "theoretical_loss": 3.5185336390333273, + "tokens_seen": 1488191488 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002773420260782347, + "loss": 2.7176, + "theoretical_loss": 3.5185199571103807, + "tokens_seen": 1488257024 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002773319959879639, + "loss": 2.7242, + "theoretical_loss": 3.518506275958599, + "tokens_seen": 1488322560 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027732196589769307, + "loss": 2.5117, + "theoretical_loss": 3.5184925955779036, + "tokens_seen": 1488388096 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027731193580742225, + "loss": 2.7704, + "theoretical_loss": 3.5184789159682177, + "tokens_seen": 1488453632 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002773019057171515, + "loss": 2.639, + "theoretical_loss": 3.518465237129464, + "tokens_seen": 1488519168 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002772918756268806, + "loss": 2.5032, + "theoretical_loss": 3.518451559061565, + "tokens_seen": 1488584704 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027728184553660985, + "loss": 2.7552, + "theoretical_loss": 3.5184378817644424, + "tokens_seen": 1488650240 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027727181544633903, + "loss": 2.7989, + "theoretical_loss": 3.5184242052380204, + "tokens_seen": 1488715776 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002772617853560682, + "loss": 2.7634, + "theoretical_loss": 3.5184105294822206, + "tokens_seen": 1488781312 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002772517552657974, + "loss": 2.6477, + "theoretical_loss": 3.5183968544969657, + "tokens_seen": 1488846848 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027724172517552657, + "loss": 2.7019, + "theoretical_loss": 3.5183831802821794, + "tokens_seen": 1488912384 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027723169508525575, + "loss": 2.8042, + "theoretical_loss": 3.518369506837783, + "tokens_seen": 1488977920 + }, + { + "epoch": 0.45, + "learning_rate": 0.000277221664994985, + "loss": 2.9483, + "theoretical_loss": 3.5183558341637, + "tokens_seen": 1489043456 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002772116349047141, + "loss": 2.61, + "theoretical_loss": 3.5183421622598527, + "tokens_seen": 1489108992 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027720160481444335, + "loss": 2.6234, + "theoretical_loss": 3.518328491126164, + "tokens_seen": 1489174528 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002771915747241725, + "loss": 2.6492, + "theoretical_loss": 3.5183148207625567, + "tokens_seen": 1489240064 + }, + { + "epoch": 0.45, + "objective/train/docs_used": 844519, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.9994213581085205, + "objective/train/theoretical_loss": 3.518301151168954, + "objective/train/tokens_used": 1509765600, + "theoretical_loss": 3.518301151168954, + "tokens_seen": 1489305600 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002771815446339017, + "loss": 2.839, + "theoretical_loss": 3.518301151168954, + "tokens_seen": 1489305600 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002771715145436309, + "loss": 2.6383, + "theoretical_loss": 3.5182874823452774, + "tokens_seen": 1489371136 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002771614844533601, + "loss": 2.7612, + "theoretical_loss": 3.518273814291451, + "tokens_seen": 1489436672 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027715145436308926, + "loss": 2.7621, + "theoretical_loss": 3.518260147007396, + "tokens_seen": 1489502208 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002771414242728185, + "loss": 2.6795, + "theoretical_loss": 3.5182464804930373, + "tokens_seen": 1489567744 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002771313941825476, + "loss": 2.7338, + "theoretical_loss": 3.5182328147482957, + "tokens_seen": 1489633280 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027712136409227685, + "loss": 2.7346, + "theoretical_loss": 3.5182191497730955, + "tokens_seen": 1489698816 + }, + { + "epoch": 0.45, + "learning_rate": 0.000277111334002006, + "loss": 2.5702, + "theoretical_loss": 3.5182054855673583, + "tokens_seen": 1489764352 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002771013039117352, + "loss": 2.6421, + "theoretical_loss": 3.5181918221310076, + "tokens_seen": 1489829888 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002770912738214644, + "loss": 2.8654, + "theoretical_loss": 3.5181781594639663, + "tokens_seen": 1489895424 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002770812437311936, + "loss": 2.8158, + "theoretical_loss": 3.518164497566157, + "tokens_seen": 1489960960 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027707121364092276, + "loss": 2.7524, + "theoretical_loss": 3.5181508364375023, + "tokens_seen": 1490026496 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027706118355065194, + "loss": 2.5381, + "theoretical_loss": 3.518137176077926, + "tokens_seen": 1490092032 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002770511534603811, + "loss": 2.7, + "theoretical_loss": 3.51812351648735, + "tokens_seen": 1490157568 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027704112337011036, + "loss": 2.6231, + "theoretical_loss": 3.518109857665697, + "tokens_seen": 1490223104 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002770310932798395, + "loss": 2.6034, + "theoretical_loss": 3.518096199612891, + "tokens_seen": 1490288640 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002770210631895687, + "loss": 2.4269, + "theoretical_loss": 3.5180825423288544, + "tokens_seen": 1490354176 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002770110330992979, + "loss": 2.535, + "theoretical_loss": 3.51806888581351, + "tokens_seen": 1490419712 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002770010030090271, + "loss": 2.7618, + "theoretical_loss": 3.518055230066781, + "tokens_seen": 1490485248 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002769909729187563, + "loss": 2.5156, + "theoretical_loss": 3.51804157508859, + "tokens_seen": 1490550784 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027698094282848544, + "loss": 2.4475, + "theoretical_loss": 3.5180279208788603, + "tokens_seen": 1490616320 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002769709127382147, + "loss": 2.5123, + "theoretical_loss": 3.5180142674375148, + "tokens_seen": 1490681856 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027696088264794386, + "loss": 2.903, + "theoretical_loss": 3.5180006147644756, + "tokens_seen": 1490747392 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027695085255767304, + "loss": 2.4743, + "theoretical_loss": 3.517986962859667, + "tokens_seen": 1490812928 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002769408224674022, + "loss": 2.7047, + "theoretical_loss": 3.5179733117230123, + "tokens_seen": 1490878464 + }, + { + "epoch": 0.45, + "objective/train/docs_used": 845874, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.5407445430755615, + "objective/train/theoretical_loss": 3.517959661354433, + "objective/train/tokens_used": 1511404000, + "theoretical_loss": 3.517959661354433, + "tokens_seen": 1490944000 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002769307923771314, + "loss": 2.6104, + "theoretical_loss": 3.517959661354433, + "tokens_seen": 1490944000 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002769207622868606, + "loss": 2.5006, + "theoretical_loss": 3.5179460117538524, + "tokens_seen": 1491009536 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002769107321965898, + "loss": 2.5745, + "theoretical_loss": 3.5179323629211945, + "tokens_seen": 1491075072 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027690070210631895, + "loss": 2.6263, + "theoretical_loss": 3.517918714856382, + "tokens_seen": 1491140608 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002768906720160482, + "loss": 2.9346, + "theoretical_loss": 3.5179050675593375, + "tokens_seen": 1491206144 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002768806419257773, + "loss": 2.827, + "theoretical_loss": 3.517891421029984, + "tokens_seen": 1491271680 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027687061183550655, + "loss": 2.837, + "theoretical_loss": 3.5178777752682455, + "tokens_seen": 1491337216 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002768605817452357, + "loss": 2.7251, + "theoretical_loss": 3.5178641302740443, + "tokens_seen": 1491402752 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002768505516549649, + "loss": 2.8094, + "theoretical_loss": 3.517850486047304, + "tokens_seen": 1491468288 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002768405215646941, + "loss": 2.8271, + "theoretical_loss": 3.517836842587947, + "tokens_seen": 1491533824 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027683049147442327, + "loss": 2.6593, + "theoretical_loss": 3.517823199895897, + "tokens_seen": 1491599360 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027682046138415245, + "loss": 2.536, + "theoretical_loss": 3.517809557971077, + "tokens_seen": 1491664896 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002768104312938817, + "loss": 2.7988, + "theoretical_loss": 3.51779591681341, + "tokens_seen": 1491730432 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002768004012036108, + "loss": 2.6265, + "theoretical_loss": 3.5177822764228193, + "tokens_seen": 1491795968 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027679037111334005, + "loss": 2.8356, + "theoretical_loss": 3.5177686367992287, + "tokens_seen": 1491861504 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027678034102306923, + "loss": 2.8025, + "theoretical_loss": 3.51775499794256, + "tokens_seen": 1491927040 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002767703109327984, + "loss": 2.4556, + "theoretical_loss": 3.517741359852738, + "tokens_seen": 1491992576 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002767602808425276, + "loss": 2.5701, + "theoretical_loss": 3.517727722529684, + "tokens_seen": 1492058112 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027675025075225677, + "loss": 2.7147, + "theoretical_loss": 3.517714085973323, + "tokens_seen": 1492123648 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027674022066198595, + "loss": 2.4684, + "theoretical_loss": 3.517700450183577, + "tokens_seen": 1492189184 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002767301905717152, + "loss": 2.8393, + "theoretical_loss": 3.5176868151603697, + "tokens_seen": 1492254720 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002767201604814443, + "loss": 2.5406, + "theoretical_loss": 3.5176731809036252, + "tokens_seen": 1492320256 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027671013039117355, + "loss": 2.6433, + "theoretical_loss": 3.517659547413265, + "tokens_seen": 1492385792 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002767001003009027, + "loss": 2.81, + "theoretical_loss": 3.517645914689214, + "tokens_seen": 1492451328 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002766900702106319, + "loss": 2.6093, + "theoretical_loss": 3.5176322827313946, + "tokens_seen": 1492516864 + }, + { + "epoch": 0.45, + "objective/train/docs_used": 846561, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.568142890930176, + "objective/train/theoretical_loss": 3.5176186515397303, + "objective/train/tokens_used": 1513042400, + "theoretical_loss": 3.5176186515397303, + "tokens_seen": 1492582400 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002766800401203611, + "loss": 2.5709, + "theoretical_loss": 3.5176186515397303, + "tokens_seen": 1492582400 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002766700100300903, + "loss": 2.6604, + "theoretical_loss": 3.517605021114144, + "tokens_seen": 1492647936 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027665997993981946, + "loss": 2.6366, + "theoretical_loss": 3.51759139145456, + "tokens_seen": 1492713472 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002766499498495487, + "loss": 2.5809, + "theoretical_loss": 3.5175777625609004, + "tokens_seen": 1492779008 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002766399197592778, + "loss": 2.6301, + "theoretical_loss": 3.5175641344330892, + "tokens_seen": 1492844544 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027662988966900705, + "loss": 2.6955, + "theoretical_loss": 3.51755050707105, + "tokens_seen": 1492910080 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002766198595787362, + "loss": 2.7078, + "theoretical_loss": 3.517536880474706, + "tokens_seen": 1492975616 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002766098294884654, + "loss": 2.6849, + "theoretical_loss": 3.51752325464398, + "tokens_seen": 1493041152 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002765997993981946, + "loss": 2.6355, + "theoretical_loss": 3.5175096295787958, + "tokens_seen": 1493106688 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002765897693079238, + "loss": 2.8213, + "theoretical_loss": 3.517496005279077, + "tokens_seen": 1493172224 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027657973921765296, + "loss": 2.7427, + "theoretical_loss": 3.5174823817447467, + "tokens_seen": 1493237760 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027656970912738214, + "loss": 2.6997, + "theoretical_loss": 3.5174687589757285, + "tokens_seen": 1493303296 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002765596790371113, + "loss": 2.5672, + "theoretical_loss": 3.517455136971946, + "tokens_seen": 1493368832 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027654964894684056, + "loss": 2.463, + "theoretical_loss": 3.5174415157333216, + "tokens_seen": 1493434368 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002765396188565697, + "loss": 2.655, + "theoretical_loss": 3.5174278952597806, + "tokens_seen": 1493499904 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002765295887662989, + "loss": 2.5403, + "theoretical_loss": 3.5174142755512445, + "tokens_seen": 1493565440 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027651955867602805, + "loss": 2.741, + "theoretical_loss": 3.517400656607638, + "tokens_seen": 1493630976 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002765095285857573, + "loss": 2.794, + "theoretical_loss": 3.5173870384288843, + "tokens_seen": 1493696512 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027649949849548646, + "loss": 2.6246, + "theoretical_loss": 3.5173734210149066, + "tokens_seen": 1493762048 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027648946840521564, + "loss": 2.6515, + "theoretical_loss": 3.5173598043656282, + "tokens_seen": 1493827584 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002764794383149448, + "loss": 2.8013, + "theoretical_loss": 3.517346188480974, + "tokens_seen": 1493893120 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027646940822467406, + "loss": 2.5686, + "theoretical_loss": 3.517332573360866, + "tokens_seen": 1493958656 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002764593781344032, + "loss": 2.7098, + "theoretical_loss": 3.517318959005228, + "tokens_seen": 1494024192 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002764493480441324, + "loss": 2.5605, + "theoretical_loss": 3.517305345413984, + "tokens_seen": 1494089728 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027643931795386155, + "loss": 2.6674, + "theoretical_loss": 3.517291732587058, + "tokens_seen": 1494155264 + }, + { + "epoch": 0.45, + "objective/train/docs_used": 848095, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.015695571899414, + "objective/train/theoretical_loss": 3.517278120524372, + "objective/train/tokens_used": 1514680800, + "theoretical_loss": 3.517278120524372, + "tokens_seen": 1494220800 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002764292878635908, + "loss": 2.7895, + "theoretical_loss": 3.517278120524372, + "tokens_seen": 1494220800 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027641925777331997, + "loss": 2.9269, + "theoretical_loss": 3.517264509225851, + "tokens_seen": 1494286336 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027640922768304915, + "loss": 2.7599, + "theoretical_loss": 3.517250898691418, + "tokens_seen": 1494351872 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027639919759277833, + "loss": 2.644, + "theoretical_loss": 3.517237288920997, + "tokens_seen": 1494417408 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002763891675025075, + "loss": 2.6497, + "theoretical_loss": 3.517223679914511, + "tokens_seen": 1494482944 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002763791374122367, + "loss": 2.6683, + "theoretical_loss": 3.5172100716718844, + "tokens_seen": 1494548480 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002763691073219659, + "loss": 2.8978, + "theoretical_loss": 3.5171964641930398, + "tokens_seen": 1494614016 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027635907723169505, + "loss": 2.6628, + "theoretical_loss": 3.517182857477902, + "tokens_seen": 1494679552 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002763490471414243, + "loss": 2.5669, + "theoretical_loss": 3.5171692515263935, + "tokens_seen": 1494745088 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002763390170511534, + "loss": 2.5354, + "theoretical_loss": 3.5171556463384386, + "tokens_seen": 1494810624 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027632898696088265, + "loss": 2.808, + "theoretical_loss": 3.5171420419139614, + "tokens_seen": 1494876160 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027631895687061183, + "loss": 2.5778, + "theoretical_loss": 3.5171284382528847, + "tokens_seen": 1494941696 + }, + { + "epoch": 0.45, + "learning_rate": 0.000276308926780341, + "loss": 2.5639, + "theoretical_loss": 3.517114835355133, + "tokens_seen": 1495007232 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002762988966900702, + "loss": 2.697, + "theoretical_loss": 3.5171012332206293, + "tokens_seen": 1495072768 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027628886659979943, + "loss": 2.759, + "theoretical_loss": 3.517087631849298, + "tokens_seen": 1495138304 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027627883650952856, + "loss": 2.6099, + "theoretical_loss": 3.517074031241062, + "tokens_seen": 1495203840 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002762688064192578, + "loss": 2.6761, + "theoretical_loss": 3.5170604313958456, + "tokens_seen": 1495269376 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027625877632898697, + "loss": 2.5862, + "theoretical_loss": 3.5170468323135724, + "tokens_seen": 1495334912 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027624874623871615, + "loss": 2.5378, + "theoretical_loss": 3.517033233994167, + "tokens_seen": 1495400448 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002762387161484454, + "loss": 2.809, + "theoretical_loss": 3.5170196364375514, + "tokens_seen": 1495465984 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002762286860581745, + "loss": 2.6776, + "theoretical_loss": 3.5170060396436504, + "tokens_seen": 1495531520 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027621865596790375, + "loss": 2.497, + "theoretical_loss": 3.5169924436123887, + "tokens_seen": 1495597056 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002762086258776329, + "loss": 2.7444, + "theoretical_loss": 3.5169788483436886, + "tokens_seen": 1495662592 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002761985957873621, + "loss": 2.8884, + "theoretical_loss": 3.5169652538374745, + "tokens_seen": 1495728128 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002761885656970913, + "loss": 2.3662, + "theoretical_loss": 3.51695166009367, + "tokens_seen": 1495793664 + }, + { + "epoch": 0.45, + "objective/train/docs_used": 848766, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.433872938156128, + "objective/train/theoretical_loss": 3.5169380671122, + "objective/train/tokens_used": 1516319200, + "theoretical_loss": 3.5169380671122, + "tokens_seen": 1495859200 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002761785356068205, + "loss": 2.8104, + "theoretical_loss": 3.5169380671122, + "tokens_seen": 1495859200 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027616850551654966, + "loss": 2.7316, + "theoretical_loss": 3.516924474892987, + "tokens_seen": 1495924736 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002761584754262789, + "loss": 2.4721, + "theoretical_loss": 3.5169108834359557, + "tokens_seen": 1495990272 + }, + { + "epoch": 0.45, + "learning_rate": 0.000276148445336008, + "loss": 2.4274, + "theoretical_loss": 3.5168972927410294, + "tokens_seen": 1496055808 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027613841524573725, + "loss": 2.533, + "theoretical_loss": 3.5168837028081326, + "tokens_seen": 1496121344 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002761283851554664, + "loss": 2.7172, + "theoretical_loss": 3.5168701136371885, + "tokens_seen": 1496186880 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002761183550651956, + "loss": 2.7499, + "theoretical_loss": 3.516856525228121, + "tokens_seen": 1496252416 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002761083249749248, + "loss": 2.5816, + "theoretical_loss": 3.516842937580855, + "tokens_seen": 1496317952 + }, + { + "epoch": 0.45, + "learning_rate": 0.000276098294884654, + "loss": 2.6538, + "theoretical_loss": 3.516829350695314, + "tokens_seen": 1496383488 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027608826479438316, + "loss": 2.52, + "theoretical_loss": 3.5168157645714215, + "tokens_seen": 1496449024 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027607823470411234, + "loss": 2.8843, + "theoretical_loss": 3.516802179209101, + "tokens_seen": 1496514560 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002760682046138415, + "loss": 2.7433, + "theoretical_loss": 3.516788594608278, + "tokens_seen": 1496580096 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027605817452357076, + "loss": 2.6969, + "theoretical_loss": 3.5167750107688756, + "tokens_seen": 1496645632 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002760481444332999, + "loss": 2.5173, + "theoretical_loss": 3.5167614276908177, + "tokens_seen": 1496711168 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002760381143430291, + "loss": 2.6249, + "theoretical_loss": 3.5167478453740286, + "tokens_seen": 1496776704 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027602808425275825, + "loss": 2.4074, + "theoretical_loss": 3.5167342638184316, + "tokens_seen": 1496842240 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002760180541624875, + "loss": 2.7976, + "theoretical_loss": 3.5167206830239515, + "tokens_seen": 1496907776 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027600802407221666, + "loss": 2.4929, + "theoretical_loss": 3.5167071029905124, + "tokens_seen": 1496973312 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027599799398194584, + "loss": 2.7588, + "theoretical_loss": 3.516693523718038, + "tokens_seen": 1497038848 + }, + { + "epoch": 0.45, + "learning_rate": 0.000275987963891675, + "loss": 2.6733, + "theoretical_loss": 3.5166799452064517, + "tokens_seen": 1497104384 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027597793380140426, + "loss": 2.8508, + "theoretical_loss": 3.5166663674556786, + "tokens_seen": 1497169920 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002759679037111334, + "loss": 2.7249, + "theoretical_loss": 3.5166527904656424, + "tokens_seen": 1497235456 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002759578736208626, + "loss": 2.7324, + "theoretical_loss": 3.5166392142362675, + "tokens_seen": 1497300992 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027594784353059175, + "loss": 2.7466, + "theoretical_loss": 3.516625638767477, + "tokens_seen": 1497366528 + }, + { + "epoch": 0.45, + "learning_rate": 0.000275937813440321, + "loss": 2.7154, + "theoretical_loss": 3.516612064059196, + "tokens_seen": 1497432064 + }, + { + "epoch": 0.45, + "objective/train/docs_used": 849888, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.092176675796509, + "objective/train/theoretical_loss": 3.516598490111348, + "objective/train/tokens_used": 1517957600, + "theoretical_loss": 3.516598490111348, + "tokens_seen": 1497497600 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027592778335005017, + "loss": 2.524, + "theoretical_loss": 3.516598490111348, + "tokens_seen": 1497497600 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027591775325977935, + "loss": 2.5878, + "theoretical_loss": 3.516584916923858, + "tokens_seen": 1497563136 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027590772316950853, + "loss": 2.6479, + "theoretical_loss": 3.516571344496649, + "tokens_seen": 1497628672 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002758976930792377, + "loss": 2.5324, + "theoretical_loss": 3.516557772829646, + "tokens_seen": 1497694208 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002758876629889669, + "loss": 2.7623, + "theoretical_loss": 3.516544201922773, + "tokens_seen": 1497759744 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002758776328986961, + "loss": 2.6869, + "theoretical_loss": 3.5165306317759537, + "tokens_seen": 1497825280 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027586760280842525, + "loss": 2.6903, + "theoretical_loss": 3.516517062389112, + "tokens_seen": 1497890816 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002758575727181545, + "loss": 2.69, + "theoretical_loss": 3.5165034937621735, + "tokens_seen": 1497956352 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002758475426278836, + "loss": 2.8369, + "theoretical_loss": 3.5164899258950615, + "tokens_seen": 1498021888 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027583751253761285, + "loss": 2.7037, + "theoretical_loss": 3.5164763587877, + "tokens_seen": 1498087424 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027582748244734203, + "loss": 2.6691, + "theoretical_loss": 3.516462792440014, + "tokens_seen": 1498152960 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002758174523570712, + "loss": 2.5442, + "theoretical_loss": 3.5164492268519267, + "tokens_seen": 1498218496 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002758074222668004, + "loss": 2.8095, + "theoretical_loss": 3.516435662023363, + "tokens_seen": 1498284032 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027579739217652963, + "loss": 2.6089, + "theoretical_loss": 3.5164220979542478, + "tokens_seen": 1498349568 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027578736208625876, + "loss": 2.6948, + "theoretical_loss": 3.5164085346445035, + "tokens_seen": 1498415104 + }, + { + "epoch": 0.45, + "learning_rate": 0.000275777331995988, + "loss": 2.5956, + "theoretical_loss": 3.516394972094056, + "tokens_seen": 1498480640 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002757673019057171, + "loss": 2.5102, + "theoretical_loss": 3.516381410302829, + "tokens_seen": 1498546176 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027575727181544635, + "loss": 2.492, + "theoretical_loss": 3.5163678492707473, + "tokens_seen": 1498611712 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027574724172517554, + "loss": 2.6877, + "theoretical_loss": 3.5163542889977344, + "tokens_seen": 1498677248 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002757372116349047, + "loss": 2.5613, + "theoretical_loss": 3.516340729483715, + "tokens_seen": 1498742784 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002757271815446339, + "loss": 2.7028, + "theoretical_loss": 3.516327170728614, + "tokens_seen": 1498808320 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002757171514543631, + "loss": 2.647, + "theoretical_loss": 3.5163136127323544, + "tokens_seen": 1498873856 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027570712136409226, + "loss": 2.6637, + "theoretical_loss": 3.516300055494862, + "tokens_seen": 1498939392 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002756970912738215, + "loss": 2.695, + "theoretical_loss": 3.5162864990160596, + "tokens_seen": 1499004928 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002756870611835506, + "loss": 2.5432, + "theoretical_loss": 3.5162729432958733, + "tokens_seen": 1499070464 + }, + { + "epoch": 0.45, + "objective/train/docs_used": 850610, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.009878396987915, + "objective/train/theoretical_loss": 3.516259388334227, + "objective/train/tokens_used": 1519596000, + "theoretical_loss": 3.516259388334227, + "tokens_seen": 1499136000 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027567703109327986, + "loss": 2.7602, + "theoretical_loss": 3.516259388334227, + "tokens_seen": 1499136000 + }, + { + "epoch": 0.45, + "learning_rate": 0.000275667001003009, + "loss": 2.6018, + "theoretical_loss": 3.516245834131044, + "tokens_seen": 1499201536 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002756569709127382, + "loss": 2.7047, + "theoretical_loss": 3.5162322806862494, + "tokens_seen": 1499267072 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002756469408224674, + "loss": 2.7506, + "theoretical_loss": 3.516218727999768, + "tokens_seen": 1499332608 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002756369107321966, + "loss": 2.5771, + "theoretical_loss": 3.516205176071524, + "tokens_seen": 1499398144 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027562688064192576, + "loss": 2.6063, + "theoretical_loss": 3.5161916249014413, + "tokens_seen": 1499463680 + }, + { + "epoch": 0.45, + "learning_rate": 0.000275616850551655, + "loss": 2.5575, + "theoretical_loss": 3.5161780744894453, + "tokens_seen": 1499529216 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002756068204613841, + "loss": 2.7908, + "theoretical_loss": 3.51616452483546, + "tokens_seen": 1499594752 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027559679037111336, + "loss": 2.6801, + "theoretical_loss": 3.5161509759394094, + "tokens_seen": 1499660288 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002755867602808425, + "loss": 2.5796, + "theoretical_loss": 3.516137427801219, + "tokens_seen": 1499725824 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002755767301905717, + "loss": 2.4648, + "theoretical_loss": 3.516123880420812, + "tokens_seen": 1499791360 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002755667001003009, + "loss": 2.5282, + "theoretical_loss": 3.5161103337981143, + "tokens_seen": 1499856896 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002755566700100301, + "loss": 2.6489, + "theoretical_loss": 3.5160967879330496, + "tokens_seen": 1499922432 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027554663991975927, + "loss": 2.5872, + "theoretical_loss": 3.516083242825542, + "tokens_seen": 1499987968 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027553660982948845, + "loss": 2.5772, + "theoretical_loss": 3.5160696984755173, + "tokens_seen": 1500053504 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027552657973921763, + "loss": 2.6289, + "theoretical_loss": 3.5160561548828992, + "tokens_seen": 1500119040 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027551654964894686, + "loss": 2.4419, + "theoretical_loss": 3.5160426120476123, + "tokens_seen": 1500184576 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027550651955867604, + "loss": 2.6465, + "theoretical_loss": 3.5160290699695813, + "tokens_seen": 1500250112 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002754964894684052, + "loss": 2.601, + "theoretical_loss": 3.5160155286487313, + "tokens_seen": 1500315648 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027548645937813446, + "loss": 2.8819, + "theoretical_loss": 3.5160019880849855, + "tokens_seen": 1500381184 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002754764292878636, + "loss": 2.6742, + "theoretical_loss": 3.5159884482782697, + "tokens_seen": 1500446720 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002754663991975928, + "loss": 2.77, + "theoretical_loss": 3.5159749092285084, + "tokens_seen": 1500512256 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027545636910732195, + "loss": 2.8821, + "theoretical_loss": 3.5159613709356257, + "tokens_seen": 1500577792 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002754463390170512, + "loss": 2.3357, + "theoretical_loss": 3.5159478333995464, + "tokens_seen": 1500643328 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027543630892678037, + "loss": 2.7119, + "theoretical_loss": 3.5159342966201956, + "tokens_seen": 1500708864 + }, + { + "epoch": 0.45, + "objective/train/docs_used": 851359, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.721400022506714, + "objective/train/theoretical_loss": 3.515920760597498, + "objective/train/tokens_used": 1521234400, + "theoretical_loss": 3.515920760597498, + "tokens_seen": 1500774400 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027542627883650955, + "loss": 2.6106, + "theoretical_loss": 3.515920760597498, + "tokens_seen": 1500774400 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027541624874623873, + "loss": 2.6244, + "theoretical_loss": 3.515907225331377, + "tokens_seen": 1500839936 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002754062186559679, + "loss": 2.8506, + "theoretical_loss": 3.515893690821759, + "tokens_seen": 1500905472 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002753961885656971, + "loss": 2.6425, + "theoretical_loss": 3.515880157068567, + "tokens_seen": 1500971008 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002753861584754263, + "loss": 2.7474, + "theoretical_loss": 3.5158666240717276, + "tokens_seen": 1501036544 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027537612838515545, + "loss": 2.5016, + "theoretical_loss": 3.515853091831164, + "tokens_seen": 1501102080 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002753660982948847, + "loss": 2.6351, + "theoretical_loss": 3.5158395603468016, + "tokens_seen": 1501167616 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002753560682046138, + "loss": 2.6063, + "theoretical_loss": 3.515826029618564, + "tokens_seen": 1501233152 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027534603811434305, + "loss": 2.3158, + "theoretical_loss": 3.515812499646378, + "tokens_seen": 1501298688 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027533600802407223, + "loss": 2.9154, + "theoretical_loss": 3.5157989704301666, + "tokens_seen": 1501364224 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002753259779338014, + "loss": 2.627, + "theoretical_loss": 3.5157854419698555, + "tokens_seen": 1501429760 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002753159478435306, + "loss": 2.5561, + "theoretical_loss": 3.515771914265369, + "tokens_seen": 1501495296 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027530591775325983, + "loss": 2.7326, + "theoretical_loss": 3.515758387316632, + "tokens_seen": 1501560832 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027529588766298896, + "loss": 2.6103, + "theoretical_loss": 3.51574486112357, + "tokens_seen": 1501626368 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002752858575727182, + "loss": 2.7574, + "theoretical_loss": 3.5157313356861066, + "tokens_seen": 1501691904 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002752758274824473, + "loss": 2.6951, + "theoretical_loss": 3.5157178110041674, + "tokens_seen": 1501757440 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027526579739217655, + "loss": 2.3809, + "theoretical_loss": 3.5157042870776767, + "tokens_seen": 1501822976 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027525576730190574, + "loss": 2.7752, + "theoretical_loss": 3.5156907639065595, + "tokens_seen": 1501888512 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002752457372116349, + "loss": 2.829, + "theoretical_loss": 3.5156772414907413, + "tokens_seen": 1501954048 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002752357071213641, + "loss": 2.7324, + "theoretical_loss": 3.515663719830146, + "tokens_seen": 1502019584 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002752256770310933, + "loss": 2.8054, + "theoretical_loss": 3.5156501989246993, + "tokens_seen": 1502085120 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027521564694082246, + "loss": 2.877, + "theoretical_loss": 3.5156366787743254, + "tokens_seen": 1502150656 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002752056168505517, + "loss": 2.8076, + "theoretical_loss": 3.5156231593789498, + "tokens_seen": 1502216192 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002751955867602808, + "loss": 2.6426, + "theoretical_loss": 3.5156096407384974, + "tokens_seen": 1502281728 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027518555667001006, + "loss": 2.5991, + "theoretical_loss": 3.515596122852892, + "tokens_seen": 1502347264 + }, + { + "epoch": 0.46, + "objective/train/docs_used": 852603, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.734255313873291, + "objective/train/theoretical_loss": 3.51558260572206, + "objective/train/tokens_used": 1522872800, + "theoretical_loss": 3.51558260572206, + "tokens_seen": 1502412800 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002751755265797392, + "loss": 2.5651, + "theoretical_loss": 3.51558260572206, + "tokens_seen": 1502412800 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002751654964894684, + "loss": 2.7124, + "theoretical_loss": 3.5155690893459255, + "tokens_seen": 1502478336 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002751554663991976, + "loss": 2.6942, + "theoretical_loss": 3.5155555737244133, + "tokens_seen": 1502543872 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002751454363089268, + "loss": 2.7556, + "theoretical_loss": 3.515542058857449, + "tokens_seen": 1502609408 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027513540621865596, + "loss": 2.8027, + "theoretical_loss": 3.5155285447449574, + "tokens_seen": 1502674944 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002751253761283852, + "loss": 2.808, + "theoretical_loss": 3.515515031386863, + "tokens_seen": 1502740480 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002751153460381143, + "loss": 2.3835, + "theoretical_loss": 3.5155015187830916, + "tokens_seen": 1502806016 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027510531594784356, + "loss": 2.7302, + "theoretical_loss": 3.515488006933567, + "tokens_seen": 1502871552 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002750952858575727, + "loss": 3.022, + "theoretical_loss": 3.515474495838216, + "tokens_seen": 1502937088 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002750852557673019, + "loss": 2.693, + "theoretical_loss": 3.5154609854969614, + "tokens_seen": 1503002624 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002750752256770311, + "loss": 2.4871, + "theoretical_loss": 3.5154474759097303, + "tokens_seen": 1503068160 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002750651955867603, + "loss": 2.8187, + "theoretical_loss": 3.5154339670764463, + "tokens_seen": 1503133696 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027505516549648947, + "loss": 2.5561, + "theoretical_loss": 3.5154204589970353, + "tokens_seen": 1503199232 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027504513540621865, + "loss": 2.7276, + "theoretical_loss": 3.515406951671422, + "tokens_seen": 1503264768 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027503510531594783, + "loss": 2.7071, + "theoretical_loss": 3.5153934450995314, + "tokens_seen": 1503330304 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027502507522567706, + "loss": 2.6191, + "theoretical_loss": 3.5153799392812886, + "tokens_seen": 1503395840 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002750150451354062, + "loss": 2.7768, + "theoretical_loss": 3.5153664342166193, + "tokens_seen": 1503461376 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002750050150451354, + "loss": 2.6842, + "theoretical_loss": 3.515352929905448, + "tokens_seen": 1503526912 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027499498495486455, + "loss": 2.6337, + "theoretical_loss": 3.5153394263476994, + "tokens_seen": 1503592448 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002749849548645938, + "loss": 2.792, + "theoretical_loss": 3.5153259235432994, + "tokens_seen": 1503657984 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027497492477432297, + "loss": 2.5864, + "theoretical_loss": 3.5153124214921734, + "tokens_seen": 1503723520 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027496489468405215, + "loss": 2.6668, + "theoretical_loss": 3.5152989201942457, + "tokens_seen": 1503789056 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027495486459378133, + "loss": 2.7186, + "theoretical_loss": 3.515285419649442, + "tokens_seen": 1503854592 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027494483450351057, + "loss": 2.7249, + "theoretical_loss": 3.515271919857687, + "tokens_seen": 1503920128 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002749348044132397, + "loss": 2.579, + "theoretical_loss": 3.515258420818906, + "tokens_seen": 1503985664 + }, + { + "epoch": 0.46, + "objective/train/docs_used": 853123, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.5433168411254883, + "objective/train/theoretical_loss": 3.515244922533025, + "objective/train/tokens_used": 1524511200, + "theoretical_loss": 3.515244922533025, + "tokens_seen": 1504051200 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027492477432296893, + "loss": 2.7134, + "theoretical_loss": 3.515244922533025, + "tokens_seen": 1504051200 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027491474423269806, + "loss": 2.9569, + "theoretical_loss": 3.5152314249999685, + "tokens_seen": 1504116736 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002749047141424273, + "loss": 2.6596, + "theoretical_loss": 3.5152179282196614, + "tokens_seen": 1504182272 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027489468405215647, + "loss": 2.7867, + "theoretical_loss": 3.5152044321920295, + "tokens_seen": 1504247808 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027488465396188565, + "loss": 2.753, + "theoretical_loss": 3.5151909369169982, + "tokens_seen": 1504313344 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027487462387161483, + "loss": 2.7246, + "theoretical_loss": 3.515177442394492, + "tokens_seen": 1504378880 + }, + { + "epoch": 0.46, + "learning_rate": 0.000274864593781344, + "loss": 2.6819, + "theoretical_loss": 3.5151639486244366, + "tokens_seen": 1504444416 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002748545636910732, + "loss": 2.6418, + "theoretical_loss": 3.515150455606757, + "tokens_seen": 1504509952 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027484453360080243, + "loss": 2.6443, + "theoretical_loss": 3.515136963341379, + "tokens_seen": 1504575488 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027483450351053156, + "loss": 2.6846, + "theoretical_loss": 3.515123471828228, + "tokens_seen": 1504641024 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002748244734202608, + "loss": 2.656, + "theoretical_loss": 3.515109981067228, + "tokens_seen": 1504706560 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027481444332999, + "loss": 2.4364, + "theoretical_loss": 3.5150964910583062, + "tokens_seen": 1504772096 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027480441323971916, + "loss": 2.502, + "theoretical_loss": 3.5150830018013863, + "tokens_seen": 1504837632 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027479438314944834, + "loss": 2.7061, + "theoretical_loss": 3.5150695132963943, + "tokens_seen": 1504903168 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002747843530591775, + "loss": 2.8568, + "theoretical_loss": 3.515056025543256, + "tokens_seen": 1504968704 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027477432296890675, + "loss": 2.4768, + "theoretical_loss": 3.5150425385418957, + "tokens_seen": 1505034240 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027476429287863594, + "loss": 2.8712, + "theoretical_loss": 3.515029052292239, + "tokens_seen": 1505099776 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002747542627883651, + "loss": 2.7145, + "theoretical_loss": 3.5150155667942125, + "tokens_seen": 1505165312 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002747442326980943, + "loss": 2.6821, + "theoretical_loss": 3.51500208204774, + "tokens_seen": 1505230848 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002747342026078235, + "loss": 2.6218, + "theoretical_loss": 3.514988598052748, + "tokens_seen": 1505296384 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027472417251755266, + "loss": 2.7157, + "theoretical_loss": 3.5149751148091615, + "tokens_seen": 1505361920 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002747141424272819, + "loss": 2.6936, + "theoretical_loss": 3.514961632316906, + "tokens_seen": 1505427456 + }, + { + "epoch": 0.46, + "learning_rate": 0.000274704112337011, + "loss": 2.5154, + "theoretical_loss": 3.5149481505759064, + "tokens_seen": 1505492992 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027469408224674026, + "loss": 2.6483, + "theoretical_loss": 3.514934669586089, + "tokens_seen": 1505558528 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002746840521564694, + "loss": 2.7648, + "theoretical_loss": 3.5149211893473784, + "tokens_seen": 1505624064 + }, + { + "epoch": 0.46, + "objective/train/docs_used": 854391, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.84373140335083, + "objective/train/theoretical_loss": 3.514907709859701, + "objective/train/tokens_used": 1526149600, + "theoretical_loss": 3.514907709859701, + "tokens_seen": 1505689600 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002746740220661986, + "loss": 2.8778, + "theoretical_loss": 3.514907709859701, + "tokens_seen": 1505689600 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002746639919759278, + "loss": 2.5202, + "theoretical_loss": 3.514894231122981, + "tokens_seen": 1505755136 + }, + { + "epoch": 0.46, + "learning_rate": 0.000274653961885657, + "loss": 2.3917, + "theoretical_loss": 3.5148807531371453, + "tokens_seen": 1505820672 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027464393179538616, + "loss": 2.7856, + "theoretical_loss": 3.5148672759021182, + "tokens_seen": 1505886208 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002746339017051154, + "loss": 2.5378, + "theoretical_loss": 3.514853799417826, + "tokens_seen": 1505951744 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002746238716148445, + "loss": 2.6395, + "theoretical_loss": 3.514840323684194, + "tokens_seen": 1506017280 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027461384152457376, + "loss": 2.6833, + "theoretical_loss": 3.514826848701148, + "tokens_seen": 1506082816 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002746038114343029, + "loss": 2.4757, + "theoretical_loss": 3.5148133744686127, + "tokens_seen": 1506148352 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002745937813440321, + "loss": 2.8487, + "theoretical_loss": 3.5147999009865143, + "tokens_seen": 1506213888 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002745837512537613, + "loss": 2.5251, + "theoretical_loss": 3.5147864282547783, + "tokens_seen": 1506279424 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002745737211634905, + "loss": 2.6661, + "theoretical_loss": 3.51477295627333, + "tokens_seen": 1506344960 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027456369107321967, + "loss": 2.6582, + "theoretical_loss": 3.514759485042095, + "tokens_seen": 1506410496 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027455366098294885, + "loss": 2.478, + "theoretical_loss": 3.514746014560999, + "tokens_seen": 1506476032 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027454363089267803, + "loss": 2.9389, + "theoretical_loss": 3.514732544829968, + "tokens_seen": 1506541568 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027453360080240726, + "loss": 2.8072, + "theoretical_loss": 3.5147190758489275, + "tokens_seen": 1506607104 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002745235707121364, + "loss": 2.759, + "theoretical_loss": 3.5147056076178025, + "tokens_seen": 1506672640 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002745135406218656, + "loss": 2.8, + "theoretical_loss": 3.514692140136519, + "tokens_seen": 1506738176 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027450351053159475, + "loss": 2.812, + "theoretical_loss": 3.514678673405003, + "tokens_seen": 1506803712 + }, + { + "epoch": 0.46, + "learning_rate": 0.000274493480441324, + "loss": 2.6487, + "theoretical_loss": 3.514665207423179, + "tokens_seen": 1506869248 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027448345035105317, + "loss": 2.6764, + "theoretical_loss": 3.514651742190974, + "tokens_seen": 1506934784 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027447342026078235, + "loss": 2.5886, + "theoretical_loss": 3.514638277708313, + "tokens_seen": 1507000320 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027446339017051153, + "loss": 2.8353, + "theoretical_loss": 3.5146248139751215, + "tokens_seen": 1507065856 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027445336008024077, + "loss": 2.5263, + "theoretical_loss": 3.514611350991326, + "tokens_seen": 1507131392 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002744433299899699, + "loss": 2.8117, + "theoretical_loss": 3.5145978887568514, + "tokens_seen": 1507196928 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027443329989969913, + "loss": 2.5862, + "theoretical_loss": 3.5145844272716236, + "tokens_seen": 1507262464 + }, + { + "epoch": 0.46, + "objective/train/docs_used": 855086, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.842468500137329, + "objective/train/theoretical_loss": 3.514570966535569, + "objective/train/tokens_used": 1527788000, + "theoretical_loss": 3.514570966535569, + "tokens_seen": 1507328000 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027442326980942826, + "loss": 2.7316, + "theoretical_loss": 3.514570966535569, + "tokens_seen": 1507328000 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002744132397191575, + "loss": 2.6764, + "theoretical_loss": 3.5145575065486123, + "tokens_seen": 1507393536 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027440320962888667, + "loss": 2.7781, + "theoretical_loss": 3.51454404731068, + "tokens_seen": 1507459072 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027439317953861585, + "loss": 2.8031, + "theoretical_loss": 3.5145305888216973, + "tokens_seen": 1507524608 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027438314944834503, + "loss": 2.8419, + "theoretical_loss": 3.5145171310815906, + "tokens_seen": 1507590144 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002743731193580742, + "loss": 2.8283, + "theoretical_loss": 3.5145036740902853, + "tokens_seen": 1507655680 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002743630892678034, + "loss": 2.704, + "theoretical_loss": 3.514490217847707, + "tokens_seen": 1507721216 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027435305917753263, + "loss": 2.6473, + "theoretical_loss": 3.5144767623537816, + "tokens_seen": 1507786752 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027434302908726176, + "loss": 2.7408, + "theoretical_loss": 3.514463307608435, + "tokens_seen": 1507852288 + }, + { + "epoch": 0.46, + "learning_rate": 0.000274332998996991, + "loss": 2.5923, + "theoretical_loss": 3.514449853611594, + "tokens_seen": 1507917824 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002743229689067202, + "loss": 2.652, + "theoretical_loss": 3.5144364003631825, + "tokens_seen": 1507983360 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027431293881644936, + "loss": 2.6363, + "theoretical_loss": 3.5144229478631273, + "tokens_seen": 1508048896 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027430290872617854, + "loss": 2.5846, + "theoretical_loss": 3.5144094961113543, + "tokens_seen": 1508114432 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002742928786359077, + "loss": 2.6099, + "theoretical_loss": 3.51439604510779, + "tokens_seen": 1508179968 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002742828485456369, + "loss": 2.5966, + "theoretical_loss": 3.514382594852359, + "tokens_seen": 1508245504 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027427281845536614, + "loss": 2.7736, + "theoretical_loss": 3.514369145344988, + "tokens_seen": 1508311040 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027426278836509526, + "loss": 2.6587, + "theoretical_loss": 3.5143556965856027, + "tokens_seen": 1508376576 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002742527582748245, + "loss": 2.689, + "theoretical_loss": 3.514342248574129, + "tokens_seen": 1508442112 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002742427281845536, + "loss": 2.5502, + "theoretical_loss": 3.5143288013104925, + "tokens_seen": 1508507648 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027423269809428286, + "loss": 2.8115, + "theoretical_loss": 3.51431535479462, + "tokens_seen": 1508573184 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027422266800401204, + "loss": 2.6116, + "theoretical_loss": 3.514301909026437, + "tokens_seen": 1508638720 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002742126379137412, + "loss": 2.8545, + "theoretical_loss": 3.5142884640058685, + "tokens_seen": 1508704256 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002742026078234704, + "loss": 2.6625, + "theoretical_loss": 3.5142750197328416, + "tokens_seen": 1508769792 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002741925777331996, + "loss": 2.6697, + "theoretical_loss": 3.514261576207282, + "tokens_seen": 1508835328 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027418254764292877, + "loss": 2.5927, + "theoretical_loss": 3.514248133429116, + "tokens_seen": 1508900864 + }, + { + "epoch": 0.46, + "objective/train/docs_used": 856541, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6383204460144043, + "objective/train/theoretical_loss": 3.514234691398269, + "objective/train/tokens_used": 1529426400, + "theoretical_loss": 3.514234691398269, + "tokens_seen": 1508966400 + }, + { + "epoch": 0.46, + "learning_rate": 0.000274172517552658, + "loss": 2.5819, + "theoretical_loss": 3.514234691398269, + "tokens_seen": 1508966400 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027416248746238713, + "loss": 2.6488, + "theoretical_loss": 3.514221250114667, + "tokens_seen": 1509031936 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027415245737211636, + "loss": 2.6351, + "theoretical_loss": 3.5142078095782363, + "tokens_seen": 1509097472 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027414242728184554, + "loss": 2.7215, + "theoretical_loss": 3.514194369788903, + "tokens_seen": 1509163008 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002741323971915747, + "loss": 2.7079, + "theoretical_loss": 3.514180930746593, + "tokens_seen": 1509228544 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002741223671013039, + "loss": 2.6891, + "theoretical_loss": 3.5141674924512323, + "tokens_seen": 1509294080 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002741123370110331, + "loss": 2.7016, + "theoretical_loss": 3.514154054902747, + "tokens_seen": 1509359616 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027410230692076227, + "loss": 2.8811, + "theoretical_loss": 3.514140618101063, + "tokens_seen": 1509425152 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002740922768304915, + "loss": 2.6444, + "theoretical_loss": 3.5141271820461064, + "tokens_seen": 1509490688 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027408224674022063, + "loss": 2.9188, + "theoretical_loss": 3.514113746737804, + "tokens_seen": 1509556224 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027407221664994987, + "loss": 2.4282, + "theoretical_loss": 3.5141003121760805, + "tokens_seen": 1509621760 + }, + { + "epoch": 0.46, + "learning_rate": 0.000274062186559679, + "loss": 2.592, + "theoretical_loss": 3.514086878360863, + "tokens_seen": 1509687296 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027405215646940823, + "loss": 2.6211, + "theoretical_loss": 3.5140734452920777, + "tokens_seen": 1509752832 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002740421263791374, + "loss": 2.7014, + "theoretical_loss": 3.514060012969651, + "tokens_seen": 1509818368 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002740320962888666, + "loss": 2.7434, + "theoretical_loss": 3.5140465813935076, + "tokens_seen": 1509883904 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002740220661985958, + "loss": 2.5182, + "theoretical_loss": 3.5140331505635745, + "tokens_seen": 1509949440 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027401203610832495, + "loss": 2.7326, + "theoretical_loss": 3.5140197204797783, + "tokens_seen": 1510014976 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002740020060180542, + "loss": 2.6692, + "theoretical_loss": 3.514006291142045, + "tokens_seen": 1510080512 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027399197592778337, + "loss": 2.6411, + "theoretical_loss": 3.5139928625503, + "tokens_seen": 1510146048 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027398194583751255, + "loss": 2.5794, + "theoretical_loss": 3.5139794347044706, + "tokens_seen": 1510211584 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027397191574724173, + "loss": 2.5461, + "theoretical_loss": 3.513966007604482, + "tokens_seen": 1510277120 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027396188565697097, + "loss": 2.9139, + "theoretical_loss": 3.5139525812502606, + "tokens_seen": 1510342656 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002739518555667001, + "loss": 2.777, + "theoretical_loss": 3.513939155641733, + "tokens_seen": 1510408192 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027394182547642933, + "loss": 2.5475, + "theoretical_loss": 3.5139257307788254, + "tokens_seen": 1510473728 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027393179538615846, + "loss": 2.768, + "theoretical_loss": 3.5139123066614646, + "tokens_seen": 1510539264 + }, + { + "epoch": 0.46, + "objective/train/docs_used": 857299, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.346788167953491, + "objective/train/theoretical_loss": 3.513898883289575, + "objective/train/tokens_used": 1531064800, + "theoretical_loss": 3.513898883289575, + "tokens_seen": 1510604800 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002739217652958877, + "loss": 2.6421, + "theoretical_loss": 3.513898883289575, + "tokens_seen": 1510604800 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027391173520561687, + "loss": 2.8865, + "theoretical_loss": 3.5138854606630847, + "tokens_seen": 1510670336 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027390170511534605, + "loss": 2.823, + "theoretical_loss": 3.5138720387819196, + "tokens_seen": 1510735872 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027389167502507524, + "loss": 2.4932, + "theoretical_loss": 3.5138586176460054, + "tokens_seen": 1510801408 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002738816449348044, + "loss": 2.5777, + "theoretical_loss": 3.513845197255269, + "tokens_seen": 1510866944 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002738716148445336, + "loss": 2.366, + "theoretical_loss": 3.513831777609636, + "tokens_seen": 1510932480 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027386158475426283, + "loss": 2.5397, + "theoretical_loss": 3.513818358709033, + "tokens_seen": 1510998016 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027385155466399196, + "loss": 2.6347, + "theoretical_loss": 3.5138049405533867, + "tokens_seen": 1511063552 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002738415245737212, + "loss": 2.481, + "theoretical_loss": 3.5137915231426233, + "tokens_seen": 1511129088 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002738314944834504, + "loss": 2.3306, + "theoretical_loss": 3.5137781064766687, + "tokens_seen": 1511194624 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027382146439317956, + "loss": 2.7997, + "theoretical_loss": 3.51376469055545, + "tokens_seen": 1511260160 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027381143430290874, + "loss": 2.802, + "theoretical_loss": 3.513751275378893, + "tokens_seen": 1511325696 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002738014042126379, + "loss": 2.7645, + "theoretical_loss": 3.5137378609469243, + "tokens_seen": 1511391232 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002737913741223671, + "loss": 2.4316, + "theoretical_loss": 3.51372444725947, + "tokens_seen": 1511456768 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027378134403209634, + "loss": 2.808, + "theoretical_loss": 3.513711034316457, + "tokens_seen": 1511522304 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027377131394182546, + "loss": 2.6881, + "theoretical_loss": 3.513697622117811, + "tokens_seen": 1511587840 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002737612838515547, + "loss": 2.5267, + "theoretical_loss": 3.5136842106634596, + "tokens_seen": 1511653376 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002737512537612838, + "loss": 2.7464, + "theoretical_loss": 3.513670799953328, + "tokens_seen": 1511718912 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027374122367101306, + "loss": 2.546, + "theoretical_loss": 3.513657389987343, + "tokens_seen": 1511784448 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027373119358074224, + "loss": 2.8224, + "theoretical_loss": 3.5136439807654316, + "tokens_seen": 1511849984 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002737211634904714, + "loss": 2.5474, + "theoretical_loss": 3.5136305722875196, + "tokens_seen": 1511915520 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002737111334002006, + "loss": 2.4044, + "theoretical_loss": 3.5136171645535335, + "tokens_seen": 1511981056 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002737011033099298, + "loss": 2.6914, + "theoretical_loss": 3.5136037575634003, + "tokens_seen": 1512046592 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027369107321965897, + "loss": 2.5204, + "theoretical_loss": 3.5135903513170463, + "tokens_seen": 1512112128 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002736810431293882, + "loss": 2.817, + "theoretical_loss": 3.513576945814398, + "tokens_seen": 1512177664 + }, + { + "epoch": 0.46, + "objective/train/docs_used": 858636, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.408207893371582, + "objective/train/theoretical_loss": 3.5135635410553814, + "objective/train/tokens_used": 1532703200, + "theoretical_loss": 3.5135635410553814, + "tokens_seen": 1512243200 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027367101303911733, + "loss": 2.6043, + "theoretical_loss": 3.5135635410553814, + "tokens_seen": 1512243200 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027366098294884656, + "loss": 2.3423, + "theoretical_loss": 3.513550137039924, + "tokens_seen": 1512308736 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027365095285857574, + "loss": 2.6209, + "theoretical_loss": 3.513536733767951, + "tokens_seen": 1512374272 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002736409227683049, + "loss": 2.7409, + "theoretical_loss": 3.5135233312393903, + "tokens_seen": 1512439808 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002736308926780341, + "loss": 2.6988, + "theoretical_loss": 3.513509929454168, + "tokens_seen": 1512505344 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002736208625877633, + "loss": 2.7564, + "theoretical_loss": 3.51349652841221, + "tokens_seen": 1512570880 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027361083249749247, + "loss": 2.6872, + "theoretical_loss": 3.5134831281134433, + "tokens_seen": 1512636416 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002736008024072217, + "loss": 2.6209, + "theoretical_loss": 3.513469728557795, + "tokens_seen": 1512701952 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027359077231695083, + "loss": 2.6654, + "theoretical_loss": 3.5134563297451917, + "tokens_seen": 1512767488 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027358074222668007, + "loss": 2.6248, + "theoretical_loss": 3.5134429316755593, + "tokens_seen": 1512833024 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002735707121364092, + "loss": 2.8361, + "theoretical_loss": 3.513429534348824, + "tokens_seen": 1512898560 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027356068204613843, + "loss": 2.8181, + "theoretical_loss": 3.5134161377649145, + "tokens_seen": 1512964096 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002735506519558676, + "loss": 2.839, + "theoretical_loss": 3.5134027419237555, + "tokens_seen": 1513029632 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002735406218655968, + "loss": 2.6038, + "theoretical_loss": 3.513389346825274, + "tokens_seen": 1513095168 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027353059177532597, + "loss": 2.8029, + "theoretical_loss": 3.513375952469397, + "tokens_seen": 1513160704 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027352056168505515, + "loss": 2.5179, + "theoretical_loss": 3.513362558856051, + "tokens_seen": 1513226240 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027351053159478433, + "loss": 2.7949, + "theoretical_loss": 3.5133491659851632, + "tokens_seen": 1513291776 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027350050150451357, + "loss": 2.6138, + "theoretical_loss": 3.5133357738566597, + "tokens_seen": 1513357312 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002734904714142427, + "loss": 2.5222, + "theoretical_loss": 3.5133223824704674, + "tokens_seen": 1513422848 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027348044132397193, + "loss": 2.83, + "theoretical_loss": 3.513308991826513, + "tokens_seen": 1513488384 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002734704112337011, + "loss": 2.5002, + "theoretical_loss": 3.513295601924723, + "tokens_seen": 1513553920 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002734603811434303, + "loss": 2.7781, + "theoretical_loss": 3.513282212765024, + "tokens_seen": 1513619456 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002734503510531595, + "loss": 2.6923, + "theoretical_loss": 3.5132688243473438, + "tokens_seen": 1513684992 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027344032096288866, + "loss": 2.6671, + "theoretical_loss": 3.5132554366716082, + "tokens_seen": 1513750528 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027343029087261784, + "loss": 2.6488, + "theoretical_loss": 3.5132420497377437, + "tokens_seen": 1513816064 + }, + { + "epoch": 0.46, + "objective/train/docs_used": 859135, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.2216038703918457, + "objective/train/theoretical_loss": 3.513228663545678, + "objective/train/tokens_used": 1534341600, + "theoretical_loss": 3.513228663545678, + "tokens_seen": 1513881600 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002734202607823471, + "loss": 2.805, + "theoretical_loss": 3.513228663545678, + "tokens_seen": 1513881600 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002734102306920762, + "loss": 2.5326, + "theoretical_loss": 3.5132152780953376, + "tokens_seen": 1513947136 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027340020060180544, + "loss": 2.6744, + "theoretical_loss": 3.513201893386649, + "tokens_seen": 1514012672 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027339017051153456, + "loss": 2.6168, + "theoretical_loss": 3.513188509419539, + "tokens_seen": 1514078208 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002733801404212638, + "loss": 2.6905, + "theoretical_loss": 3.513175126193935, + "tokens_seen": 1514143744 + }, + { + "epoch": 0.46, + "learning_rate": 0.000273370110330993, + "loss": 2.7873, + "theoretical_loss": 3.5131617437097633, + "tokens_seen": 1514209280 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027336008024072216, + "loss": 2.542, + "theoretical_loss": 3.5131483619669504, + "tokens_seen": 1514274816 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027335005015045134, + "loss": 2.5798, + "theoretical_loss": 3.513134980965424, + "tokens_seen": 1514340352 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002733400200601806, + "loss": 2.8032, + "theoretical_loss": 3.5131216007051105, + "tokens_seen": 1514405888 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002733299899699097, + "loss": 2.6284, + "theoretical_loss": 3.513108221185937, + "tokens_seen": 1514471424 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027331995987963894, + "loss": 2.7706, + "theoretical_loss": 3.5130948424078294, + "tokens_seen": 1514536960 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027330992978936807, + "loss": 2.6015, + "theoretical_loss": 3.5130814643707162, + "tokens_seen": 1514602496 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002732998996990973, + "loss": 2.5041, + "theoretical_loss": 3.5130680870745232, + "tokens_seen": 1514668032 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002732898696088265, + "loss": 2.6051, + "theoretical_loss": 3.5130547105191776, + "tokens_seen": 1514733568 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027327983951855566, + "loss": 2.4931, + "theoretical_loss": 3.5130413347046066, + "tokens_seen": 1514799104 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002732698094282849, + "loss": 2.7411, + "theoretical_loss": 3.5130279596307363, + "tokens_seen": 1514864640 + }, + { + "epoch": 0.46, + "learning_rate": 0.000273259779338014, + "loss": 2.6018, + "theoretical_loss": 3.513014585297494, + "tokens_seen": 1514930176 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027324974924774326, + "loss": 2.7278, + "theoretical_loss": 3.5130012117048075, + "tokens_seen": 1514995712 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027323971915747244, + "loss": 2.8988, + "theoretical_loss": 3.5129878388526032, + "tokens_seen": 1515061248 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002732296890672016, + "loss": 2.6159, + "theoretical_loss": 3.5129744667408076, + "tokens_seen": 1515126784 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002732196589769308, + "loss": 2.6634, + "theoretical_loss": 3.5129610953693478, + "tokens_seen": 1515192320 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027320962888666, + "loss": 2.6032, + "theoretical_loss": 3.5129477247381518, + "tokens_seen": 1515257856 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027319959879638917, + "loss": 2.6416, + "theoretical_loss": 3.512934354847145, + "tokens_seen": 1515323392 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002731895687061184, + "loss": 2.5813, + "theoretical_loss": 3.5129209856962555, + "tokens_seen": 1515388928 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027317953861584753, + "loss": 2.6828, + "theoretical_loss": 3.5129076172854106, + "tokens_seen": 1515454464 + }, + { + "epoch": 0.46, + "objective/train/docs_used": 860397, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.273402214050293, + "objective/train/theoretical_loss": 3.5128942496145363, + "objective/train/tokens_used": 1535980000, + "theoretical_loss": 3.5128942496145363, + "tokens_seen": 1515520000 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027316950852557676, + "loss": 2.3424, + "theoretical_loss": 3.5128942496145363, + "tokens_seen": 1515520000 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027315947843530594, + "loss": 2.8683, + "theoretical_loss": 3.5128808826835605, + "tokens_seen": 1515585536 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002731494483450351, + "loss": 2.4136, + "theoretical_loss": 3.5128675164924097, + "tokens_seen": 1515651072 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002731394182547643, + "loss": 2.7566, + "theoretical_loss": 3.5128541510410116, + "tokens_seen": 1515716608 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002731293881644935, + "loss": 2.6959, + "theoretical_loss": 3.5128407863292925, + "tokens_seen": 1515782144 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027311935807422267, + "loss": 2.6938, + "theoretical_loss": 3.51282742235718, + "tokens_seen": 1515847680 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002731093279839519, + "loss": 2.7473, + "theoretical_loss": 3.512814059124601, + "tokens_seen": 1515913216 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027309929789368103, + "loss": 2.838, + "theoretical_loss": 3.5128006966314826, + "tokens_seen": 1515978752 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027308926780341027, + "loss": 2.6701, + "theoretical_loss": 3.5127873348777516, + "tokens_seen": 1516044288 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002730792377131394, + "loss": 2.5956, + "theoretical_loss": 3.512773973863336, + "tokens_seen": 1516109824 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027306920762286863, + "loss": 2.657, + "theoretical_loss": 3.5127606135881626, + "tokens_seen": 1516175360 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002730591775325978, + "loss": 2.5572, + "theoretical_loss": 3.512747254052158, + "tokens_seen": 1516240896 + }, + { + "epoch": 0.46, + "learning_rate": 0.000273049147442327, + "loss": 2.6373, + "theoretical_loss": 3.51273389525525, + "tokens_seen": 1516306432 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027303911735205617, + "loss": 2.4931, + "theoretical_loss": 3.512720537197365, + "tokens_seen": 1516371968 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027302908726178535, + "loss": 2.775, + "theoretical_loss": 3.512707179878431, + "tokens_seen": 1516437504 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027301905717151453, + "loss": 2.7688, + "theoretical_loss": 3.512693823298375, + "tokens_seen": 1516503040 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027300902708124377, + "loss": 2.4166, + "theoretical_loss": 3.512680467457124, + "tokens_seen": 1516568576 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002729989969909729, + "loss": 2.3449, + "theoretical_loss": 3.5126671123546056, + "tokens_seen": 1516634112 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027298896690070213, + "loss": 2.7545, + "theoretical_loss": 3.5126537579907464, + "tokens_seen": 1516699648 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002729789368104313, + "loss": 2.7385, + "theoretical_loss": 3.5126404043654738, + "tokens_seen": 1516765184 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002729689067201605, + "loss": 2.7298, + "theoretical_loss": 3.5126270514787157, + "tokens_seen": 1516830720 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002729588766298897, + "loss": 2.6336, + "theoretical_loss": 3.5126136993303985, + "tokens_seen": 1516896256 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027294884653961886, + "loss": 2.6572, + "theoretical_loss": 3.5126003479204497, + "tokens_seen": 1516961792 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027293881644934804, + "loss": 2.6037, + "theoretical_loss": 3.512586997248797, + "tokens_seen": 1517027328 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002729287863590773, + "loss": 2.7679, + "theoretical_loss": 3.5125736473153673, + "tokens_seen": 1517092864 + }, + { + "epoch": 0.46, + "objective/train/docs_used": 860980, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.677279233932495, + "objective/train/theoretical_loss": 3.5125602981200874, + "objective/train/tokens_used": 1537618400, + "theoretical_loss": 3.5125602981200874, + "tokens_seen": 1517158400 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002729187562688064, + "loss": 2.6296, + "theoretical_loss": 3.5125602981200874, + "tokens_seen": 1517158400 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027290872617853564, + "loss": 2.672, + "theoretical_loss": 3.5125469496628856, + "tokens_seen": 1517223936 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027289869608826476, + "loss": 2.7707, + "theoretical_loss": 3.512533601943689, + "tokens_seen": 1517289472 + }, + { + "epoch": 0.46, + "learning_rate": 0.000272888665997994, + "loss": 2.5975, + "theoretical_loss": 3.512520254962424, + "tokens_seen": 1517355008 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002728786359077232, + "loss": 2.5897, + "theoretical_loss": 3.5125069087190193, + "tokens_seen": 1517420544 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027286860581745236, + "loss": 2.6605, + "theoretical_loss": 3.5124935632134013, + "tokens_seen": 1517486080 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027285857572718154, + "loss": 2.5295, + "theoretical_loss": 3.5124802184454973, + "tokens_seen": 1517551616 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002728485456369108, + "loss": 2.6369, + "theoretical_loss": 3.512466874415235, + "tokens_seen": 1517617152 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002728385155466399, + "loss": 2.6559, + "theoretical_loss": 3.512453531122542, + "tokens_seen": 1517682688 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027282848545636914, + "loss": 2.4541, + "theoretical_loss": 3.512440188567346, + "tokens_seen": 1517748224 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027281845536609827, + "loss": 2.7089, + "theoretical_loss": 3.512426846749573, + "tokens_seen": 1517813760 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002728084252758275, + "loss": 2.3099, + "theoretical_loss": 3.512413505669152, + "tokens_seen": 1517879296 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002727983951855567, + "loss": 2.5183, + "theoretical_loss": 3.5124001653260093, + "tokens_seen": 1517944832 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027278836509528586, + "loss": 2.8327, + "theoretical_loss": 3.512386825720072, + "tokens_seen": 1518010368 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027277833500501504, + "loss": 2.6961, + "theoretical_loss": 3.5123734868512693, + "tokens_seen": 1518075904 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002727683049147442, + "loss": 2.7532, + "theoretical_loss": 3.512360148719527, + "tokens_seen": 1518141440 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002727582748244734, + "loss": 2.5912, + "theoretical_loss": 3.5123468113247736, + "tokens_seen": 1518206976 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027274824473420264, + "loss": 2.7968, + "theoretical_loss": 3.5123334746669355, + "tokens_seen": 1518272512 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027273821464393177, + "loss": 2.6578, + "theoretical_loss": 3.512320138745941, + "tokens_seen": 1518338048 + }, + { + "epoch": 0.46, + "learning_rate": 0.000272728184553661, + "loss": 2.6945, + "theoretical_loss": 3.5123068035617173, + "tokens_seen": 1518403584 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027271815446339013, + "loss": 2.5733, + "theoretical_loss": 3.512293469114192, + "tokens_seen": 1518469120 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027270812437311937, + "loss": 2.7666, + "theoretical_loss": 3.5122801354032926, + "tokens_seen": 1518534656 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027269809428284855, + "loss": 2.5965, + "theoretical_loss": 3.5122668024289467, + "tokens_seen": 1518600192 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027268806419257773, + "loss": 2.6326, + "theoretical_loss": 3.5122534701910815, + "tokens_seen": 1518665728 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002726780341023069, + "loss": 2.8596, + "theoretical_loss": 3.512240138689625, + "tokens_seen": 1518731264 + }, + { + "epoch": 0.46, + "objective/train/docs_used": 862373, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.665384531021118, + "objective/train/theoretical_loss": 3.5122268079245043, + "objective/train/tokens_used": 1539256800, + "theoretical_loss": 3.5122268079245043, + "tokens_seen": 1518796800 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027266800401203614, + "loss": 2.6643, + "theoretical_loss": 3.5122268079245043, + "tokens_seen": 1518796800 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027265797392176527, + "loss": 2.3891, + "theoretical_loss": 3.5122134778956475, + "tokens_seen": 1518862336 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002726479438314945, + "loss": 2.4346, + "theoretical_loss": 3.5122001486029815, + "tokens_seen": 1518927872 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027263791374122363, + "loss": 2.5085, + "theoretical_loss": 3.512186820046434, + "tokens_seen": 1518993408 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027262788365095287, + "loss": 2.6479, + "theoretical_loss": 3.512173492225933, + "tokens_seen": 1519058944 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027261785356068205, + "loss": 2.4931, + "theoretical_loss": 3.512160165141406, + "tokens_seen": 1519124480 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027260782347041123, + "loss": 2.6941, + "theoretical_loss": 3.5121468387927806, + "tokens_seen": 1519190016 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002725977933801404, + "loss": 2.3883, + "theoretical_loss": 3.5121335131799847, + "tokens_seen": 1519255552 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002725877632898696, + "loss": 2.5411, + "theoretical_loss": 3.512120188302945, + "tokens_seen": 1519321088 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002725777331995988, + "loss": 2.7081, + "theoretical_loss": 3.5121068641615896, + "tokens_seen": 1519386624 + }, + { + "epoch": 0.46, + "learning_rate": 0.000272567703109328, + "loss": 2.7638, + "theoretical_loss": 3.512093540755847, + "tokens_seen": 1519452160 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027255767301905714, + "loss": 2.6385, + "theoretical_loss": 3.5120802180856434, + "tokens_seen": 1519517696 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027254764292878637, + "loss": 2.3653, + "theoretical_loss": 3.512066896150907, + "tokens_seen": 1519583232 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002725376128385155, + "loss": 2.9798, + "theoretical_loss": 3.5120535749515667, + "tokens_seen": 1519648768 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027252758274824473, + "loss": 2.8561, + "theoretical_loss": 3.5120402544875486, + "tokens_seen": 1519714304 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027251755265797397, + "loss": 2.7738, + "theoretical_loss": 3.512026934758781, + "tokens_seen": 1519779840 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002725075225677031, + "loss": 2.3047, + "theoretical_loss": 3.5120136157651913, + "tokens_seen": 1519845376 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027249749247743233, + "loss": 2.772, + "theoretical_loss": 3.5120002975067077, + "tokens_seen": 1519910912 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002724874623871615, + "loss": 2.756, + "theoretical_loss": 3.5119869799832584, + "tokens_seen": 1519976448 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002724774322968907, + "loss": 2.6451, + "theoretical_loss": 3.5119736631947696, + "tokens_seen": 1520041984 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002724674022066199, + "loss": 2.8718, + "theoretical_loss": 3.5119603471411702, + "tokens_seen": 1520107520 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027245737211634906, + "loss": 2.6381, + "theoretical_loss": 3.511947031822388, + "tokens_seen": 1520173056 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027244734202607824, + "loss": 2.6063, + "theoretical_loss": 3.5119337172383505, + "tokens_seen": 1520238592 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002724373119358075, + "loss": 2.4883, + "theoretical_loss": 3.5119204033889853, + "tokens_seen": 1520304128 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002724272818455366, + "loss": 2.764, + "theoretical_loss": 3.51190709027422, + "tokens_seen": 1520369664 + }, + { + "epoch": 0.46, + "objective/train/docs_used": 862945, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7052602767944336, + "objective/train/theoretical_loss": 3.5118937778939836, + "objective/train/tokens_used": 1540895200, + "theoretical_loss": 3.5118937778939836, + "tokens_seen": 1520435200 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027241725175526584, + "loss": 2.598, + "theoretical_loss": 3.5118937778939836, + "tokens_seen": 1520435200 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027240722166499496, + "loss": 2.8251, + "theoretical_loss": 3.5118804662482024, + "tokens_seen": 1520500736 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002723971915747242, + "loss": 2.7346, + "theoretical_loss": 3.5118671553368053, + "tokens_seen": 1520566272 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002723871614844534, + "loss": 2.4015, + "theoretical_loss": 3.5118538451597194, + "tokens_seen": 1520631808 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027237713139418256, + "loss": 2.7183, + "theoretical_loss": 3.511840535716873, + "tokens_seen": 1520697344 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027236710130391174, + "loss": 2.275, + "theoretical_loss": 3.511827227008194, + "tokens_seen": 1520762880 + }, + { + "epoch": 0.46, + "learning_rate": 0.000272357071213641, + "loss": 2.6667, + "theoretical_loss": 3.51181391903361, + "tokens_seen": 1520828416 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002723470411233701, + "loss": 2.8877, + "theoretical_loss": 3.511800611793049, + "tokens_seen": 1520893952 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027233701103309934, + "loss": 2.51, + "theoretical_loss": 3.5117873052864392, + "tokens_seen": 1520959488 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027232698094282847, + "loss": 2.676, + "theoretical_loss": 3.511773999513708, + "tokens_seen": 1521025024 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002723169508525577, + "loss": 2.6427, + "theoretical_loss": 3.5117606944747832, + "tokens_seen": 1521090560 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002723069207622869, + "loss": 2.6071, + "theoretical_loss": 3.5117473901695933, + "tokens_seen": 1521156096 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027229689067201606, + "loss": 2.5001, + "theoretical_loss": 3.511734086598066, + "tokens_seen": 1521221632 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027228686058174524, + "loss": 2.6152, + "theoretical_loss": 3.511720783760129, + "tokens_seen": 1521287168 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002722768304914744, + "loss": 2.8559, + "theoretical_loss": 3.5117074816557103, + "tokens_seen": 1521352704 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002722668004012036, + "loss": 2.6342, + "theoretical_loss": 3.511694180284738, + "tokens_seen": 1521418240 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027225677031093284, + "loss": 2.735, + "theoretical_loss": 3.51168087964714, + "tokens_seen": 1521483776 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027224674022066197, + "loss": 2.439, + "theoretical_loss": 3.511667579742845, + "tokens_seen": 1521549312 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002722367101303912, + "loss": 2.8825, + "theoretical_loss": 3.5116542805717796, + "tokens_seen": 1521614848 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027222668004012033, + "loss": 2.7682, + "theoretical_loss": 3.5116409821338728, + "tokens_seen": 1521680384 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027221664994984957, + "loss": 2.6962, + "theoretical_loss": 3.5116276844290524, + "tokens_seen": 1521745920 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027220661985957875, + "loss": 3.0633, + "theoretical_loss": 3.511614387457246, + "tokens_seen": 1521811456 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027219658976930793, + "loss": 2.7238, + "theoretical_loss": 3.511601091218382, + "tokens_seen": 1521876992 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002721865596790371, + "loss": 2.674, + "theoretical_loss": 3.511587795712389, + "tokens_seen": 1521942528 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027217652958876635, + "loss": 2.7451, + "theoretical_loss": 3.5115745009391937, + "tokens_seen": 1522008064 + }, + { + "epoch": 0.46, + "objective/train/docs_used": 863367, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.753692388534546, + "objective/train/theoretical_loss": 3.5115612068987256, + "objective/train/tokens_used": 1542533600, + "theoretical_loss": 3.5115612068987256, + "tokens_seen": 1522073600 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027216649949849547, + "loss": 2.8243, + "theoretical_loss": 3.5115612068987256, + "tokens_seen": 1522073600 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002721564694082247, + "loss": 2.5438, + "theoretical_loss": 3.5115479135909116, + "tokens_seen": 1522139136 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027214643931795383, + "loss": 2.4777, + "theoretical_loss": 3.5115346210156804, + "tokens_seen": 1522204672 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027213640922768307, + "loss": 2.632, + "theoretical_loss": 3.51152132917296, + "tokens_seen": 1522270208 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027212637913741225, + "loss": 2.5149, + "theoretical_loss": 3.5115080380626784, + "tokens_seen": 1522335744 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027211634904714143, + "loss": 2.6074, + "theoretical_loss": 3.5114947476847638, + "tokens_seen": 1522401280 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002721063189568706, + "loss": 2.7528, + "theoretical_loss": 3.5114814580391442, + "tokens_seen": 1522466816 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002720962888665998, + "loss": 2.7586, + "theoretical_loss": 3.5114681691257483, + "tokens_seen": 1522532352 + }, + { + "epoch": 0.46, + "learning_rate": 0.000272086258776329, + "loss": 2.7323, + "theoretical_loss": 3.5114548809445036, + "tokens_seen": 1522597888 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002720762286860582, + "loss": 2.5154, + "theoretical_loss": 3.511441593495338, + "tokens_seen": 1522663424 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027206619859578734, + "loss": 2.4858, + "theoretical_loss": 3.51142830677818, + "tokens_seen": 1522728960 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027205616850551657, + "loss": 2.6845, + "theoretical_loss": 3.511415020792959, + "tokens_seen": 1522794496 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002720461384152457, + "loss": 2.5089, + "theoretical_loss": 3.511401735539601, + "tokens_seen": 1522860032 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027203610832497493, + "loss": 2.9385, + "theoretical_loss": 3.5113884510180355, + "tokens_seen": 1522925568 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002720260782347041, + "loss": 2.7577, + "theoretical_loss": 3.51137516722819, + "tokens_seen": 1522991104 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002720160481444333, + "loss": 2.4122, + "theoretical_loss": 3.5113618841699936, + "tokens_seen": 1523056640 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002720060180541625, + "loss": 2.8687, + "theoretical_loss": 3.511348601843374, + "tokens_seen": 1523122176 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002719959879638917, + "loss": 2.5549, + "theoretical_loss": 3.51133532024826, + "tokens_seen": 1523187712 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027198595787362084, + "loss": 2.5542, + "theoretical_loss": 3.511322039384578, + "tokens_seen": 1523253248 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002719759277833501, + "loss": 2.6586, + "theoretical_loss": 3.511308759252259, + "tokens_seen": 1523318784 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002719658976930792, + "loss": 2.6887, + "theoretical_loss": 3.5112954798512286, + "tokens_seen": 1523384320 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027195586760280844, + "loss": 2.6896, + "theoretical_loss": 3.5112822011814173, + "tokens_seen": 1523449856 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002719458375125376, + "loss": 2.5033, + "theoretical_loss": 3.511268923242752, + "tokens_seen": 1523515392 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002719358074222668, + "loss": 2.4208, + "theoretical_loss": 3.5112556460351616, + "tokens_seen": 1523580928 + }, + { + "epoch": 0.46, + "learning_rate": 0.000271925777331996, + "loss": 2.851, + "theoretical_loss": 3.511242369558574, + "tokens_seen": 1523646464 + }, + { + "epoch": 0.46, + "objective/train/docs_used": 864510, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7547850608825684, + "objective/train/theoretical_loss": 3.5112290938129176, + "objective/train/tokens_used": 1544172000, + "theoretical_loss": 3.5112290938129176, + "tokens_seen": 1523712000 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027191574724172516, + "loss": 2.7514, + "theoretical_loss": 3.5112290938129176, + "tokens_seen": 1523712000 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027190571715145434, + "loss": 2.5057, + "theoretical_loss": 3.5112158187981213, + "tokens_seen": 1523777536 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002718956870611836, + "loss": 2.7995, + "theoretical_loss": 3.511202544514113, + "tokens_seen": 1523843072 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002718856569709127, + "loss": 2.7713, + "theoretical_loss": 3.5111892709608203, + "tokens_seen": 1523908608 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027187562688064194, + "loss": 2.6661, + "theoretical_loss": 3.5111759981381727, + "tokens_seen": 1523974144 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027186559679037107, + "loss": 2.5033, + "theoretical_loss": 3.5111627260460985, + "tokens_seen": 1524039680 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002718555667001003, + "loss": 2.5633, + "theoretical_loss": 3.511149454684525, + "tokens_seen": 1524105216 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002718455366098295, + "loss": 2.682, + "theoretical_loss": 3.511136184053382, + "tokens_seen": 1524170752 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027183550651955867, + "loss": 2.5702, + "theoretical_loss": 3.5111229141525966, + "tokens_seen": 1524236288 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027182547642928785, + "loss": 2.9351, + "theoretical_loss": 3.511109644982098, + "tokens_seen": 1524301824 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002718154463390171, + "loss": 2.5775, + "theoretical_loss": 3.511096376541815, + "tokens_seen": 1524367360 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002718054162487462, + "loss": 2.4679, + "theoretical_loss": 3.5110831088316745, + "tokens_seen": 1524432896 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027179538615847544, + "loss": 2.6322, + "theoretical_loss": 3.511069841851606, + "tokens_seen": 1524498432 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027178535606820457, + "loss": 2.7708, + "theoretical_loss": 3.511056575601539, + "tokens_seen": 1524563968 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002717753259779338, + "loss": 2.4231, + "theoretical_loss": 3.5110433100813996, + "tokens_seen": 1524629504 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027176529588766304, + "loss": 2.4412, + "theoretical_loss": 3.5110300452911174, + "tokens_seen": 1524695040 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027175526579739217, + "loss": 2.6643, + "theoretical_loss": 3.511016781230621, + "tokens_seen": 1524760576 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002717452357071214, + "loss": 2.6002, + "theoretical_loss": 3.511003517899839, + "tokens_seen": 1524826112 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027173520561685053, + "loss": 2.8063, + "theoretical_loss": 3.5109902552987, + "tokens_seen": 1524891648 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027172517552657977, + "loss": 2.3955, + "theoretical_loss": 3.510976993427132, + "tokens_seen": 1524957184 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027171514543630895, + "loss": 2.5937, + "theoretical_loss": 3.510963732285063, + "tokens_seen": 1525022720 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027170511534603813, + "loss": 2.779, + "theoretical_loss": 3.510950471872423, + "tokens_seen": 1525088256 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002716950852557673, + "loss": 2.4343, + "theoretical_loss": 3.51093721218914, + "tokens_seen": 1525153792 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027168505516549655, + "loss": 2.3207, + "theoretical_loss": 3.5109239532351415, + "tokens_seen": 1525219328 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027167502507522567, + "loss": 2.614, + "theoretical_loss": 3.5109106950103577, + "tokens_seen": 1525284864 + }, + { + "epoch": 0.46, + "objective/train/docs_used": 865194, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.894483804702759, + "objective/train/theoretical_loss": 3.5108974375147155, + "objective/train/tokens_used": 1545810400, + "theoretical_loss": 3.5108974375147155, + "tokens_seen": 1525350400 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002716649949849549, + "loss": 2.8039, + "theoretical_loss": 3.5108974375147155, + "tokens_seen": 1525350400 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027165496489468403, + "loss": 2.5759, + "theoretical_loss": 3.510884180748145, + "tokens_seen": 1525415936 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027164493480441327, + "loss": 2.4914, + "theoretical_loss": 3.5108709247105736, + "tokens_seen": 1525481472 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027163490471414245, + "loss": 3.0196, + "theoretical_loss": 3.510857669401931, + "tokens_seen": 1525547008 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027162487462387163, + "loss": 2.5921, + "theoretical_loss": 3.5108444148221443, + "tokens_seen": 1525612544 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002716148445336008, + "loss": 2.6311, + "theoretical_loss": 3.5108311609711436, + "tokens_seen": 1525678080 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027160481444333, + "loss": 2.6551, + "theoretical_loss": 3.510817907848857, + "tokens_seen": 1525743616 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002715947843530592, + "loss": 2.4967, + "theoretical_loss": 3.510804655455213, + "tokens_seen": 1525809152 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002715847542627884, + "loss": 2.5649, + "theoretical_loss": 3.51079140379014, + "tokens_seen": 1525874688 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027157472417251754, + "loss": 2.642, + "theoretical_loss": 3.5107781528535673, + "tokens_seen": 1525940224 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002715646940822468, + "loss": 2.582, + "theoretical_loss": 3.5107649026454233, + "tokens_seen": 1526005760 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002715546639919759, + "loss": 2.588, + "theoretical_loss": 3.510751653165636, + "tokens_seen": 1526071296 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027154463390170514, + "loss": 2.7201, + "theoretical_loss": 3.510738404414136, + "tokens_seen": 1526136832 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002715346038114343, + "loss": 2.7459, + "theoretical_loss": 3.5107251563908495, + "tokens_seen": 1526202368 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002715245737211635, + "loss": 2.621, + "theoretical_loss": 3.510711909095707, + "tokens_seen": 1526267904 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002715145436308927, + "loss": 2.5769, + "theoretical_loss": 3.5106986625286365, + "tokens_seen": 1526333440 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002715045135406219, + "loss": 2.6575, + "theoretical_loss": 3.510685416689567, + "tokens_seen": 1526398976 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027149448345035104, + "loss": 2.6203, + "theoretical_loss": 3.5106721715784266, + "tokens_seen": 1526464512 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002714844533600803, + "loss": 2.4005, + "theoretical_loss": 3.510658927195145, + "tokens_seen": 1526530048 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002714744232698094, + "loss": 2.691, + "theoretical_loss": 3.5106456835396505, + "tokens_seen": 1526595584 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027146439317953864, + "loss": 2.6977, + "theoretical_loss": 3.5106324406118716, + "tokens_seen": 1526661120 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002714543630892678, + "loss": 2.6096, + "theoretical_loss": 3.510619198411738, + "tokens_seen": 1526726656 + }, + { + "epoch": 0.46, + "learning_rate": 0.000271444332998997, + "loss": 2.5677, + "theoretical_loss": 3.5106059569391768, + "tokens_seen": 1526792192 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002714343029087262, + "loss": 2.8643, + "theoretical_loss": 3.510592716194118, + "tokens_seen": 1526857728 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027142427281845536, + "loss": 2.7059, + "theoretical_loss": 3.510579476176491, + "tokens_seen": 1526923264 + }, + { + "epoch": 0.46, + "objective/train/docs_used": 866480, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.9824302196502686, + "objective/train/theoretical_loss": 3.5105662368862234, + "objective/train/tokens_used": 1547448800, + "theoretical_loss": 3.5105662368862234, + "tokens_seen": 1526988800 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027141424272818454, + "loss": 2.8261, + "theoretical_loss": 3.5105662368862234, + "tokens_seen": 1526988800 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002714042126379138, + "loss": 2.7503, + "theoretical_loss": 3.510552998323244, + "tokens_seen": 1527054336 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002713941825476429, + "loss": 2.434, + "theoretical_loss": 3.5105397604874833, + "tokens_seen": 1527119872 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027138415245737214, + "loss": 2.6653, + "theoretical_loss": 3.510526523378868, + "tokens_seen": 1527185408 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027137412236710127, + "loss": 2.6938, + "theoretical_loss": 3.510513286997328, + "tokens_seen": 1527250944 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002713640922768305, + "loss": 2.6539, + "theoretical_loss": 3.5105000513427926, + "tokens_seen": 1527316480 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002713540621865597, + "loss": 2.4706, + "theoretical_loss": 3.5104868164151894, + "tokens_seen": 1527382016 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027134403209628887, + "loss": 2.5544, + "theoretical_loss": 3.5104735822144484, + "tokens_seen": 1527447552 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027133400200601805, + "loss": 2.7014, + "theoretical_loss": 3.510460348740498, + "tokens_seen": 1527513088 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002713239719157473, + "loss": 2.5719, + "theoretical_loss": 3.5104471159932675, + "tokens_seen": 1527578624 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002713139418254764, + "loss": 2.7148, + "theoretical_loss": 3.510433883972686, + "tokens_seen": 1527644160 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027130391173520564, + "loss": 2.7908, + "theoretical_loss": 3.510420652678681, + "tokens_seen": 1527709696 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027129388164493477, + "loss": 2.7047, + "theoretical_loss": 3.510407422111183, + "tokens_seen": 1527775232 + }, + { + "epoch": 0.46, + "learning_rate": 0.000271283851554664, + "loss": 2.7796, + "theoretical_loss": 3.5103941922701205, + "tokens_seen": 1527840768 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002712738214643932, + "loss": 2.6356, + "theoretical_loss": 3.5103809631554217, + "tokens_seen": 1527906304 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027126379137412237, + "loss": 2.6364, + "theoretical_loss": 3.5103677347670166, + "tokens_seen": 1527971840 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027125376128385155, + "loss": 2.6625, + "theoretical_loss": 3.510354507104834, + "tokens_seen": 1528037376 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027124373119358073, + "loss": 2.3966, + "theoretical_loss": 3.5103412801688023, + "tokens_seen": 1528102912 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002712337011033099, + "loss": 2.5465, + "theoretical_loss": 3.510328053958851, + "tokens_seen": 1528168448 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027122367101303915, + "loss": 2.6086, + "theoretical_loss": 3.5103148284749093, + "tokens_seen": 1528233984 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002712136409227683, + "loss": 2.7276, + "theoretical_loss": 3.5103016037169055, + "tokens_seen": 1528299520 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002712036108324975, + "loss": 2.5843, + "theoretical_loss": 3.510288379684769, + "tokens_seen": 1528365056 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002711935807422267, + "loss": 2.5088, + "theoretical_loss": 3.510275156378429, + "tokens_seen": 1528430592 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027118355065195587, + "loss": 2.5216, + "theoretical_loss": 3.510261933797814, + "tokens_seen": 1528496128 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027117352056168505, + "loss": 2.6404, + "theoretical_loss": 3.510248711942854, + "tokens_seen": 1528561664 + }, + { + "epoch": 0.46, + "objective/train/docs_used": 867112, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.8659591674804688, + "objective/train/theoretical_loss": 3.5102354908134776, + "objective/train/tokens_used": 1549087200, + "theoretical_loss": 3.5102354908134776, + "tokens_seen": 1528627200 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027116349047141423, + "loss": 2.7753, + "theoretical_loss": 3.5102354908134776, + "tokens_seen": 1528627200 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002711534603811434, + "loss": 2.6257, + "theoretical_loss": 3.510222270409613, + "tokens_seen": 1528692736 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027114343029087265, + "loss": 2.6825, + "theoretical_loss": 3.5102090507311905, + "tokens_seen": 1528758272 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002711334002006018, + "loss": 2.6841, + "theoretical_loss": 3.510195831778139, + "tokens_seen": 1528823808 + }, + { + "epoch": 0.46, + "learning_rate": 0.000271123370110331, + "loss": 2.6866, + "theoretical_loss": 3.5101826135503873, + "tokens_seen": 1528889344 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027111334002006014, + "loss": 2.7496, + "theoretical_loss": 3.5101693960478646, + "tokens_seen": 1528954880 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002711033099297894, + "loss": 2.7529, + "theoretical_loss": 3.5101561792705, + "tokens_seen": 1529020416 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027109327983951856, + "loss": 2.6561, + "theoretical_loss": 3.5101429632182226, + "tokens_seen": 1529085952 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027108324974924774, + "loss": 2.6239, + "theoretical_loss": 3.5101297478909617, + "tokens_seen": 1529151488 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002710732196589769, + "loss": 2.5443, + "theoretical_loss": 3.510116533288646, + "tokens_seen": 1529217024 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002710631895687061, + "loss": 2.6816, + "theoretical_loss": 3.5101033194112055, + "tokens_seen": 1529282560 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002710531594784353, + "loss": 2.5328, + "theoretical_loss": 3.510090106258569, + "tokens_seen": 1529348096 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002710431293881645, + "loss": 2.5171, + "theoretical_loss": 3.5100768938306652, + "tokens_seen": 1529413632 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027103309929789364, + "loss": 2.5695, + "theoretical_loss": 3.5100636821274236, + "tokens_seen": 1529479168 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002710230692076229, + "loss": 2.4543, + "theoretical_loss": 3.510050471148774, + "tokens_seen": 1529544704 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002710130391173521, + "loss": 2.5014, + "theoretical_loss": 3.510037260894645, + "tokens_seen": 1529610240 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027100300902708124, + "loss": 2.5403, + "theoretical_loss": 3.5100240513649656, + "tokens_seen": 1529675776 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002709929789368105, + "loss": 2.5143, + "theoretical_loss": 3.510010842559666, + "tokens_seen": 1529741312 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002709829488465396, + "loss": 2.3705, + "theoretical_loss": 3.5099976344786743, + "tokens_seen": 1529806848 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027097291875626884, + "loss": 2.5136, + "theoretical_loss": 3.5099844271219203, + "tokens_seen": 1529872384 + }, + { + "epoch": 0.46, + "learning_rate": 0.000270962888665998, + "loss": 2.6365, + "theoretical_loss": 3.5099712204893327, + "tokens_seen": 1529937920 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002709528585757272, + "loss": 2.5817, + "theoretical_loss": 3.5099580145808424, + "tokens_seen": 1530003456 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002709428284854564, + "loss": 2.3495, + "theoretical_loss": 3.509944809396377, + "tokens_seen": 1530068992 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027093279839518556, + "loss": 2.6275, + "theoretical_loss": 3.5099316049358666, + "tokens_seen": 1530134528 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027092276830491474, + "loss": 2.7594, + "theoretical_loss": 3.50991840119924, + "tokens_seen": 1530200064 + }, + { + "epoch": 0.46, + "objective/train/docs_used": 868323, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.711040735244751, + "objective/train/theoretical_loss": 3.5099051981864275, + "objective/train/tokens_used": 1550725600, + "theoretical_loss": 3.5099051981864275, + "tokens_seen": 1530265600 + }, + { + "epoch": 0.46, + "learning_rate": 0.000270912738214644, + "loss": 2.7843, + "theoretical_loss": 3.5099051981864275, + "tokens_seen": 1530265600 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002709027081243731, + "loss": 2.81, + "theoretical_loss": 3.509891995897357, + "tokens_seen": 1530331136 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027089267803410234, + "loss": 2.474, + "theoretical_loss": 3.509878794331959, + "tokens_seen": 1530396672 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027088264794383147, + "loss": 2.7413, + "theoretical_loss": 3.5098655934901624, + "tokens_seen": 1530462208 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002708726178535607, + "loss": 2.6657, + "theoretical_loss": 3.5098523933718964, + "tokens_seen": 1530527744 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002708625877632899, + "loss": 2.4665, + "theoretical_loss": 3.5098391939770908, + "tokens_seen": 1530593280 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027085255767301907, + "loss": 2.7353, + "theoretical_loss": 3.5098259953056745, + "tokens_seen": 1530658816 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027084252758274825, + "loss": 2.5655, + "theoretical_loss": 3.5098127973575775, + "tokens_seen": 1530724352 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002708324974924775, + "loss": 2.5701, + "theoretical_loss": 3.5097996001327285, + "tokens_seen": 1530789888 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002708224674022066, + "loss": 2.6626, + "theoretical_loss": 3.509786403631057, + "tokens_seen": 1530855424 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027081243731193584, + "loss": 2.5452, + "theoretical_loss": 3.509773207852493, + "tokens_seen": 1530920960 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027080240722166497, + "loss": 2.7451, + "theoretical_loss": 3.5097600127969657, + "tokens_seen": 1530986496 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002707923771313942, + "loss": 2.668, + "theoretical_loss": 3.509746818464404, + "tokens_seen": 1531052032 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002707823470411234, + "loss": 2.497, + "theoretical_loss": 3.5097336248547375, + "tokens_seen": 1531117568 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027077231695085257, + "loss": 2.6907, + "theoretical_loss": 3.5097204319678967, + "tokens_seen": 1531183104 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027076228686058175, + "loss": 2.6726, + "theoretical_loss": 3.5097072398038094, + "tokens_seen": 1531248640 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027075225677031093, + "loss": 2.799, + "theoretical_loss": 3.509694048362406, + "tokens_seen": 1531314176 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002707422266800401, + "loss": 2.5713, + "theoretical_loss": 3.509680857643617, + "tokens_seen": 1531379712 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027073219658976935, + "loss": 2.6292, + "theoretical_loss": 3.5096676676473697, + "tokens_seen": 1531445248 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002707221664994985, + "loss": 2.5998, + "theoretical_loss": 3.5096544783735952, + "tokens_seen": 1531510784 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002707121364092277, + "loss": 2.5795, + "theoretical_loss": 3.509641289822222, + "tokens_seen": 1531576320 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002707021063189569, + "loss": 2.8189, + "theoretical_loss": 3.5096281019931803, + "tokens_seen": 1531641856 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027069207622868607, + "loss": 2.5381, + "theoretical_loss": 3.5096149148864, + "tokens_seen": 1531707392 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027068204613841525, + "loss": 2.7031, + "theoretical_loss": 3.5096017285018095, + "tokens_seen": 1531772928 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027067201604814443, + "loss": 2.5673, + "theoretical_loss": 3.5095885428393396, + "tokens_seen": 1531838464 + }, + { + "epoch": 0.46, + "objective/train/docs_used": 869121, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.4047632217407227, + "objective/train/theoretical_loss": 3.5095753578989184, + "objective/train/tokens_used": 1552364000, + "theoretical_loss": 3.5095753578989184, + "tokens_seen": 1531904000 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002706619859578736, + "loss": 2.6565, + "theoretical_loss": 3.5095753578989184, + "tokens_seen": 1531904000 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027065195586760285, + "loss": 2.567, + "theoretical_loss": 3.509562173680477, + "tokens_seen": 1531969536 + }, + { + "epoch": 0.46, + "learning_rate": 0.000270641925777332, + "loss": 2.7269, + "theoretical_loss": 3.5095489901839443, + "tokens_seen": 1532035072 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002706318956870612, + "loss": 2.4268, + "theoretical_loss": 3.5095358074092493, + "tokens_seen": 1532100608 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027062186559679034, + "loss": 2.6777, + "theoretical_loss": 3.5095226253563228, + "tokens_seen": 1532166144 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002706118355065196, + "loss": 2.7631, + "theoretical_loss": 3.5095094440250936, + "tokens_seen": 1532231680 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027060180541624876, + "loss": 2.5253, + "theoretical_loss": 3.5094962634154916, + "tokens_seen": 1532297216 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027059177532597794, + "loss": 2.6229, + "theoretical_loss": 3.5094830835274458, + "tokens_seen": 1532362752 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002705817452357071, + "loss": 2.3646, + "theoretical_loss": 3.509469904360887, + "tokens_seen": 1532428288 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002705717151454363, + "loss": 2.5702, + "theoretical_loss": 3.509456725915744, + "tokens_seen": 1532493824 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002705616850551655, + "loss": 2.611, + "theoretical_loss": 3.509443548191947, + "tokens_seen": 1532559360 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002705516549648947, + "loss": 2.6369, + "theoretical_loss": 3.509430371189425, + "tokens_seen": 1532624896 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027054162487462384, + "loss": 2.6186, + "theoretical_loss": 3.5094171949081083, + "tokens_seen": 1532690432 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002705315947843531, + "loss": 2.6781, + "theoretical_loss": 3.509404019347926, + "tokens_seen": 1532755968 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027052156469408226, + "loss": 2.6966, + "theoretical_loss": 3.5093908445088084, + "tokens_seen": 1532821504 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027051153460381144, + "loss": 2.4321, + "theoretical_loss": 3.509377670390685, + "tokens_seen": 1532887040 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002705015045135406, + "loss": 2.6418, + "theoretical_loss": 3.5093644969934856, + "tokens_seen": 1532952576 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002704914744232698, + "loss": 2.9097, + "theoretical_loss": 3.50935132431714, + "tokens_seen": 1533018112 + }, + { + "epoch": 0.46, + "learning_rate": 0.000270481444332999, + "loss": 2.6073, + "theoretical_loss": 3.509338152361577, + "tokens_seen": 1533083648 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002704714142427282, + "loss": 2.2719, + "theoretical_loss": 3.5093249811267277, + "tokens_seen": 1533149184 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027046138415245735, + "loss": 2.8109, + "theoretical_loss": 3.5093118106125214, + "tokens_seen": 1533214720 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002704513540621866, + "loss": 2.6481, + "theoretical_loss": 3.509298640818888, + "tokens_seen": 1533280256 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002704413239719157, + "loss": 2.6129, + "theoretical_loss": 3.5092854717457564, + "tokens_seen": 1533345792 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027043129388164494, + "loss": 2.6885, + "theoretical_loss": 3.5092723033930566, + "tokens_seen": 1533411328 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002704212637913741, + "loss": 2.5937, + "theoretical_loss": 3.5092591357607197, + "tokens_seen": 1533476864 + }, + { + "epoch": 0.46, + "objective/train/docs_used": 869825, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.3891470432281494, + "objective/train/theoretical_loss": 3.509245968848674, + "objective/train/tokens_used": 1554002400, + "theoretical_loss": 3.509245968848674, + "tokens_seen": 1533542400 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002704112337011033, + "loss": 2.5347, + "theoretical_loss": 3.509245968848674, + "tokens_seen": 1533542400 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002704012036108325, + "loss": 2.5087, + "theoretical_loss": 3.5092328026568507, + "tokens_seen": 1533607936 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027039117352056167, + "loss": 2.6024, + "theoretical_loss": 3.5092196371851783, + "tokens_seen": 1533673472 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027038114343029085, + "loss": 2.6018, + "theoretical_loss": 3.509206472433587, + "tokens_seen": 1533739008 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002703711133400201, + "loss": 2.8623, + "theoretical_loss": 3.5091933084020077, + "tokens_seen": 1533804544 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002703610832497492, + "loss": 2.7243, + "theoretical_loss": 3.509180145090369, + "tokens_seen": 1533870080 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027035105315947845, + "loss": 2.4806, + "theoretical_loss": 3.509166982498601, + "tokens_seen": 1533935616 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027034102306920763, + "loss": 2.6781, + "theoretical_loss": 3.5091538206266337, + "tokens_seen": 1534001152 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002703309929789368, + "loss": 2.5875, + "theoretical_loss": 3.5091406594743972, + "tokens_seen": 1534066688 + }, + { + "epoch": 0.46, + "learning_rate": 0.000270320962888666, + "loss": 2.6666, + "theoretical_loss": 3.5091274990418215, + "tokens_seen": 1534132224 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027031093279839517, + "loss": 2.5112, + "theoretical_loss": 3.5091143393288355, + "tokens_seen": 1534197760 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027030090270812435, + "loss": 2.634, + "theoretical_loss": 3.509101180335371, + "tokens_seen": 1534263296 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002702908726178536, + "loss": 2.5239, + "theoretical_loss": 3.509088022061356, + "tokens_seen": 1534328832 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002702808425275827, + "loss": 2.6371, + "theoretical_loss": 3.5090748645067213, + "tokens_seen": 1534394368 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027027081243731195, + "loss": 2.484, + "theoretical_loss": 3.509061707671397, + "tokens_seen": 1534459904 + }, + { + "epoch": 0.47, + "learning_rate": 0.00027026078234704113, + "loss": 2.6945, + "theoretical_loss": 3.5090485515553134, + "tokens_seen": 1534525440 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002702507522567703, + "loss": 2.678, + "theoretical_loss": 3.5090353961583993, + "tokens_seen": 1534590976 + }, + { + "epoch": 0.47, + "learning_rate": 0.00027024072216649955, + "loss": 2.737, + "theoretical_loss": 3.509022241480585, + "tokens_seen": 1534656512 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002702306920762287, + "loss": 2.6117, + "theoretical_loss": 3.509009087521801, + "tokens_seen": 1534722048 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002702206619859579, + "loss": 2.6539, + "theoretical_loss": 3.5089959342819776, + "tokens_seen": 1534787584 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002702106318956871, + "loss": 2.5156, + "theoretical_loss": 3.508982781761044, + "tokens_seen": 1534853120 + }, + { + "epoch": 0.47, + "learning_rate": 0.00027020060180541627, + "loss": 2.7063, + "theoretical_loss": 3.50896962995893, + "tokens_seen": 1534918656 + }, + { + "epoch": 0.47, + "learning_rate": 0.00027019057171514545, + "loss": 2.6048, + "theoretical_loss": 3.5089564788755667, + "tokens_seen": 1534984192 + }, + { + "epoch": 0.47, + "learning_rate": 0.00027018054162487463, + "loss": 2.7402, + "theoretical_loss": 3.5089433285108838, + "tokens_seen": 1535049728 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002701705115346038, + "loss": 2.4767, + "theoretical_loss": 3.5089301788648104, + "tokens_seen": 1535115264 + }, + { + "epoch": 0.47, + "objective/train/docs_used": 871111, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.4902398586273193, + "objective/train/theoretical_loss": 3.5089170299372774, + "objective/train/tokens_used": 1555640800, + "theoretical_loss": 3.5089170299372774, + "tokens_seen": 1535180800 + }, + { + "epoch": 0.47, + "learning_rate": 0.00027016048144433305, + "loss": 2.501, + "theoretical_loss": 3.5089170299372774, + "tokens_seen": 1535180800 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002701504513540622, + "loss": 2.678, + "theoretical_loss": 3.508903881728215, + "tokens_seen": 1535246336 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002701404212637914, + "loss": 2.6975, + "theoretical_loss": 3.5088907342375526, + "tokens_seen": 1535311872 + }, + { + "epoch": 0.47, + "learning_rate": 0.00027013039117352054, + "loss": 2.4735, + "theoretical_loss": 3.5088775874652214, + "tokens_seen": 1535377408 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002701203610832498, + "loss": 2.8302, + "theoretical_loss": 3.5088644414111503, + "tokens_seen": 1535442944 + }, + { + "epoch": 0.47, + "learning_rate": 0.00027011033099297896, + "loss": 2.6955, + "theoretical_loss": 3.50885129607527, + "tokens_seen": 1535508480 + }, + { + "epoch": 0.47, + "learning_rate": 0.00027010030090270814, + "loss": 2.7225, + "theoretical_loss": 3.5088381514575104, + "tokens_seen": 1535574016 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002700902708124373, + "loss": 2.4598, + "theoretical_loss": 3.5088250075578014, + "tokens_seen": 1535639552 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002700802407221665, + "loss": 2.4976, + "theoretical_loss": 3.508811864376074, + "tokens_seen": 1535705088 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002700702106318957, + "loss": 2.7821, + "theoretical_loss": 3.5087987219122576, + "tokens_seen": 1535770624 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002700601805416249, + "loss": 2.5636, + "theoretical_loss": 3.5087855801662826, + "tokens_seen": 1535836160 + }, + { + "epoch": 0.47, + "learning_rate": 0.00027005015045135404, + "loss": 2.7662, + "theoretical_loss": 3.5087724391380792, + "tokens_seen": 1535901696 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002700401203610833, + "loss": 2.5276, + "theoretical_loss": 3.5087592988275773, + "tokens_seen": 1535967232 + }, + { + "epoch": 0.47, + "learning_rate": 0.00027003009027081246, + "loss": 2.7484, + "theoretical_loss": 3.5087461592347076, + "tokens_seen": 1536032768 + }, + { + "epoch": 0.47, + "learning_rate": 0.00027002006018054164, + "loss": 2.4088, + "theoretical_loss": 3.5087330203594, + "tokens_seen": 1536098304 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002700100300902708, + "loss": 2.7465, + "theoretical_loss": 3.508719882201584, + "tokens_seen": 1536163840 + }, + { + "epoch": 0.47, + "learning_rate": 0.00027, + "loss": 2.5433, + "theoretical_loss": 3.5087067447611906, + "tokens_seen": 1536229376 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002699899699097292, + "loss": 2.642, + "theoretical_loss": 3.5086936080381506, + "tokens_seen": 1536294912 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002699799398194584, + "loss": 2.7059, + "theoretical_loss": 3.5086804720323927, + "tokens_seen": 1536360448 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026996990972918755, + "loss": 2.4946, + "theoretical_loss": 3.5086673367438483, + "tokens_seen": 1536425984 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002699598796389168, + "loss": 2.4166, + "theoretical_loss": 3.5086542021724476, + "tokens_seen": 1536491520 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002699498495486459, + "loss": 2.6881, + "theoretical_loss": 3.50864106831812, + "tokens_seen": 1536557056 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026993981945837514, + "loss": 2.6221, + "theoretical_loss": 3.508627935180797, + "tokens_seen": 1536622592 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002699297893681043, + "loss": 2.5263, + "theoretical_loss": 3.508614802760408, + "tokens_seen": 1536688128 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002699197592778335, + "loss": 2.7051, + "theoretical_loss": 3.5086016710568835, + "tokens_seen": 1536753664 + }, + { + "epoch": 0.47, + "objective/train/docs_used": 871860, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.5156991481781006, + "objective/train/theoretical_loss": 3.5085885400701535, + "objective/train/tokens_used": 1557279200, + "theoretical_loss": 3.5085885400701535, + "tokens_seen": 1536819200 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002699097291875627, + "loss": 2.7643, + "theoretical_loss": 3.5085885400701535, + "tokens_seen": 1536819200 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026989969909729187, + "loss": 2.5637, + "theoretical_loss": 3.508575409800149, + "tokens_seen": 1536884736 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026988966900702105, + "loss": 2.7466, + "theoretical_loss": 3.5085622802467995, + "tokens_seen": 1536950272 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002698796389167503, + "loss": 2.3715, + "theoretical_loss": 3.508549151410037, + "tokens_seen": 1537015808 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002698696088264794, + "loss": 2.7149, + "theoretical_loss": 3.508536023289789, + "tokens_seen": 1537081344 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026985957873620865, + "loss": 2.5531, + "theoretical_loss": 3.5085228958859886, + "tokens_seen": 1537146880 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026984954864593783, + "loss": 2.7664, + "theoretical_loss": 3.5085097691985645, + "tokens_seen": 1537212416 + }, + { + "epoch": 0.47, + "learning_rate": 0.000269839518555667, + "loss": 2.4226, + "theoretical_loss": 3.508496643227448, + "tokens_seen": 1537277952 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002698294884653962, + "loss": 2.6892, + "theoretical_loss": 3.508483517972569, + "tokens_seen": 1537343488 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026981945837512537, + "loss": 2.6959, + "theoretical_loss": 3.5084703934338575, + "tokens_seen": 1537409024 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026980942828485455, + "loss": 2.6946, + "theoretical_loss": 3.5084572696112444, + "tokens_seen": 1537474560 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002697993981945838, + "loss": 2.3024, + "theoretical_loss": 3.5084441465046607, + "tokens_seen": 1537540096 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002697893681043129, + "loss": 2.4168, + "theoretical_loss": 3.508431024114035, + "tokens_seen": 1537605632 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026977933801404215, + "loss": 2.6279, + "theoretical_loss": 3.5084179024393, + "tokens_seen": 1537671168 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002697693079237713, + "loss": 2.5607, + "theoretical_loss": 3.508404781480385, + "tokens_seen": 1537736704 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002697592778335005, + "loss": 2.692, + "theoretical_loss": 3.50839166123722, + "tokens_seen": 1537802240 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002697492477432297, + "loss": 2.9005, + "theoretical_loss": 3.508378541709736, + "tokens_seen": 1537867776 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002697392176529589, + "loss": 2.7201, + "theoretical_loss": 3.508365422897864, + "tokens_seen": 1537933312 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026972918756268806, + "loss": 2.4686, + "theoretical_loss": 3.508352304801533, + "tokens_seen": 1537998848 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002697191574724173, + "loss": 2.3366, + "theoretical_loss": 3.508339187420675, + "tokens_seen": 1538064384 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002697091273821464, + "loss": 2.7771, + "theoretical_loss": 3.5083260707552193, + "tokens_seen": 1538129920 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026969909729187565, + "loss": 2.496, + "theoretical_loss": 3.5083129548050973, + "tokens_seen": 1538195456 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002696890672016048, + "loss": 2.6483, + "theoretical_loss": 3.508299839570239, + "tokens_seen": 1538260992 + }, + { + "epoch": 0.47, + "learning_rate": 0.000269679037111334, + "loss": 2.6134, + "theoretical_loss": 3.508286725050575, + "tokens_seen": 1538326528 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002696690070210632, + "loss": 2.6032, + "theoretical_loss": 3.5082736112460364, + "tokens_seen": 1538392064 + }, + { + "epoch": 0.47, + "objective/train/docs_used": 873345, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.3517513275146484, + "objective/train/theoretical_loss": 3.5082604981565533, + "objective/train/tokens_used": 1558917600, + "theoretical_loss": 3.5082604981565533, + "tokens_seen": 1538457600 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002696589769307924, + "loss": 2.6716, + "theoretical_loss": 3.5082604981565533, + "tokens_seen": 1538457600 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026964894684052156, + "loss": 2.5355, + "theoretical_loss": 3.508247385782056, + "tokens_seen": 1538523136 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026963891675025074, + "loss": 2.632, + "theoretical_loss": 3.508234274122475, + "tokens_seen": 1538588672 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002696288866599799, + "loss": 2.6904, + "theoretical_loss": 3.5082211631777414, + "tokens_seen": 1538654208 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026961885656970916, + "loss": 2.4571, + "theoretical_loss": 3.5082080529477855, + "tokens_seen": 1538719744 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002696088264794383, + "loss": 2.3093, + "theoretical_loss": 3.5081949434325383, + "tokens_seen": 1538785280 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002695987963891675, + "loss": 2.618, + "theoretical_loss": 3.5081818346319293, + "tokens_seen": 1538850816 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026958876629889665, + "loss": 2.753, + "theoretical_loss": 3.5081687265458905, + "tokens_seen": 1538916352 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002695787362086259, + "loss": 2.5434, + "theoretical_loss": 3.5081556191743517, + "tokens_seen": 1538981888 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026956870611835506, + "loss": 2.6772, + "theoretical_loss": 3.5081425125172436, + "tokens_seen": 1539047424 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026955867602808424, + "loss": 2.7859, + "theoretical_loss": 3.508129406574497, + "tokens_seen": 1539112960 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002695486459378134, + "loss": 2.5648, + "theoretical_loss": 3.5081163013460426, + "tokens_seen": 1539178496 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026953861584754266, + "loss": 2.6376, + "theoretical_loss": 3.5081031968318106, + "tokens_seen": 1539244032 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002695285857572718, + "loss": 2.5485, + "theoretical_loss": 3.5080900930317327, + "tokens_seen": 1539309568 + }, + { + "epoch": 0.47, + "learning_rate": 0.000269518555667001, + "loss": 2.7848, + "theoretical_loss": 3.508076989945738, + "tokens_seen": 1539375104 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002695085255767302, + "loss": 2.5069, + "theoretical_loss": 3.5080638875737584, + "tokens_seen": 1539440640 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002694984954864594, + "loss": 2.4509, + "theoretical_loss": 3.5080507859157244, + "tokens_seen": 1539506176 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002694884653961886, + "loss": 2.5489, + "theoretical_loss": 3.5080376849715664, + "tokens_seen": 1539571712 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026947843530591775, + "loss": 2.6621, + "theoretical_loss": 3.5080245847412153, + "tokens_seen": 1539637248 + }, + { + "epoch": 0.47, + "learning_rate": 0.000269468405215647, + "loss": 2.7986, + "theoretical_loss": 3.5080114852246025, + "tokens_seen": 1539702784 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002694583751253761, + "loss": 2.8523, + "theoretical_loss": 3.507998386421657, + "tokens_seen": 1539768320 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026944834503510534, + "loss": 2.5764, + "theoretical_loss": 3.5079852883323115, + "tokens_seen": 1539833856 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002694383149448345, + "loss": 2.5184, + "theoretical_loss": 3.5079721909564956, + "tokens_seen": 1539899392 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002694282848545637, + "loss": 2.5757, + "theoretical_loss": 3.50795909429414, + "tokens_seen": 1539964928 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002694182547642929, + "loss": 2.6813, + "theoretical_loss": 3.5079459983451757, + "tokens_seen": 1540030464 + }, + { + "epoch": 0.47, + "objective/train/docs_used": 874032, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7814881801605225, + "objective/train/theoretical_loss": 3.507932903109534, + "objective/train/tokens_used": 1560556000, + "theoretical_loss": 3.507932903109534, + "tokens_seen": 1540096000 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026940822467402207, + "loss": 2.5366, + "theoretical_loss": 3.507932903109534, + "tokens_seen": 1540096000 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026939819458375125, + "loss": 2.6113, + "theoretical_loss": 3.5079198085871455, + "tokens_seen": 1540161536 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002693881644934805, + "loss": 2.5866, + "theoretical_loss": 3.50790671477794, + "tokens_seen": 1540227072 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002693781344032096, + "loss": 2.6616, + "theoretical_loss": 3.50789362168185, + "tokens_seen": 1540292608 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026936810431293885, + "loss": 2.7373, + "theoretical_loss": 3.507880529298805, + "tokens_seen": 1540358144 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026935807422266803, + "loss": 2.4436, + "theoretical_loss": 3.507867437628736, + "tokens_seen": 1540423680 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002693480441323972, + "loss": 2.9462, + "theoretical_loss": 3.507854346671575, + "tokens_seen": 1540489216 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002693380140421264, + "loss": 2.5578, + "theoretical_loss": 3.507841256427251, + "tokens_seen": 1540554752 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026932798395185557, + "loss": 2.5172, + "theoretical_loss": 3.5078281668956963, + "tokens_seen": 1540620288 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026931795386158475, + "loss": 2.6261, + "theoretical_loss": 3.507815078076841, + "tokens_seen": 1540685824 + }, + { + "epoch": 0.47, + "learning_rate": 0.000269307923771314, + "loss": 2.9126, + "theoretical_loss": 3.5078019899706163, + "tokens_seen": 1540751360 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002692978936810431, + "loss": 2.677, + "theoretical_loss": 3.5077889025769533, + "tokens_seen": 1540816896 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026928786359077235, + "loss": 2.4633, + "theoretical_loss": 3.5077758158957826, + "tokens_seen": 1540882432 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002692778335005015, + "loss": 2.573, + "theoretical_loss": 3.507762729927035, + "tokens_seen": 1540947968 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002692678034102307, + "loss": 2.4887, + "theoretical_loss": 3.5077496446706418, + "tokens_seen": 1541013504 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002692577733199599, + "loss": 2.608, + "theoretical_loss": 3.5077365601265336, + "tokens_seen": 1541079040 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002692477432296891, + "loss": 2.732, + "theoretical_loss": 3.5077234762946414, + "tokens_seen": 1541144576 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026923771313941826, + "loss": 2.5873, + "theoretical_loss": 3.5077103931748965, + "tokens_seen": 1541210112 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002692276830491475, + "loss": 2.638, + "theoretical_loss": 3.507697310767229, + "tokens_seen": 1541275648 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002692176529588766, + "loss": 2.6657, + "theoretical_loss": 3.507684229071571, + "tokens_seen": 1541341184 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026920762286860585, + "loss": 2.778, + "theoretical_loss": 3.5076711480878524, + "tokens_seen": 1541406720 + }, + { + "epoch": 0.47, + "learning_rate": 0.000269197592778335, + "loss": 2.5782, + "theoretical_loss": 3.5076580678160054, + "tokens_seen": 1541472256 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002691875626880642, + "loss": 2.7724, + "theoretical_loss": 3.5076449882559597, + "tokens_seen": 1541537792 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002691775325977934, + "loss": 2.5657, + "theoretical_loss": 3.507631909407647, + "tokens_seen": 1541603328 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002691675025075226, + "loss": 2.7338, + "theoretical_loss": 3.5076188312709986, + "tokens_seen": 1541668864 + }, + { + "epoch": 0.47, + "objective/train/docs_used": 874753, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.3264641761779785, + "objective/train/theoretical_loss": 3.5076057538459446, + "objective/train/tokens_used": 1562194400, + "theoretical_loss": 3.5076057538459446, + "tokens_seen": 1541734400 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026915747241725176, + "loss": 2.5034, + "theoretical_loss": 3.5076057538459446, + "tokens_seen": 1541734400 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026914744232698094, + "loss": 2.53, + "theoretical_loss": 3.5075926771324166, + "tokens_seen": 1541799936 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002691374122367101, + "loss": 2.4873, + "theoretical_loss": 3.507579601130346, + "tokens_seen": 1541865472 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026912738214643936, + "loss": 2.7544, + "theoretical_loss": 3.5075665258396636, + "tokens_seen": 1541931008 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002691173520561685, + "loss": 2.6133, + "theoretical_loss": 3.5075534512603, + "tokens_seen": 1541996544 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002691073219658977, + "loss": 2.6763, + "theoretical_loss": 3.5075403773921865, + "tokens_seen": 1542062080 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026909729187562685, + "loss": 2.6611, + "theoretical_loss": 3.5075273042352544, + "tokens_seen": 1542127616 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002690872617853561, + "loss": 2.6221, + "theoretical_loss": 3.507514231789435, + "tokens_seen": 1542193152 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026907723169508526, + "loss": 2.5379, + "theoretical_loss": 3.507501160054659, + "tokens_seen": 1542258688 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026906720160481444, + "loss": 2.7017, + "theoretical_loss": 3.5074880890308577, + "tokens_seen": 1542324224 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002690571715145436, + "loss": 2.6782, + "theoretical_loss": 3.5074750187179617, + "tokens_seen": 1542389760 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026904714142427286, + "loss": 2.4061, + "theoretical_loss": 3.507461949115903, + "tokens_seen": 1542455296 + }, + { + "epoch": 0.47, + "learning_rate": 0.000269037111334002, + "loss": 2.481, + "theoretical_loss": 3.5074488802246115, + "tokens_seen": 1542520832 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002690270812437312, + "loss": 2.6896, + "theoretical_loss": 3.5074358120440206, + "tokens_seen": 1542586368 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026901705115346035, + "loss": 2.3587, + "theoretical_loss": 3.5074227445740584, + "tokens_seen": 1542651904 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002690070210631896, + "loss": 2.282, + "theoretical_loss": 3.5074096778146586, + "tokens_seen": 1542717440 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026899699097291877, + "loss": 2.6044, + "theoretical_loss": 3.507396611765752, + "tokens_seen": 1542782976 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026898696088264795, + "loss": 2.9122, + "theoretical_loss": 3.5073835464272682, + "tokens_seen": 1542848512 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026897693079237713, + "loss": 2.6776, + "theoretical_loss": 3.50737048179914, + "tokens_seen": 1542914048 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002689669007021063, + "loss": 2.6363, + "theoretical_loss": 3.5073574178812974, + "tokens_seen": 1542979584 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002689568706118355, + "loss": 2.8523, + "theoretical_loss": 3.5073443546736733, + "tokens_seen": 1543045120 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002689468405215647, + "loss": 2.6437, + "theoretical_loss": 3.507331292176197, + "tokens_seen": 1543110656 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026893681043129385, + "loss": 2.7208, + "theoretical_loss": 3.507318230388801, + "tokens_seen": 1543176192 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002689267803410231, + "loss": 2.5591, + "theoretical_loss": 3.507305169311416, + "tokens_seen": 1543241728 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002689167502507522, + "loss": 2.6801, + "theoretical_loss": 3.507292108943974, + "tokens_seen": 1543307264 + }, + { + "epoch": 0.47, + "objective/train/docs_used": 875990, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.1270933151245117, + "objective/train/theoretical_loss": 3.5072790492864057, + "objective/train/tokens_used": 1563832800, + "theoretical_loss": 3.5072790492864057, + "tokens_seen": 1543372800 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026890672016048145, + "loss": 2.8653, + "theoretical_loss": 3.5072790492864057, + "tokens_seen": 1543372800 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026889669007021063, + "loss": 2.9012, + "theoretical_loss": 3.5072659903386416, + "tokens_seen": 1543438336 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002688866599799398, + "loss": 2.852, + "theoretical_loss": 3.5072529321006147, + "tokens_seen": 1543503872 + }, + { + "epoch": 0.47, + "learning_rate": 0.000268876629889669, + "loss": 2.6837, + "theoretical_loss": 3.507239874572255, + "tokens_seen": 1543569408 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026886659979939823, + "loss": 2.6397, + "theoretical_loss": 3.5072268177534935, + "tokens_seen": 1543634944 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026885656970912736, + "loss": 2.6115, + "theoretical_loss": 3.5072137616442625, + "tokens_seen": 1543700480 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002688465396188566, + "loss": 2.6432, + "theoretical_loss": 3.5072007062444936, + "tokens_seen": 1543766016 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002688365095285857, + "loss": 2.4367, + "theoretical_loss": 3.5071876515541174, + "tokens_seen": 1543831552 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002688365095285857, + "loss": 2.7342, + "theoretical_loss": 3.507174597573065, + "tokens_seen": 1543897088 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026882647943831495, + "loss": 2.6594, + "theoretical_loss": 3.507161544301268, + "tokens_seen": 1543962624 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026881644934804413, + "loss": 2.5378, + "theoretical_loss": 3.5071484917386586, + "tokens_seen": 1544028160 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002688064192577733, + "loss": 2.8145, + "theoretical_loss": 3.5071354398851673, + "tokens_seen": 1544093696 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002687963891675025, + "loss": 2.6075, + "theoretical_loss": 3.507122388740725, + "tokens_seen": 1544159232 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002687863590772317, + "loss": 2.7772, + "theoretical_loss": 3.5071093383052645, + "tokens_seen": 1544224768 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026877632898696086, + "loss": 2.6651, + "theoretical_loss": 3.5070962885787154, + "tokens_seen": 1544290304 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002687662988966901, + "loss": 2.6603, + "theoretical_loss": 3.507083239561011, + "tokens_seen": 1544355840 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002687562688064193, + "loss": 2.5861, + "theoretical_loss": 3.507070191252082, + "tokens_seen": 1544421376 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026874623871614846, + "loss": 2.6028, + "theoretical_loss": 3.507057143651859, + "tokens_seen": 1544486912 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002687362086258777, + "loss": 2.8573, + "theoretical_loss": 3.507044096760275, + "tokens_seen": 1544552448 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002687261785356068, + "loss": 2.5564, + "theoretical_loss": 3.5070310505772597, + "tokens_seen": 1544617984 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026871614844533605, + "loss": 2.7542, + "theoretical_loss": 3.5070180051027458, + "tokens_seen": 1544683520 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002687061183550652, + "loss": 2.7218, + "theoretical_loss": 3.5070049603366646, + "tokens_seen": 1544749056 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002686960882647944, + "loss": 2.5865, + "theoretical_loss": 3.506991916278947, + "tokens_seen": 1544814592 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002686860581745236, + "loss": 2.7045, + "theoretical_loss": 3.5069788729295253, + "tokens_seen": 1544880128 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002686760280842528, + "loss": 2.5319, + "theoretical_loss": 3.5069658302883298, + "tokens_seen": 1544945664 + }, + { + "epoch": 0.47, + "objective/train/docs_used": 876526, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.5547046661376953, + "objective/train/theoretical_loss": 3.5069527883552936, + "objective/train/tokens_used": 1565471200, + "theoretical_loss": 3.5069527883552936, + "tokens_seen": 1545011200 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026866599799398196, + "loss": 2.9034, + "theoretical_loss": 3.5069527883552936, + "tokens_seen": 1545011200 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026865596790371114, + "loss": 2.7457, + "theoretical_loss": 3.506939747130347, + "tokens_seen": 1545076736 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002686459378134403, + "loss": 2.7905, + "theoretical_loss": 3.506926706613422, + "tokens_seen": 1545142272 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026863590772316956, + "loss": 2.5075, + "theoretical_loss": 3.50691366680445, + "tokens_seen": 1545207808 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002686258776328987, + "loss": 2.7384, + "theoretical_loss": 3.506900627703362, + "tokens_seen": 1545273344 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002686158475426279, + "loss": 2.6256, + "theoretical_loss": 3.5068875893100913, + "tokens_seen": 1545338880 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026860581745235705, + "loss": 2.3252, + "theoretical_loss": 3.506874551624567, + "tokens_seen": 1545404416 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002685957873620863, + "loss": 2.5677, + "theoretical_loss": 3.506861514646723, + "tokens_seen": 1545469952 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026858575727181546, + "loss": 2.418, + "theoretical_loss": 3.5068484783764893, + "tokens_seen": 1545535488 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026857572718154464, + "loss": 2.5654, + "theoretical_loss": 3.5068354428137978, + "tokens_seen": 1545601024 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002685656970912738, + "loss": 2.3617, + "theoretical_loss": 3.5068224079585812, + "tokens_seen": 1545666560 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026855566700100306, + "loss": 2.6082, + "theoretical_loss": 3.5068093738107695, + "tokens_seen": 1545732096 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002685456369107322, + "loss": 2.5198, + "theoretical_loss": 3.506796340370295, + "tokens_seen": 1545797632 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002685356068204614, + "loss": 2.7052, + "theoretical_loss": 3.5067833076370896, + "tokens_seen": 1545863168 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026852557673019055, + "loss": 2.6513, + "theoretical_loss": 3.506770275611085, + "tokens_seen": 1545928704 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002685155466399198, + "loss": 2.4439, + "theoretical_loss": 3.5067572442922126, + "tokens_seen": 1545994240 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026850551654964897, + "loss": 2.3904, + "theoretical_loss": 3.5067442136804035, + "tokens_seen": 1546059776 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026849548645937815, + "loss": 2.6958, + "theoretical_loss": 3.50673118377559, + "tokens_seen": 1546125312 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026848545636910733, + "loss": 2.4947, + "theoretical_loss": 3.5067181545777037, + "tokens_seen": 1546190848 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002684754262788365, + "loss": 2.7375, + "theoretical_loss": 3.5067051260866764, + "tokens_seen": 1546256384 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002684653961885657, + "loss": 2.837, + "theoretical_loss": 3.50669209830244, + "tokens_seen": 1546321920 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002684553660982949, + "loss": 2.5413, + "theoretical_loss": 3.5066790712249256, + "tokens_seen": 1546387456 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026844533600802405, + "loss": 2.6354, + "theoretical_loss": 3.5066660448540645, + "tokens_seen": 1546452992 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002684353059177533, + "loss": 2.5961, + "theoretical_loss": 3.50665301918979, + "tokens_seen": 1546518528 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002684353059177533, + "loss": 2.7453, + "theoretical_loss": 3.5066399942320325, + "tokens_seen": 1546584064 + }, + { + "epoch": 0.47, + "objective/train/docs_used": 877732, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.2810020446777344, + "objective/train/theoretical_loss": 3.506626969980724, + "objective/train/tokens_used": 1567109600, + "theoretical_loss": 3.506626969980724, + "tokens_seen": 1546649600 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002684252758274824, + "loss": 2.5022, + "theoretical_loss": 3.506626969980724, + "tokens_seen": 1546649600 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026841524573721165, + "loss": 2.7434, + "theoretical_loss": 3.506613946435797, + "tokens_seen": 1546715136 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026840521564694083, + "loss": 2.6172, + "theoretical_loss": 3.5066009235971825, + "tokens_seen": 1546780672 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026839518555667, + "loss": 2.7468, + "theoretical_loss": 3.506587901464812, + "tokens_seen": 1546846208 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002683851554663992, + "loss": 2.5373, + "theoretical_loss": 3.5065748800386176, + "tokens_seen": 1546911744 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026837512537612843, + "loss": 2.7588, + "theoretical_loss": 3.506561859318532, + "tokens_seen": 1546977280 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026836509528585756, + "loss": 2.6691, + "theoretical_loss": 3.506548839304486, + "tokens_seen": 1547042816 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002683550651955868, + "loss": 2.8414, + "theoretical_loss": 3.506535819996411, + "tokens_seen": 1547108352 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002683450351053159, + "loss": 2.6825, + "theoretical_loss": 3.50652280139424, + "tokens_seen": 1547173888 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026833500501504515, + "loss": 2.8223, + "theoretical_loss": 3.506509783497904, + "tokens_seen": 1547239424 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026832497492477433, + "loss": 2.8127, + "theoretical_loss": 3.506496766307335, + "tokens_seen": 1547304960 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002683149448345035, + "loss": 2.7559, + "theoretical_loss": 3.5064837498224657, + "tokens_seen": 1547370496 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002683049147442327, + "loss": 2.4971, + "theoretical_loss": 3.5064707340432264, + "tokens_seen": 1547436032 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002682948846539619, + "loss": 2.7106, + "theoretical_loss": 3.5064577189695503, + "tokens_seen": 1547501568 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026828485456369106, + "loss": 2.4608, + "theoretical_loss": 3.5064447046013685, + "tokens_seen": 1547567104 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002682748244734203, + "loss": 2.6151, + "theoretical_loss": 3.506431690938613, + "tokens_seen": 1547632640 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002682647943831494, + "loss": 2.7374, + "theoretical_loss": 3.506418677981216, + "tokens_seen": 1547698176 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026825476429287866, + "loss": 2.6634, + "theoretical_loss": 3.5064056657291096, + "tokens_seen": 1547763712 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002682447342026078, + "loss": 2.789, + "theoretical_loss": 3.506392654182225, + "tokens_seen": 1547829248 + }, + { + "epoch": 0.47, + "learning_rate": 0.000268234704112337, + "loss": 2.63, + "theoretical_loss": 3.5063796433404946, + "tokens_seen": 1547894784 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002682246740220662, + "loss": 2.8251, + "theoretical_loss": 3.50636663320385, + "tokens_seen": 1547960320 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002682146439317954, + "loss": 2.6498, + "theoretical_loss": 3.506353623772223, + "tokens_seen": 1548025856 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026820461384152456, + "loss": 2.4327, + "theoretical_loss": 3.5063406150455467, + "tokens_seen": 1548091392 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002681945837512538, + "loss": 2.7826, + "theoretical_loss": 3.506327607023752, + "tokens_seen": 1548156928 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002681845536609829, + "loss": 2.709, + "theoretical_loss": 3.50631459970677, + "tokens_seen": 1548222464 + }, + { + "epoch": 0.47, + "objective/train/docs_used": 878530, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.854560136795044, + "objective/train/theoretical_loss": 3.5063015930945354, + "objective/train/tokens_used": 1568748000, + "theoretical_loss": 3.5063015930945354, + "tokens_seen": 1548288000 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026817452357071216, + "loss": 2.7018, + "theoretical_loss": 3.5063015930945354, + "tokens_seen": 1548288000 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002681644934804413, + "loss": 2.5975, + "theoretical_loss": 3.506288587186978, + "tokens_seen": 1548353536 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002681544633901705, + "loss": 2.6415, + "theoretical_loss": 3.50627558198403, + "tokens_seen": 1548419072 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002681444332998997, + "loss": 2.5517, + "theoretical_loss": 3.506262577485624, + "tokens_seen": 1548484608 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002681444332998997, + "loss": 2.6193, + "theoretical_loss": 3.506249573691692, + "tokens_seen": 1548550144 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002681344032096289, + "loss": 2.7581, + "theoretical_loss": 3.506236570602166, + "tokens_seen": 1548615680 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026812437311935807, + "loss": 2.588, + "theoretical_loss": 3.5062235682169773, + "tokens_seen": 1548681216 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026811434302908725, + "loss": 2.8013, + "theoretical_loss": 3.5062105665360592, + "tokens_seen": 1548746752 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026810431293881643, + "loss": 2.6225, + "theoretical_loss": 3.5061975655593423, + "tokens_seen": 1548812288 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026809428284854566, + "loss": 2.6403, + "theoretical_loss": 3.50618456528676, + "tokens_seen": 1548877824 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002680842527582748, + "loss": 2.831, + "theoretical_loss": 3.5061715657182435, + "tokens_seen": 1548943360 + }, + { + "epoch": 0.47, + "learning_rate": 0.000268074222668004, + "loss": 2.585, + "theoretical_loss": 3.506158566853725, + "tokens_seen": 1549008896 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026806419257773315, + "loss": 2.8504, + "theoretical_loss": 3.506145568693137, + "tokens_seen": 1549074432 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002680541624874624, + "loss": 2.3203, + "theoretical_loss": 3.506132571236412, + "tokens_seen": 1549139968 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026804413239719157, + "loss": 2.4639, + "theoretical_loss": 3.5061195744834808, + "tokens_seen": 1549205504 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026803410230692075, + "loss": 2.4918, + "theoretical_loss": 3.5061065784342764, + "tokens_seen": 1549271040 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026802407221664993, + "loss": 2.5981, + "theoretical_loss": 3.506093583088731, + "tokens_seen": 1549336576 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026801404212637917, + "loss": 2.8474, + "theoretical_loss": 3.5060805884467756, + "tokens_seen": 1549402112 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026800401203610835, + "loss": 2.6992, + "theoretical_loss": 3.5060675945083437, + "tokens_seen": 1549467648 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026799398194583753, + "loss": 2.6719, + "theoretical_loss": 3.5060546012733673, + "tokens_seen": 1549533184 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002679839518555667, + "loss": 2.6879, + "theoretical_loss": 3.506041608741778, + "tokens_seen": 1549598720 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002679739217652959, + "loss": 2.6404, + "theoretical_loss": 3.5060286169135084, + "tokens_seen": 1549664256 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002679638916750251, + "loss": 2.9387, + "theoretical_loss": 3.5060156257884905, + "tokens_seen": 1549729792 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026795386158475425, + "loss": 2.6224, + "theoretical_loss": 3.5060026353666562, + "tokens_seen": 1549795328 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002679438314944835, + "loss": 2.7271, + "theoretical_loss": 3.5059896456479382, + "tokens_seen": 1549860864 + }, + { + "epoch": 0.47, + "objective/train/docs_used": 879841, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.9379489421844482, + "objective/train/theoretical_loss": 3.505976656632269, + "objective/train/tokens_used": 1570386400, + "theoretical_loss": 3.505976656632269, + "tokens_seen": 1549926400 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002679338014042126, + "loss": 2.578, + "theoretical_loss": 3.505976656632269, + "tokens_seen": 1549926400 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026792377131394185, + "loss": 2.5411, + "theoretical_loss": 3.5059636683195796, + "tokens_seen": 1549991936 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026791374122367103, + "loss": 2.7893, + "theoretical_loss": 3.5059506807098035, + "tokens_seen": 1550057472 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002679037111334002, + "loss": 2.8206, + "theoretical_loss": 3.505937693802872, + "tokens_seen": 1550123008 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002678936810431294, + "loss": 2.5987, + "theoretical_loss": 3.505924707598718, + "tokens_seen": 1550188544 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026788365095285863, + "loss": 2.6916, + "theoretical_loss": 3.5059117220972738, + "tokens_seen": 1550254080 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026787362086258776, + "loss": 2.6769, + "theoretical_loss": 3.5058987372984713, + "tokens_seen": 1550319616 + }, + { + "epoch": 0.47, + "learning_rate": 0.000267863590772317, + "loss": 2.5113, + "theoretical_loss": 3.5058857532022425, + "tokens_seen": 1550385152 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002678535606820461, + "loss": 2.6285, + "theoretical_loss": 3.50587276980852, + "tokens_seen": 1550450688 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026784353059177535, + "loss": 2.797, + "theoretical_loss": 3.5058597871172372, + "tokens_seen": 1550516224 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026783350050150453, + "loss": 2.3696, + "theoretical_loss": 3.5058468051283245, + "tokens_seen": 1550581760 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002678234704112337, + "loss": 2.5797, + "theoretical_loss": 3.5058338238417157, + "tokens_seen": 1550647296 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002678134403209629, + "loss": 2.5292, + "theoretical_loss": 3.505820843257342, + "tokens_seen": 1550712832 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002678034102306921, + "loss": 2.8502, + "theoretical_loss": 3.5058078633751366, + "tokens_seen": 1550778368 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026779338014042126, + "loss": 2.5625, + "theoretical_loss": 3.505794884195031, + "tokens_seen": 1550843904 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002677833500501505, + "loss": 2.6469, + "theoretical_loss": 3.5057819057169586, + "tokens_seen": 1550909440 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002677733199598796, + "loss": 2.8036, + "theoretical_loss": 3.505768927940851, + "tokens_seen": 1550974976 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026776328986960886, + "loss": 2.5229, + "theoretical_loss": 3.505755950866641, + "tokens_seen": 1551040512 + }, + { + "epoch": 0.47, + "learning_rate": 0.000267753259779338, + "loss": 2.7097, + "theoretical_loss": 3.505742974494261, + "tokens_seen": 1551106048 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002677432296890672, + "loss": 2.5701, + "theoretical_loss": 3.5057299988236426, + "tokens_seen": 1551171584 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002677331995987964, + "loss": 2.7407, + "theoretical_loss": 3.505717023854719, + "tokens_seen": 1551237120 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002677231695085256, + "loss": 2.729, + "theoretical_loss": 3.5057040495874228, + "tokens_seen": 1551302656 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026771313941825476, + "loss": 2.687, + "theoretical_loss": 3.5056910760216855, + "tokens_seen": 1551368192 + }, + { + "epoch": 0.47, + "learning_rate": 0.000267703109327984, + "loss": 2.7082, + "theoretical_loss": 3.5056781031574404, + "tokens_seen": 1551433728 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002676930792377131, + "loss": 2.6538, + "theoretical_loss": 3.505665130994619, + "tokens_seen": 1551499264 + }, + { + "epoch": 0.47, + "objective/train/docs_used": 880523, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.479295253753662, + "objective/train/theoretical_loss": 3.505652159533155, + "objective/train/tokens_used": 1572024800, + "theoretical_loss": 3.505652159533155, + "tokens_seen": 1551564800 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026768304914744236, + "loss": 2.6367, + "theoretical_loss": 3.505652159533155, + "tokens_seen": 1551564800 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002676730190571715, + "loss": 2.7375, + "theoretical_loss": 3.5056391887729803, + "tokens_seen": 1551630336 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002676629889669007, + "loss": 2.6114, + "theoretical_loss": 3.505626218714027, + "tokens_seen": 1551695872 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002676529588766299, + "loss": 2.4883, + "theoretical_loss": 3.5056132493562275, + "tokens_seen": 1551761408 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002676429287863591, + "loss": 2.4734, + "theoretical_loss": 3.5056002806995155, + "tokens_seen": 1551826944 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026763289869608827, + "loss": 2.3787, + "theoretical_loss": 3.5055873127438217, + "tokens_seen": 1551892480 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026762286860581745, + "loss": 2.8983, + "theoretical_loss": 3.50557434548908, + "tokens_seen": 1551958016 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026761283851554663, + "loss": 2.7443, + "theoretical_loss": 3.505561378935223, + "tokens_seen": 1552023552 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026760280842527586, + "loss": 2.5498, + "theoretical_loss": 3.5055484130821815, + "tokens_seen": 1552089088 + }, + { + "epoch": 0.47, + "learning_rate": 0.000267592778335005, + "loss": 2.5412, + "theoretical_loss": 3.5055354479298897, + "tokens_seen": 1552154624 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002675827482447342, + "loss": 2.6835, + "theoretical_loss": 3.5055224834782797, + "tokens_seen": 1552220160 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026757271815446335, + "loss": 2.8905, + "theoretical_loss": 3.5055095197272843, + "tokens_seen": 1552285696 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002675626880641926, + "loss": 2.7525, + "theoretical_loss": 3.5054965566768352, + "tokens_seen": 1552351232 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026755265797392177, + "loss": 2.505, + "theoretical_loss": 3.505483594326866, + "tokens_seen": 1552416768 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026754262788365095, + "loss": 2.7643, + "theoretical_loss": 3.5054706326773086, + "tokens_seen": 1552482304 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026753259779338013, + "loss": 2.4957, + "theoretical_loss": 3.5054576717280956, + "tokens_seen": 1552547840 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026752256770310937, + "loss": 2.7663, + "theoretical_loss": 3.50544471147916, + "tokens_seen": 1552613376 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002675125376128385, + "loss": 2.5019, + "theoretical_loss": 3.505431751930434, + "tokens_seen": 1552678912 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026750250752256773, + "loss": 2.6132, + "theoretical_loss": 3.505418793081851, + "tokens_seen": 1552744448 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026749247743229686, + "loss": 2.6286, + "theoretical_loss": 3.5054058349333426, + "tokens_seen": 1552809984 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002674824473420261, + "loss": 2.8158, + "theoretical_loss": 3.5053928774848417, + "tokens_seen": 1552875520 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026747241725175527, + "loss": 2.7978, + "theoretical_loss": 3.5053799207362815, + "tokens_seen": 1552941056 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026746238716148445, + "loss": 2.6513, + "theoretical_loss": 3.5053669646875942, + "tokens_seen": 1553006592 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026745235707121363, + "loss": 2.7977, + "theoretical_loss": 3.5053540093387126, + "tokens_seen": 1553072128 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002674423269809428, + "loss": 2.58, + "theoretical_loss": 3.50534105468957, + "tokens_seen": 1553137664 + }, + { + "epoch": 0.47, + "objective/train/docs_used": 881131, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7973225116729736, + "objective/train/theoretical_loss": 3.505328100740097, + "objective/train/tokens_used": 1573663200, + "theoretical_loss": 3.505328100740097, + "tokens_seen": 1553203200 + }, + { + "epoch": 0.47, + "learning_rate": 0.000267432296890672, + "loss": 2.7726, + "theoretical_loss": 3.505328100740097, + "tokens_seen": 1553203200 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026742226680040123, + "loss": 2.5386, + "theoretical_loss": 3.5053151474902284, + "tokens_seen": 1553268736 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026741223671013036, + "loss": 2.6382, + "theoretical_loss": 3.505302194939896, + "tokens_seen": 1553334272 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002674022066198596, + "loss": 2.6111, + "theoretical_loss": 3.505289243089033, + "tokens_seen": 1553399808 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002673921765295888, + "loss": 2.532, + "theoretical_loss": 3.5052762919375717, + "tokens_seen": 1553465344 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026738214643931796, + "loss": 2.8618, + "theoretical_loss": 3.5052633414854446, + "tokens_seen": 1553530880 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026737211634904714, + "loss": 2.6006, + "theoretical_loss": 3.5052503917325852, + "tokens_seen": 1553596416 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002673620862587763, + "loss": 2.7615, + "theoretical_loss": 3.505237442678926, + "tokens_seen": 1553661952 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002673520561685055, + "loss": 2.7135, + "theoretical_loss": 3.5052244943243993, + "tokens_seen": 1553727488 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026734202607823474, + "loss": 2.6707, + "theoretical_loss": 3.505211546668938, + "tokens_seen": 1553793024 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026733199598796386, + "loss": 2.5746, + "theoretical_loss": 3.5051985997124757, + "tokens_seen": 1553858560 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002673219658976931, + "loss": 2.6357, + "theoretical_loss": 3.5051856534549444, + "tokens_seen": 1553924096 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002673119358074222, + "loss": 2.6948, + "theoretical_loss": 3.505172707896276, + "tokens_seen": 1553989632 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026730190571715146, + "loss": 2.7391, + "theoretical_loss": 3.5051597630364055, + "tokens_seen": 1554055168 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026729187562688064, + "loss": 2.6263, + "theoretical_loss": 3.505146818875264, + "tokens_seen": 1554120704 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002672818455366098, + "loss": 2.7992, + "theoretical_loss": 3.5051338754127848, + "tokens_seen": 1554186240 + }, + { + "epoch": 0.47, + "learning_rate": 0.000267271815446339, + "loss": 2.7001, + "theoretical_loss": 3.505120932648901, + "tokens_seen": 1554251776 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002672617853560682, + "loss": 2.6657, + "theoretical_loss": 3.505107990583545, + "tokens_seen": 1554317312 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002672517552657974, + "loss": 2.6599, + "theoretical_loss": 3.50509504921665, + "tokens_seen": 1554382848 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002672417251755266, + "loss": 2.525, + "theoretical_loss": 3.505082108548149, + "tokens_seen": 1554448384 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002672316950852558, + "loss": 2.7486, + "theoretical_loss": 3.5050691685779745, + "tokens_seen": 1554513920 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026722166499498496, + "loss": 2.8148, + "theoretical_loss": 3.505056229306059, + "tokens_seen": 1554579456 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002672116349047142, + "loss": 2.6569, + "theoretical_loss": 3.5050432907323366, + "tokens_seen": 1554644992 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002672016048144433, + "loss": 2.5502, + "theoretical_loss": 3.505030352856739, + "tokens_seen": 1554710528 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026719157472417256, + "loss": 2.757, + "theoretical_loss": 3.5050174156791996, + "tokens_seen": 1554776064 + }, + { + "epoch": 0.47, + "objective/train/docs_used": 882511, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7652337551116943, + "objective/train/theoretical_loss": 3.505004479199651, + "objective/train/tokens_used": 1575301600, + "theoretical_loss": 3.505004479199651, + "tokens_seen": 1554841600 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002671815446339017, + "loss": 2.4742, + "theoretical_loss": 3.505004479199651, + "tokens_seen": 1554841600 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002671715145436309, + "loss": 2.5903, + "theoretical_loss": 3.504991543418027, + "tokens_seen": 1554907136 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002671614844533601, + "loss": 2.3719, + "theoretical_loss": 3.50497860833426, + "tokens_seen": 1554972672 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002671514543630893, + "loss": 2.2801, + "theoretical_loss": 3.5049656739482824, + "tokens_seen": 1555038208 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026714142427281847, + "loss": 2.6529, + "theoretical_loss": 3.504952740260028, + "tokens_seen": 1555103744 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026713139418254765, + "loss": 2.4929, + "theoretical_loss": 3.504939807269429, + "tokens_seen": 1555169280 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026712136409227683, + "loss": 2.5887, + "theoretical_loss": 3.5049268749764186, + "tokens_seen": 1555234816 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026711133400200606, + "loss": 2.4504, + "theoretical_loss": 3.504913943380931, + "tokens_seen": 1555300352 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002671013039117352, + "loss": 2.6124, + "theoretical_loss": 3.5049010124828968, + "tokens_seen": 1555365888 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002670912738214644, + "loss": 2.5725, + "theoretical_loss": 3.5048880822822515, + "tokens_seen": 1555431424 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026708124373119355, + "loss": 2.651, + "theoretical_loss": 3.5048751527789266, + "tokens_seen": 1555496960 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002670712136409228, + "loss": 2.629, + "theoretical_loss": 3.504862223972855, + "tokens_seen": 1555562496 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026706118355065197, + "loss": 2.6126, + "theoretical_loss": 3.5048492958639708, + "tokens_seen": 1555628032 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026705115346038115, + "loss": 2.6937, + "theoretical_loss": 3.5048363684522057, + "tokens_seen": 1555693568 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026704112337011033, + "loss": 2.6486, + "theoretical_loss": 3.504823441737494, + "tokens_seen": 1555759104 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026703109327983957, + "loss": 2.6347, + "theoretical_loss": 3.504810515719768, + "tokens_seen": 1555824640 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002670210631895687, + "loss": 2.6718, + "theoretical_loss": 3.5047975903989617, + "tokens_seen": 1555890176 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026701103309929793, + "loss": 2.5597, + "theoretical_loss": 3.5047846657750066, + "tokens_seen": 1555955712 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026700100300902706, + "loss": 2.842, + "theoretical_loss": 3.504771741847837, + "tokens_seen": 1556021248 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002669909729187563, + "loss": 2.6668, + "theoretical_loss": 3.5047588186173853, + "tokens_seen": 1556086784 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026698094282848547, + "loss": 2.682, + "theoretical_loss": 3.5047458960835853, + "tokens_seen": 1556152320 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026697091273821465, + "loss": 2.9121, + "theoretical_loss": 3.5047329742463695, + "tokens_seen": 1556217856 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026696088264794383, + "loss": 2.6262, + "theoretical_loss": 3.5047200531056713, + "tokens_seen": 1556283392 + }, + { + "epoch": 0.47, + "learning_rate": 0.000266950852557673, + "loss": 2.7765, + "theoretical_loss": 3.504707132661424, + "tokens_seen": 1556348928 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002669408224674022, + "loss": 2.3933, + "theoretical_loss": 3.5046942129135603, + "tokens_seen": 1556414464 + }, + { + "epoch": 0.47, + "objective/train/docs_used": 882978, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.572951555252075, + "objective/train/theoretical_loss": 3.5046812938620135, + "objective/train/tokens_used": 1576940000, + "theoretical_loss": 3.5046812938620135, + "tokens_seen": 1556480000 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026693079237713143, + "loss": 2.7363, + "theoretical_loss": 3.5046812938620135, + "tokens_seen": 1556480000 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026692076228686056, + "loss": 2.7876, + "theoretical_loss": 3.504668375506717, + "tokens_seen": 1556545536 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002669107321965898, + "loss": 2.4722, + "theoretical_loss": 3.504655457847604, + "tokens_seen": 1556611072 + }, + { + "epoch": 0.47, + "learning_rate": 0.000266900702106319, + "loss": 2.6556, + "theoretical_loss": 3.504642540884607, + "tokens_seen": 1556676608 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026689067201604816, + "loss": 2.6797, + "theoretical_loss": 3.5046296246176594, + "tokens_seen": 1556742144 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026688064192577734, + "loss": 2.5893, + "theoretical_loss": 3.5046167090466955, + "tokens_seen": 1556807680 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002668706118355065, + "loss": 2.7034, + "theoretical_loss": 3.504603794171647, + "tokens_seen": 1556873216 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002668605817452357, + "loss": 2.637, + "theoretical_loss": 3.504590879992448, + "tokens_seen": 1556938752 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026685055165496494, + "loss": 2.788, + "theoretical_loss": 3.5045779665090313, + "tokens_seen": 1557004288 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026684052156469406, + "loss": 2.5739, + "theoretical_loss": 3.5045650537213304, + "tokens_seen": 1557069824 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002668304914744233, + "loss": 2.7429, + "theoretical_loss": 3.5045521416292784, + "tokens_seen": 1557135360 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002668204613841524, + "loss": 2.8136, + "theoretical_loss": 3.5045392302328087, + "tokens_seen": 1557200896 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026681043129388166, + "loss": 2.6863, + "theoretical_loss": 3.5045263195318546, + "tokens_seen": 1557266432 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026680040120361084, + "loss": 2.6553, + "theoretical_loss": 3.5045134095263486, + "tokens_seen": 1557331968 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026679037111334, + "loss": 2.6953, + "theoretical_loss": 3.504500500216225, + "tokens_seen": 1557397504 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002667803410230692, + "loss": 2.6387, + "theoretical_loss": 3.504487591601417, + "tokens_seen": 1557463040 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002667703109327984, + "loss": 2.4644, + "theoretical_loss": 3.504474683681857, + "tokens_seen": 1557528576 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026676028084252757, + "loss": 2.6149, + "theoretical_loss": 3.504461776457479, + "tokens_seen": 1557594112 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002667502507522568, + "loss": 2.8233, + "theoretical_loss": 3.504448869928216, + "tokens_seen": 1557659648 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026674022066198593, + "loss": 2.7498, + "theoretical_loss": 3.504435964094002, + "tokens_seen": 1557725184 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026673019057171516, + "loss": 2.7899, + "theoretical_loss": 3.504423058954769, + "tokens_seen": 1557790720 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026672016048144434, + "loss": 2.6994, + "theoretical_loss": 3.504410154510452, + "tokens_seen": 1557856256 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002667101303911735, + "loss": 2.7488, + "theoretical_loss": 3.5043972507609835, + "tokens_seen": 1557921792 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002667001003009027, + "loss": 2.8226, + "theoretical_loss": 3.5043843477062966, + "tokens_seen": 1557987328 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002666900702106319, + "loss": 2.6302, + "theoretical_loss": 3.504371445346325, + "tokens_seen": 1558052864 + }, + { + "epoch": 0.47, + "objective/train/docs_used": 884109, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7029547691345215, + "objective/train/theoretical_loss": 3.5043585436810014, + "objective/train/tokens_used": 1578578400, + "theoretical_loss": 3.5043585436810014, + "tokens_seen": 1558118400 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026668004012036107, + "loss": 2.8241, + "theoretical_loss": 3.5043585436810014, + "tokens_seen": 1558118400 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002666700100300903, + "loss": 2.7298, + "theoretical_loss": 3.504345642710261, + "tokens_seen": 1558183936 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026665997993981943, + "loss": 2.524, + "theoretical_loss": 3.504332742434035, + "tokens_seen": 1558249472 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026664994984954867, + "loss": 2.458, + "theoretical_loss": 3.504319842852258, + "tokens_seen": 1558315008 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002666399197592778, + "loss": 2.6819, + "theoretical_loss": 3.5043069439648633, + "tokens_seen": 1558380544 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026662988966900703, + "loss": 2.3976, + "theoretical_loss": 3.5042940457717844, + "tokens_seen": 1558446080 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002666198595787362, + "loss": 2.4241, + "theoretical_loss": 3.5042811482729546, + "tokens_seen": 1558511616 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002666098294884654, + "loss": 2.5834, + "theoretical_loss": 3.5042682514683072, + "tokens_seen": 1558577152 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026659979939819457, + "loss": 2.4591, + "theoretical_loss": 3.5042553553577758, + "tokens_seen": 1558642688 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026658976930792375, + "loss": 2.6336, + "theoretical_loss": 3.5042424599412936, + "tokens_seen": 1558708224 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026657973921765293, + "loss": 2.4418, + "theoretical_loss": 3.504229565218794, + "tokens_seen": 1558773760 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026656970912738217, + "loss": 2.8897, + "theoretical_loss": 3.5042166711902114, + "tokens_seen": 1558839296 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002665596790371113, + "loss": 2.6208, + "theoretical_loss": 3.504203777855478, + "tokens_seen": 1558904832 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026654964894684053, + "loss": 2.8536, + "theoretical_loss": 3.5041908852145287, + "tokens_seen": 1558970368 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002665396188565697, + "loss": 2.678, + "theoretical_loss": 3.5041779932672963, + "tokens_seen": 1559035904 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002665295887662989, + "loss": 2.6219, + "theoretical_loss": 3.5041651020137135, + "tokens_seen": 1559101440 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002665195586760281, + "loss": 2.7704, + "theoretical_loss": 3.5041522114537154, + "tokens_seen": 1559166976 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026650952858575726, + "loss": 2.5061, + "theoretical_loss": 3.504139321587234, + "tokens_seen": 1559232512 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002664994984954865, + "loss": 2.8058, + "theoretical_loss": 3.504126432414204, + "tokens_seen": 1559298048 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026648946840521567, + "loss": 2.8879, + "theoretical_loss": 3.504113543934558, + "tokens_seen": 1559363584 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026647943831494485, + "loss": 2.6543, + "theoretical_loss": 3.5041006561482306, + "tokens_seen": 1559429120 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026646940822467403, + "loss": 2.7267, + "theoretical_loss": 3.504087769055155, + "tokens_seen": 1559494656 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002664593781344032, + "loss": 2.7472, + "theoretical_loss": 3.504074882655264, + "tokens_seen": 1559560192 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002664493480441324, + "loss": 2.3249, + "theoretical_loss": 3.5040619969484923, + "tokens_seen": 1559625728 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026643931795386163, + "loss": 2.7736, + "theoretical_loss": 3.5040491119347728, + "tokens_seen": 1559691264 + }, + { + "epoch": 0.47, + "objective/train/docs_used": 884823, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.30165958404541, + "objective/train/theoretical_loss": 3.5040362276140393, + "objective/train/tokens_used": 1580216800, + "theoretical_loss": 3.5040362276140393, + "tokens_seen": 1559756800 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026642928786359076, + "loss": 2.6031, + "theoretical_loss": 3.5040362276140393, + "tokens_seen": 1559756800 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026641925777332, + "loss": 2.6888, + "theoretical_loss": 3.5040233439862254, + "tokens_seen": 1559822336 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002664092276830492, + "loss": 2.659, + "theoretical_loss": 3.504010461051265, + "tokens_seen": 1559887872 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026639919759277836, + "loss": 2.597, + "theoretical_loss": 3.5039975788090914, + "tokens_seen": 1559953408 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026638916750250754, + "loss": 2.6734, + "theoretical_loss": 3.503984697259638, + "tokens_seen": 1560018944 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002663791374122367, + "loss": 2.4815, + "theoretical_loss": 3.503971816402839, + "tokens_seen": 1560084480 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002663691073219659, + "loss": 2.6672, + "theoretical_loss": 3.503958936238628, + "tokens_seen": 1560150016 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026635907723169514, + "loss": 2.5021, + "theoretical_loss": 3.503946056766938, + "tokens_seen": 1560215552 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026634904714142426, + "loss": 2.7134, + "theoretical_loss": 3.5039331779877037, + "tokens_seen": 1560281088 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002663390170511535, + "loss": 2.5799, + "theoretical_loss": 3.503920299900858, + "tokens_seen": 1560346624 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002663289869608826, + "loss": 2.7724, + "theoretical_loss": 3.5039074225063347, + "tokens_seen": 1560412160 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026631895687061186, + "loss": 2.6521, + "theoretical_loss": 3.5038945458040684, + "tokens_seen": 1560477696 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026630892678034104, + "loss": 2.7392, + "theoretical_loss": 3.5038816697939916, + "tokens_seen": 1560543232 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002662988966900702, + "loss": 2.7369, + "theoretical_loss": 3.5038687944760385, + "tokens_seen": 1560608768 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002662888665997994, + "loss": 2.6859, + "theoretical_loss": 3.503855919850143, + "tokens_seen": 1560674304 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002662788365095286, + "loss": 2.7329, + "theoretical_loss": 3.5038430459162386, + "tokens_seen": 1560739840 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026626880641925777, + "loss": 2.7825, + "theoretical_loss": 3.503830172674259, + "tokens_seen": 1560805376 + }, + { + "epoch": 0.47, + "learning_rate": 0.000266258776328987, + "loss": 2.7725, + "theoretical_loss": 3.503817300124138, + "tokens_seen": 1560870912 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026624874623871613, + "loss": 2.5991, + "theoretical_loss": 3.50380442826581, + "tokens_seen": 1560936448 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026623871614844536, + "loss": 2.8068, + "theoretical_loss": 3.503791557099208, + "tokens_seen": 1561001984 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026622868605817454, + "loss": 2.8045, + "theoretical_loss": 3.503778686624266, + "tokens_seen": 1561067520 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002662186559679037, + "loss": 2.4958, + "theoretical_loss": 3.503765816840918, + "tokens_seen": 1561133056 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002662086258776329, + "loss": 2.4953, + "theoretical_loss": 3.5037529477490974, + "tokens_seen": 1561198592 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002661985957873621, + "loss": 2.3831, + "theoretical_loss": 3.5037400793487383, + "tokens_seen": 1561264128 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026618856569709127, + "loss": 2.7529, + "theoretical_loss": 3.5037272116397746, + "tokens_seen": 1561329664 + }, + { + "epoch": 0.47, + "objective/train/docs_used": 885915, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.1612162590026855, + "objective/train/theoretical_loss": 3.50371434462214, + "objective/train/tokens_used": 1581855200, + "theoretical_loss": 3.50371434462214, + "tokens_seen": 1561395200 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002661785356068205, + "loss": 2.5664, + "theoretical_loss": 3.50371434462214, + "tokens_seen": 1561395200 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026616850551654963, + "loss": 2.5367, + "theoretical_loss": 3.5037014782957683, + "tokens_seen": 1561460736 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026615847542627887, + "loss": 2.7683, + "theoretical_loss": 3.5036886126605937, + "tokens_seen": 1561526272 + }, + { + "epoch": 0.47, + "learning_rate": 0.000266148445336008, + "loss": 2.7742, + "theoretical_loss": 3.5036757477165494, + "tokens_seen": 1561591808 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026613841524573723, + "loss": 2.7019, + "theoretical_loss": 3.5036628834635697, + "tokens_seen": 1561657344 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002661283851554664, + "loss": 2.6511, + "theoretical_loss": 3.5036500199015883, + "tokens_seen": 1561722880 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002661183550651956, + "loss": 2.6826, + "theoretical_loss": 3.50363715703054, + "tokens_seen": 1561788416 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026610832497492477, + "loss": 2.7834, + "theoretical_loss": 3.503624294850357, + "tokens_seen": 1561853952 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026609829488465395, + "loss": 2.5064, + "theoretical_loss": 3.5036114333609745, + "tokens_seen": 1561919488 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026608826479438313, + "loss": 2.6448, + "theoretical_loss": 3.5035985725623258, + "tokens_seen": 1561985024 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026607823470411237, + "loss": 2.5006, + "theoretical_loss": 3.5035857124543455, + "tokens_seen": 1562050560 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002660682046138415, + "loss": 2.7908, + "theoretical_loss": 3.503572853036967, + "tokens_seen": 1562116096 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026605817452357073, + "loss": 2.6785, + "theoretical_loss": 3.503559994310124, + "tokens_seen": 1562181632 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002660481444332999, + "loss": 2.7311, + "theoretical_loss": 3.503547136273751, + "tokens_seen": 1562247168 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002660381143430291, + "loss": 2.7428, + "theoretical_loss": 3.5035342789277824, + "tokens_seen": 1562312704 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002660280842527583, + "loss": 2.5247, + "theoretical_loss": 3.5035214222721507, + "tokens_seen": 1562378240 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026601805416248746, + "loss": 2.7014, + "theoretical_loss": 3.503508566306791, + "tokens_seen": 1562443776 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026600802407221664, + "loss": 2.4614, + "theoretical_loss": 3.5034957110316367, + "tokens_seen": 1562509312 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026599799398194587, + "loss": 2.6065, + "theoretical_loss": 3.503482856446622, + "tokens_seen": 1562574848 + }, + { + "epoch": 0.47, + "learning_rate": 0.000265987963891675, + "loss": 2.8561, + "theoretical_loss": 3.5034700025516816, + "tokens_seen": 1562640384 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026597793380140423, + "loss": 2.5597, + "theoretical_loss": 3.5034571493467483, + "tokens_seen": 1562705920 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026596790371113336, + "loss": 2.721, + "theoretical_loss": 3.5034442968317574, + "tokens_seen": 1562771456 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002659578736208626, + "loss": 2.5856, + "theoretical_loss": 3.5034314450066417, + "tokens_seen": 1562836992 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002659478435305918, + "loss": 2.8039, + "theoretical_loss": 3.503418593871336, + "tokens_seen": 1562902528 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026593781344032096, + "loss": 2.5467, + "theoretical_loss": 3.503405743425774, + "tokens_seen": 1562968064 + }, + { + "epoch": 0.47, + "objective/train/docs_used": 886454, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.3615882396698, + "objective/train/theoretical_loss": 3.5033928936698904, + "objective/train/tokens_used": 1583493600, + "theoretical_loss": 3.5033928936698904, + "tokens_seen": 1563033600 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026592778335005014, + "loss": 2.4235, + "theoretical_loss": 3.5033928936698904, + "tokens_seen": 1563033600 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002659177532597794, + "loss": 2.5397, + "theoretical_loss": 3.5033800446036185, + "tokens_seen": 1563099136 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002659077231695085, + "loss": 2.5323, + "theoretical_loss": 3.5033671962268924, + "tokens_seen": 1563164672 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026589769307923774, + "loss": 2.5386, + "theoretical_loss": 3.5033543485396472, + "tokens_seen": 1563230208 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026588766298896686, + "loss": 2.7464, + "theoretical_loss": 3.503341501541816, + "tokens_seen": 1563295744 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002658776328986961, + "loss": 2.6199, + "theoretical_loss": 3.5033286552333327, + "tokens_seen": 1563361280 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002658676028084253, + "loss": 2.6369, + "theoretical_loss": 3.503315809614132, + "tokens_seen": 1563426816 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026585757271815446, + "loss": 2.7709, + "theoretical_loss": 3.5033029646841483, + "tokens_seen": 1563492352 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026584754262788364, + "loss": 2.8897, + "theoretical_loss": 3.5032901204433156, + "tokens_seen": 1563557888 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002658375125376128, + "loss": 2.8508, + "theoretical_loss": 3.503277276891567, + "tokens_seen": 1563623424 + }, + { + "epoch": 0.47, + "learning_rate": 0.000265827482447342, + "loss": 2.7037, + "theoretical_loss": 3.5032644340288375, + "tokens_seen": 1563688960 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026581745235707124, + "loss": 2.6463, + "theoretical_loss": 3.5032515918550615, + "tokens_seen": 1563754496 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026580742226680037, + "loss": 2.668, + "theoretical_loss": 3.503238750370173, + "tokens_seen": 1563820032 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002657973921765296, + "loss": 2.6335, + "theoretical_loss": 3.503225909574106, + "tokens_seen": 1563885568 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026578736208625873, + "loss": 2.8101, + "theoretical_loss": 3.5032130694667947, + "tokens_seen": 1563951104 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026577733199598797, + "loss": 2.5864, + "theoretical_loss": 3.503200230048173, + "tokens_seen": 1564016640 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026576730190571715, + "loss": 2.6987, + "theoretical_loss": 3.5031873913181757, + "tokens_seen": 1564082176 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026575727181544633, + "loss": 2.6508, + "theoretical_loss": 3.503174553276737, + "tokens_seen": 1564147712 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026574724172517556, + "loss": 2.8672, + "theoretical_loss": 3.5031617159237904, + "tokens_seen": 1564213248 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026573721163490474, + "loss": 2.8442, + "theoretical_loss": 3.503148879259271, + "tokens_seen": 1564278784 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002657271815446339, + "loss": 2.7294, + "theoretical_loss": 3.5031360432831127, + "tokens_seen": 1564344320 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002657171514543631, + "loss": 2.7183, + "theoretical_loss": 3.5031232079952495, + "tokens_seen": 1564409856 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002657071213640923, + "loss": 2.9104, + "theoretical_loss": 3.5031103733956157, + "tokens_seen": 1564475392 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026569709127382147, + "loss": 2.7364, + "theoretical_loss": 3.5030975394841457, + "tokens_seen": 1564540928 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002656870611835507, + "loss": 2.6928, + "theoretical_loss": 3.5030847062607746, + "tokens_seen": 1564606464 + }, + { + "epoch": 0.47, + "objective/train/docs_used": 887471, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6693336963653564, + "objective/train/theoretical_loss": 3.5030718737254354, + "objective/train/tokens_used": 1585132000, + "theoretical_loss": 3.5030718737254354, + "tokens_seen": 1564672000 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026567703109327983, + "loss": 2.6895, + "theoretical_loss": 3.5030718737254354, + "tokens_seen": 1564672000 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026566700100300907, + "loss": 2.5607, + "theoretical_loss": 3.5030590418780623, + "tokens_seen": 1564737536 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002656569709127382, + "loss": 2.8389, + "theoretical_loss": 3.503046210718591, + "tokens_seen": 1564803072 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026564694082246743, + "loss": 2.599, + "theoretical_loss": 3.503033380246955, + "tokens_seen": 1564868608 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002656369107321966, + "loss": 2.7831, + "theoretical_loss": 3.5030205504630882, + "tokens_seen": 1564934144 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002656268806419258, + "loss": 2.7136, + "theoretical_loss": 3.5030077213669255, + "tokens_seen": 1564999680 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026561685055165497, + "loss": 2.8276, + "theoretical_loss": 3.5029948929584016, + "tokens_seen": 1565065216 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026560682046138415, + "loss": 2.69, + "theoretical_loss": 3.5029820652374495, + "tokens_seen": 1565130752 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026559679037111333, + "loss": 2.7486, + "theoretical_loss": 3.502969238204005, + "tokens_seen": 1565196288 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026558676028084257, + "loss": 2.9877, + "theoretical_loss": 3.502956411858002, + "tokens_seen": 1565261824 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002655767301905717, + "loss": 2.7252, + "theoretical_loss": 3.5029435861993745, + "tokens_seen": 1565327360 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026556670010030093, + "loss": 2.7503, + "theoretical_loss": 3.502930761228057, + "tokens_seen": 1565392896 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002655566700100301, + "loss": 2.6569, + "theoretical_loss": 3.5029179369439847, + "tokens_seen": 1565458432 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002655466399197593, + "loss": 2.9175, + "theoretical_loss": 3.5029051133470905, + "tokens_seen": 1565523968 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002655366098294885, + "loss": 2.7855, + "theoretical_loss": 3.5028922904373103, + "tokens_seen": 1565589504 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026552657973921766, + "loss": 2.7238, + "theoretical_loss": 3.5028794682145774, + "tokens_seen": 1565655040 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026551654964894684, + "loss": 2.9808, + "theoretical_loss": 3.5028666466788274, + "tokens_seen": 1565720576 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026550651955867607, + "loss": 2.8105, + "theoretical_loss": 3.5028538258299937, + "tokens_seen": 1565786112 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002654964894684052, + "loss": 2.4349, + "theoretical_loss": 3.502841005668011, + "tokens_seen": 1565851648 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026548645937813443, + "loss": 2.8218, + "theoretical_loss": 3.502828186192814, + "tokens_seen": 1565917184 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026547642928786356, + "loss": 2.6792, + "theoretical_loss": 3.502815367404337, + "tokens_seen": 1565982720 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002654663991975928, + "loss": 2.5439, + "theoretical_loss": 3.5028025493025146, + "tokens_seen": 1566048256 + }, + { + "epoch": 0.47, + "learning_rate": 0.000265456369107322, + "loss": 2.6763, + "theoretical_loss": 3.5027897318872814, + "tokens_seen": 1566113792 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026544633901705116, + "loss": 2.5056, + "theoretical_loss": 3.502776915158571, + "tokens_seen": 1566179328 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026543630892678034, + "loss": 2.5619, + "theoretical_loss": 3.5027640991163196, + "tokens_seen": 1566244864 + }, + { + "epoch": 0.47, + "objective/train/docs_used": 888126, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7966861724853516, + "objective/train/theoretical_loss": 3.50275128376046, + "objective/train/tokens_used": 1586770400, + "theoretical_loss": 3.50275128376046, + "tokens_seen": 1566310400 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002654262788365096, + "loss": 2.7252, + "theoretical_loss": 3.50275128376046, + "tokens_seen": 1566310400 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002654162487462387, + "loss": 2.6509, + "theoretical_loss": 3.502738469090928, + "tokens_seen": 1566375936 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026540621865596794, + "loss": 2.5746, + "theoretical_loss": 3.502725655107657, + "tokens_seen": 1566441472 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026539618856569707, + "loss": 2.6297, + "theoretical_loss": 3.502712841810582, + "tokens_seen": 1566507008 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002653861584754263, + "loss": 2.5507, + "theoretical_loss": 3.502700029199638, + "tokens_seen": 1566572544 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002653761283851555, + "loss": 2.7078, + "theoretical_loss": 3.5026872172747594, + "tokens_seen": 1566638080 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026536609829488466, + "loss": 2.6675, + "theoretical_loss": 3.5026744060358803, + "tokens_seen": 1566703616 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026535606820461384, + "loss": 2.626, + "theoretical_loss": 3.5026615954829357, + "tokens_seen": 1566769152 + }, + { + "epoch": 0.47, + "learning_rate": 0.000265346038114343, + "loss": 2.6753, + "theoretical_loss": 3.50264878561586, + "tokens_seen": 1566834688 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002653360080240722, + "loss": 2.7148, + "theoretical_loss": 3.502635976434588, + "tokens_seen": 1566900224 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026532597793380144, + "loss": 2.6467, + "theoretical_loss": 3.5026231679390536, + "tokens_seen": 1566965760 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026531594784353057, + "loss": 2.4947, + "theoretical_loss": 3.5026103601291925, + "tokens_seen": 1567031296 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002653059177532598, + "loss": 2.8709, + "theoretical_loss": 3.5025975530049385, + "tokens_seen": 1567096832 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026529588766298893, + "loss": 2.6367, + "theoretical_loss": 3.5025847465662268, + "tokens_seen": 1567162368 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026528585757271817, + "loss": 2.5024, + "theoretical_loss": 3.5025719408129916, + "tokens_seen": 1567227904 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026527582748244735, + "loss": 2.614, + "theoretical_loss": 3.5025591357451673, + "tokens_seen": 1567293440 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026526579739217653, + "loss": 2.7796, + "theoretical_loss": 3.5025463313626894, + "tokens_seen": 1567358976 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002652557673019057, + "loss": 2.4553, + "theoretical_loss": 3.502533527665492, + "tokens_seen": 1567424512 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026524573721163494, + "loss": 2.7518, + "theoretical_loss": 3.5025207246535097, + "tokens_seen": 1567490048 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026523570712136407, + "loss": 2.5752, + "theoretical_loss": 3.5025079223266777, + "tokens_seen": 1567555584 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002652256770310933, + "loss": 2.9062, + "theoretical_loss": 3.50249512068493, + "tokens_seen": 1567621120 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026521564694082243, + "loss": 2.6521, + "theoretical_loss": 3.502482319728202, + "tokens_seen": 1567686656 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026520561685055167, + "loss": 2.6597, + "theoretical_loss": 3.5024695194564277, + "tokens_seen": 1567752192 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026519558676028085, + "loss": 2.7621, + "theoretical_loss": 3.502456719869542, + "tokens_seen": 1567817728 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026518555667001003, + "loss": 2.4398, + "theoretical_loss": 3.5024439209674805, + "tokens_seen": 1567883264 + }, + { + "epoch": 0.48, + "objective/train/docs_used": 889384, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.324211359024048, + "objective/train/theoretical_loss": 3.5024311227501768, + "objective/train/tokens_used": 1588408800, + "theoretical_loss": 3.5024311227501768, + "tokens_seen": 1567948800 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002651755265797392, + "loss": 2.6573, + "theoretical_loss": 3.5024311227501768, + "tokens_seen": 1567948800 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002651654964894684, + "loss": 2.6958, + "theoretical_loss": 3.5024183252175662, + "tokens_seen": 1568014336 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002651554663991976, + "loss": 2.6041, + "theoretical_loss": 3.502405528369583, + "tokens_seen": 1568079872 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002651454363089268, + "loss": 2.6194, + "theoretical_loss": 3.5023927322061637, + "tokens_seen": 1568145408 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026513540621865594, + "loss": 2.6206, + "theoretical_loss": 3.50237993672724, + "tokens_seen": 1568210944 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026512537612838517, + "loss": 2.706, + "theoretical_loss": 3.5023671419327496, + "tokens_seen": 1568276480 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002651153460381143, + "loss": 2.6996, + "theoretical_loss": 3.5023543478226253, + "tokens_seen": 1568342016 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026510531594784353, + "loss": 2.5707, + "theoretical_loss": 3.5023415543968026, + "tokens_seen": 1568407552 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002650952858575727, + "loss": 2.6157, + "theoretical_loss": 3.5023287616552166, + "tokens_seen": 1568473088 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002650852557673019, + "loss": 2.7471, + "theoretical_loss": 3.5023159695978023, + "tokens_seen": 1568538624 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002650752256770311, + "loss": 2.7747, + "theoretical_loss": 3.5023031782244933, + "tokens_seen": 1568604160 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002650651955867603, + "loss": 2.5033, + "theoretical_loss": 3.502290387535226, + "tokens_seen": 1568669696 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026505516549648944, + "loss": 2.6006, + "theoretical_loss": 3.5022775975299343, + "tokens_seen": 1568735232 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002650451354062187, + "loss": 2.6026, + "theoretical_loss": 3.502264808208553, + "tokens_seen": 1568800768 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002650351053159478, + "loss": 2.6164, + "theoretical_loss": 3.502252019571017, + "tokens_seen": 1568866304 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026502507522567704, + "loss": 2.5897, + "theoretical_loss": 3.5022392316172617, + "tokens_seen": 1568931840 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002650150451354062, + "loss": 2.665, + "theoretical_loss": 3.502226444347222, + "tokens_seen": 1568997376 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002650050150451354, + "loss": 2.6655, + "theoretical_loss": 3.502213657760832, + "tokens_seen": 1569062912 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026499498495486464, + "loss": 2.585, + "theoretical_loss": 3.502200871858027, + "tokens_seen": 1569128448 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026498495486459376, + "loss": 2.7316, + "theoretical_loss": 3.5021880866387423, + "tokens_seen": 1569193984 + }, + { + "epoch": 0.48, + "learning_rate": 0.000264974924774323, + "loss": 2.6172, + "theoretical_loss": 3.5021753021029123, + "tokens_seen": 1569259520 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002649648946840522, + "loss": 2.6228, + "theoretical_loss": 3.502162518250472, + "tokens_seen": 1569325056 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026495486459378136, + "loss": 2.5903, + "theoretical_loss": 3.5021497350813564, + "tokens_seen": 1569390592 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026494483450351054, + "loss": 2.9013, + "theoretical_loss": 3.5021369525955004, + "tokens_seen": 1569456128 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002649348044132398, + "loss": 2.6631, + "theoretical_loss": 3.5021241707928397, + "tokens_seen": 1569521664 + }, + { + "epoch": 0.48, + "objective/train/docs_used": 890057, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.0533785820007324, + "objective/train/theoretical_loss": 3.5021113896733076, + "objective/train/tokens_used": 1590047200, + "theoretical_loss": 3.5021113896733076, + "tokens_seen": 1569587200 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002649247743229689, + "loss": 2.8308, + "theoretical_loss": 3.5021113896733076, + "tokens_seen": 1569587200 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026491474423269814, + "loss": 2.7446, + "theoretical_loss": 3.5020986092368407, + "tokens_seen": 1569652736 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026490471414242727, + "loss": 2.774, + "theoretical_loss": 3.5020858294833728, + "tokens_seen": 1569718272 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002648946840521565, + "loss": 2.7005, + "theoretical_loss": 3.5020730504128395, + "tokens_seen": 1569783808 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002648846539618857, + "loss": 2.7592, + "theoretical_loss": 3.5020602720251763, + "tokens_seen": 1569849344 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026487462387161486, + "loss": 2.5057, + "theoretical_loss": 3.5020474943203173, + "tokens_seen": 1569914880 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026486459378134404, + "loss": 2.5614, + "theoretical_loss": 3.502034717298198, + "tokens_seen": 1569980416 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002648545636910732, + "loss": 2.8723, + "theoretical_loss": 3.5020219409587527, + "tokens_seen": 1570045952 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002648445336008024, + "loss": 2.9036, + "theoretical_loss": 3.502009165301917, + "tokens_seen": 1570111488 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026483450351053164, + "loss": 2.6188, + "theoretical_loss": 3.5019963903276263, + "tokens_seen": 1570177024 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026482447342026077, + "loss": 2.6846, + "theoretical_loss": 3.5019836160358153, + "tokens_seen": 1570242560 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026481444332999, + "loss": 2.6081, + "theoretical_loss": 3.5019708424264184, + "tokens_seen": 1570308096 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026480441323971913, + "loss": 2.6515, + "theoretical_loss": 3.501958069499372, + "tokens_seen": 1570373632 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026479438314944837, + "loss": 2.8164, + "theoretical_loss": 3.5019452972546103, + "tokens_seen": 1570439168 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026478435305917755, + "loss": 2.5991, + "theoretical_loss": 3.5019325256920686, + "tokens_seen": 1570504704 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026477432296890673, + "loss": 2.5419, + "theoretical_loss": 3.5019197548116816, + "tokens_seen": 1570570240 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002647642928786359, + "loss": 2.8089, + "theoretical_loss": 3.501906984613385, + "tokens_seen": 1570635776 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026475426278836514, + "loss": 2.5921, + "theoretical_loss": 3.5018942150971135, + "tokens_seen": 1570701312 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026474423269809427, + "loss": 2.6271, + "theoretical_loss": 3.5018814462628027, + "tokens_seen": 1570766848 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002647342026078235, + "loss": 2.6823, + "theoretical_loss": 3.5018686781103874, + "tokens_seen": 1570832384 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026472417251755263, + "loss": 2.6426, + "theoretical_loss": 3.5018559106398026, + "tokens_seen": 1570897920 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026471414242728187, + "loss": 2.8259, + "theoretical_loss": 3.501843143850983, + "tokens_seen": 1570963456 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026470411233701105, + "loss": 2.6911, + "theoretical_loss": 3.5018303777438646, + "tokens_seen": 1571028992 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026469408224674023, + "loss": 2.7221, + "theoretical_loss": 3.501817612318383, + "tokens_seen": 1571094528 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002646840521564694, + "loss": 2.813, + "theoretical_loss": 3.5018048475744723, + "tokens_seen": 1571160064 + }, + { + "epoch": 0.48, + "objective/train/docs_used": 891281, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.1199958324432373, + "objective/train/theoretical_loss": 3.501792083512068, + "objective/train/tokens_used": 1591685600, + "theoretical_loss": 3.501792083512068, + "tokens_seen": 1571225600 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002646740220661986, + "loss": 2.8728, + "theoretical_loss": 3.501792083512068, + "tokens_seen": 1571225600 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002646639919759278, + "loss": 2.5942, + "theoretical_loss": 3.5017793201311056, + "tokens_seen": 1571291136 + }, + { + "epoch": 0.48, + "learning_rate": 0.000264653961885657, + "loss": 2.6648, + "theoretical_loss": 3.5017665574315195, + "tokens_seen": 1571356672 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026464393179538614, + "loss": 2.592, + "theoretical_loss": 3.5017537954132463, + "tokens_seen": 1571422208 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026463390170511537, + "loss": 2.7513, + "theoretical_loss": 3.5017410340762196, + "tokens_seen": 1571487744 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002646238716148445, + "loss": 2.7748, + "theoretical_loss": 3.501728273420376, + "tokens_seen": 1571553280 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026461384152457373, + "loss": 2.7513, + "theoretical_loss": 3.5017155134456495, + "tokens_seen": 1571618816 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002646038114343029, + "loss": 2.6692, + "theoretical_loss": 3.5017027541519763, + "tokens_seen": 1571684352 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002645937813440321, + "loss": 2.7646, + "theoretical_loss": 3.5016899955392917, + "tokens_seen": 1571749888 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002645837512537613, + "loss": 2.6459, + "theoretical_loss": 3.50167723760753, + "tokens_seen": 1571815424 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002645737211634905, + "loss": 2.7537, + "theoretical_loss": 3.5016644803566277, + "tokens_seen": 1571880960 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026456369107321964, + "loss": 2.808, + "theoretical_loss": 3.501651723786519, + "tokens_seen": 1571946496 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002645536609829489, + "loss": 2.5749, + "theoretical_loss": 3.50163896789714, + "tokens_seen": 1572012032 + }, + { + "epoch": 0.48, + "learning_rate": 0.000264543630892678, + "loss": 2.6917, + "theoretical_loss": 3.5016262126884254, + "tokens_seen": 1572077568 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026453360080240724, + "loss": 2.8162, + "theoretical_loss": 3.5016134581603104, + "tokens_seen": 1572143104 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002645235707121364, + "loss": 2.583, + "theoretical_loss": 3.5016007043127315, + "tokens_seen": 1572208640 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002645135406218656, + "loss": 2.7966, + "theoretical_loss": 3.5015879511456225, + "tokens_seen": 1572274176 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002645035105315948, + "loss": 2.6936, + "theoretical_loss": 3.50157519865892, + "tokens_seen": 1572339712 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026449348044132396, + "loss": 2.7127, + "theoretical_loss": 3.5015624468525584, + "tokens_seen": 1572405248 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026448345035105314, + "loss": 2.6371, + "theoretical_loss": 3.5015496957264736, + "tokens_seen": 1572470784 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002644734202607824, + "loss": 2.6332, + "theoretical_loss": 3.5015369452806, + "tokens_seen": 1572536320 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002644633901705115, + "loss": 2.7522, + "theoretical_loss": 3.5015241955148744, + "tokens_seen": 1572601856 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026445336008024074, + "loss": 2.5881, + "theoretical_loss": 3.501511446429232, + "tokens_seen": 1572667392 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026444332998996987, + "loss": 2.7153, + "theoretical_loss": 3.5014986980236067, + "tokens_seen": 1572732928 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002644332998996991, + "loss": 2.5926, + "theoretical_loss": 3.5014859502979356, + "tokens_seen": 1572798464 + }, + { + "epoch": 0.48, + "objective/train/docs_used": 891900, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7055470943450928, + "objective/train/theoretical_loss": 3.501473203252153, + "objective/train/tokens_used": 1593324000, + "theoretical_loss": 3.501473203252153, + "tokens_seen": 1572864000 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002644232698094283, + "loss": 2.669, + "theoretical_loss": 3.501473203252153, + "tokens_seen": 1572864000 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026441323971915747, + "loss": 2.763, + "theoretical_loss": 3.501460456886195, + "tokens_seen": 1572929536 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026440320962888665, + "loss": 2.7568, + "theoretical_loss": 3.5014477111999964, + "tokens_seen": 1572995072 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002643931795386159, + "loss": 2.9153, + "theoretical_loss": 3.5014349661934934, + "tokens_seen": 1573060608 + }, + { + "epoch": 0.48, + "learning_rate": 0.000264383149448345, + "loss": 2.6459, + "theoretical_loss": 3.501422221866621, + "tokens_seen": 1573126144 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026437311935807424, + "loss": 2.7664, + "theoretical_loss": 3.5014094782193146, + "tokens_seen": 1573191680 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026436308926780337, + "loss": 2.7188, + "theoretical_loss": 3.501396735251509, + "tokens_seen": 1573257216 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002643530591775326, + "loss": 2.7762, + "theoretical_loss": 3.501383992963141, + "tokens_seen": 1573322752 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002643430290872618, + "loss": 2.7941, + "theoretical_loss": 3.5013712513541453, + "tokens_seen": 1573388288 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026433299899699097, + "loss": 2.6778, + "theoretical_loss": 3.501358510424458, + "tokens_seen": 1573453824 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026432296890672015, + "loss": 2.7274, + "theoretical_loss": 3.501345770174013, + "tokens_seen": 1573519360 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026431293881644933, + "loss": 2.7025, + "theoretical_loss": 3.501333030602748, + "tokens_seen": 1573584896 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002643029087261785, + "loss": 2.7404, + "theoretical_loss": 3.501320291710597, + "tokens_seen": 1573650432 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026429287863590775, + "loss": 2.8442, + "theoretical_loss": 3.501307553497496, + "tokens_seen": 1573715968 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002642828485456369, + "loss": 2.8384, + "theoretical_loss": 3.501294815963381, + "tokens_seen": 1573781504 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002642728184553661, + "loss": 2.7089, + "theoretical_loss": 3.5012820791081865, + "tokens_seen": 1573847040 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026426278836509534, + "loss": 2.683, + "theoretical_loss": 3.501269342931848, + "tokens_seen": 1573912576 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026425275827482447, + "loss": 2.4976, + "theoretical_loss": 3.5012566074343026, + "tokens_seen": 1573978112 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002642427281845537, + "loss": 2.6649, + "theoretical_loss": 3.5012438726154844, + "tokens_seen": 1574043648 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026423269809428283, + "loss": 2.5737, + "theoretical_loss": 3.501231138475329, + "tokens_seen": 1574109184 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026422266800401207, + "loss": 2.8145, + "theoretical_loss": 3.5012184050137733, + "tokens_seen": 1574174720 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026421263791374125, + "loss": 2.2724, + "theoretical_loss": 3.5012056722307516, + "tokens_seen": 1574240256 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026420260782347043, + "loss": 2.6868, + "theoretical_loss": 3.5011929401261996, + "tokens_seen": 1574305792 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002641925777331996, + "loss": 2.9098, + "theoretical_loss": 3.5011802087000534, + "tokens_seen": 1574371328 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002641825476429288, + "loss": 2.6891, + "theoretical_loss": 3.501167477952249, + "tokens_seen": 1574436864 + }, + { + "epoch": 0.48, + "objective/train/docs_used": 893256, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.4908218383789062, + "objective/train/theoretical_loss": 3.5011547478827207, + "objective/train/tokens_used": 1594962400, + "theoretical_loss": 3.5011547478827207, + "tokens_seen": 1574502400 + }, + { + "epoch": 0.48, + "learning_rate": 0.000264172517552658, + "loss": 2.4707, + "theoretical_loss": 3.5011547478827207, + "tokens_seen": 1574502400 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002641624874623872, + "loss": 2.6072, + "theoretical_loss": 3.5011420184914055, + "tokens_seen": 1574567936 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026415245737211634, + "loss": 2.6627, + "theoretical_loss": 3.501129289778238, + "tokens_seen": 1574633472 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026414242728184557, + "loss": 2.7859, + "theoretical_loss": 3.5011165617431543, + "tokens_seen": 1574699008 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002641323971915747, + "loss": 2.9986, + "theoretical_loss": 3.50110383438609, + "tokens_seen": 1574764544 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026412236710130393, + "loss": 2.9619, + "theoretical_loss": 3.501091107706981, + "tokens_seen": 1574830080 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002641123370110331, + "loss": 2.74, + "theoretical_loss": 3.5010783817057627, + "tokens_seen": 1574895616 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002641023069207623, + "loss": 2.7225, + "theoretical_loss": 3.501065656382371, + "tokens_seen": 1574961152 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002640922768304915, + "loss": 2.781, + "theoretical_loss": 3.5010529317367416, + "tokens_seen": 1575026688 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002640822467402207, + "loss": 2.5774, + "theoretical_loss": 3.5010402077688094, + "tokens_seen": 1575092224 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026407221664994984, + "loss": 2.7135, + "theoretical_loss": 3.501027484478511, + "tokens_seen": 1575157760 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002640621865596791, + "loss": 2.7949, + "theoretical_loss": 3.5010147618657825, + "tokens_seen": 1575223296 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002640521564694082, + "loss": 2.705, + "theoretical_loss": 3.501002039930558, + "tokens_seen": 1575288832 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026404212637913744, + "loss": 2.4863, + "theoretical_loss": 3.500989318672775, + "tokens_seen": 1575354368 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002640320962888666, + "loss": 2.6126, + "theoretical_loss": 3.5009765980923686, + "tokens_seen": 1575419904 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002640220661985958, + "loss": 2.886, + "theoretical_loss": 3.5009638781892742, + "tokens_seen": 1575485440 + }, + { + "epoch": 0.48, + "learning_rate": 0.000264012036108325, + "loss": 2.5331, + "theoretical_loss": 3.500951158963428, + "tokens_seen": 1575550976 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026400200601805416, + "loss": 2.5459, + "theoretical_loss": 3.500938440414765, + "tokens_seen": 1575616512 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026399197592778334, + "loss": 2.5582, + "theoretical_loss": 3.500925722543222, + "tokens_seen": 1575682048 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002639819458375126, + "loss": 2.9556, + "theoretical_loss": 3.5009130053487345, + "tokens_seen": 1575747584 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002639719157472417, + "loss": 2.7089, + "theoretical_loss": 3.500900288831238, + "tokens_seen": 1575813120 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026396188565697094, + "loss": 2.6036, + "theoretical_loss": 3.5008875729906688, + "tokens_seen": 1575878656 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026395185556670007, + "loss": 2.6857, + "theoretical_loss": 3.500874857826962, + "tokens_seen": 1575944192 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002639418254764293, + "loss": 2.7287, + "theoretical_loss": 3.500862143340054, + "tokens_seen": 1576009728 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002639317953861585, + "loss": 2.7285, + "theoretical_loss": 3.50084942952988, + "tokens_seen": 1576075264 + }, + { + "epoch": 0.48, + "objective/train/docs_used": 893728, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.182720422744751, + "objective/train/theoretical_loss": 3.500836716396377, + "objective/train/tokens_used": 1596600800, + "theoretical_loss": 3.500836716396377, + "tokens_seen": 1576140800 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026392176529588767, + "loss": 2.8196, + "theoretical_loss": 3.500836716396377, + "tokens_seen": 1576140800 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026391173520561685, + "loss": 2.6356, + "theoretical_loss": 3.5008240039394796, + "tokens_seen": 1576206336 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002639017051153461, + "loss": 2.506, + "theoretical_loss": 3.5008112921591246, + "tokens_seen": 1576271872 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002638916750250752, + "loss": 2.7407, + "theoretical_loss": 3.5007985810552475, + "tokens_seen": 1576337408 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026388164493480444, + "loss": 2.8055, + "theoretical_loss": 3.5007858706277837, + "tokens_seen": 1576402944 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026387161484453357, + "loss": 2.6331, + "theoretical_loss": 3.50077316087667, + "tokens_seen": 1576468480 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002638615847542628, + "loss": 2.7424, + "theoretical_loss": 3.500760451801842, + "tokens_seen": 1576534016 + }, + { + "epoch": 0.48, + "learning_rate": 0.000263851554663992, + "loss": 2.6155, + "theoretical_loss": 3.5007477434032346, + "tokens_seen": 1576599552 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026384152457372117, + "loss": 2.499, + "theoretical_loss": 3.500735035680785, + "tokens_seen": 1576665088 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026383149448345035, + "loss": 2.6684, + "theoretical_loss": 3.5007223286344287, + "tokens_seen": 1576730624 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026382146439317953, + "loss": 2.6054, + "theoretical_loss": 3.500709622264102, + "tokens_seen": 1576796160 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002638114343029087, + "loss": 2.5362, + "theoretical_loss": 3.50069691656974, + "tokens_seen": 1576861696 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026380140421263795, + "loss": 2.7158, + "theoretical_loss": 3.5006842115512793, + "tokens_seen": 1576927232 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002637913741223671, + "loss": 2.7259, + "theoretical_loss": 3.500671507208655, + "tokens_seen": 1576992768 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002637813440320963, + "loss": 2.622, + "theoretical_loss": 3.5006588035418043, + "tokens_seen": 1577058304 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002637713139418255, + "loss": 2.6125, + "theoretical_loss": 3.5006461005506626, + "tokens_seen": 1577123840 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026376128385155467, + "loss": 2.6768, + "theoretical_loss": 3.5006333982351663, + "tokens_seen": 1577189376 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026375125376128385, + "loss": 2.6811, + "theoretical_loss": 3.5006206965952504, + "tokens_seen": 1577254912 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026374122367101303, + "loss": 2.5918, + "theoretical_loss": 3.500607995630851, + "tokens_seen": 1577320448 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002637311935807422, + "loss": 2.7375, + "theoretical_loss": 3.5005952953419053, + "tokens_seen": 1577385984 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026372116349047145, + "loss": 2.6595, + "theoretical_loss": 3.5005825957283485, + "tokens_seen": 1577451520 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002637111334002006, + "loss": 2.6494, + "theoretical_loss": 3.5005698967901164, + "tokens_seen": 1577517056 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002637011033099298, + "loss": 2.5599, + "theoretical_loss": 3.5005571985271455, + "tokens_seen": 1577582592 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026369107321965894, + "loss": 2.6759, + "theoretical_loss": 3.5005445009393723, + "tokens_seen": 1577648128 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002636810431293882, + "loss": 2.8208, + "theoretical_loss": 3.5005318040267315, + "tokens_seen": 1577713664 + }, + { + "epoch": 0.48, + "objective/train/docs_used": 894313, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.0645577907562256, + "objective/train/theoretical_loss": 3.50051910778916, + "objective/train/tokens_used": 1598239200, + "theoretical_loss": 3.50051910778916, + "tokens_seen": 1577779200 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026367101303911736, + "loss": 2.8037, + "theoretical_loss": 3.50051910778916, + "tokens_seen": 1577779200 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026366098294884654, + "loss": 2.7788, + "theoretical_loss": 3.500506412226594, + "tokens_seen": 1577844736 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002636509528585757, + "loss": 2.8521, + "theoretical_loss": 3.500493717338969, + "tokens_seen": 1577910272 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002636409227683049, + "loss": 2.9867, + "theoretical_loss": 3.5004810231262216, + "tokens_seen": 1577975808 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002636308926780341, + "loss": 2.6917, + "theoretical_loss": 3.500468329588288, + "tokens_seen": 1578041344 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002636208625877633, + "loss": 2.7894, + "theoretical_loss": 3.5004556367251043, + "tokens_seen": 1578106880 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026361083249749244, + "loss": 2.7319, + "theoretical_loss": 3.500442944536606, + "tokens_seen": 1578172416 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002636008024072217, + "loss": 2.7371, + "theoretical_loss": 3.500430253022729, + "tokens_seen": 1578237952 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026359077231695086, + "loss": 2.4953, + "theoretical_loss": 3.5004175621834106, + "tokens_seen": 1578303488 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026358074222668004, + "loss": 2.5183, + "theoretical_loss": 3.5004048720185863, + "tokens_seen": 1578369024 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002635707121364092, + "loss": 2.8705, + "theoretical_loss": 3.5003921825281923, + "tokens_seen": 1578434560 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002635606820461384, + "loss": 2.6175, + "theoretical_loss": 3.500379493712165, + "tokens_seen": 1578500096 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002635506519558676, + "loss": 2.7729, + "theoretical_loss": 3.50036680557044, + "tokens_seen": 1578565632 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002635406218655968, + "loss": 2.8186, + "theoretical_loss": 3.500354118102954, + "tokens_seen": 1578631168 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026353059177532595, + "loss": 2.5573, + "theoretical_loss": 3.500341431309643, + "tokens_seen": 1578696704 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002635205616850552, + "loss": 2.7043, + "theoretical_loss": 3.5003287451904423, + "tokens_seen": 1578762240 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026351053159478436, + "loss": 2.6805, + "theoretical_loss": 3.5003160597452903, + "tokens_seen": 1578827776 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026350050150451354, + "loss": 2.7161, + "theoretical_loss": 3.500303374974121, + "tokens_seen": 1578893312 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002634904714142428, + "loss": 2.5517, + "theoretical_loss": 3.500290690876872, + "tokens_seen": 1578958848 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002634804413239719, + "loss": 2.6471, + "theoretical_loss": 3.5002780074534785, + "tokens_seen": 1579024384 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026347041123370114, + "loss": 2.953, + "theoretical_loss": 3.5002653247038777, + "tokens_seen": 1579089920 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026346038114343027, + "loss": 2.7035, + "theoretical_loss": 3.5002526426280047, + "tokens_seen": 1579155456 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002634503510531595, + "loss": 2.6186, + "theoretical_loss": 3.5002399612257973, + "tokens_seen": 1579220992 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002634403209628887, + "loss": 2.5354, + "theoretical_loss": 3.5002272804971906, + "tokens_seen": 1579286528 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026343029087261787, + "loss": 2.6999, + "theoretical_loss": 3.5002146004421215, + "tokens_seen": 1579352064 + }, + { + "epoch": 0.48, + "objective/train/docs_used": 895615, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.528836488723755, + "objective/train/theoretical_loss": 3.5002019210605253, + "objective/train/tokens_used": 1599877600, + "theoretical_loss": 3.5002019210605253, + "tokens_seen": 1579417600 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026342026078234705, + "loss": 2.5951, + "theoretical_loss": 3.5002019210605253, + "tokens_seen": 1579417600 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002634102306920763, + "loss": 2.6115, + "theoretical_loss": 3.5001892423523397, + "tokens_seen": 1579483136 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002634002006018054, + "loss": 2.7122, + "theoretical_loss": 3.5001765643174996, + "tokens_seen": 1579548672 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026339017051153464, + "loss": 2.6496, + "theoretical_loss": 3.500163886955942, + "tokens_seen": 1579614208 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026338014042126377, + "loss": 2.7969, + "theoretical_loss": 3.500151210267604, + "tokens_seen": 1579679744 + }, + { + "epoch": 0.48, + "learning_rate": 0.000263370110330993, + "loss": 2.5556, + "theoretical_loss": 3.5001385342524203, + "tokens_seen": 1579745280 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002633600802407222, + "loss": 2.7371, + "theoretical_loss": 3.5001258589103283, + "tokens_seen": 1579810816 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026335005015045137, + "loss": 2.7449, + "theoretical_loss": 3.500113184241264, + "tokens_seen": 1579876352 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026334002006018055, + "loss": 2.7537, + "theoretical_loss": 3.5001005102451637, + "tokens_seen": 1579941888 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026332998996990973, + "loss": 2.475, + "theoretical_loss": 3.500087836921964, + "tokens_seen": 1580007424 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002633199598796389, + "loss": 2.784, + "theoretical_loss": 3.500075164271601, + "tokens_seen": 1580072960 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026330992978936815, + "loss": 2.841, + "theoretical_loss": 3.500062492294011, + "tokens_seen": 1580138496 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002632998996990973, + "loss": 2.7577, + "theoretical_loss": 3.5000498209891306, + "tokens_seen": 1580204032 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002632898696088265, + "loss": 2.7016, + "theoretical_loss": 3.5000371503568966, + "tokens_seen": 1580269568 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002632798395185557, + "loss": 2.8021, + "theoretical_loss": 3.5000244803972453, + "tokens_seen": 1580335104 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026326980942828487, + "loss": 2.7488, + "theoretical_loss": 3.500011811110112, + "tokens_seen": 1580400640 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026325977933801405, + "loss": 2.6692, + "theoretical_loss": 3.499999142495434, + "tokens_seen": 1580466176 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026324974924774323, + "loss": 2.6125, + "theoretical_loss": 3.499986474553148, + "tokens_seen": 1580531712 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002632397191574724, + "loss": 2.6299, + "theoretical_loss": 3.49997380728319, + "tokens_seen": 1580597248 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026322968906720165, + "loss": 2.4835, + "theoretical_loss": 3.499961140685496, + "tokens_seen": 1580662784 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002632196589769308, + "loss": 2.7466, + "theoretical_loss": 3.4999484747600036, + "tokens_seen": 1580728320 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026320962888666, + "loss": 2.8991, + "theoretical_loss": 3.499935809506648, + "tokens_seen": 1580793856 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026319959879638914, + "loss": 2.8255, + "theoretical_loss": 3.499923144925367, + "tokens_seen": 1580859392 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002631895687061184, + "loss": 2.6409, + "theoretical_loss": 3.499910481016096, + "tokens_seen": 1580924928 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026317953861584756, + "loss": 2.5723, + "theoretical_loss": 3.4998978177787716, + "tokens_seen": 1580990464 + }, + { + "epoch": 0.48, + "objective/train/docs_used": 896135, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.372347593307495, + "objective/train/theoretical_loss": 3.499885155213331, + "objective/train/tokens_used": 1601516000, + "theoretical_loss": 3.499885155213331, + "tokens_seen": 1581056000 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026316950852557674, + "loss": 2.5965, + "theoretical_loss": 3.499885155213331, + "tokens_seen": 1581056000 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002631594784353059, + "loss": 2.8718, + "theoretical_loss": 3.4998724933197103, + "tokens_seen": 1581121536 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002631494483450351, + "loss": 2.7437, + "theoretical_loss": 3.4998598320978456, + "tokens_seen": 1581187072 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002631394182547643, + "loss": 2.7977, + "theoretical_loss": 3.4998471715476738, + "tokens_seen": 1581252608 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002631293881644935, + "loss": 2.6126, + "theoretical_loss": 3.4998345116691314, + "tokens_seen": 1581318144 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026311935807422264, + "loss": 2.561, + "theoretical_loss": 3.499821852462155, + "tokens_seen": 1581383680 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002631093279839519, + "loss": 2.5534, + "theoretical_loss": 3.499809193926681, + "tokens_seen": 1581449216 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026309929789368106, + "loss": 2.7194, + "theoretical_loss": 3.4997965360626466, + "tokens_seen": 1581514752 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026308926780341024, + "loss": 2.4544, + "theoretical_loss": 3.4997838788699873, + "tokens_seen": 1581580288 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002630792377131394, + "loss": 3.0615, + "theoretical_loss": 3.49977122234864, + "tokens_seen": 1581645824 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002630692076228686, + "loss": 2.8, + "theoretical_loss": 3.499758566498542, + "tokens_seen": 1581711360 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002630591775325978, + "loss": 2.7348, + "theoretical_loss": 3.499745911319629, + "tokens_seen": 1581776896 + }, + { + "epoch": 0.48, + "learning_rate": 0.000263049147442327, + "loss": 2.641, + "theoretical_loss": 3.499733256811838, + "tokens_seen": 1581842432 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026303911735205615, + "loss": 2.836, + "theoretical_loss": 3.4997206029751053, + "tokens_seen": 1581907968 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002630290872617854, + "loss": 2.8477, + "theoretical_loss": 3.4997079498093684, + "tokens_seen": 1581973504 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002630190571715145, + "loss": 2.7052, + "theoretical_loss": 3.499695297314563, + "tokens_seen": 1582039040 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026300902708124374, + "loss": 2.459, + "theoretical_loss": 3.4996826454906262, + "tokens_seen": 1582104576 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002629989969909729, + "loss": 2.9837, + "theoretical_loss": 3.499669994337494, + "tokens_seen": 1582170112 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002629889669007021, + "loss": 2.7521, + "theoretical_loss": 3.499657343855104, + "tokens_seen": 1582235648 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002629789368104313, + "loss": 2.415, + "theoretical_loss": 3.499644694043392, + "tokens_seen": 1582301184 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026296890672016047, + "loss": 2.7402, + "theoretical_loss": 3.4996320449022953, + "tokens_seen": 1582366720 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026295887662988965, + "loss": 2.8285, + "theoretical_loss": 3.4996193964317506, + "tokens_seen": 1582432256 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002629488465396189, + "loss": 2.6737, + "theoretical_loss": 3.4996067486316935, + "tokens_seen": 1582497792 + }, + { + "epoch": 0.48, + "learning_rate": 0.000262938816449348, + "loss": 2.6805, + "theoretical_loss": 3.499594101502062, + "tokens_seen": 1582563328 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026292878635907725, + "loss": 2.8038, + "theoretical_loss": 3.499581455042793, + "tokens_seen": 1582628864 + }, + { + "epoch": 0.48, + "objective/train/docs_used": 897089, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.389763116836548, + "objective/train/theoretical_loss": 3.4995688092538213, + "objective/train/tokens_used": 1603154400, + "theoretical_loss": 3.4995688092538213, + "tokens_seen": 1582694400 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026291875626880643, + "loss": 2.7733, + "theoretical_loss": 3.4995688092538213, + "tokens_seen": 1582694400 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002629087261785356, + "loss": 2.9535, + "theoretical_loss": 3.499556164135085, + "tokens_seen": 1582759936 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002628986960882648, + "loss": 2.7526, + "theoretical_loss": 3.4995435196865214, + "tokens_seen": 1582825472 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026288866599799397, + "loss": 2.6486, + "theoretical_loss": 3.499530875908066, + "tokens_seen": 1582891008 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026287863590772315, + "loss": 2.5385, + "theoretical_loss": 3.499518232799656, + "tokens_seen": 1582956544 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002628686058174524, + "loss": 2.6386, + "theoretical_loss": 3.4995055903612284, + "tokens_seen": 1583022080 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002628585757271815, + "loss": 2.7523, + "theoretical_loss": 3.49949294859272, + "tokens_seen": 1583087616 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026284854563691075, + "loss": 2.746, + "theoretical_loss": 3.499480307494067, + "tokens_seen": 1583153152 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002628385155466399, + "loss": 3.0892, + "theoretical_loss": 3.4994676670652067, + "tokens_seen": 1583218688 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002628284854563691, + "loss": 2.8331, + "theoretical_loss": 3.499455027306076, + "tokens_seen": 1583284224 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002628184553660983, + "loss": 2.8276, + "theoretical_loss": 3.49944238821661, + "tokens_seen": 1583349760 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002628084252758275, + "loss": 2.731, + "theoretical_loss": 3.4994297497967484, + "tokens_seen": 1583415296 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026279839518555666, + "loss": 2.7001, + "theoretical_loss": 3.4994171120464257, + "tokens_seen": 1583480832 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002627883650952859, + "loss": 2.7547, + "theoretical_loss": 3.49940447496558, + "tokens_seen": 1583546368 + }, + { + "epoch": 0.48, + "learning_rate": 0.000262778335005015, + "loss": 2.6488, + "theoretical_loss": 3.499391838554147, + "tokens_seen": 1583611904 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026276830491474425, + "loss": 2.7237, + "theoretical_loss": 3.499379202812065, + "tokens_seen": 1583677440 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026275827482447343, + "loss": 2.656, + "theoretical_loss": 3.49936656773927, + "tokens_seen": 1583742976 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002627482447342026, + "loss": 2.7162, + "theoretical_loss": 3.499353933335698, + "tokens_seen": 1583808512 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026273821464393185, + "loss": 2.8421, + "theoretical_loss": 3.4993412996012876, + "tokens_seen": 1583874048 + }, + { + "epoch": 0.48, + "learning_rate": 0.000262728184553661, + "loss": 2.8592, + "theoretical_loss": 3.4993286665359746, + "tokens_seen": 1583939584 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002627181544633902, + "loss": 2.645, + "theoretical_loss": 3.4993160341396963, + "tokens_seen": 1584005120 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026270812437311934, + "loss": 2.8319, + "theoretical_loss": 3.499303402412389, + "tokens_seen": 1584070656 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002626980942828486, + "loss": 2.4982, + "theoretical_loss": 3.4992907713539907, + "tokens_seen": 1584136192 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026268806419257776, + "loss": 2.7888, + "theoretical_loss": 3.499278140964437, + "tokens_seen": 1584201728 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026267803410230694, + "loss": 2.9605, + "theoretical_loss": 3.4992655112436655, + "tokens_seen": 1584267264 + }, + { + "epoch": 0.48, + "objective/train/docs_used": 897735, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.918968677520752, + "objective/train/theoretical_loss": 3.4992528821916133, + "objective/train/tokens_used": 1604792800, + "theoretical_loss": 3.4992528821916133, + "tokens_seen": 1584332800 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002626680040120361, + "loss": 2.7962, + "theoretical_loss": 3.4992528821916133, + "tokens_seen": 1584332800 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002626579739217653, + "loss": 2.9258, + "theoretical_loss": 3.4992402538082166, + "tokens_seen": 1584398336 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002626479438314945, + "loss": 2.7381, + "theoretical_loss": 3.4992276260934134, + "tokens_seen": 1584463872 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002626379137412237, + "loss": 2.8392, + "theoretical_loss": 3.49921499904714, + "tokens_seen": 1584529408 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026262788365095284, + "loss": 2.6746, + "theoretical_loss": 3.499202372669333, + "tokens_seen": 1584594944 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002626178535606821, + "loss": 2.6837, + "theoretical_loss": 3.4991897469599307, + "tokens_seen": 1584660480 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026260782347041126, + "loss": 2.7665, + "theoretical_loss": 3.4991771219188683, + "tokens_seen": 1584726016 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026259779338014044, + "loss": 2.9386, + "theoretical_loss": 3.4991644975460843, + "tokens_seen": 1584791552 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002625877632898696, + "loss": 2.7503, + "theoretical_loss": 3.4991518738415146, + "tokens_seen": 1584857088 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002625777331995988, + "loss": 2.5844, + "theoretical_loss": 3.499139250805097, + "tokens_seen": 1584922624 + }, + { + "epoch": 0.48, + "learning_rate": 0.000262567703109328, + "loss": 2.7778, + "theoretical_loss": 3.499126628436768, + "tokens_seen": 1584988160 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002625576730190572, + "loss": 2.7445, + "theoretical_loss": 3.4991140067364648, + "tokens_seen": 1585053696 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026254764292878635, + "loss": 2.9071, + "theoretical_loss": 3.499101385704124, + "tokens_seen": 1585119232 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002625376128385156, + "loss": 2.6512, + "theoretical_loss": 3.499088765339684, + "tokens_seen": 1585184768 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002625275827482447, + "loss": 2.4614, + "theoretical_loss": 3.4990761456430803, + "tokens_seen": 1585250304 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026251755265797394, + "loss": 2.5917, + "theoretical_loss": 3.4990635266142505, + "tokens_seen": 1585315840 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002625075225677031, + "loss": 2.6636, + "theoretical_loss": 3.4990509082531323, + "tokens_seen": 1585381376 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002624974924774323, + "loss": 2.8031, + "theoretical_loss": 3.4990382905596618, + "tokens_seen": 1585446912 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002624874623871615, + "loss": 2.3497, + "theoretical_loss": 3.4990256735337764, + "tokens_seen": 1585512448 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026247743229689067, + "loss": 2.8636, + "theoretical_loss": 3.4990130571754134, + "tokens_seen": 1585577984 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026246740220661985, + "loss": 2.635, + "theoretical_loss": 3.49900044148451, + "tokens_seen": 1585643520 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002624573721163491, + "loss": 2.8336, + "theoretical_loss": 3.4989878264610024, + "tokens_seen": 1585709056 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002624473420260782, + "loss": 2.7977, + "theoretical_loss": 3.4989752121048285, + "tokens_seen": 1585774592 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026243731193580745, + "loss": 2.7406, + "theoretical_loss": 3.4989625984159254, + "tokens_seen": 1585840128 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026242728184553663, + "loss": 2.6479, + "theoretical_loss": 3.4989499853942307, + "tokens_seen": 1585905664 + }, + { + "epoch": 0.48, + "objective/train/docs_used": 899130, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.167370557785034, + "objective/train/theoretical_loss": 3.49893737303968, + "objective/train/tokens_used": 1606431200, + "theoretical_loss": 3.49893737303968, + "tokens_seen": 1585971200 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002624172517552658, + "loss": 2.7708, + "theoretical_loss": 3.49893737303968, + "tokens_seen": 1585971200 + }, + { + "epoch": 0.48, + "learning_rate": 0.000262407221664995, + "loss": 2.5952, + "theoretical_loss": 3.498924761352212, + "tokens_seen": 1586036736 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026239719157472417, + "loss": 2.7297, + "theoretical_loss": 3.498912150331763, + "tokens_seen": 1586102272 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026238716148445335, + "loss": 2.6205, + "theoretical_loss": 3.4988995399782707, + "tokens_seen": 1586167808 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002623771313941826, + "loss": 2.9432, + "theoretical_loss": 3.498886930291672, + "tokens_seen": 1586233344 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002623671013039117, + "loss": 3.05, + "theoretical_loss": 3.498874321271904, + "tokens_seen": 1586298880 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026235707121364095, + "loss": 2.7488, + "theoretical_loss": 3.4988617129189037, + "tokens_seen": 1586364416 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002623470411233701, + "loss": 2.6221, + "theoretical_loss": 3.4988491052326087, + "tokens_seen": 1586429952 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002623370110330993, + "loss": 2.6648, + "theoretical_loss": 3.498836498212956, + "tokens_seen": 1586495488 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002623269809428285, + "loss": 2.6171, + "theoretical_loss": 3.498823891859883, + "tokens_seen": 1586561024 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002623169508525577, + "loss": 2.6433, + "theoretical_loss": 3.4988112861733267, + "tokens_seen": 1586626560 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026230692076228686, + "loss": 2.7704, + "theoretical_loss": 3.498798681153225, + "tokens_seen": 1586692096 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002622968906720161, + "loss": 2.8534, + "theoretical_loss": 3.4987860767995143, + "tokens_seen": 1586757632 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002622868605817452, + "loss": 2.6374, + "theoretical_loss": 3.4987734731121316, + "tokens_seen": 1586823168 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026227683049147445, + "loss": 2.5578, + "theoretical_loss": 3.498760870091015, + "tokens_seen": 1586888704 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002622668004012036, + "loss": 2.9796, + "theoretical_loss": 3.4987482677361017, + "tokens_seen": 1586954240 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002622567703109328, + "loss": 2.5739, + "theoretical_loss": 3.4987356660473283, + "tokens_seen": 1587019776 + }, + { + "epoch": 0.48, + "learning_rate": 0.000262246740220662, + "loss": 2.6819, + "theoretical_loss": 3.498723065024633, + "tokens_seen": 1587085312 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002622367101303912, + "loss": 2.6012, + "theoretical_loss": 3.498710464667952, + "tokens_seen": 1587150848 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026222668004012036, + "loss": 2.7009, + "theoretical_loss": 3.4986978649772236, + "tokens_seen": 1587216384 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026221664994984954, + "loss": 2.662, + "theoretical_loss": 3.4986852659523846, + "tokens_seen": 1587281920 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002622066198595787, + "loss": 2.6132, + "theoretical_loss": 3.4986726675933726, + "tokens_seen": 1587347456 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026219658976930796, + "loss": 2.5783, + "theoretical_loss": 3.498660069900124, + "tokens_seen": 1587412992 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002621865596790371, + "loss": 2.8344, + "theoretical_loss": 3.4986474728725776, + "tokens_seen": 1587478528 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002621765295887663, + "loss": 2.9317, + "theoretical_loss": 3.4986348765106703, + "tokens_seen": 1587544064 + }, + { + "epoch": 0.48, + "objective/train/docs_used": 899656, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.1017861366271973, + "objective/train/theoretical_loss": 3.4986222808143386, + "objective/train/tokens_used": 1608069600, + "theoretical_loss": 3.4986222808143386, + "tokens_seen": 1587609600 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026216649949849545, + "loss": 2.957, + "theoretical_loss": 3.4986222808143386, + "tokens_seen": 1587609600 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002621564694082247, + "loss": 2.6009, + "theoretical_loss": 3.4986096857835207, + "tokens_seen": 1587675136 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026214643931795386, + "loss": 2.5252, + "theoretical_loss": 3.4985970914181532, + "tokens_seen": 1587740672 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026213640922768304, + "loss": 2.7977, + "theoretical_loss": 3.4985844977181744, + "tokens_seen": 1587806208 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002621263791374122, + "loss": 2.8498, + "theoretical_loss": 3.4985719046835215, + "tokens_seen": 1587871744 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026211634904714146, + "loss": 2.5914, + "theoretical_loss": 3.498559312314131, + "tokens_seen": 1587937280 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002621063189568706, + "loss": 2.4409, + "theoretical_loss": 3.4985467206099417, + "tokens_seen": 1588002816 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002620962888665998, + "loss": 2.5218, + "theoretical_loss": 3.4985341295708903, + "tokens_seen": 1588068352 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026208625877632895, + "loss": 2.5151, + "theoretical_loss": 3.4985215391969136, + "tokens_seen": 1588133888 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002620762286860582, + "loss": 2.6292, + "theoretical_loss": 3.4985089494879498, + "tokens_seen": 1588199424 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026206619859578737, + "loss": 2.7699, + "theoretical_loss": 3.4984963604439363, + "tokens_seen": 1588264960 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026205616850551655, + "loss": 2.5576, + "theoretical_loss": 3.49848377206481, + "tokens_seen": 1588330496 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026204613841524573, + "loss": 2.7754, + "theoretical_loss": 3.4984711843505094, + "tokens_seen": 1588396032 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002620361083249749, + "loss": 2.8126, + "theoretical_loss": 3.4984585973009708, + "tokens_seen": 1588461568 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002620260782347041, + "loss": 2.622, + "theoretical_loss": 3.4984460109161324, + "tokens_seen": 1588527104 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002620160481444333, + "loss": 2.8635, + "theoretical_loss": 3.4984334251959313, + "tokens_seen": 1588592640 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002620060180541625, + "loss": 2.8456, + "theoretical_loss": 3.498420840140305, + "tokens_seen": 1588658176 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002619959879638917, + "loss": 2.6958, + "theoretical_loss": 3.498408255749191, + "tokens_seen": 1588723712 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026198595787362087, + "loss": 2.7256, + "theoretical_loss": 3.4983956720225278, + "tokens_seen": 1588789248 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026197592778335005, + "loss": 2.577, + "theoretical_loss": 3.498383088960251, + "tokens_seen": 1588854784 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002619658976930793, + "loss": 2.5495, + "theoretical_loss": 3.4983705065623, + "tokens_seen": 1588920320 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002619558676028084, + "loss": 2.6142, + "theoretical_loss": 3.4983579248286114, + "tokens_seen": 1588985856 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026194583751253765, + "loss": 2.8967, + "theoretical_loss": 3.4983453437591225, + "tokens_seen": 1589051392 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026193580742226683, + "loss": 2.5934, + "theoretical_loss": 3.498332763353771, + "tokens_seen": 1589116928 + }, + { + "epoch": 0.48, + "learning_rate": 0.000261925777331996, + "loss": 2.7573, + "theoretical_loss": 3.498320183612495, + "tokens_seen": 1589182464 + }, + { + "epoch": 0.48, + "objective/train/docs_used": 900745, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7677981853485107, + "objective/train/theoretical_loss": 3.4983076045352317, + "objective/train/tokens_used": 1609708000, + "theoretical_loss": 3.4983076045352317, + "tokens_seen": 1589248000 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002619157472417252, + "loss": 2.8303, + "theoretical_loss": 3.4983076045352317, + "tokens_seen": 1589248000 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026190571715145437, + "loss": 2.6996, + "theoretical_loss": 3.4982950261219186, + "tokens_seen": 1589313536 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026189568706118355, + "loss": 2.6302, + "theoretical_loss": 3.4982824483724935, + "tokens_seen": 1589379072 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002618856569709128, + "loss": 2.6755, + "theoretical_loss": 3.498269871286894, + "tokens_seen": 1589444608 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002618756268806419, + "loss": 2.7171, + "theoretical_loss": 3.4982572948650565, + "tokens_seen": 1589510144 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026186559679037115, + "loss": 2.5872, + "theoretical_loss": 3.4982447191069204, + "tokens_seen": 1589575680 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002618555667001003, + "loss": 2.7924, + "theoretical_loss": 3.4982321440124227, + "tokens_seen": 1589641216 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002618455366098295, + "loss": 2.6166, + "theoretical_loss": 3.498219569581501, + "tokens_seen": 1589706752 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002618355065195587, + "loss": 2.5861, + "theoretical_loss": 3.4982069958140922, + "tokens_seen": 1589772288 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002618254764292879, + "loss": 2.6168, + "theoretical_loss": 3.4981944227101347, + "tokens_seen": 1589837824 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026181544633901706, + "loss": 2.5718, + "theoretical_loss": 3.498181850269566, + "tokens_seen": 1589903360 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002618054162487463, + "loss": 2.7838, + "theoretical_loss": 3.498169278492324, + "tokens_seen": 1589968896 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002617953861584754, + "loss": 2.7223, + "theoretical_loss": 3.498156707378346, + "tokens_seen": 1590034432 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026178535606820465, + "loss": 2.4258, + "theoretical_loss": 3.4981441369275696, + "tokens_seen": 1590099968 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002617753259779338, + "loss": 2.6129, + "theoretical_loss": 3.4981315671399327, + "tokens_seen": 1590165504 + }, + { + "epoch": 0.48, + "learning_rate": 0.000261765295887663, + "loss": 2.675, + "theoretical_loss": 3.498118998015373, + "tokens_seen": 1590231040 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002617552657973922, + "loss": 2.6444, + "theoretical_loss": 3.498106429553828, + "tokens_seen": 1590296576 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002617452357071214, + "loss": 2.6565, + "theoretical_loss": 3.498093861755236, + "tokens_seen": 1590362112 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026173520561685056, + "loss": 2.9966, + "theoretical_loss": 3.4980812946195337, + "tokens_seen": 1590427648 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026172517552657974, + "loss": 2.9415, + "theoretical_loss": 3.4980687281466594, + "tokens_seen": 1590493184 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002617151454363089, + "loss": 2.813, + "theoretical_loss": 3.4980561623365514, + "tokens_seen": 1590558720 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026170511534603816, + "loss": 2.4639, + "theoretical_loss": 3.498043597189146, + "tokens_seen": 1590624256 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002616950852557673, + "loss": 2.7421, + "theoretical_loss": 3.4980310327043824, + "tokens_seen": 1590689792 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002616850551654965, + "loss": 2.6982, + "theoretical_loss": 3.498018468882198, + "tokens_seen": 1590755328 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026167502507522565, + "loss": 2.3779, + "theoretical_loss": 3.49800590572253, + "tokens_seen": 1590820864 + }, + { + "epoch": 0.48, + "objective/train/docs_used": 902130, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.575000047683716, + "objective/train/theoretical_loss": 3.4979933432253163, + "objective/train/tokens_used": 1611346400, + "theoretical_loss": 3.4979933432253163, + "tokens_seen": 1590886400 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002616649949849549, + "loss": 2.6366, + "theoretical_loss": 3.4979933432253163, + "tokens_seen": 1590886400 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026165496489468406, + "loss": 2.5783, + "theoretical_loss": 3.497980781390495, + "tokens_seen": 1590951936 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026164493480441324, + "loss": 2.7549, + "theoretical_loss": 3.497968220218004, + "tokens_seen": 1591017472 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002616349047141424, + "loss": 2.4876, + "theoretical_loss": 3.49795565970778, + "tokens_seen": 1591083008 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026162487462387166, + "loss": 2.5434, + "theoretical_loss": 3.4979430998597625, + "tokens_seen": 1591148544 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002616148445336008, + "loss": 2.5552, + "theoretical_loss": 3.4979305406738885, + "tokens_seen": 1591214080 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026160481444333, + "loss": 2.7422, + "theoretical_loss": 3.4979179821500956, + "tokens_seen": 1591279616 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026159478435305915, + "loss": 2.9047, + "theoretical_loss": 3.497905424288322, + "tokens_seen": 1591345152 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002615847542627884, + "loss": 2.702, + "theoretical_loss": 3.497892867088505, + "tokens_seen": 1591410688 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026157472417251757, + "loss": 2.7172, + "theoretical_loss": 3.4978803105505833, + "tokens_seen": 1591476224 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026156469408224675, + "loss": 2.8041, + "theoretical_loss": 3.497867754674494, + "tokens_seen": 1591541760 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026155466399197593, + "loss": 2.6831, + "theoretical_loss": 3.4978551994601754, + "tokens_seen": 1591607296 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002615446339017051, + "loss": 2.5987, + "theoretical_loss": 3.497842644907565, + "tokens_seen": 1591672832 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002615346038114343, + "loss": 2.6653, + "theoretical_loss": 3.4978300910166014, + "tokens_seen": 1591738368 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002615245737211635, + "loss": 2.8068, + "theoretical_loss": 3.4978175377872214, + "tokens_seen": 1591803904 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026151454363089265, + "loss": 2.7481, + "theoretical_loss": 3.497804985219364, + "tokens_seen": 1591869440 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002615045135406219, + "loss": 2.3972, + "theoretical_loss": 3.4977924333129664, + "tokens_seen": 1591934976 + }, + { + "epoch": 0.48, + "learning_rate": 0.000261494483450351, + "loss": 2.6956, + "theoretical_loss": 3.4977798820679666, + "tokens_seen": 1592000512 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026148445336008025, + "loss": 2.6453, + "theoretical_loss": 3.4977673314843027, + "tokens_seen": 1592066048 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026147442326980943, + "loss": 2.5616, + "theoretical_loss": 3.4977547815619126, + "tokens_seen": 1592131584 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002614643931795386, + "loss": 2.3382, + "theoretical_loss": 3.497742232300734, + "tokens_seen": 1592197120 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002614543630892678, + "loss": 2.6802, + "theoretical_loss": 3.4977296837007055, + "tokens_seen": 1592262656 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026144433299899703, + "loss": 2.8063, + "theoretical_loss": 3.4977171357617642, + "tokens_seen": 1592328192 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026143430290872616, + "loss": 2.8384, + "theoretical_loss": 3.497704588483849, + "tokens_seen": 1592393728 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002614242728184554, + "loss": 2.7395, + "theoretical_loss": 3.497692041866897, + "tokens_seen": 1592459264 + }, + { + "epoch": 0.48, + "objective/train/docs_used": 902721, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7940666675567627, + "objective/train/theoretical_loss": 3.4976794959108464, + "objective/train/tokens_used": 1612984800, + "theoretical_loss": 3.4976794959108464, + "tokens_seen": 1592524800 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002614142427281845, + "loss": 2.7981, + "theoretical_loss": 3.4976794959108464, + "tokens_seen": 1592524800 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026140421263791375, + "loss": 2.8462, + "theoretical_loss": 3.4976669506156357, + "tokens_seen": 1592590336 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026139418254764293, + "loss": 2.8098, + "theoretical_loss": 3.4976544059812023, + "tokens_seen": 1592655872 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002613841524573721, + "loss": 2.7418, + "theoretical_loss": 3.4976418620074847, + "tokens_seen": 1592721408 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002613741223671013, + "loss": 2.5639, + "theoretical_loss": 3.49762931869442, + "tokens_seen": 1592786944 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002613640922768305, + "loss": 2.7883, + "theoretical_loss": 3.497616776041948, + "tokens_seen": 1592852480 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026135406218655966, + "loss": 2.6127, + "theoretical_loss": 3.497604234050005, + "tokens_seen": 1592918016 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002613440320962889, + "loss": 2.808, + "theoretical_loss": 3.4975916927185295, + "tokens_seen": 1592983552 + }, + { + "epoch": 0.48, + "learning_rate": 0.000261334002006018, + "loss": 2.766, + "theoretical_loss": 3.49757915204746, + "tokens_seen": 1593049088 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026132397191574726, + "loss": 2.652, + "theoretical_loss": 3.497566612036734, + "tokens_seen": 1593114624 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002613139418254764, + "loss": 2.6829, + "theoretical_loss": 3.49755407268629, + "tokens_seen": 1593180160 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002613039117352056, + "loss": 2.6869, + "theoretical_loss": 3.497541533996066, + "tokens_seen": 1593245696 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002612938816449348, + "loss": 2.7348, + "theoretical_loss": 3.497528995966, + "tokens_seen": 1593311232 + }, + { + "epoch": 0.48, + "learning_rate": 0.000261283851554664, + "loss": 2.6006, + "theoretical_loss": 3.4975164585960297, + "tokens_seen": 1593376768 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026127382146439316, + "loss": 2.7989, + "theoretical_loss": 3.497503921886094, + "tokens_seen": 1593442304 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002612637913741224, + "loss": 2.991, + "theoretical_loss": 3.4974913858361303, + "tokens_seen": 1593507840 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002612537612838516, + "loss": 2.6996, + "theoretical_loss": 3.497478850446077, + "tokens_seen": 1593573376 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026124373119358076, + "loss": 2.6661, + "theoretical_loss": 3.4974663157158727, + "tokens_seen": 1593638912 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026123370110330994, + "loss": 2.6068, + "theoretical_loss": 3.497453781645455, + "tokens_seen": 1593704448 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002612236710130391, + "loss": 2.5688, + "theoretical_loss": 3.497441248234762, + "tokens_seen": 1593769984 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026121364092276836, + "loss": 2.6439, + "theoretical_loss": 3.4974287154837316, + "tokens_seen": 1593835520 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002612036108324975, + "loss": 2.6737, + "theoretical_loss": 3.4974161833923025, + "tokens_seen": 1593901056 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002611935807422267, + "loss": 2.5887, + "theoretical_loss": 3.497403651960413, + "tokens_seen": 1593966592 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026118355065195585, + "loss": 2.5174, + "theoretical_loss": 3.4973911211880004, + "tokens_seen": 1594032128 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002611735205616851, + "loss": 2.6723, + "theoretical_loss": 3.4973785910750035, + "tokens_seen": 1594097664 + }, + { + "epoch": 0.48, + "objective/train/docs_used": 903500, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.48885178565979, + "objective/train/theoretical_loss": 3.497366061621361, + "objective/train/tokens_used": 1614623200, + "theoretical_loss": 3.497366061621361, + "tokens_seen": 1594163200 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026116349047141426, + "loss": 2.7086, + "theoretical_loss": 3.497366061621361, + "tokens_seen": 1594163200 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026115346038114344, + "loss": 2.4913, + "theoretical_loss": 3.4973535328270104, + "tokens_seen": 1594228736 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002611434302908726, + "loss": 2.6544, + "theoretical_loss": 3.4973410046918896, + "tokens_seen": 1594294272 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026113340020060186, + "loss": 2.5319, + "theoretical_loss": 3.4973284772159374, + "tokens_seen": 1594359808 + }, + { + "epoch": 0.48, + "learning_rate": 0.000261123370110331, + "loss": 2.7298, + "theoretical_loss": 3.4973159503990923, + "tokens_seen": 1594425344 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002611133400200602, + "loss": 2.6915, + "theoretical_loss": 3.4973034242412915, + "tokens_seen": 1594490880 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026110330992978935, + "loss": 2.7708, + "theoretical_loss": 3.4972908987424747, + "tokens_seen": 1594556416 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002610932798395186, + "loss": 2.5125, + "theoretical_loss": 3.4972783739025783, + "tokens_seen": 1594621952 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026108324974924777, + "loss": 2.8099, + "theoretical_loss": 3.497265849721542, + "tokens_seen": 1594687488 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026107321965897695, + "loss": 2.5683, + "theoretical_loss": 3.4972533261993037, + "tokens_seen": 1594753024 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026106318956870613, + "loss": 2.6534, + "theoretical_loss": 3.497240803335802, + "tokens_seen": 1594818560 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002610531594784353, + "loss": 2.6545, + "theoretical_loss": 3.4972282811309743, + "tokens_seen": 1594884096 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002610431293881645, + "loss": 2.9213, + "theoretical_loss": 3.4972157595847593, + "tokens_seen": 1594949632 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002610330992978937, + "loss": 2.6616, + "theoretical_loss": 3.4972032386970957, + "tokens_seen": 1595015168 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026102306920762285, + "loss": 2.7885, + "theoretical_loss": 3.4971907184679214, + "tokens_seen": 1595080704 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002610130391173521, + "loss": 2.8479, + "theoretical_loss": 3.4971781988971746, + "tokens_seen": 1595146240 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002610030090270812, + "loss": 2.4684, + "theoretical_loss": 3.497165679984794, + "tokens_seen": 1595211776 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026099297893681045, + "loss": 2.7109, + "theoretical_loss": 3.4971531617307177, + "tokens_seen": 1595277312 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026098294884653963, + "loss": 2.4211, + "theoretical_loss": 3.497140644134884, + "tokens_seen": 1595342848 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002609729187562688, + "loss": 2.8292, + "theoretical_loss": 3.4971281271972314, + "tokens_seen": 1595408384 + }, + { + "epoch": 0.48, + "learning_rate": 0.000260962888665998, + "loss": 2.5297, + "theoretical_loss": 3.4971156109176986, + "tokens_seen": 1595473920 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026095285857572723, + "loss": 2.6044, + "theoretical_loss": 3.4971030952962234, + "tokens_seen": 1595539456 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026094282848545636, + "loss": 2.8063, + "theoretical_loss": 3.497090580332744, + "tokens_seen": 1595604992 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002609327983951856, + "loss": 2.8419, + "theoretical_loss": 3.4970780660271994, + "tokens_seen": 1595670528 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002609227683049147, + "loss": 2.7503, + "theoretical_loss": 3.497065552379528, + "tokens_seen": 1595736064 + }, + { + "epoch": 0.48, + "objective/train/docs_used": 904004, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.718827247619629, + "objective/train/theoretical_loss": 3.4970530393896673, + "objective/train/tokens_used": 1616261600, + "theoretical_loss": 3.4970530393896673, + "tokens_seen": 1595801600 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026091273821464395, + "loss": 2.7564, + "theoretical_loss": 3.4970530393896673, + "tokens_seen": 1595801600 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026090270812437313, + "loss": 2.8144, + "theoretical_loss": 3.497040527057557, + "tokens_seen": 1595867136 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002608926780341023, + "loss": 2.5914, + "theoretical_loss": 3.497028015383134, + "tokens_seen": 1595932672 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002608826479438315, + "loss": 2.3493, + "theoretical_loss": 3.4970155043663382, + "tokens_seen": 1595998208 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002608726178535607, + "loss": 2.7136, + "theoretical_loss": 3.4970029940071075, + "tokens_seen": 1596063744 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026086258776328986, + "loss": 2.8225, + "theoretical_loss": 3.49699048430538, + "tokens_seen": 1596129280 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002608525576730191, + "loss": 2.713, + "theoretical_loss": 3.4969779752610943, + "tokens_seen": 1596194816 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002608425275827482, + "loss": 2.7492, + "theoretical_loss": 3.4969654668741894, + "tokens_seen": 1596260352 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026083249749247746, + "loss": 2.6386, + "theoretical_loss": 3.496952959144603, + "tokens_seen": 1596325888 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002608224674022066, + "loss": 2.7848, + "theoretical_loss": 3.496940452072274, + "tokens_seen": 1596391424 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002608124373119358, + "loss": 2.7021, + "theoretical_loss": 3.4969279456571405, + "tokens_seen": 1596456960 + }, + { + "epoch": 0.48, + "learning_rate": 0.000260802407221665, + "loss": 2.7307, + "theoretical_loss": 3.4969154398991416, + "tokens_seen": 1596522496 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002607923771313942, + "loss": 2.7602, + "theoretical_loss": 3.4969029347982152, + "tokens_seen": 1596588032 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026078234704112336, + "loss": 2.6061, + "theoretical_loss": 3.4968904303543002, + "tokens_seen": 1596653568 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002607723169508526, + "loss": 2.8404, + "theoretical_loss": 3.4968779265673353, + "tokens_seen": 1596719104 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002607622868605817, + "loss": 2.7151, + "theoretical_loss": 3.4968654234372583, + "tokens_seen": 1596784640 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026075225677031096, + "loss": 2.8818, + "theoretical_loss": 3.4968529209640087, + "tokens_seen": 1596850176 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002607422266800401, + "loss": 2.6638, + "theoretical_loss": 3.4968404191475235, + "tokens_seen": 1596915712 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002607321965897693, + "loss": 2.5644, + "theoretical_loss": 3.4968279179877433, + "tokens_seen": 1596981248 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002607221664994985, + "loss": 2.7131, + "theoretical_loss": 3.496815417484605, + "tokens_seen": 1597046784 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002607121364092277, + "loss": 2.6615, + "theoretical_loss": 3.496802917638048, + "tokens_seen": 1597112320 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026070210631895687, + "loss": 2.579, + "theoretical_loss": 3.4967904184480103, + "tokens_seen": 1597177856 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026069207622868605, + "loss": 2.6922, + "theoretical_loss": 3.4967779199144307, + "tokens_seen": 1597243392 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026068204613841523, + "loss": 2.4864, + "theoretical_loss": 3.4967654220372486, + "tokens_seen": 1597308928 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026067201604814446, + "loss": 2.654, + "theoretical_loss": 3.496752924816401, + "tokens_seen": 1597374464 + }, + { + "epoch": 0.48, + "objective/train/docs_used": 904558, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.35654354095459, + "objective/train/theoretical_loss": 3.496740428251828, + "objective/train/tokens_used": 1617900000, + "theoretical_loss": 3.496740428251828, + "tokens_seen": 1597440000 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002606619859578736, + "loss": 2.6972, + "theoretical_loss": 3.496740428251828, + "tokens_seen": 1597440000 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002606519558676028, + "loss": 2.558, + "theoretical_loss": 3.4967279323434677, + "tokens_seen": 1597505536 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026064192577733195, + "loss": 2.8316, + "theoretical_loss": 3.4967154370912583, + "tokens_seen": 1597571072 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002606318956870612, + "loss": 2.7249, + "theoretical_loss": 3.4967029424951392, + "tokens_seen": 1597636608 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026062186559679037, + "loss": 2.7114, + "theoretical_loss": 3.4966904485550483, + "tokens_seen": 1597702144 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026061183550651955, + "loss": 2.7889, + "theoretical_loss": 3.4966779552709246, + "tokens_seen": 1597767680 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026060180541624873, + "loss": 2.5386, + "theoretical_loss": 3.496665462642707, + "tokens_seen": 1597833216 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026059177532597797, + "loss": 2.7449, + "theoretical_loss": 3.4966529706703335, + "tokens_seen": 1597898752 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002605817452357071, + "loss": 2.6227, + "theoretical_loss": 3.4966404793537436, + "tokens_seen": 1597964288 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026057171514543633, + "loss": 2.5227, + "theoretical_loss": 3.4966279886928753, + "tokens_seen": 1598029824 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026056168505516546, + "loss": 2.8561, + "theoretical_loss": 3.4966154986876674, + "tokens_seen": 1598095360 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002605516549648947, + "loss": 2.768, + "theoretical_loss": 3.4966030093380587, + "tokens_seen": 1598160896 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026054162487462387, + "loss": 2.7105, + "theoretical_loss": 3.496590520643988, + "tokens_seen": 1598226432 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026053159478435305, + "loss": 2.9612, + "theoretical_loss": 3.496578032605394, + "tokens_seen": 1598291968 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026052156469408223, + "loss": 2.7647, + "theoretical_loss": 3.4965655452222153, + "tokens_seen": 1598357504 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002605115346038114, + "loss": 2.7009, + "theoretical_loss": 3.4965530584943907, + "tokens_seen": 1598423040 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026050150451354065, + "loss": 2.9298, + "theoretical_loss": 3.496540572421859, + "tokens_seen": 1598488576 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026049147442326983, + "loss": 2.7208, + "theoretical_loss": 3.4965280870045587, + "tokens_seen": 1598554112 + }, + { + "epoch": 0.48, + "learning_rate": 0.000260481444332999, + "loss": 2.6483, + "theoretical_loss": 3.496515602242429, + "tokens_seen": 1598619648 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002604714142427282, + "loss": 2.7861, + "theoretical_loss": 3.4965031181354083, + "tokens_seen": 1598685184 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026046138415245743, + "loss": 2.8704, + "theoretical_loss": 3.4964906346834352, + "tokens_seen": 1598750720 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026045135406218656, + "loss": 2.7265, + "theoretical_loss": 3.496478151886449, + "tokens_seen": 1598816256 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002604413239719158, + "loss": 2.759, + "theoretical_loss": 3.4964656697443885, + "tokens_seen": 1598881792 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002604312938816449, + "loss": 2.725, + "theoretical_loss": 3.496453188257192, + "tokens_seen": 1598947328 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026042126379137415, + "loss": 2.7788, + "theoretical_loss": 3.4964407074247985, + "tokens_seen": 1599012864 + }, + { + "epoch": 0.48, + "objective/train/docs_used": 905731, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.9062352180480957, + "objective/train/theoretical_loss": 3.4964282272471467, + "objective/train/tokens_used": 1619538400, + "theoretical_loss": 3.4964282272471467, + "tokens_seen": 1599078400 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026041123370110333, + "loss": 2.698, + "theoretical_loss": 3.4964282272471467, + "tokens_seen": 1599078400 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002604012036108325, + "loss": 2.7577, + "theoretical_loss": 3.4964157477241757, + "tokens_seen": 1599143936 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002603911735205617, + "loss": 2.688, + "theoretical_loss": 3.4964032688558246, + "tokens_seen": 1599209472 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002603811434302909, + "loss": 2.4558, + "theoretical_loss": 3.4963907906420313, + "tokens_seen": 1599275008 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026037111334002006, + "loss": 2.8439, + "theoretical_loss": 3.4963783130827353, + "tokens_seen": 1599340544 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002603610832497493, + "loss": 2.6674, + "theoretical_loss": 3.4963658361778753, + "tokens_seen": 1599406080 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002603510531594784, + "loss": 2.946, + "theoretical_loss": 3.49635335992739, + "tokens_seen": 1599471616 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026034102306920766, + "loss": 2.6006, + "theoretical_loss": 3.496340884331219, + "tokens_seen": 1599537152 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002603309929789368, + "loss": 2.697, + "theoretical_loss": 3.4963284093893003, + "tokens_seen": 1599602688 + }, + { + "epoch": 0.48, + "learning_rate": 0.000260320962888666, + "loss": 2.8307, + "theoretical_loss": 3.4963159351015736, + "tokens_seen": 1599668224 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002603109327983952, + "loss": 2.4871, + "theoretical_loss": 3.496303461467977, + "tokens_seen": 1599733760 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002603009027081244, + "loss": 2.6679, + "theoretical_loss": 3.4962909884884494, + "tokens_seen": 1599799296 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026029087261785356, + "loss": 2.5279, + "theoretical_loss": 3.4962785161629304, + "tokens_seen": 1599864832 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002602808425275828, + "loss": 2.7653, + "theoretical_loss": 3.4962660444913585, + "tokens_seen": 1599930368 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002602708124373119, + "loss": 2.7988, + "theoretical_loss": 3.4962535734736733, + "tokens_seen": 1599995904 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026026078234704116, + "loss": 2.8359, + "theoretical_loss": 3.496241103109812, + "tokens_seen": 1600061440 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002602507522567703, + "loss": 2.5393, + "theoretical_loss": 3.4962286333997152, + "tokens_seen": 1600126976 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002602407221664995, + "loss": 2.757, + "theoretical_loss": 3.4962161643433216, + "tokens_seen": 1600192512 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002602306920762287, + "loss": 2.6657, + "theoretical_loss": 3.4962036959405696, + "tokens_seen": 1600258048 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002602206619859579, + "loss": 2.683, + "theoretical_loss": 3.4961912281913987, + "tokens_seen": 1600323584 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026021063189568707, + "loss": 2.764, + "theoretical_loss": 3.496178761095747, + "tokens_seen": 1600389120 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026020060180541625, + "loss": 2.4939, + "theoretical_loss": 3.4961662946535546, + "tokens_seen": 1600454656 + }, + { + "epoch": 0.49, + "learning_rate": 0.00026019057171514543, + "loss": 2.6931, + "theoretical_loss": 3.49615382886476, + "tokens_seen": 1600520192 + }, + { + "epoch": 0.49, + "learning_rate": 0.00026018054162487466, + "loss": 2.6397, + "theoretical_loss": 3.496141363729302, + "tokens_seen": 1600585728 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002601705115346038, + "loss": 2.831, + "theoretical_loss": 3.49612889924712, + "tokens_seen": 1600651264 + }, + { + "epoch": 0.49, + "objective/train/docs_used": 906398, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.4906368255615234, + "objective/train/theoretical_loss": 3.496116435418153, + "objective/train/tokens_used": 1621176800, + "theoretical_loss": 3.496116435418153, + "tokens_seen": 1600716800 + }, + { + "epoch": 0.49, + "learning_rate": 0.000260160481444333, + "loss": 2.4815, + "theoretical_loss": 3.496116435418153, + "tokens_seen": 1600716800 + }, + { + "epoch": 0.49, + "learning_rate": 0.00026015045135406215, + "loss": 2.6906, + "theoretical_loss": 3.4961039722423397, + "tokens_seen": 1600782336 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002601404212637914, + "loss": 2.7122, + "theoretical_loss": 3.4960915097196192, + "tokens_seen": 1600847872 + }, + { + "epoch": 0.49, + "learning_rate": 0.00026013039117352057, + "loss": 2.8328, + "theoretical_loss": 3.49607904784993, + "tokens_seen": 1600913408 + }, + { + "epoch": 0.49, + "learning_rate": 0.00026012036108324975, + "loss": 2.7021, + "theoretical_loss": 3.496066586633213, + "tokens_seen": 1600978944 + }, + { + "epoch": 0.49, + "learning_rate": 0.00026011033099297893, + "loss": 2.5841, + "theoretical_loss": 3.4960541260694056, + "tokens_seen": 1601044480 + }, + { + "epoch": 0.49, + "learning_rate": 0.00026010030090270817, + "loss": 2.9424, + "theoretical_loss": 3.496041666158447, + "tokens_seen": 1601110016 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002600902708124373, + "loss": 2.8346, + "theoretical_loss": 3.4960292069002765, + "tokens_seen": 1601175552 + }, + { + "epoch": 0.49, + "learning_rate": 0.00026008024072216653, + "loss": 2.7453, + "theoretical_loss": 3.496016748294834, + "tokens_seen": 1601241088 + }, + { + "epoch": 0.49, + "learning_rate": 0.00026007021063189566, + "loss": 2.8551, + "theoretical_loss": 3.496004290342057, + "tokens_seen": 1601306624 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002600601805416249, + "loss": 2.8126, + "theoretical_loss": 3.495991833041886, + "tokens_seen": 1601372160 + }, + { + "epoch": 0.49, + "learning_rate": 0.00026005015045135407, + "loss": 2.5068, + "theoretical_loss": 3.49597937639426, + "tokens_seen": 1601437696 + }, + { + "epoch": 0.49, + "learning_rate": 0.00026004012036108325, + "loss": 2.7105, + "theoretical_loss": 3.495966920399117, + "tokens_seen": 1601503232 + }, + { + "epoch": 0.49, + "learning_rate": 0.00026003009027081243, + "loss": 2.6331, + "theoretical_loss": 3.495954465056397, + "tokens_seen": 1601568768 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002600200601805416, + "loss": 2.7384, + "theoretical_loss": 3.495942010366039, + "tokens_seen": 1601634304 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002600100300902708, + "loss": 2.7402, + "theoretical_loss": 3.4959295563279817, + "tokens_seen": 1601699840 + }, + { + "epoch": 0.49, + "learning_rate": 0.00026000000000000003, + "loss": 2.8806, + "theoretical_loss": 3.495917102942165, + "tokens_seen": 1601765376 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025998996990972916, + "loss": 2.7246, + "theoretical_loss": 3.4959046502085274, + "tokens_seen": 1601830912 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002599799398194584, + "loss": 2.727, + "theoretical_loss": 3.4958921981270086, + "tokens_seen": 1601896448 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002599699097291876, + "loss": 2.8253, + "theoretical_loss": 3.4958797466975478, + "tokens_seen": 1601961984 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025995987963891676, + "loss": 2.8698, + "theoretical_loss": 3.4958672959200836, + "tokens_seen": 1602027520 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025994984954864594, + "loss": 2.694, + "theoretical_loss": 3.4958548457945553, + "tokens_seen": 1602093056 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002599398194583751, + "loss": 2.6891, + "theoretical_loss": 3.495842396320903, + "tokens_seen": 1602158592 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002599297893681043, + "loss": 2.7865, + "theoretical_loss": 3.4958299474990646, + "tokens_seen": 1602224128 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025991975927783353, + "loss": 2.7245, + "theoretical_loss": 3.4958174993289797, + "tokens_seen": 1602289664 + }, + { + "epoch": 0.49, + "objective/train/docs_used": 907690, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.3768420219421387, + "objective/train/theoretical_loss": 3.495805051810588, + "objective/train/tokens_used": 1622815200, + "theoretical_loss": 3.495805051810588, + "tokens_seen": 1602355200 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025990972918756266, + "loss": 2.7582, + "theoretical_loss": 3.495805051810588, + "tokens_seen": 1602355200 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002598996990972919, + "loss": 2.6375, + "theoretical_loss": 3.4957926049438286, + "tokens_seen": 1602420736 + }, + { + "epoch": 0.49, + "learning_rate": 0.000259889669007021, + "loss": 2.7981, + "theoretical_loss": 3.495780158728641, + "tokens_seen": 1602486272 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025987963891675026, + "loss": 2.6378, + "theoretical_loss": 3.4957677131649634, + "tokens_seen": 1602551808 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025986960882647944, + "loss": 2.7292, + "theoretical_loss": 3.4957552682527355, + "tokens_seen": 1602617344 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002598595787362086, + "loss": 2.6314, + "theoretical_loss": 3.4957428239918973, + "tokens_seen": 1602682880 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002598495486459378, + "loss": 2.5375, + "theoretical_loss": 3.4957303803823874, + "tokens_seen": 1602748416 + }, + { + "epoch": 0.49, + "learning_rate": 0.000259839518555667, + "loss": 2.684, + "theoretical_loss": 3.4957179374241454, + "tokens_seen": 1602813952 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025982948846539616, + "loss": 2.694, + "theoretical_loss": 3.49570549511711, + "tokens_seen": 1602879488 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002598194583751254, + "loss": 2.6162, + "theoretical_loss": 3.4956930534612214, + "tokens_seen": 1602945024 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002598094282848545, + "loss": 2.6484, + "theoretical_loss": 3.495680612456418, + "tokens_seen": 1603010560 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025979939819458376, + "loss": 2.7721, + "theoretical_loss": 3.49566817210264, + "tokens_seen": 1603076096 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025978936810431294, + "loss": 2.5568, + "theoretical_loss": 3.4956557323998254, + "tokens_seen": 1603141632 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002597793380140421, + "loss": 2.6392, + "theoretical_loss": 3.495643293347915, + "tokens_seen": 1603207168 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002597693079237713, + "loss": 2.908, + "theoretical_loss": 3.4956308549468478, + "tokens_seen": 1603272704 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002597592778335005, + "loss": 2.5908, + "theoretical_loss": 3.495618417196562, + "tokens_seen": 1603338240 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002597492477432297, + "loss": 2.7014, + "theoretical_loss": 3.4956059800969985, + "tokens_seen": 1603403776 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002597392176529589, + "loss": 2.5717, + "theoretical_loss": 3.4955935436480954, + "tokens_seen": 1603469312 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002597291875626881, + "loss": 2.6985, + "theoretical_loss": 3.4955811078497927, + "tokens_seen": 1603534848 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025971915747241727, + "loss": 2.722, + "theoretical_loss": 3.49556867270203, + "tokens_seen": 1603600384 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025970912738214645, + "loss": 2.7127, + "theoretical_loss": 3.4955562382047463, + "tokens_seen": 1603665920 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025969909729187563, + "loss": 2.6843, + "theoretical_loss": 3.4955438043578813, + "tokens_seen": 1603731456 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025968906720160486, + "loss": 2.7583, + "theoretical_loss": 3.495531371161374, + "tokens_seen": 1603796992 + }, + { + "epoch": 0.49, + "learning_rate": 0.000259679037111334, + "loss": 2.7646, + "theoretical_loss": 3.495518938615164, + "tokens_seen": 1603862528 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002596690070210632, + "loss": 2.6325, + "theoretical_loss": 3.4955065067191904, + "tokens_seen": 1603928064 + }, + { + "epoch": 0.49, + "objective/train/docs_used": 908320, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.8385825157165527, + "objective/train/theoretical_loss": 3.4954940754733936, + "objective/train/tokens_used": 1624453600, + "theoretical_loss": 3.4954940754733936, + "tokens_seen": 1603993600 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025965897693079235, + "loss": 2.6791, + "theoretical_loss": 3.4954940754733936, + "tokens_seen": 1603993600 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002596489468405216, + "loss": 2.8119, + "theoretical_loss": 3.4954816448777115, + "tokens_seen": 1604059136 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025963891675025077, + "loss": 2.7004, + "theoretical_loss": 3.495469214932085, + "tokens_seen": 1604124672 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025962888665997995, + "loss": 2.6099, + "theoretical_loss": 3.4954567856364527, + "tokens_seen": 1604190208 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025961885656970913, + "loss": 2.66, + "theoretical_loss": 3.4954443569907543, + "tokens_seen": 1604255744 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025960882647943837, + "loss": 2.6533, + "theoretical_loss": 3.49543192899493, + "tokens_seen": 1604321280 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002595987963891675, + "loss": 2.7317, + "theoretical_loss": 3.495419501648918, + "tokens_seen": 1604386816 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025958876629889673, + "loss": 2.56, + "theoretical_loss": 3.495407074952658, + "tokens_seen": 1604452352 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025957873620862586, + "loss": 2.8694, + "theoretical_loss": 3.4953946489060903, + "tokens_seen": 1604517888 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002595687061183551, + "loss": 2.715, + "theoretical_loss": 3.4953822235091536, + "tokens_seen": 1604583424 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025955867602808427, + "loss": 2.5571, + "theoretical_loss": 3.495369798761788, + "tokens_seen": 1604648960 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025954864593781345, + "loss": 2.5318, + "theoretical_loss": 3.4953573746639326, + "tokens_seen": 1604714496 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025953861584754263, + "loss": 2.7921, + "theoretical_loss": 3.4953449512155275, + "tokens_seen": 1604780032 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002595285857572718, + "loss": 2.6441, + "theoretical_loss": 3.4953325284165113, + "tokens_seen": 1604845568 + }, + { + "epoch": 0.49, + "learning_rate": 0.000259518555667001, + "loss": 2.7661, + "theoretical_loss": 3.495320106266824, + "tokens_seen": 1604911104 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025950852557673023, + "loss": 2.4536, + "theoretical_loss": 3.4953076847664057, + "tokens_seen": 1604976640 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025949849548645936, + "loss": 2.6355, + "theoretical_loss": 3.495295263915195, + "tokens_seen": 1605042176 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002594884653961886, + "loss": 2.7312, + "theoretical_loss": 3.4952828437131322, + "tokens_seen": 1605107712 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002594784353059178, + "loss": 2.6637, + "theoretical_loss": 3.4952704241601564, + "tokens_seen": 1605173248 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025946840521564696, + "loss": 2.8363, + "theoretical_loss": 3.4952580052562077, + "tokens_seen": 1605238784 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025945837512537614, + "loss": 2.7213, + "theoretical_loss": 3.4952455870012247, + "tokens_seen": 1605304320 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002594483450351053, + "loss": 2.5354, + "theoretical_loss": 3.495233169395148, + "tokens_seen": 1605369856 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002594383149448345, + "loss": 2.7725, + "theoretical_loss": 3.4952207524379166, + "tokens_seen": 1605435392 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025942828485456373, + "loss": 2.5619, + "theoretical_loss": 3.495208336129471, + "tokens_seen": 1605500928 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025941825476429286, + "loss": 2.723, + "theoretical_loss": 3.4951959204697496, + "tokens_seen": 1605566464 + }, + { + "epoch": 0.49, + "objective/train/docs_used": 909776, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.455014705657959, + "objective/train/theoretical_loss": 3.4951835054586926, + "objective/train/tokens_used": 1626092000, + "theoretical_loss": 3.4951835054586926, + "tokens_seen": 1605632000 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002594082246740221, + "loss": 2.6595, + "theoretical_loss": 3.4951835054586926, + "tokens_seen": 1605632000 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002593981945837512, + "loss": 2.4715, + "theoretical_loss": 3.49517109109624, + "tokens_seen": 1605697536 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025938816449348046, + "loss": 2.8225, + "theoretical_loss": 3.4951586773823307, + "tokens_seen": 1605763072 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025937813440320964, + "loss": 2.4891, + "theoretical_loss": 3.4951462643169053, + "tokens_seen": 1605828608 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002593681043129388, + "loss": 2.3947, + "theoretical_loss": 3.4951338518999027, + "tokens_seen": 1605894144 + }, + { + "epoch": 0.49, + "learning_rate": 0.000259358074222668, + "loss": 2.6206, + "theoretical_loss": 3.4951214401312622, + "tokens_seen": 1605959680 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002593480441323972, + "loss": 2.505, + "theoretical_loss": 3.4951090290109246, + "tokens_seen": 1606025216 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025933801404212636, + "loss": 2.473, + "theoretical_loss": 3.495096618538829, + "tokens_seen": 1606090752 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002593279839518556, + "loss": 2.4977, + "theoretical_loss": 3.495084208714915, + "tokens_seen": 1606156288 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002593179538615847, + "loss": 2.5361, + "theoretical_loss": 3.4950717995391223, + "tokens_seen": 1606221824 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025930792377131396, + "loss": 2.6969, + "theoretical_loss": 3.4950593910113907, + "tokens_seen": 1606287360 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025929789368104314, + "loss": 2.4686, + "theoretical_loss": 3.49504698313166, + "tokens_seen": 1606352896 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002592878635907723, + "loss": 2.7281, + "theoretical_loss": 3.4950345758998704, + "tokens_seen": 1606418432 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002592778335005015, + "loss": 2.7121, + "theoretical_loss": 3.4950221693159604, + "tokens_seen": 1606483968 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002592678034102307, + "loss": 2.648, + "theoretical_loss": 3.495009763379871, + "tokens_seen": 1606549504 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025925777331995987, + "loss": 2.5698, + "theoretical_loss": 3.494997358091541, + "tokens_seen": 1606615040 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002592477432296891, + "loss": 2.517, + "theoretical_loss": 3.4949849534509108, + "tokens_seen": 1606680576 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025923771313941823, + "loss": 2.5637, + "theoretical_loss": 3.49497254945792, + "tokens_seen": 1606746112 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025922768304914747, + "loss": 2.6876, + "theoretical_loss": 3.4949601461125077, + "tokens_seen": 1606811648 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002592176529588766, + "loss": 2.9125, + "theoretical_loss": 3.494947743414615, + "tokens_seen": 1606877184 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025920762286860583, + "loss": 2.7833, + "theoretical_loss": 3.4949353413641804, + "tokens_seen": 1606942720 + }, + { + "epoch": 0.49, + "learning_rate": 0.000259197592778335, + "loss": 2.7121, + "theoretical_loss": 3.4949229399611443, + "tokens_seen": 1607008256 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002591875626880642, + "loss": 2.5735, + "theoretical_loss": 3.494910539205447, + "tokens_seen": 1607073792 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025917753259779337, + "loss": 2.5359, + "theoretical_loss": 3.4948981390970273, + "tokens_seen": 1607139328 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025916750250752255, + "loss": 2.7378, + "theoretical_loss": 3.4948857396358255, + "tokens_seen": 1607204864 + }, + { + "epoch": 0.49, + "objective/train/docs_used": 910493, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.0588538646698, + "objective/train/theoretical_loss": 3.4948733408217816, + "objective/train/tokens_used": 1627730400, + "theoretical_loss": 3.4948733408217816, + "tokens_seen": 1607270400 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025915747241725173, + "loss": 2.8445, + "theoretical_loss": 3.4948733408217816, + "tokens_seen": 1607270400 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025914744232698097, + "loss": 2.7285, + "theoretical_loss": 3.494860942654835, + "tokens_seen": 1607335936 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002591374122367101, + "loss": 2.6857, + "theoretical_loss": 3.4948485451349263, + "tokens_seen": 1607401472 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025912738214643933, + "loss": 2.9187, + "theoretical_loss": 3.494836148261995, + "tokens_seen": 1607467008 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002591173520561685, + "loss": 2.8852, + "theoretical_loss": 3.4948237520359804, + "tokens_seen": 1607532544 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002591073219658977, + "loss": 2.5443, + "theoretical_loss": 3.4948113564568226, + "tokens_seen": 1607598080 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002590972918756269, + "loss": 2.5121, + "theoretical_loss": 3.494798961524462, + "tokens_seen": 1607663616 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025908726178535606, + "loss": 2.469, + "theoretical_loss": 3.4947865672388376, + "tokens_seen": 1607729152 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025907723169508524, + "loss": 2.8426, + "theoretical_loss": 3.4947741735998905, + "tokens_seen": 1607794688 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025906720160481447, + "loss": 2.9496, + "theoretical_loss": 3.49476178060756, + "tokens_seen": 1607860224 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002590571715145436, + "loss": 2.7124, + "theoretical_loss": 3.494749388261786, + "tokens_seen": 1607925760 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025904714142427283, + "loss": 2.2566, + "theoretical_loss": 3.4947369965625077, + "tokens_seen": 1607991296 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025903711133400196, + "loss": 2.4619, + "theoretical_loss": 3.494724605509666, + "tokens_seen": 1608056832 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002590270812437312, + "loss": 2.669, + "theoretical_loss": 3.494712215103201, + "tokens_seen": 1608122368 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002590170511534604, + "loss": 2.6423, + "theoretical_loss": 3.494699825343052, + "tokens_seen": 1608187904 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025900702106318956, + "loss": 2.6727, + "theoretical_loss": 3.494687436229159, + "tokens_seen": 1608253440 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002589969909729188, + "loss": 2.7075, + "theoretical_loss": 3.4946750477614623, + "tokens_seen": 1608318976 + }, + { + "epoch": 0.49, + "learning_rate": 0.000258986960882648, + "loss": 2.6667, + "theoretical_loss": 3.494662659939901, + "tokens_seen": 1608384512 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025897693079237716, + "loss": 2.6771, + "theoretical_loss": 3.4946502727644164, + "tokens_seen": 1608450048 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025896690070210634, + "loss": 2.7289, + "theoretical_loss": 3.4946378862349476, + "tokens_seen": 1608515584 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002589568706118355, + "loss": 2.6809, + "theoretical_loss": 3.494625500351435, + "tokens_seen": 1608581120 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002589468405215647, + "loss": 2.6902, + "theoretical_loss": 3.4946131151138182, + "tokens_seen": 1608646656 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025893681043129393, + "loss": 2.6293, + "theoretical_loss": 3.4946007305220377, + "tokens_seen": 1608712192 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025892678034102306, + "loss": 2.5661, + "theoretical_loss": 3.494588346576033, + "tokens_seen": 1608777728 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002589167502507523, + "loss": 2.8871, + "theoretical_loss": 3.4945759632757443, + "tokens_seen": 1608843264 + }, + { + "epoch": 0.49, + "objective/train/docs_used": 911872, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.2851009368896484, + "objective/train/theoretical_loss": 3.4945635806211115, + "objective/train/tokens_used": 1629368800, + "theoretical_loss": 3.4945635806211115, + "tokens_seen": 1608908800 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002589067201604814, + "loss": 2.6762, + "theoretical_loss": 3.4945635806211115, + "tokens_seen": 1608908800 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025889669007021066, + "loss": 2.6382, + "theoretical_loss": 3.494551198612075, + "tokens_seen": 1608974336 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025888665997993984, + "loss": 2.4714, + "theoretical_loss": 3.494538817248575, + "tokens_seen": 1609039872 + }, + { + "epoch": 0.49, + "learning_rate": 0.000258876629889669, + "loss": 2.6774, + "theoretical_loss": 3.494526436530551, + "tokens_seen": 1609105408 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002588665997993982, + "loss": 2.7979, + "theoretical_loss": 3.494514056457943, + "tokens_seen": 1609170944 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002588565697091274, + "loss": 2.7973, + "theoretical_loss": 3.4945016770306916, + "tokens_seen": 1609236480 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025884653961885657, + "loss": 2.7139, + "theoretical_loss": 3.4944892982487366, + "tokens_seen": 1609302016 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002588365095285858, + "loss": 2.3827, + "theoretical_loss": 3.4944769201120183, + "tokens_seen": 1609367552 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025882647943831493, + "loss": 2.5488, + "theoretical_loss": 3.4944645426204763, + "tokens_seen": 1609433088 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025881644934804416, + "loss": 2.5725, + "theoretical_loss": 3.4944521657740513, + "tokens_seen": 1609498624 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025880641925777334, + "loss": 2.8059, + "theoretical_loss": 3.494439789572683, + "tokens_seen": 1609564160 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002587963891675025, + "loss": 2.7357, + "theoretical_loss": 3.494427414016312, + "tokens_seen": 1609629696 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002587863590772317, + "loss": 2.5872, + "theoretical_loss": 3.494415039104877, + "tokens_seen": 1609695232 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002587763289869609, + "loss": 2.7997, + "theoretical_loss": 3.4944026648383204, + "tokens_seen": 1609760768 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025876629889669007, + "loss": 2.7805, + "theoretical_loss": 3.4943902912165807, + "tokens_seen": 1609826304 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002587562688064193, + "loss": 2.7639, + "theoretical_loss": 3.494377918239598, + "tokens_seen": 1609891840 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025874623871614843, + "loss": 2.6294, + "theoretical_loss": 3.4943655459073133, + "tokens_seen": 1609957376 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025873620862587767, + "loss": 2.7067, + "theoretical_loss": 3.4943531742196665, + "tokens_seen": 1610022912 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002587261785356068, + "loss": 2.5636, + "theoretical_loss": 3.4943408031765983, + "tokens_seen": 1610088448 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025871614844533603, + "loss": 2.7722, + "theoretical_loss": 3.4943284327780475, + "tokens_seen": 1610153984 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002587061183550652, + "loss": 2.6834, + "theoretical_loss": 3.4943160630239554, + "tokens_seen": 1610219520 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002586960882647944, + "loss": 2.6903, + "theoretical_loss": 3.4943036939142615, + "tokens_seen": 1610285056 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025868605817452357, + "loss": 2.7809, + "theoretical_loss": 3.4942913254489065, + "tokens_seen": 1610350592 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025867602808425275, + "loss": 2.5931, + "theoretical_loss": 3.494278957627831, + "tokens_seen": 1610416128 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025866599799398193, + "loss": 2.5278, + "theoretical_loss": 3.494266590450974, + "tokens_seen": 1610481664 + }, + { + "epoch": 0.49, + "objective/train/docs_used": 912654, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.9397103786468506, + "objective/train/theoretical_loss": 3.494254223918277, + "objective/train/tokens_used": 1631007200, + "theoretical_loss": 3.494254223918277, + "tokens_seen": 1610547200 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025865596790371117, + "loss": 2.7136, + "theoretical_loss": 3.494254223918277, + "tokens_seen": 1610547200 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002586459378134403, + "loss": 2.5268, + "theoretical_loss": 3.4942418580296795, + "tokens_seen": 1610612736 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025863590772316953, + "loss": 2.5062, + "theoretical_loss": 3.494229492785122, + "tokens_seen": 1610678272 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002586258776328987, + "loss": 2.527, + "theoretical_loss": 3.4942171281845447, + "tokens_seen": 1610743808 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002586158475426279, + "loss": 2.7881, + "theoretical_loss": 3.494204764227887, + "tokens_seen": 1610809344 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002586058174523571, + "loss": 2.5718, + "theoretical_loss": 3.4941924009150913, + "tokens_seen": 1610874880 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025859578736208626, + "loss": 2.5051, + "theoretical_loss": 3.494180038246096, + "tokens_seen": 1610940416 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025858575727181544, + "loss": 2.6178, + "theoretical_loss": 3.494167676220842, + "tokens_seen": 1611005952 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025857572718154467, + "loss": 2.6661, + "theoretical_loss": 3.4941553148392694, + "tokens_seen": 1611071488 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002585656970912738, + "loss": 2.5595, + "theoretical_loss": 3.494142954101319, + "tokens_seen": 1611137024 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025855566700100303, + "loss": 2.6576, + "theoretical_loss": 3.4941305940069305, + "tokens_seen": 1611202560 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025854563691073216, + "loss": 2.7774, + "theoretical_loss": 3.494118234556044, + "tokens_seen": 1611268096 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002585356068204614, + "loss": 2.6667, + "theoretical_loss": 3.4941058757486014, + "tokens_seen": 1611333632 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002585255767301906, + "loss": 2.524, + "theoretical_loss": 3.494093517584542, + "tokens_seen": 1611399168 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025851554663991976, + "loss": 2.8198, + "theoretical_loss": 3.4940811600638053, + "tokens_seen": 1611464704 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025850551654964894, + "loss": 2.7214, + "theoretical_loss": 3.4940688031863325, + "tokens_seen": 1611530240 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002584954864593782, + "loss": 2.7012, + "theoretical_loss": 3.4940564469520643, + "tokens_seen": 1611595776 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002584854563691073, + "loss": 2.7192, + "theoretical_loss": 3.4940440913609407, + "tokens_seen": 1611661312 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025847542627883654, + "loss": 2.6198, + "theoretical_loss": 3.494031736412902, + "tokens_seen": 1611726848 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025846539618856566, + "loss": 2.7778, + "theoretical_loss": 3.494019382107888, + "tokens_seen": 1611792384 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002584553660982949, + "loss": 2.6047, + "theoretical_loss": 3.49400702844584, + "tokens_seen": 1611857920 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002584453360080241, + "loss": 2.9519, + "theoretical_loss": 3.4939946754266984, + "tokens_seen": 1611923456 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025843530591775326, + "loss": 2.6439, + "theoretical_loss": 3.4939823230504037, + "tokens_seen": 1611988992 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025842527582748244, + "loss": 2.7908, + "theoretical_loss": 3.4939699713168952, + "tokens_seen": 1612054528 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002584152457372116, + "loss": 2.8517, + "theoretical_loss": 3.493957620226115, + "tokens_seen": 1612120064 + }, + { + "epoch": 0.49, + "objective/train/docs_used": 913987, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.94030499458313, + "objective/train/theoretical_loss": 3.493945269778002, + "objective/train/tokens_used": 1632645600, + "theoretical_loss": 3.493945269778002, + "tokens_seen": 1612185600 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002584052156469408, + "loss": 2.6486, + "theoretical_loss": 3.493945269778002, + "tokens_seen": 1612185600 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025839518555667004, + "loss": 2.6309, + "theoretical_loss": 3.4939329199724964, + "tokens_seen": 1612251136 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025838515546639917, + "loss": 2.5463, + "theoretical_loss": 3.4939205708095407, + "tokens_seen": 1612316672 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002583751253761284, + "loss": 2.51, + "theoretical_loss": 3.4939082222890736, + "tokens_seen": 1612382208 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025836509528585753, + "loss": 2.6653, + "theoretical_loss": 3.493895874411036, + "tokens_seen": 1612447744 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025835506519558677, + "loss": 2.7192, + "theoretical_loss": 3.4938835271753685, + "tokens_seen": 1612513280 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025834503510531595, + "loss": 2.7144, + "theoretical_loss": 3.493871180582012, + "tokens_seen": 1612578816 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025833500501504513, + "loss": 2.5606, + "theoretical_loss": 3.493858834630906, + "tokens_seen": 1612644352 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002583249749247743, + "loss": 2.6408, + "theoretical_loss": 3.4938464893219914, + "tokens_seen": 1612709888 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025831494483450354, + "loss": 2.4812, + "theoretical_loss": 3.4938341446552093, + "tokens_seen": 1612775424 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025830491474423267, + "loss": 2.8445, + "theoretical_loss": 3.4938218006304993, + "tokens_seen": 1612840960 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002582948846539619, + "loss": 2.4342, + "theoretical_loss": 3.493809457247803, + "tokens_seen": 1612906496 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025828485456369103, + "loss": 2.5199, + "theoretical_loss": 3.4937971145070597, + "tokens_seen": 1612972032 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025827482447342027, + "loss": 2.6946, + "theoretical_loss": 3.4937847724082105, + "tokens_seen": 1613037568 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025826479438314945, + "loss": 2.4772, + "theoretical_loss": 3.493772430951196, + "tokens_seen": 1613103104 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025825476429287863, + "loss": 2.5588, + "theoretical_loss": 3.4937600901359565, + "tokens_seen": 1613168640 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025824473420260787, + "loss": 2.3718, + "theoretical_loss": 3.493747749962433, + "tokens_seen": 1613234176 + }, + { + "epoch": 0.49, + "learning_rate": 0.000258234704112337, + "loss": 2.8618, + "theoretical_loss": 3.4937354104305656, + "tokens_seen": 1613299712 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025822467402206623, + "loss": 2.8316, + "theoretical_loss": 3.4937230715402956, + "tokens_seen": 1613365248 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002582146439317954, + "loss": 2.6758, + "theoretical_loss": 3.4937107332915622, + "tokens_seen": 1613430784 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002582046138415246, + "loss": 2.5044, + "theoretical_loss": 3.4936983956843073, + "tokens_seen": 1613496320 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025819458375125377, + "loss": 2.8687, + "theoretical_loss": 3.4936860587184713, + "tokens_seen": 1613561856 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025818455366098295, + "loss": 2.7244, + "theoretical_loss": 3.4936737223939938, + "tokens_seen": 1613627392 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025817452357071213, + "loss": 2.621, + "theoretical_loss": 3.4936613867108166, + "tokens_seen": 1613692928 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025816449348044137, + "loss": 2.512, + "theoretical_loss": 3.4936490516688803, + "tokens_seen": 1613758464 + }, + { + "epoch": 0.49, + "objective/train/docs_used": 914763, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.854111671447754, + "objective/train/theoretical_loss": 3.4936367172681244, + "objective/train/tokens_used": 1634284000, + "theoretical_loss": 3.4936367172681244, + "tokens_seen": 1613824000 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002581544633901705, + "loss": 2.6701, + "theoretical_loss": 3.4936367172681244, + "tokens_seen": 1613824000 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025814443329989973, + "loss": 2.7177, + "theoretical_loss": 3.4936243835084904, + "tokens_seen": 1613889536 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002581344032096289, + "loss": 2.6409, + "theoretical_loss": 3.493612050389919, + "tokens_seen": 1613955072 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002581243731193581, + "loss": 2.8581, + "theoretical_loss": 3.4935997179123506, + "tokens_seen": 1614020608 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002581143430290873, + "loss": 2.5797, + "theoretical_loss": 3.493587386075726, + "tokens_seen": 1614086144 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025810431293881646, + "loss": 2.5372, + "theoretical_loss": 3.493575054879985, + "tokens_seen": 1614151680 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025809428284854564, + "loss": 2.5583, + "theoretical_loss": 3.49356272432507, + "tokens_seen": 1614217216 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025808425275827487, + "loss": 2.6498, + "theoretical_loss": 3.49355039441092, + "tokens_seen": 1614282752 + }, + { + "epoch": 0.49, + "learning_rate": 0.000258074222668004, + "loss": 2.7567, + "theoretical_loss": 3.493538065137477, + "tokens_seen": 1614348288 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025806419257773323, + "loss": 2.8447, + "theoretical_loss": 3.4935257365046803, + "tokens_seen": 1614413824 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025805416248746236, + "loss": 2.6613, + "theoretical_loss": 3.493513408512472, + "tokens_seen": 1614479360 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002580441323971916, + "loss": 2.7107, + "theoretical_loss": 3.493501081160792, + "tokens_seen": 1614544896 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002580341023069208, + "loss": 2.7432, + "theoretical_loss": 3.493488754449581, + "tokens_seen": 1614610432 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025802407221664996, + "loss": 2.6289, + "theoretical_loss": 3.4934764283787807, + "tokens_seen": 1614675968 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025801404212637914, + "loss": 2.76, + "theoretical_loss": 3.493464102948331, + "tokens_seen": 1614741504 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002580040120361084, + "loss": 2.7724, + "theoretical_loss": 3.493451778158172, + "tokens_seen": 1614807040 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002579939819458375, + "loss": 2.5807, + "theoretical_loss": 3.493439454008246, + "tokens_seen": 1614872576 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025798395185556674, + "loss": 2.6596, + "theoretical_loss": 3.4934271304984925, + "tokens_seen": 1614938112 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025797392176529586, + "loss": 2.6594, + "theoretical_loss": 3.493414807628853, + "tokens_seen": 1615003648 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002579638916750251, + "loss": 2.4078, + "theoretical_loss": 3.493402485399268, + "tokens_seen": 1615069184 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002579538615847543, + "loss": 2.3958, + "theoretical_loss": 3.493390163809678, + "tokens_seen": 1615134720 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025794383149448346, + "loss": 2.637, + "theoretical_loss": 3.493377842860024, + "tokens_seen": 1615200256 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025793380140421264, + "loss": 2.7606, + "theoretical_loss": 3.4933655225502473, + "tokens_seen": 1615265792 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002579237713139418, + "loss": 2.663, + "theoretical_loss": 3.4933532028802885, + "tokens_seen": 1615331328 + }, + { + "epoch": 0.49, + "learning_rate": 0.000257913741223671, + "loss": 2.6237, + "theoretical_loss": 3.493340883850088, + "tokens_seen": 1615396864 + }, + { + "epoch": 0.49, + "objective/train/docs_used": 916021, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.5929806232452393, + "objective/train/theoretical_loss": 3.4933285654595867, + "objective/train/tokens_used": 1635922400, + "theoretical_loss": 3.4933285654595867, + "tokens_seen": 1615462400 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025790371113340024, + "loss": 2.714, + "theoretical_loss": 3.4933285654595867, + "tokens_seen": 1615462400 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025789368104312937, + "loss": 2.5591, + "theoretical_loss": 3.4933162477087256, + "tokens_seen": 1615527936 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002578836509528586, + "loss": 2.6897, + "theoretical_loss": 3.493303930597446, + "tokens_seen": 1615593472 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025787362086258773, + "loss": 2.8127, + "theoretical_loss": 3.493291614125688, + "tokens_seen": 1615659008 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025786359077231697, + "loss": 2.5633, + "theoretical_loss": 3.493279298293392, + "tokens_seen": 1615724544 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025785356068204615, + "loss": 2.4377, + "theoretical_loss": 3.4932669831005008, + "tokens_seen": 1615790080 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025784353059177533, + "loss": 2.5933, + "theoretical_loss": 3.493254668546953, + "tokens_seen": 1615855616 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002578335005015045, + "loss": 2.6613, + "theoretical_loss": 3.4932423546326907, + "tokens_seen": 1615921152 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025782347041123374, + "loss": 2.4152, + "theoretical_loss": 3.4932300413576556, + "tokens_seen": 1615986688 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025781344032096287, + "loss": 2.8931, + "theoretical_loss": 3.4932177287217865, + "tokens_seen": 1616052224 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002578034102306921, + "loss": 2.7547, + "theoretical_loss": 3.4932054167250257, + "tokens_seen": 1616117760 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025779338014042123, + "loss": 2.8457, + "theoretical_loss": 3.4931931053673146, + "tokens_seen": 1616183296 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025778335005015047, + "loss": 2.6618, + "theoretical_loss": 3.4931807946485924, + "tokens_seen": 1616248832 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025777331995987965, + "loss": 2.6792, + "theoretical_loss": 3.4931684845688014, + "tokens_seen": 1616314368 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025776328986960883, + "loss": 2.6712, + "theoretical_loss": 3.4931561751278823, + "tokens_seen": 1616379904 + }, + { + "epoch": 0.49, + "learning_rate": 0.000257753259779338, + "loss": 2.6282, + "theoretical_loss": 3.4931438663257754, + "tokens_seen": 1616445440 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002577432296890672, + "loss": 2.8687, + "theoretical_loss": 3.4931315581624225, + "tokens_seen": 1616510976 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002577331995987964, + "loss": 2.5065, + "theoretical_loss": 3.493119250637764, + "tokens_seen": 1616576512 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002577231695085256, + "loss": 2.6321, + "theoretical_loss": 3.493106943751741, + "tokens_seen": 1616642048 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025771313941825474, + "loss": 2.5872, + "theoretical_loss": 3.493094637504295, + "tokens_seen": 1616707584 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025770310932798397, + "loss": 2.5382, + "theoretical_loss": 3.4930823318953657, + "tokens_seen": 1616773120 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002576930792377131, + "loss": 2.7278, + "theoretical_loss": 3.4930700269248955, + "tokens_seen": 1616838656 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025768304914744233, + "loss": 2.6329, + "theoretical_loss": 3.493057722592824, + "tokens_seen": 1616904192 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002576730190571715, + "loss": 2.754, + "theoretical_loss": 3.493045418899094, + "tokens_seen": 1616969728 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002576629889669007, + "loss": 2.555, + "theoretical_loss": 3.493033115843645, + "tokens_seen": 1617035264 + }, + { + "epoch": 0.49, + "objective/train/docs_used": 916782, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.5034728050231934, + "objective/train/theoretical_loss": 3.4930208134264182, + "objective/train/tokens_used": 1637560800, + "theoretical_loss": 3.4930208134264182, + "tokens_seen": 1617100800 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002576529588766299, + "loss": 2.6427, + "theoretical_loss": 3.4930208134264182, + "tokens_seen": 1617100800 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002576429287863591, + "loss": 2.6308, + "theoretical_loss": 3.4930085116473553, + "tokens_seen": 1617166336 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025763289869608824, + "loss": 2.9618, + "theoretical_loss": 3.492996210506397, + "tokens_seen": 1617231872 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002576228686058175, + "loss": 2.4546, + "theoretical_loss": 3.4929839100034847, + "tokens_seen": 1617297408 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002576128385155466, + "loss": 2.6034, + "theoretical_loss": 3.4929716101385586, + "tokens_seen": 1617362944 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025760280842527584, + "loss": 2.6479, + "theoretical_loss": 3.4929593109115604, + "tokens_seen": 1617428480 + }, + { + "epoch": 0.49, + "learning_rate": 0.000257592778335005, + "loss": 2.6991, + "theoretical_loss": 3.492947012322431, + "tokens_seen": 1617494016 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002575827482447342, + "loss": 2.607, + "theoretical_loss": 3.4929347143711116, + "tokens_seen": 1617559552 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002575727181544634, + "loss": 2.4884, + "theoretical_loss": 3.492922417057543, + "tokens_seen": 1617625088 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025756268806419256, + "loss": 2.7523, + "theoretical_loss": 3.4929101203816666, + "tokens_seen": 1617690624 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025755265797392174, + "loss": 2.4583, + "theoretical_loss": 3.492897824343423, + "tokens_seen": 1617756160 + }, + { + "epoch": 0.49, + "learning_rate": 0.000257542627883651, + "loss": 2.6694, + "theoretical_loss": 3.492885528942754, + "tokens_seen": 1617821696 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002575325977933801, + "loss": 2.6321, + "theoretical_loss": 3.4928732341796005, + "tokens_seen": 1617887232 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025752256770310934, + "loss": 2.6658, + "theoretical_loss": 3.4928609400539035, + "tokens_seen": 1617952768 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025751253761283847, + "loss": 2.7629, + "theoretical_loss": 3.4928486465656037, + "tokens_seen": 1618018304 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002575025075225677, + "loss": 2.6484, + "theoretical_loss": 3.492836353714643, + "tokens_seen": 1618083840 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025749247743229694, + "loss": 2.7616, + "theoretical_loss": 3.4928240615009623, + "tokens_seen": 1618149376 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025748244734202606, + "loss": 2.9032, + "theoretical_loss": 3.4928117699245025, + "tokens_seen": 1618214912 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002574724172517553, + "loss": 2.682, + "theoretical_loss": 3.492799478985205, + "tokens_seen": 1618280448 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002574623871614845, + "loss": 2.5735, + "theoretical_loss": 3.492787188683011, + "tokens_seen": 1618345984 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025745235707121366, + "loss": 2.6248, + "theoretical_loss": 3.4927748990178613, + "tokens_seen": 1618411520 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025744232698094284, + "loss": 2.4098, + "theoretical_loss": 3.492762609989698, + "tokens_seen": 1618477056 + }, + { + "epoch": 0.49, + "learning_rate": 0.000257432296890672, + "loss": 2.543, + "theoretical_loss": 3.4927503215984608, + "tokens_seen": 1618542592 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002574222668004012, + "loss": 2.5331, + "theoretical_loss": 3.492738033844092, + "tokens_seen": 1618608128 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025741223671013044, + "loss": 2.6237, + "theoretical_loss": 3.492725746726533, + "tokens_seen": 1618673664 + }, + { + "epoch": 0.49, + "objective/train/docs_used": 918149, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.4079349040985107, + "objective/train/theoretical_loss": 3.4927134602457244, + "objective/train/tokens_used": 1639199200, + "theoretical_loss": 3.4927134602457244, + "tokens_seen": 1618739200 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025740220661985957, + "loss": 2.7202, + "theoretical_loss": 3.4927134602457244, + "tokens_seen": 1618739200 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002573921765295888, + "loss": 2.5934, + "theoretical_loss": 3.492701174401608, + "tokens_seen": 1618804736 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025738214643931793, + "loss": 2.7045, + "theoretical_loss": 3.492688889194124, + "tokens_seen": 1618870272 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025737211634904717, + "loss": 2.6588, + "theoretical_loss": 3.492676604623215, + "tokens_seen": 1618935808 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025736208625877635, + "loss": 2.6202, + "theoretical_loss": 3.492664320688821, + "tokens_seen": 1619001344 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025735205616850553, + "loss": 2.7169, + "theoretical_loss": 3.492652037390884, + "tokens_seen": 1619066880 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002573420260782347, + "loss": 2.7203, + "theoretical_loss": 3.4926397547293444, + "tokens_seen": 1619132416 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025733199598796394, + "loss": 2.6263, + "theoretical_loss": 3.492627472704145, + "tokens_seen": 1619197952 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025732196589769307, + "loss": 2.5592, + "theoretical_loss": 3.4926151913152257, + "tokens_seen": 1619263488 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002573119358074223, + "loss": 2.7398, + "theoretical_loss": 3.4926029105625283, + "tokens_seen": 1619329024 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025730190571715143, + "loss": 2.6113, + "theoretical_loss": 3.492590630445995, + "tokens_seen": 1619394560 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025729187562688067, + "loss": 2.608, + "theoretical_loss": 3.492578350965565, + "tokens_seen": 1619460096 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025728184553660985, + "loss": 2.6739, + "theoretical_loss": 3.492566072121181, + "tokens_seen": 1619525632 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025727181544633903, + "loss": 2.3546, + "theoretical_loss": 3.492553793912785, + "tokens_seen": 1619591168 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002572617853560682, + "loss": 2.5358, + "theoretical_loss": 3.4925415163403164, + "tokens_seen": 1619656704 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002572517552657974, + "loss": 2.615, + "theoretical_loss": 3.4925292394037184, + "tokens_seen": 1619722240 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002572417251755266, + "loss": 2.8158, + "theoretical_loss": 3.492516963102931, + "tokens_seen": 1619787776 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002572316950852558, + "loss": 2.5732, + "theoretical_loss": 3.492504687437896, + "tokens_seen": 1619853312 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025722166499498494, + "loss": 2.5895, + "theoretical_loss": 3.4924924124085557, + "tokens_seen": 1619918848 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025721163490471417, + "loss": 2.6298, + "theoretical_loss": 3.49248013801485, + "tokens_seen": 1619984384 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002572016048144433, + "loss": 2.7212, + "theoretical_loss": 3.4924678642567204, + "tokens_seen": 1620049920 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025719157472417253, + "loss": 2.7494, + "theoretical_loss": 3.4924555911341093, + "tokens_seen": 1620115456 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002571815446339017, + "loss": 2.5823, + "theoretical_loss": 3.4924433186469575, + "tokens_seen": 1620180992 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002571715145436309, + "loss": 2.5979, + "theoretical_loss": 3.492431046795206, + "tokens_seen": 1620246528 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002571614844533601, + "loss": 2.6941, + "theoretical_loss": 3.4924187755787974, + "tokens_seen": 1620312064 + }, + { + "epoch": 0.49, + "objective/train/docs_used": 918561, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.5720138549804688, + "objective/train/theoretical_loss": 3.492406504997672, + "objective/train/tokens_used": 1640837600, + "theoretical_loss": 3.492406504997672, + "tokens_seen": 1620377600 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002571514543630893, + "loss": 2.576, + "theoretical_loss": 3.492406504997672, + "tokens_seen": 1620377600 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025714142427281844, + "loss": 2.7793, + "theoretical_loss": 3.4923942350517714, + "tokens_seen": 1620443136 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002571313941825477, + "loss": 2.4806, + "theoretical_loss": 3.4923819657410373, + "tokens_seen": 1620508672 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002571213640922768, + "loss": 2.645, + "theoretical_loss": 3.492369697065411, + "tokens_seen": 1620574208 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025711133400200604, + "loss": 2.7524, + "theoretical_loss": 3.4923574290248336, + "tokens_seen": 1620639744 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002571013039117352, + "loss": 2.6595, + "theoretical_loss": 3.492345161619248, + "tokens_seen": 1620705280 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002570912738214644, + "loss": 2.4945, + "theoretical_loss": 3.4923328948485937, + "tokens_seen": 1620770816 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002570812437311936, + "loss": 2.6996, + "theoretical_loss": 3.492320628712813, + "tokens_seen": 1620836352 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025707121364092276, + "loss": 2.7047, + "theoretical_loss": 3.4923083632118477, + "tokens_seen": 1620901888 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025706118355065194, + "loss": 2.6321, + "theoretical_loss": 3.492296098345639, + "tokens_seen": 1620967424 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002570511534603812, + "loss": 2.6665, + "theoretical_loss": 3.4922838341141285, + "tokens_seen": 1621032960 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002570411233701103, + "loss": 2.6462, + "theoretical_loss": 3.492271570517257, + "tokens_seen": 1621098496 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025703109327983954, + "loss": 2.7351, + "theoretical_loss": 3.492259307554967, + "tokens_seen": 1621164032 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025702106318956867, + "loss": 2.6509, + "theoretical_loss": 3.4922470452272, + "tokens_seen": 1621229568 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002570110330992979, + "loss": 2.5674, + "theoretical_loss": 3.4922347835338967, + "tokens_seen": 1621295104 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002570010030090271, + "loss": 2.8379, + "theoretical_loss": 3.492222522474999, + "tokens_seen": 1621360640 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025699097291875626, + "loss": 2.8726, + "theoretical_loss": 3.492210262050449, + "tokens_seen": 1621426176 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025698094282848545, + "loss": 2.5967, + "theoretical_loss": 3.492198002260187, + "tokens_seen": 1621491712 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002569709127382147, + "loss": 2.4966, + "theoretical_loss": 3.4921857431041556, + "tokens_seen": 1621557248 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002569608826479438, + "loss": 2.4404, + "theoretical_loss": 3.4921734845822963, + "tokens_seen": 1621622784 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025695085255767304, + "loss": 2.6908, + "theoretical_loss": 3.4921612266945496, + "tokens_seen": 1621688320 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025694082246740217, + "loss": 2.6526, + "theoretical_loss": 3.4921489694408585, + "tokens_seen": 1621753856 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002569307923771314, + "loss": 2.7901, + "theoretical_loss": 3.4921367128211642, + "tokens_seen": 1621819392 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002569207622868606, + "loss": 2.4908, + "theoretical_loss": 3.4921244568354077, + "tokens_seen": 1621884928 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025691073219658977, + "loss": 2.7237, + "theoretical_loss": 3.492112201483531, + "tokens_seen": 1621950464 + }, + { + "epoch": 0.49, + "objective/train/docs_used": 919775, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.619884490966797, + "objective/train/theoretical_loss": 3.4920999467654754, + "objective/train/tokens_used": 1642476000, + "theoretical_loss": 3.4920999467654754, + "tokens_seen": 1622016000 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025690070210631895, + "loss": 2.731, + "theoretical_loss": 3.4920999467654754, + "tokens_seen": 1622016000 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025689067201604813, + "loss": 2.7186, + "theoretical_loss": 3.4920876926811832, + "tokens_seen": 1622081536 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002568806419257773, + "loss": 2.4039, + "theoretical_loss": 3.4920754392305957, + "tokens_seen": 1622147072 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025687061183550655, + "loss": 2.6673, + "theoretical_loss": 3.492063186413654, + "tokens_seen": 1622212608 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002568605817452357, + "loss": 2.6949, + "theoretical_loss": 3.4920509342303, + "tokens_seen": 1622278144 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002568505516549649, + "loss": 2.5707, + "theoretical_loss": 3.492038682680476, + "tokens_seen": 1622343680 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025684052156469404, + "loss": 2.8497, + "theoretical_loss": 3.492026431764123, + "tokens_seen": 1622409216 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025683049147442327, + "loss": 2.594, + "theoretical_loss": 3.4920141814811827, + "tokens_seen": 1622474752 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025682046138415245, + "loss": 2.8176, + "theoretical_loss": 3.492001931831597, + "tokens_seen": 1622540288 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025681043129388163, + "loss": 2.7861, + "theoretical_loss": 3.4919896828153076, + "tokens_seen": 1622605824 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002568004012036108, + "loss": 2.6437, + "theoretical_loss": 3.491977434432256, + "tokens_seen": 1622671360 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025679037111334005, + "loss": 2.5926, + "theoretical_loss": 3.4919651866823833, + "tokens_seen": 1622736896 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002567803410230692, + "loss": 2.7258, + "theoretical_loss": 3.4919529395656324, + "tokens_seen": 1622802432 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002567703109327984, + "loss": 2.7952, + "theoretical_loss": 3.4919406930819443, + "tokens_seen": 1622867968 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025676028084252754, + "loss": 2.907, + "theoretical_loss": 3.4919284472312606, + "tokens_seen": 1622933504 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002567502507522568, + "loss": 2.6042, + "theoretical_loss": 3.491916202013524, + "tokens_seen": 1622999040 + }, + { + "epoch": 0.49, + "learning_rate": 0.000256740220661986, + "loss": 2.7306, + "theoretical_loss": 3.4919039574286748, + "tokens_seen": 1623064576 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025673019057171514, + "loss": 2.7198, + "theoretical_loss": 3.4918917134766554, + "tokens_seen": 1623130112 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025672016048144437, + "loss": 2.5197, + "theoretical_loss": 3.491879470157408, + "tokens_seen": 1623195648 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002567101303911735, + "loss": 2.6842, + "theoretical_loss": 3.4918672274708737, + "tokens_seen": 1623261184 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025670010030090273, + "loss": 2.5378, + "theoretical_loss": 3.4918549854169947, + "tokens_seen": 1623326720 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002566900702106319, + "loss": 2.6526, + "theoretical_loss": 3.4918427439957123, + "tokens_seen": 1623392256 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002566800401203611, + "loss": 2.7002, + "theoretical_loss": 3.4918305032069687, + "tokens_seen": 1623457792 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002566700100300903, + "loss": 2.6275, + "theoretical_loss": 3.4918182630507055, + "tokens_seen": 1623523328 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002566599799398195, + "loss": 2.4708, + "theoretical_loss": 3.4918060235268644, + "tokens_seen": 1623588864 + }, + { + "epoch": 0.49, + "objective/train/docs_used": 920322, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.834566831588745, + "objective/train/theoretical_loss": 3.4917937846353873, + "objective/train/tokens_used": 1644114400, + "theoretical_loss": 3.4917937846353873, + "tokens_seen": 1623654400 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025664994984954864, + "loss": 2.6362, + "theoretical_loss": 3.4917937846353873, + "tokens_seen": 1623654400 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002566399197592779, + "loss": 2.8142, + "theoretical_loss": 3.491781546376216, + "tokens_seen": 1623719936 + }, + { + "epoch": 0.49, + "learning_rate": 0.000256629889669007, + "loss": 2.4013, + "theoretical_loss": 3.4917693087492925, + "tokens_seen": 1623785472 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025661985957873624, + "loss": 2.667, + "theoretical_loss": 3.491757071754558, + "tokens_seen": 1623851008 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002566098294884654, + "loss": 2.6514, + "theoretical_loss": 3.4917448353919553, + "tokens_seen": 1623916544 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002565997993981946, + "loss": 2.6361, + "theoretical_loss": 3.4917325996614257, + "tokens_seen": 1623982080 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002565897693079238, + "loss": 2.5313, + "theoretical_loss": 3.4917203645629105, + "tokens_seen": 1624047616 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025657973921765296, + "loss": 2.6645, + "theoretical_loss": 3.4917081300963524, + "tokens_seen": 1624113152 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025656970912738214, + "loss": 2.8972, + "theoretical_loss": 3.4916958962616933, + "tokens_seen": 1624178688 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002565596790371114, + "loss": 2.5212, + "theoretical_loss": 3.4916836630588746, + "tokens_seen": 1624244224 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002565496489468405, + "loss": 2.4382, + "theoretical_loss": 3.491671430487838, + "tokens_seen": 1624309760 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025653961885656974, + "loss": 2.4731, + "theoretical_loss": 3.491659198548526, + "tokens_seen": 1624375296 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025652958876629887, + "loss": 2.6795, + "theoretical_loss": 3.49164696724088, + "tokens_seen": 1624440832 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002565195586760281, + "loss": 2.5595, + "theoretical_loss": 3.4916347365648424, + "tokens_seen": 1624506368 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002565095285857573, + "loss": 2.7219, + "theoretical_loss": 3.491622506520354, + "tokens_seen": 1624571904 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025649949849548647, + "loss": 2.6734, + "theoretical_loss": 3.491610277107358, + "tokens_seen": 1624637440 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025648946840521565, + "loss": 2.6619, + "theoretical_loss": 3.4915980483257965, + "tokens_seen": 1624702976 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002564794383149449, + "loss": 3.0292, + "theoretical_loss": 3.49158582017561, + "tokens_seen": 1624768512 + }, + { + "epoch": 0.49, + "learning_rate": 0.000256469408224674, + "loss": 2.603, + "theoretical_loss": 3.4915735926567413, + "tokens_seen": 1624834048 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025645937813440324, + "loss": 2.6598, + "theoretical_loss": 3.4915613657691322, + "tokens_seen": 1624899584 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025644934804413237, + "loss": 2.7168, + "theoretical_loss": 3.4915491395127245, + "tokens_seen": 1624965120 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002564393179538616, + "loss": 2.5567, + "theoretical_loss": 3.491536913887461, + "tokens_seen": 1625030656 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002564292878635908, + "loss": 2.7828, + "theoretical_loss": 3.491524688893282, + "tokens_seen": 1625096192 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025641925777331997, + "loss": 2.7466, + "theoretical_loss": 3.4915124645301314, + "tokens_seen": 1625161728 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025640922768304915, + "loss": 2.6878, + "theoretical_loss": 3.49150024079795, + "tokens_seen": 1625227264 + }, + { + "epoch": 0.49, + "objective/train/docs_used": 921030, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.4457623958587646, + "objective/train/theoretical_loss": 3.49148801769668, + "objective/train/tokens_used": 1645752800, + "theoretical_loss": 3.49148801769668, + "tokens_seen": 1625292800 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025639919759277833, + "loss": 2.6703, + "theoretical_loss": 3.49148801769668, + "tokens_seen": 1625292800 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002563891675025075, + "loss": 2.6605, + "theoretical_loss": 3.4914757952262634, + "tokens_seen": 1625358336 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025637913741223675, + "loss": 2.7137, + "theoretical_loss": 3.4914635733866426, + "tokens_seen": 1625423872 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002563691073219659, + "loss": 2.6537, + "theoretical_loss": 3.4914513521777586, + "tokens_seen": 1625489408 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002563590772316951, + "loss": 2.6922, + "theoretical_loss": 3.491439131599555, + "tokens_seen": 1625554944 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025634904714142424, + "loss": 2.558, + "theoretical_loss": 3.4914269116519723, + "tokens_seen": 1625620480 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025633901705115347, + "loss": 2.6252, + "theoretical_loss": 3.4914146923349536, + "tokens_seen": 1625686016 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025632898696088265, + "loss": 2.7521, + "theoretical_loss": 3.49140247364844, + "tokens_seen": 1625751552 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025631895687061183, + "loss": 2.6088, + "theoretical_loss": 3.491390255592374, + "tokens_seen": 1625817088 + }, + { + "epoch": 0.49, + "learning_rate": 0.000256308926780341, + "loss": 2.3685, + "theoretical_loss": 3.4913780381666983, + "tokens_seen": 1625882624 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025629889669007025, + "loss": 2.8154, + "theoretical_loss": 3.491365821371354, + "tokens_seen": 1625948160 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002562888665997994, + "loss": 2.6036, + "theoretical_loss": 3.491353605206284, + "tokens_seen": 1626013696 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002562788365095286, + "loss": 2.667, + "theoretical_loss": 3.4913413896714296, + "tokens_seen": 1626079232 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025626880641925774, + "loss": 2.7133, + "theoretical_loss": 3.491329174766733, + "tokens_seen": 1626144768 + }, + { + "epoch": 0.49, + "learning_rate": 0.000256258776328987, + "loss": 2.7865, + "theoretical_loss": 3.491316960492137, + "tokens_seen": 1626210304 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025624874623871616, + "loss": 2.6998, + "theoretical_loss": 3.4913047468475833, + "tokens_seen": 1626275840 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025623871614844534, + "loss": 2.4867, + "theoretical_loss": 3.491292533833014, + "tokens_seen": 1626341376 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002562286860581745, + "loss": 2.7245, + "theoretical_loss": 3.4912803214483707, + "tokens_seen": 1626406912 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002562186559679037, + "loss": 2.4856, + "theoretical_loss": 3.4912681096935962, + "tokens_seen": 1626472448 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002562086258776329, + "loss": 2.6834, + "theoretical_loss": 3.491255898568632, + "tokens_seen": 1626537984 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002561985957873621, + "loss": 2.5982, + "theoretical_loss": 3.4912436880734212, + "tokens_seen": 1626603520 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025618856569709124, + "loss": 2.5192, + "theoretical_loss": 3.4912314782079057, + "tokens_seen": 1626669056 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002561785356068205, + "loss": 2.635, + "theoretical_loss": 3.491219268972027, + "tokens_seen": 1626734592 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025616850551654966, + "loss": 2.4526, + "theoretical_loss": 3.4912070603657277, + "tokens_seen": 1626800128 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025615847542627884, + "loss": 2.5719, + "theoretical_loss": 3.49119485238895, + "tokens_seen": 1626865664 + }, + { + "epoch": 0.49, + "objective/train/docs_used": 922470, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.5063717365264893, + "objective/train/theoretical_loss": 3.4911826450416363, + "objective/train/tokens_used": 1647391200, + "theoretical_loss": 3.4911826450416363, + "tokens_seen": 1626931200 + }, + { + "epoch": 0.49, + "learning_rate": 0.000256148445336008, + "loss": 2.362, + "theoretical_loss": 3.4911826450416363, + "tokens_seen": 1626931200 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002561384152457372, + "loss": 2.8672, + "theoretical_loss": 3.491170438323728, + "tokens_seen": 1626996736 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002561283851554664, + "loss": 2.6896, + "theoretical_loss": 3.4911582322351684, + "tokens_seen": 1627062272 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002561183550651956, + "loss": 2.6023, + "theoretical_loss": 3.4911460267758985, + "tokens_seen": 1627127808 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025610832497492475, + "loss": 2.6118, + "theoretical_loss": 3.491133821945861, + "tokens_seen": 1627193344 + }, + { + "epoch": 0.49, + "learning_rate": 0.000256098294884654, + "loss": 2.5379, + "theoretical_loss": 3.491121617744999, + "tokens_seen": 1627258880 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002560882647943831, + "loss": 2.7467, + "theoretical_loss": 3.4911094141732537, + "tokens_seen": 1627324416 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025607823470411234, + "loss": 2.6316, + "theoretical_loss": 3.4910972112305676, + "tokens_seen": 1627389952 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002560682046138415, + "loss": 2.7317, + "theoretical_loss": 3.491085008916883, + "tokens_seen": 1627455488 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002560581745235707, + "loss": 2.7525, + "theoretical_loss": 3.4910728072321415, + "tokens_seen": 1627521024 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002560481444332999, + "loss": 2.8315, + "theoretical_loss": 3.491060606176287, + "tokens_seen": 1627586560 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025603811434302907, + "loss": 2.7077, + "theoretical_loss": 3.49104840574926, + "tokens_seen": 1627652096 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025602808425275825, + "loss": 2.7463, + "theoretical_loss": 3.491036205951003, + "tokens_seen": 1627717632 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002560180541624875, + "loss": 2.5497, + "theoretical_loss": 3.49102400678146, + "tokens_seen": 1627783168 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002560080240722166, + "loss": 2.4112, + "theoretical_loss": 3.4910118082405717, + "tokens_seen": 1627848704 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025599799398194585, + "loss": 2.6435, + "theoretical_loss": 3.4909996103282803, + "tokens_seen": 1627914240 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002559879638916751, + "loss": 2.5789, + "theoretical_loss": 3.490987413044529, + "tokens_seen": 1627979776 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002559779338014042, + "loss": 2.611, + "theoretical_loss": 3.490975216389259, + "tokens_seen": 1628045312 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025596790371113344, + "loss": 2.7127, + "theoretical_loss": 3.4909630203624142, + "tokens_seen": 1628110848 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025595787362086257, + "loss": 2.5723, + "theoretical_loss": 3.490950824963936, + "tokens_seen": 1628176384 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002559478435305918, + "loss": 3.0508, + "theoretical_loss": 3.4909386301937664, + "tokens_seen": 1628241920 + }, + { + "epoch": 0.49, + "learning_rate": 0.000255937813440321, + "loss": 2.4163, + "theoretical_loss": 3.490926436051848, + "tokens_seen": 1628307456 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025592778335005017, + "loss": 2.852, + "theoretical_loss": 3.4909142425381234, + "tokens_seen": 1628372992 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025591775325977935, + "loss": 2.7264, + "theoretical_loss": 3.4909020496525347, + "tokens_seen": 1628438528 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025590772316950853, + "loss": 2.6802, + "theoretical_loss": 3.4908898573950244, + "tokens_seen": 1628504064 + }, + { + "epoch": 0.49, + "objective/train/docs_used": 922615, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.1358861923217773, + "objective/train/theoretical_loss": 3.4908776657655354, + "objective/train/tokens_used": 1649029600, + "theoretical_loss": 3.4908776657655354, + "tokens_seen": 1628569600 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002558976930792377, + "loss": 2.7994, + "theoretical_loss": 3.4908776657655354, + "tokens_seen": 1628569600 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025588766298896695, + "loss": 2.644, + "theoretical_loss": 3.490865474764009, + "tokens_seen": 1628635136 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002558776328986961, + "loss": 2.5193, + "theoretical_loss": 3.490853284390388, + "tokens_seen": 1628700672 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002558676028084253, + "loss": 2.6006, + "theoretical_loss": 3.4908410946446153, + "tokens_seen": 1628766208 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025585757271815444, + "loss": 2.6979, + "theoretical_loss": 3.490828905526633, + "tokens_seen": 1628831744 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025584754262788367, + "loss": 2.7464, + "theoretical_loss": 3.490816717036383, + "tokens_seen": 1628897280 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025583751253761285, + "loss": 2.7604, + "theoretical_loss": 3.490804529173808, + "tokens_seen": 1628962816 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025582748244734203, + "loss": 2.5126, + "theoretical_loss": 3.4907923419388514, + "tokens_seen": 1629028352 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002558174523570712, + "loss": 2.7242, + "theoretical_loss": 3.4907801553314544, + "tokens_seen": 1629093888 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025580742226680045, + "loss": 2.7586, + "theoretical_loss": 3.4907679693515594, + "tokens_seen": 1629159424 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002557973921765296, + "loss": 2.5466, + "theoretical_loss": 3.4907557839991097, + "tokens_seen": 1629224960 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002557873620862588, + "loss": 2.9257, + "theoretical_loss": 3.4907435992740474, + "tokens_seen": 1629290496 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025577733199598794, + "loss": 2.7548, + "theoretical_loss": 3.490731415176315, + "tokens_seen": 1629356032 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002557673019057172, + "loss": 2.603, + "theoretical_loss": 3.4907192317058544, + "tokens_seen": 1629421568 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025575727181544636, + "loss": 2.749, + "theoretical_loss": 3.4907070488626086, + "tokens_seen": 1629487104 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025574724172517554, + "loss": 2.5497, + "theoretical_loss": 3.4906948666465203, + "tokens_seen": 1629552640 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002557372116349047, + "loss": 2.8098, + "theoretical_loss": 3.490682685057532, + "tokens_seen": 1629618176 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002557271815446339, + "loss": 2.5439, + "theoretical_loss": 3.490670504095586, + "tokens_seen": 1629683712 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002557171514543631, + "loss": 2.7372, + "theoretical_loss": 3.4906583237606243, + "tokens_seen": 1629749248 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002557071213640923, + "loss": 2.711, + "theoretical_loss": 3.49064614405259, + "tokens_seen": 1629814784 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025569709127382144, + "loss": 2.7814, + "theoretical_loss": 3.4906339649714258, + "tokens_seen": 1629880320 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002556870611835507, + "loss": 2.4778, + "theoretical_loss": 3.4906217865170737, + "tokens_seen": 1629945856 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025567703109327986, + "loss": 2.698, + "theoretical_loss": 3.4906096086894767, + "tokens_seen": 1630011392 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025566700100300904, + "loss": 2.6937, + "theoretical_loss": 3.4905974314885766, + "tokens_seen": 1630076928 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002556569709127382, + "loss": 2.7772, + "theoretical_loss": 3.4905852549143166, + "tokens_seen": 1630142464 + }, + { + "epoch": 0.49, + "objective/train/docs_used": 923942, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6910572052001953, + "objective/train/theoretical_loss": 3.49057307896664, + "objective/train/tokens_used": 1650668000, + "theoretical_loss": 3.49057307896664, + "tokens_seen": 1630208000 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002556469408224674, + "loss": 2.5191, + "theoretical_loss": 3.49057307896664, + "tokens_seen": 1630208000 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002556369107321966, + "loss": 2.7938, + "theoretical_loss": 3.4905609036454877, + "tokens_seen": 1630273536 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002556268806419258, + "loss": 2.6547, + "theoretical_loss": 3.490548728950803, + "tokens_seen": 1630339072 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025561685055165495, + "loss": 2.608, + "theoretical_loss": 3.4905365548825285, + "tokens_seen": 1630404608 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002556068204613842, + "loss": 2.7972, + "theoretical_loss": 3.4905243814406073, + "tokens_seen": 1630470144 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002555967903711133, + "loss": 2.1101, + "theoretical_loss": 3.490512208624981, + "tokens_seen": 1630535680 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025558676028084254, + "loss": 2.7658, + "theoretical_loss": 3.4905000364355936, + "tokens_seen": 1630601216 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002555767301905717, + "loss": 2.6385, + "theoretical_loss": 3.490487864872386, + "tokens_seen": 1630666752 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002555667001003009, + "loss": 2.6816, + "theoretical_loss": 3.4904756939353025, + "tokens_seen": 1630732288 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002555566700100301, + "loss": 2.7146, + "theoretical_loss": 3.4904635236242845, + "tokens_seen": 1630797824 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025554663991975927, + "loss": 2.6587, + "theoretical_loss": 3.4904513539392754, + "tokens_seen": 1630863360 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025553660982948845, + "loss": 2.7573, + "theoretical_loss": 3.490439184880217, + "tokens_seen": 1630928896 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002555265797392177, + "loss": 2.7101, + "theoretical_loss": 3.490427016447053, + "tokens_seen": 1630994432 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002555165496489468, + "loss": 2.7657, + "theoretical_loss": 3.490414848639725, + "tokens_seen": 1631059968 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025550651955867605, + "loss": 2.6783, + "theoretical_loss": 3.490402681458177, + "tokens_seen": 1631125504 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025549648946840523, + "loss": 2.7087, + "theoretical_loss": 3.4903905149023506, + "tokens_seen": 1631191040 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002554864593781344, + "loss": 2.9684, + "theoretical_loss": 3.4903783489721882, + "tokens_seen": 1631256576 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002554764292878636, + "loss": 2.5646, + "theoretical_loss": 3.490366183667634, + "tokens_seen": 1631322112 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025546639919759277, + "loss": 2.5757, + "theoretical_loss": 3.4903540189886284, + "tokens_seen": 1631387648 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025545636910732195, + "loss": 2.5922, + "theoretical_loss": 3.4903418549351164, + "tokens_seen": 1631453184 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002554463390170512, + "loss": 2.6842, + "theoretical_loss": 3.49032969150704, + "tokens_seen": 1631518720 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002554363089267803, + "loss": 2.6988, + "theoretical_loss": 3.490317528704341, + "tokens_seen": 1631584256 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025542627883650955, + "loss": 2.8228, + "theoretical_loss": 3.4903053665269637, + "tokens_seen": 1631649792 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002554162487462387, + "loss": 2.6516, + "theoretical_loss": 3.490293204974849, + "tokens_seen": 1631715328 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002554062186559679, + "loss": 2.6444, + "theoretical_loss": 3.490281044047941, + "tokens_seen": 1631780864 + }, + { + "epoch": 0.49, + "objective/train/docs_used": 924504, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.011815309524536, + "objective/train/theoretical_loss": 3.4902688837461824, + "objective/train/tokens_used": 1652306400, + "theoretical_loss": 3.4902688837461824, + "tokens_seen": 1631846400 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002553961885656971, + "loss": 2.8255, + "theoretical_loss": 3.4902688837461824, + "tokens_seen": 1631846400 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002553861584754263, + "loss": 2.9364, + "theoretical_loss": 3.490256724069515, + "tokens_seen": 1631911936 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025537612838515546, + "loss": 2.69, + "theoretical_loss": 3.490244565017883, + "tokens_seen": 1631977472 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025536609829488464, + "loss": 2.3849, + "theoretical_loss": 3.490232406591227, + "tokens_seen": 1632043008 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002553560682046138, + "loss": 2.7567, + "theoretical_loss": 3.4902202487894924, + "tokens_seen": 1632108544 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025534603811434305, + "loss": 2.6172, + "theoretical_loss": 3.49020809161262, + "tokens_seen": 1632174080 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002553360080240722, + "loss": 2.6428, + "theoretical_loss": 3.490195935060554, + "tokens_seen": 1632239616 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002553259779338014, + "loss": 2.8332, + "theoretical_loss": 3.490183779133236, + "tokens_seen": 1632305152 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002553159478435306, + "loss": 2.5037, + "theoretical_loss": 3.4901716238306095, + "tokens_seen": 1632370688 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002553059177532598, + "loss": 2.6729, + "theoretical_loss": 3.490159469152617, + "tokens_seen": 1632436224 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025529588766298896, + "loss": 2.4324, + "theoretical_loss": 3.4901473150992017, + "tokens_seen": 1632501760 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025528585757271814, + "loss": 2.8338, + "theoretical_loss": 3.490135161670306, + "tokens_seen": 1632567296 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002552758274824473, + "loss": 2.759, + "theoretical_loss": 3.490123008865873, + "tokens_seen": 1632632832 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025526579739217656, + "loss": 2.5953, + "theoretical_loss": 3.490110856685846, + "tokens_seen": 1632698368 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002552557673019057, + "loss": 2.4807, + "theoretical_loss": 3.4900987051301664, + "tokens_seen": 1632763904 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002552457372116349, + "loss": 2.7636, + "theoretical_loss": 3.490086554198779, + "tokens_seen": 1632829440 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002552357071213641, + "loss": 2.7314, + "theoretical_loss": 3.490074403891625, + "tokens_seen": 1632894976 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002552256770310933, + "loss": 2.5647, + "theoretical_loss": 3.490062254208649, + "tokens_seen": 1632960512 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002552156469408225, + "loss": 2.5307, + "theoretical_loss": 3.490050105149792, + "tokens_seen": 1633026048 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025520561685055164, + "loss": 2.8265, + "theoretical_loss": 3.490037956714998, + "tokens_seen": 1633091584 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002551955867602809, + "loss": 2.6253, + "theoretical_loss": 3.4900258089042095, + "tokens_seen": 1633157120 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025518555667001006, + "loss": 2.8975, + "theoretical_loss": 3.490013661717369, + "tokens_seen": 1633222656 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025517552657973924, + "loss": 2.8682, + "theoretical_loss": 3.490001515154421, + "tokens_seen": 1633288192 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002551654964894684, + "loss": 2.5715, + "theoretical_loss": 3.4899893692153072, + "tokens_seen": 1633353728 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002551554663991976, + "loss": 2.667, + "theoretical_loss": 3.4899772238999707, + "tokens_seen": 1633419264 + }, + { + "epoch": 0.49, + "objective/train/docs_used": 925849, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.2080798149108887, + "objective/train/theoretical_loss": 3.4899650792083543, + "objective/train/tokens_used": 1653944800, + "theoretical_loss": 3.4899650792083543, + "tokens_seen": 1633484800 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002551454363089268, + "loss": 2.7293, + "theoretical_loss": 3.4899650792083543, + "tokens_seen": 1633484800 + }, + { + "epoch": 0.5, + "learning_rate": 0.000255135406218656, + "loss": 2.7689, + "theoretical_loss": 3.4899529351404013, + "tokens_seen": 1633550336 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025512537612838515, + "loss": 2.563, + "theoretical_loss": 3.4899407916960543, + "tokens_seen": 1633615872 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002551153460381144, + "loss": 2.5292, + "theoretical_loss": 3.4899286488752566, + "tokens_seen": 1633681408 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002551053159478435, + "loss": 2.4033, + "theoretical_loss": 3.489916506677951, + "tokens_seen": 1633746944 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025509528585757274, + "loss": 2.7688, + "theoretical_loss": 3.4899043651040804, + "tokens_seen": 1633812480 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002550852557673019, + "loss": 2.4598, + "theoretical_loss": 3.4898922241535884, + "tokens_seen": 1633878016 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002550752256770311, + "loss": 2.5381, + "theoretical_loss": 3.489880083826417, + "tokens_seen": 1633943552 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002550651955867603, + "loss": 2.7462, + "theoretical_loss": 3.4898679441225093, + "tokens_seen": 1634009088 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025505516549648947, + "loss": 2.5542, + "theoretical_loss": 3.4898558050418096, + "tokens_seen": 1634074624 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025504513540621865, + "loss": 2.6411, + "theoretical_loss": 3.4898436665842594, + "tokens_seen": 1634140160 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002550351053159479, + "loss": 2.6913, + "theoretical_loss": 3.4898315287498023, + "tokens_seen": 1634205696 + }, + { + "epoch": 0.5, + "learning_rate": 0.000255025075225677, + "loss": 2.66, + "theoretical_loss": 3.489819391538381, + "tokens_seen": 1634271232 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025501504513540625, + "loss": 2.8295, + "theoretical_loss": 3.4898072549499397, + "tokens_seen": 1634336768 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025500501504513543, + "loss": 2.7056, + "theoretical_loss": 3.4897951189844205, + "tokens_seen": 1634402304 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002549949849548646, + "loss": 2.7261, + "theoretical_loss": 3.489782983641766, + "tokens_seen": 1634467840 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002549849548645938, + "loss": 2.841, + "theoretical_loss": 3.4897708489219204, + "tokens_seen": 1634533376 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025497492477432297, + "loss": 2.7969, + "theoretical_loss": 3.489758714824826, + "tokens_seen": 1634598912 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025496489468405215, + "loss": 2.6501, + "theoretical_loss": 3.4897465813504263, + "tokens_seen": 1634664448 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002549548645937814, + "loss": 2.6636, + "theoretical_loss": 3.489734448498664, + "tokens_seen": 1634729984 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002549448345035105, + "loss": 2.644, + "theoretical_loss": 3.489722316269482, + "tokens_seen": 1634795520 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025493480441323975, + "loss": 2.6461, + "theoretical_loss": 3.4897101846628242, + "tokens_seen": 1634861056 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002549247743229689, + "loss": 2.7274, + "theoretical_loss": 3.4896980536786333, + "tokens_seen": 1634926592 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002549147442326981, + "loss": 2.6074, + "theoretical_loss": 3.489685923316852, + "tokens_seen": 1634992128 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002549047141424273, + "loss": 2.6455, + "theoretical_loss": 3.489673793577424, + "tokens_seen": 1635057664 + }, + { + "epoch": 0.5, + "objective/train/docs_used": 926453, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.500638484954834, + "objective/train/theoretical_loss": 3.489661664460292, + "objective/train/tokens_used": 1655583200, + "theoretical_loss": 3.489661664460292, + "tokens_seen": 1635123200 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002548946840521565, + "loss": 2.6878, + "theoretical_loss": 3.489661664460292, + "tokens_seen": 1635123200 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025488465396188566, + "loss": 2.6834, + "theoretical_loss": 3.4896495359653996, + "tokens_seen": 1635188736 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025487462387161484, + "loss": 2.6627, + "theoretical_loss": 3.4896374080926895, + "tokens_seen": 1635254272 + }, + { + "epoch": 0.5, + "learning_rate": 0.000254864593781344, + "loss": 2.755, + "theoretical_loss": 3.4896252808421053, + "tokens_seen": 1635319808 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025485456369107325, + "loss": 2.5524, + "theoretical_loss": 3.4896131542135898, + "tokens_seen": 1635385344 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002548445336008024, + "loss": 2.6611, + "theoretical_loss": 3.4896010282070864, + "tokens_seen": 1635450880 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002548345035105316, + "loss": 2.6464, + "theoretical_loss": 3.489588902822538, + "tokens_seen": 1635516416 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002548244734202608, + "loss": 2.7166, + "theoretical_loss": 3.4895767780598876, + "tokens_seen": 1635581952 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025481444332999, + "loss": 2.6226, + "theoretical_loss": 3.489564653919079, + "tokens_seen": 1635647488 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025480441323971916, + "loss": 2.5761, + "theoretical_loss": 3.489552530400055, + "tokens_seen": 1635713024 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025479438314944834, + "loss": 2.697, + "theoretical_loss": 3.489540407502759, + "tokens_seen": 1635778560 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002547843530591775, + "loss": 2.5504, + "theoretical_loss": 3.489528285227134, + "tokens_seen": 1635844096 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025477432296890676, + "loss": 2.8259, + "theoretical_loss": 3.489516163573123, + "tokens_seen": 1635909632 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002547642928786359, + "loss": 2.7835, + "theoretical_loss": 3.4895040425406703, + "tokens_seen": 1635975168 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002547542627883651, + "loss": 2.6933, + "theoretical_loss": 3.4894919221297176, + "tokens_seen": 1636040704 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025474423269809425, + "loss": 2.7196, + "theoretical_loss": 3.4894798023402096, + "tokens_seen": 1636106240 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002547342026078235, + "loss": 2.735, + "theoretical_loss": 3.4894676831720886, + "tokens_seen": 1636171776 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025472417251755266, + "loss": 2.7205, + "theoretical_loss": 3.489455564625298, + "tokens_seen": 1636237312 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025471414242728184, + "loss": 2.8542, + "theoretical_loss": 3.489443446699781, + "tokens_seen": 1636302848 + }, + { + "epoch": 0.5, + "learning_rate": 0.000254704112337011, + "loss": 2.7648, + "theoretical_loss": 3.4894313293954813, + "tokens_seen": 1636368384 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025469408224674026, + "loss": 2.7497, + "theoretical_loss": 3.489419212712342, + "tokens_seen": 1636433920 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002546840521564694, + "loss": 2.8123, + "theoretical_loss": 3.4894070966503064, + "tokens_seen": 1636499456 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002546740220661986, + "loss": 2.768, + "theoretical_loss": 3.4893949812093172, + "tokens_seen": 1636564992 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025466399197592775, + "loss": 2.6197, + "theoretical_loss": 3.4893828663893185, + "tokens_seen": 1636630528 + }, + { + "epoch": 0.5, + "learning_rate": 0.000254653961885657, + "loss": 2.668, + "theoretical_loss": 3.4893707521902533, + "tokens_seen": 1636696064 + }, + { + "epoch": 0.5, + "objective/train/docs_used": 927487, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.350449562072754, + "objective/train/theoretical_loss": 3.4893586386120647, + "objective/train/tokens_used": 1657221600, + "theoretical_loss": 3.4893586386120647, + "tokens_seen": 1636761600 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025464393179538616, + "loss": 2.6807, + "theoretical_loss": 3.4893586386120647, + "tokens_seen": 1636761600 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025463390170511535, + "loss": 2.6847, + "theoretical_loss": 3.489346525654696, + "tokens_seen": 1636827136 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025462387161484453, + "loss": 2.6619, + "theoretical_loss": 3.4893344133180912, + "tokens_seen": 1636892672 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002546138415245737, + "loss": 2.8307, + "theoretical_loss": 3.4893223016021935, + "tokens_seen": 1636958208 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002546038114343029, + "loss": 2.8786, + "theoretical_loss": 3.4893101905069455, + "tokens_seen": 1637023744 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002545937813440321, + "loss": 2.7948, + "theoretical_loss": 3.489298080032291, + "tokens_seen": 1637089280 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025458375125376125, + "loss": 2.716, + "theoretical_loss": 3.4892859701781735, + "tokens_seen": 1637154816 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002545737211634905, + "loss": 2.3872, + "theoretical_loss": 3.489273860944536, + "tokens_seen": 1637220352 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002545636910732196, + "loss": 2.4703, + "theoretical_loss": 3.4892617523313225, + "tokens_seen": 1637285888 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025455366098294885, + "loss": 2.8589, + "theoretical_loss": 3.4892496443384755, + "tokens_seen": 1637351424 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025454363089267803, + "loss": 2.73, + "theoretical_loss": 3.4892375369659394, + "tokens_seen": 1637416960 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002545336008024072, + "loss": 2.7654, + "theoretical_loss": 3.489225430213656, + "tokens_seen": 1637482496 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002545235707121364, + "loss": 2.8331, + "theoretical_loss": 3.489213324081571, + "tokens_seen": 1637548032 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025451354062186563, + "loss": 2.5256, + "theoretical_loss": 3.489201218569626, + "tokens_seen": 1637613568 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002545035105315948, + "loss": 2.5646, + "theoretical_loss": 3.4891891136777646, + "tokens_seen": 1637679104 + }, + { + "epoch": 0.5, + "learning_rate": 0.000254493480441324, + "loss": 2.5526, + "theoretical_loss": 3.489177009405931, + "tokens_seen": 1637744640 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025448345035105317, + "loss": 2.7342, + "theoretical_loss": 3.4891649057540683, + "tokens_seen": 1637810176 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025447342026078235, + "loss": 2.4742, + "theoretical_loss": 3.4891528027221197, + "tokens_seen": 1637875712 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002544633901705116, + "loss": 2.7209, + "theoretical_loss": 3.489140700310029, + "tokens_seen": 1637941248 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002544533600802407, + "loss": 2.6555, + "theoretical_loss": 3.4891285985177394, + "tokens_seen": 1638006784 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025444332998996995, + "loss": 2.5405, + "theoretical_loss": 3.4891164973451945, + "tokens_seen": 1638072320 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002544332998996991, + "loss": 2.6878, + "theoretical_loss": 3.4891043967923374, + "tokens_seen": 1638137856 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002544232698094283, + "loss": 2.6315, + "theoretical_loss": 3.4890922968591123, + "tokens_seen": 1638203392 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002544132397191575, + "loss": 2.7385, + "theoretical_loss": 3.489080197545462, + "tokens_seen": 1638268928 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002544032096288867, + "loss": 2.6765, + "theoretical_loss": 3.4890680988513303, + "tokens_seen": 1638334464 + }, + { + "debugging/Self-BLEU-5": 0.19279732024941007, + "debugging/distinct-1-grams": 0.8527669172932331, + "debugging/distinct-2-grams": 0.9811776061776062, + "debugging/entropy-1-grams": 5.259420970783413, + "debugging/entropy-2-grams": 5.659296756070026, + "debugging/length": 369.4, + "debugging/num_segments": 5, + "debugging/score": 0.014449788938036645, + "debugging/score_std": 0.01297966481363859, + "epoch": 0.5, + "objective/train/docs_used": 928998, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6203629970550537, + "objective/train/theoretical_loss": 3.4890560007766607, + "objective/train/tokens_used": 1658860000, + "theoretical_loss": 3.4890560007766607, + "tokens_seen": 1638400000 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025439317953861586, + "loss": 2.5951, + "theoretical_loss": 3.4890560007766607, + "tokens_seen": 1638400000 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025438314944834504, + "loss": 2.6332, + "theoretical_loss": 3.489043903321397, + "tokens_seen": 1638465536 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002543731193580742, + "loss": 2.7229, + "theoretical_loss": 3.4890318064854817, + "tokens_seen": 1638531072 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025436308926780345, + "loss": 2.5603, + "theoretical_loss": 3.4890197102688596, + "tokens_seen": 1638596608 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002543530591775326, + "loss": 2.7071, + "theoretical_loss": 3.489007614671473, + "tokens_seen": 1638662144 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002543430290872618, + "loss": 2.5846, + "theoretical_loss": 3.4889955196932663, + "tokens_seen": 1638727680 + }, + { + "epoch": 0.5, + "learning_rate": 0.000254332998996991, + "loss": 2.6252, + "theoretical_loss": 3.488983425334183, + "tokens_seen": 1638793216 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002543229689067202, + "loss": 2.6392, + "theoretical_loss": 3.4889713315941666, + "tokens_seen": 1638858752 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025431293881644936, + "loss": 2.7603, + "theoretical_loss": 3.4889592384731603, + "tokens_seen": 1638924288 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025430290872617854, + "loss": 2.7847, + "theoretical_loss": 3.4889471459711077, + "tokens_seen": 1638989824 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002542928786359077, + "loss": 2.5729, + "theoretical_loss": 3.4889350540879533, + "tokens_seen": 1639055360 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025428284854563696, + "loss": 2.6388, + "theoretical_loss": 3.488922962823639, + "tokens_seen": 1639120896 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002542728184553661, + "loss": 2.5481, + "theoretical_loss": 3.48891087217811, + "tokens_seen": 1639186432 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002542627883650953, + "loss": 2.6182, + "theoretical_loss": 3.488898782151309, + "tokens_seen": 1639251968 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025425275827482445, + "loss": 2.647, + "theoretical_loss": 3.4888866927431796, + "tokens_seen": 1639317504 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002542427281845537, + "loss": 2.6509, + "theoretical_loss": 3.4888746039536658, + "tokens_seen": 1639383040 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025423269809428286, + "loss": 2.797, + "theoretical_loss": 3.488862515782711, + "tokens_seen": 1639448576 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025422266800401204, + "loss": 2.4674, + "theoretical_loss": 3.4888504282302595, + "tokens_seen": 1639514112 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002542126379137412, + "loss": 2.6669, + "theoretical_loss": 3.4888383412962534, + "tokens_seen": 1639579648 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025420260782347046, + "loss": 2.8805, + "theoretical_loss": 3.488826254980638, + "tokens_seen": 1639645184 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002541925777331996, + "loss": 2.7985, + "theoretical_loss": 3.4888141692833554, + "tokens_seen": 1639710720 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002541825476429288, + "loss": 2.607, + "theoretical_loss": 3.488802084204351, + "tokens_seen": 1639776256 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025417251755265795, + "loss": 2.4574, + "theoretical_loss": 3.4887899997435667, + "tokens_seen": 1639841792 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002541624874623872, + "loss": 2.4446, + "theoretical_loss": 3.488777915900947, + "tokens_seen": 1639907328 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025415245737211637, + "loss": 2.7655, + "theoretical_loss": 3.488765832676436, + "tokens_seen": 1639972864 + }, + { + "epoch": 0.5, + "objective/train/docs_used": 929588, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.856471538543701, + "objective/train/theoretical_loss": 3.488753750069977, + "objective/train/tokens_used": 1660498400, + "theoretical_loss": 3.488753750069977, + "tokens_seen": 1640038400 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025414242728184555, + "loss": 2.7719, + "theoretical_loss": 3.488753750069977, + "tokens_seen": 1640038400 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025413239719157473, + "loss": 2.5249, + "theoretical_loss": 3.488741668081513, + "tokens_seen": 1640103936 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002541223671013039, + "loss": 2.6664, + "theoretical_loss": 3.4887295867109884, + "tokens_seen": 1640169472 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002541123370110331, + "loss": 2.6055, + "theoretical_loss": 3.4887175059583475, + "tokens_seen": 1640235008 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002541023069207623, + "loss": 2.7435, + "theoretical_loss": 3.4887054258235333, + "tokens_seen": 1640300544 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025409227683049145, + "loss": 2.6253, + "theoretical_loss": 3.488693346306489, + "tokens_seen": 1640366080 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002540822467402207, + "loss": 2.6139, + "theoretical_loss": 3.4886812674071592, + "tokens_seen": 1640431616 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002540722166499498, + "loss": 2.5848, + "theoretical_loss": 3.4886691891254875, + "tokens_seen": 1640497152 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025406218655967905, + "loss": 2.7479, + "theoretical_loss": 3.488657111461417, + "tokens_seen": 1640562688 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025405215646940823, + "loss": 2.6903, + "theoretical_loss": 3.4886450344148923, + "tokens_seen": 1640628224 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002540421263791374, + "loss": 2.5902, + "theoretical_loss": 3.488632957985857, + "tokens_seen": 1640693760 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002540320962888666, + "loss": 2.3858, + "theoretical_loss": 3.488620882174254, + "tokens_seen": 1640759296 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025402206619859583, + "loss": 2.864, + "theoretical_loss": 3.4886088069800283, + "tokens_seen": 1640824832 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025401203610832495, + "loss": 2.8382, + "theoretical_loss": 3.488596732403123, + "tokens_seen": 1640890368 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002540020060180542, + "loss": 2.5899, + "theoretical_loss": 3.488584658443482, + "tokens_seen": 1640955904 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002539919759277833, + "loss": 2.4831, + "theoretical_loss": 3.488572585101049, + "tokens_seen": 1641021440 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025398194583751255, + "loss": 2.6643, + "theoretical_loss": 3.4885605123757677, + "tokens_seen": 1641086976 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025397191574724173, + "loss": 2.8251, + "theoretical_loss": 3.488548440267582, + "tokens_seen": 1641152512 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002539618856569709, + "loss": 2.7273, + "theoretical_loss": 3.4885363687764364, + "tokens_seen": 1641218048 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002539518555667001, + "loss": 2.6723, + "theoretical_loss": 3.488524297902274, + "tokens_seen": 1641283584 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002539418254764293, + "loss": 2.6961, + "theoretical_loss": 3.488512227645039, + "tokens_seen": 1641349120 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025393179538615846, + "loss": 2.8485, + "theoretical_loss": 3.4885001580046744, + "tokens_seen": 1641414656 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002539217652958877, + "loss": 2.5259, + "theoretical_loss": 3.488488088981125, + "tokens_seen": 1641480192 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002539117352056168, + "loss": 2.4502, + "theoretical_loss": 3.4884760205743337, + "tokens_seen": 1641545728 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025390170511534606, + "loss": 2.5826, + "theoretical_loss": 3.4884639527842456, + "tokens_seen": 1641611264 + }, + { + "epoch": 0.5, + "objective/train/docs_used": 930445, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.653027296066284, + "objective/train/theoretical_loss": 3.4884518856108038, + "objective/train/tokens_used": 1662136800, + "theoretical_loss": 3.4884518856108038, + "tokens_seen": 1641676800 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002538916750250752, + "loss": 2.725, + "theoretical_loss": 3.4884518856108038, + "tokens_seen": 1641676800 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002538816449348044, + "loss": 2.6095, + "theoretical_loss": 3.4884398190539523, + "tokens_seen": 1641742336 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002538716148445336, + "loss": 2.8779, + "theoretical_loss": 3.488427753113635, + "tokens_seen": 1641807872 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002538615847542628, + "loss": 2.5967, + "theoretical_loss": 3.488415687789796, + "tokens_seen": 1641873408 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025385155466399196, + "loss": 2.8424, + "theoretical_loss": 3.4884036230823785, + "tokens_seen": 1641938944 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002538415245737212, + "loss": 2.6959, + "theoretical_loss": 3.4883915589913275, + "tokens_seen": 1642004480 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002538314944834503, + "loss": 2.4441, + "theoretical_loss": 3.488379495516586, + "tokens_seen": 1642070016 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025382146439317956, + "loss": 2.7217, + "theoretical_loss": 3.488367432658098, + "tokens_seen": 1642135552 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002538114343029087, + "loss": 2.4914, + "theoretical_loss": 3.488355370415808, + "tokens_seen": 1642201088 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002538014042126379, + "loss": 2.6458, + "theoretical_loss": 3.488343308789659, + "tokens_seen": 1642266624 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002537913741223671, + "loss": 2.7323, + "theoretical_loss": 3.4883312477795965, + "tokens_seen": 1642332160 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002537813440320963, + "loss": 2.6381, + "theoretical_loss": 3.488319187385563, + "tokens_seen": 1642397696 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025377131394182546, + "loss": 2.6525, + "theoretical_loss": 3.488307127607503, + "tokens_seen": 1642463232 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025376128385155465, + "loss": 2.5494, + "theoretical_loss": 3.4882950684453604, + "tokens_seen": 1642528768 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002537512537612839, + "loss": 2.3424, + "theoretical_loss": 3.488283009899079, + "tokens_seen": 1642594304 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025374122367101306, + "loss": 2.3999, + "theoretical_loss": 3.4882709519686026, + "tokens_seen": 1642659840 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025373119358074224, + "loss": 2.4489, + "theoretical_loss": 3.488258894653876, + "tokens_seen": 1642725376 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002537211634904714, + "loss": 2.5933, + "theoretical_loss": 3.4882468379548426, + "tokens_seen": 1642790912 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025371113340020066, + "loss": 2.6437, + "theoretical_loss": 3.4882347818714465, + "tokens_seen": 1642856448 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002537011033099298, + "loss": 2.9817, + "theoretical_loss": 3.488222726403632, + "tokens_seen": 1642921984 + }, + { + "epoch": 0.5, + "learning_rate": 0.000253691073219659, + "loss": 2.55, + "theoretical_loss": 3.488210671551342, + "tokens_seen": 1642987520 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025368104312938815, + "loss": 2.7757, + "theoretical_loss": 3.4881986173145227, + "tokens_seen": 1643053056 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002536710130391174, + "loss": 2.4662, + "theoretical_loss": 3.488186563693116, + "tokens_seen": 1643118592 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025366098294884657, + "loss": 2.6378, + "theoretical_loss": 3.4881745106870667, + "tokens_seen": 1643184128 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025365095285857575, + "loss": 2.6961, + "theoretical_loss": 3.488162458296319, + "tokens_seen": 1643249664 + }, + { + "epoch": 0.5, + "objective/train/docs_used": 931444, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.5048954486846924, + "objective/train/theoretical_loss": 3.488150406520817, + "objective/train/tokens_used": 1663775200, + "theoretical_loss": 3.488150406520817, + "tokens_seen": 1643315200 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025364092276830493, + "loss": 2.4919, + "theoretical_loss": 3.488150406520817, + "tokens_seen": 1643315200 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002536308926780341, + "loss": 2.7426, + "theoretical_loss": 3.4881383553605043, + "tokens_seen": 1643380736 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002536208625877633, + "loss": 2.698, + "theoretical_loss": 3.4881263048153253, + "tokens_seen": 1643446272 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002536108324974925, + "loss": 2.5909, + "theoretical_loss": 3.4881142548852244, + "tokens_seen": 1643511808 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025360080240722165, + "loss": 2.7217, + "theoretical_loss": 3.488102205570145, + "tokens_seen": 1643577344 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002535907723169509, + "loss": 2.7635, + "theoretical_loss": 3.4880901568700313, + "tokens_seen": 1643642880 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025358074222668, + "loss": 2.6869, + "theoretical_loss": 3.488078108784828, + "tokens_seen": 1643708416 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025357071213640925, + "loss": 2.7894, + "theoretical_loss": 3.4880660613144787, + "tokens_seen": 1643773952 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025356068204613843, + "loss": 2.5939, + "theoretical_loss": 3.4880540144589274, + "tokens_seen": 1643839488 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002535506519558676, + "loss": 2.8999, + "theoretical_loss": 3.488041968218118, + "tokens_seen": 1643905024 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002535406218655968, + "loss": 2.8713, + "theoretical_loss": 3.4880299225919957, + "tokens_seen": 1643970560 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025353059177532603, + "loss": 2.7818, + "theoretical_loss": 3.488017877580504, + "tokens_seen": 1644036096 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025352056168505516, + "loss": 2.7086, + "theoretical_loss": 3.4880058331835873, + "tokens_seen": 1644101632 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002535105315947844, + "loss": 2.7381, + "theoretical_loss": 3.4879937894011888, + "tokens_seen": 1644167168 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002535005015045135, + "loss": 2.7073, + "theoretical_loss": 3.4879817462332534, + "tokens_seen": 1644232704 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025349047141424275, + "loss": 2.5454, + "theoretical_loss": 3.4879697036797257, + "tokens_seen": 1644298240 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025348044132397193, + "loss": 2.693, + "theoretical_loss": 3.4879576617405488, + "tokens_seen": 1644363776 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002534704112337011, + "loss": 2.7477, + "theoretical_loss": 3.4879456204156676, + "tokens_seen": 1644429312 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002534603811434303, + "loss": 2.7091, + "theoretical_loss": 3.487933579705026, + "tokens_seen": 1644494848 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002534503510531595, + "loss": 2.6505, + "theoretical_loss": 3.4879215396085685, + "tokens_seen": 1644560384 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025344032096288866, + "loss": 2.8136, + "theoretical_loss": 3.4879095001262392, + "tokens_seen": 1644625920 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002534302908726179, + "loss": 2.6184, + "theoretical_loss": 3.487897461257982, + "tokens_seen": 1644691456 + }, + { + "epoch": 0.5, + "learning_rate": 0.000253420260782347, + "loss": 2.6525, + "theoretical_loss": 3.487885423003741, + "tokens_seen": 1644756992 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025341023069207626, + "loss": 2.4858, + "theoretical_loss": 3.487873385363461, + "tokens_seen": 1644822528 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002534002006018054, + "loss": 2.855, + "theoretical_loss": 3.487861348337086, + "tokens_seen": 1644888064 + }, + { + "epoch": 0.5, + "objective/train/docs_used": 932028, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.790616035461426, + "objective/train/theoretical_loss": 3.4878493119245606, + "objective/train/tokens_used": 1665413600, + "theoretical_loss": 3.4878493119245606, + "tokens_seen": 1644953600 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002533901705115346, + "loss": 2.5954, + "theoretical_loss": 3.4878493119245606, + "tokens_seen": 1644953600 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002533801404212638, + "loss": 2.6956, + "theoretical_loss": 3.487837276125828, + "tokens_seen": 1645019136 + }, + { + "epoch": 0.5, + "learning_rate": 0.000253370110330993, + "loss": 2.8317, + "theoretical_loss": 3.4878252409408335, + "tokens_seen": 1645084672 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025336008024072216, + "loss": 2.6737, + "theoretical_loss": 3.4878132063695206, + "tokens_seen": 1645150208 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002533500501504514, + "loss": 2.6305, + "theoretical_loss": 3.4878011724118343, + "tokens_seen": 1645215744 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002533400200601805, + "loss": 2.6615, + "theoretical_loss": 3.487789139067718, + "tokens_seen": 1645281280 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025332998996990976, + "loss": 2.621, + "theoretical_loss": 3.487777106337117, + "tokens_seen": 1645346816 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002533199598796389, + "loss": 2.6607, + "theoretical_loss": 3.4877650742199746, + "tokens_seen": 1645412352 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002533099297893681, + "loss": 2.6226, + "theoretical_loss": 3.4877530427162355, + "tokens_seen": 1645477888 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002532998996990973, + "loss": 2.6629, + "theoretical_loss": 3.487741011825844, + "tokens_seen": 1645543424 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002532898696088265, + "loss": 2.6159, + "theoretical_loss": 3.4877289815487447, + "tokens_seen": 1645608960 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025327983951855566, + "loss": 2.7561, + "theoretical_loss": 3.487716951884882, + "tokens_seen": 1645674496 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025326980942828485, + "loss": 2.4739, + "theoretical_loss": 3.487704922834199, + "tokens_seen": 1645740032 + }, + { + "epoch": 0.5, + "learning_rate": 0.000253259779338014, + "loss": 2.8335, + "theoretical_loss": 3.487692894396641, + "tokens_seen": 1645805568 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025324974924774326, + "loss": 2.4795, + "theoretical_loss": 3.487680866572153, + "tokens_seen": 1645871104 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002532397191574724, + "loss": 2.7664, + "theoretical_loss": 3.487668839360678, + "tokens_seen": 1645936640 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002532296890672016, + "loss": 2.7457, + "theoretical_loss": 3.487656812762161, + "tokens_seen": 1646002176 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025321965897693075, + "loss": 2.5341, + "theoretical_loss": 3.4876447867765465, + "tokens_seen": 1646067712 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025320962888666, + "loss": 2.6492, + "theoretical_loss": 3.4876327614037788, + "tokens_seen": 1646133248 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025319959879638917, + "loss": 2.5726, + "theoretical_loss": 3.4876207366438017, + "tokens_seen": 1646198784 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025318956870611835, + "loss": 2.6443, + "theoretical_loss": 3.4876087124965602, + "tokens_seen": 1646264320 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025317953861584753, + "loss": 2.6376, + "theoretical_loss": 3.4875966889619985, + "tokens_seen": 1646329856 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025316950852557677, + "loss": 2.7839, + "theoretical_loss": 3.4875846660400605, + "tokens_seen": 1646395392 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002531594784353059, + "loss": 2.672, + "theoretical_loss": 3.487572643730692, + "tokens_seen": 1646460928 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025314944834503513, + "loss": 2.6667, + "theoretical_loss": 3.4875606220338358, + "tokens_seen": 1646526464 + }, + { + "epoch": 0.5, + "objective/train/docs_used": 933210, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.2621560096740723, + "objective/train/theoretical_loss": 3.487548600949437, + "objective/train/tokens_used": 1667052000, + "theoretical_loss": 3.487548600949437, + "tokens_seen": 1646592000 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025313941825476425, + "loss": 2.5969, + "theoretical_loss": 3.487548600949437, + "tokens_seen": 1646592000 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002531293881644935, + "loss": 2.5717, + "theoretical_loss": 3.4875365804774408, + "tokens_seen": 1646657536 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025311935807422267, + "loss": 2.6774, + "theoretical_loss": 3.48752456061779, + "tokens_seen": 1646723072 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025310932798395185, + "loss": 2.6748, + "theoretical_loss": 3.48751254137043, + "tokens_seen": 1646788608 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025309929789368103, + "loss": 2.8313, + "theoretical_loss": 3.4875005227353055, + "tokens_seen": 1646854144 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002530892678034102, + "loss": 2.6916, + "theoretical_loss": 3.4874885047123607, + "tokens_seen": 1646919680 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002530792377131394, + "loss": 2.609, + "theoretical_loss": 3.4874764873015396, + "tokens_seen": 1646985216 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025306920762286863, + "loss": 2.5592, + "theoretical_loss": 3.487464470502787, + "tokens_seen": 1647050752 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025305917753259776, + "loss": 2.6194, + "theoretical_loss": 3.487452454316048, + "tokens_seen": 1647116288 + }, + { + "epoch": 0.5, + "learning_rate": 0.000253049147442327, + "loss": 2.6538, + "theoretical_loss": 3.487440438741266, + "tokens_seen": 1647181824 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002530391173520562, + "loss": 2.8373, + "theoretical_loss": 3.4874284237783857, + "tokens_seen": 1647247360 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025302908726178536, + "loss": 2.4853, + "theoretical_loss": 3.487416409427352, + "tokens_seen": 1647312896 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025301905717151454, + "loss": 2.7558, + "theoretical_loss": 3.4874043956881096, + "tokens_seen": 1647378432 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002530090270812437, + "loss": 2.5474, + "theoretical_loss": 3.4873923825606026, + "tokens_seen": 1647443968 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025299899699097295, + "loss": 2.6657, + "theoretical_loss": 3.4873803700447756, + "tokens_seen": 1647509504 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025298896690070213, + "loss": 2.6233, + "theoretical_loss": 3.487368358140573, + "tokens_seen": 1647575040 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002529789368104313, + "loss": 2.3518, + "theoretical_loss": 3.4873563468479394, + "tokens_seen": 1647640576 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002529689067201605, + "loss": 2.542, + "theoretical_loss": 3.4873443361668195, + "tokens_seen": 1647706112 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002529588766298897, + "loss": 2.5682, + "theoretical_loss": 3.4873323260971576, + "tokens_seen": 1647771648 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025294884653961886, + "loss": 2.307, + "theoretical_loss": 3.4873203166388986, + "tokens_seen": 1647837184 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002529388164493481, + "loss": 2.4016, + "theoretical_loss": 3.487308307791987, + "tokens_seen": 1647902720 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002529287863590772, + "loss": 2.8442, + "theoretical_loss": 3.487296299556367, + "tokens_seen": 1647968256 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025291875626880646, + "loss": 2.6269, + "theoretical_loss": 3.4872842919319833, + "tokens_seen": 1648033792 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002529087261785356, + "loss": 2.4845, + "theoretical_loss": 3.4872722849187805, + "tokens_seen": 1648099328 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002528986960882648, + "loss": 2.6504, + "theoretical_loss": 3.4872602785167035, + "tokens_seen": 1648164864 + }, + { + "epoch": 0.5, + "objective/train/docs_used": 933788, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.8399863243103027, + "objective/train/theoretical_loss": 3.4872482727256964, + "objective/train/tokens_used": 1668690400, + "theoretical_loss": 3.4872482727256964, + "tokens_seen": 1648230400 + }, + { + "epoch": 0.5, + "learning_rate": 0.000252888665997994, + "loss": 2.7388, + "theoretical_loss": 3.4872482727256964, + "tokens_seen": 1648230400 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002528786359077232, + "loss": 2.6741, + "theoretical_loss": 3.4872362675457045, + "tokens_seen": 1648295936 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025286860581745236, + "loss": 2.5122, + "theoretical_loss": 3.487224262976672, + "tokens_seen": 1648361472 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002528585757271816, + "loss": 2.7131, + "theoretical_loss": 3.487212259018543, + "tokens_seen": 1648427008 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002528485456369107, + "loss": 2.6432, + "theoretical_loss": 3.487200255671263, + "tokens_seen": 1648492544 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025283851554663996, + "loss": 2.6599, + "theoretical_loss": 3.487188252934776, + "tokens_seen": 1648558080 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002528284854563691, + "loss": 2.5087, + "theoretical_loss": 3.4871762508090267, + "tokens_seen": 1648623616 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002528184553660983, + "loss": 2.7464, + "theoretical_loss": 3.4871642492939605, + "tokens_seen": 1648689152 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002528084252758275, + "loss": 2.6298, + "theoretical_loss": 3.4871522483895214, + "tokens_seen": 1648754688 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002527983951855567, + "loss": 2.6267, + "theoretical_loss": 3.4871402480956535, + "tokens_seen": 1648820224 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025278836509528586, + "loss": 2.8183, + "theoretical_loss": 3.487128248412303, + "tokens_seen": 1648885760 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025277833500501505, + "loss": 2.6931, + "theoretical_loss": 3.4871162493394134, + "tokens_seen": 1648951296 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002527683049147442, + "loss": 2.6051, + "theoretical_loss": 3.4871042508769294, + "tokens_seen": 1649016832 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025275827482447346, + "loss": 2.6014, + "theoretical_loss": 3.4870922530247963, + "tokens_seen": 1649082368 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002527482447342026, + "loss": 2.5651, + "theoretical_loss": 3.4870802557829585, + "tokens_seen": 1649147904 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002527382146439318, + "loss": 2.7217, + "theoretical_loss": 3.4870682591513606, + "tokens_seen": 1649213440 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025272818455366095, + "loss": 2.3497, + "theoretical_loss": 3.4870562631299475, + "tokens_seen": 1649278976 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002527181544633902, + "loss": 2.7354, + "theoretical_loss": 3.4870442677186637, + "tokens_seen": 1649344512 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025270812437311937, + "loss": 2.5851, + "theoretical_loss": 3.487032272917454, + "tokens_seen": 1649410048 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025269809428284855, + "loss": 2.5794, + "theoretical_loss": 3.4870202787262636, + "tokens_seen": 1649475584 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025268806419257773, + "loss": 2.5924, + "theoretical_loss": 3.4870082851450364, + "tokens_seen": 1649541120 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025267803410230697, + "loss": 2.2791, + "theoretical_loss": 3.4869962921737176, + "tokens_seen": 1649606656 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002526680040120361, + "loss": 2.6715, + "theoretical_loss": 3.486984299812252, + "tokens_seen": 1649672192 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025265797392176533, + "loss": 2.6692, + "theoretical_loss": 3.486972308060584, + "tokens_seen": 1649737728 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025264794383149445, + "loss": 2.7439, + "theoretical_loss": 3.486960316918659, + "tokens_seen": 1649803264 + }, + { + "epoch": 0.5, + "objective/train/docs_used": 934285, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.9445223808288574, + "objective/train/theoretical_loss": 3.4869483263864214, + "objective/train/tokens_used": 1670328800, + "theoretical_loss": 3.4869483263864214, + "tokens_seen": 1649868800 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002526379137412237, + "loss": 2.745, + "theoretical_loss": 3.4869483263864214, + "tokens_seen": 1649868800 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025262788365095287, + "loss": 2.626, + "theoretical_loss": 3.486936336463816, + "tokens_seen": 1649934336 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025261785356068205, + "loss": 2.7966, + "theoretical_loss": 3.4869243471507874, + "tokens_seen": 1649999872 + } + ], + "max_steps": 50354, + "num_train_epochs": 9223372036854775807, + "total_flos": 8.42054334676992e+17, + "trial_name": null, + "trial_params": null +}